diff --git a/contrib/intel/filter_sse2_intrinsics.c b/contrib/intel/filter_sse2_intrinsics.c index e7b317159..9de1e92d8 100644 --- a/contrib/intel/filter_sse2_intrinsics.c +++ b/contrib/intel/filter_sse2_intrinsics.c @@ -29,29 +29,41 @@ * whichever of a, b, or c is closest to p=a+b-c. */ -#ifndef PNG_NO_INTEL_SSE_3BPP -static __m128i load3(const void* p) { - png_uint_32 packed; - memcpy(&packed, p, 3); - return _mm_cvtsi32_si128(packed); -} -#endif - static __m128i load4(const void* p) { return _mm_cvtsi32_si128(*(const int*)p); } -#ifndef PNG_NO_INTEL_SSE_3BPP -static void store3(void* p, __m128i v) { - png_uint_32 packed = _mm_cvtsi128_si32(v); - memcpy(p, &packed, 3); -} -#endif - static void store4(void* p, __m128i v) { *(int*)p = _mm_cvtsi128_si32(v); } +#ifndef PNG_NO_INTEL_SSE_3BPP +static __m128i load3(const void* p) { + /* We'll load 2 bytes, then 1 byte, + * then mask them together, and finally load into SSE. + */ + const png_uint_16* p01 = p; + const png_byte* p2 = (const png_byte*)(p01+1); + + png_uint_32 v012 = (png_uint_32)(*p01) + | (png_uint_32)(*p2) << 16; + return load4(&v012); +} + +static void store3(void* p, __m128i v) { + /* We'll pull from SSE as a 32-bit int, then write + * its bottom two bytes, then its third byte. + */ + png_uint_32 v012; + store4(&v012, v); + + png_uint_16* p01 = p; + png_byte* p2 = (png_byte*)(p01+1); + *p01 = v012; + *p2 = v012 >> 16; +} +#endif + #ifndef PNG_NO_INTEL_SSE_3BPP void png_read_filter_row_sub3_sse2(png_row_infop row_info, png_bytep row, png_const_bytep prev) @@ -64,7 +76,15 @@ void png_read_filter_row_sub3_sse2(png_row_infop row_info, png_bytep row, __m128i a, d = _mm_setzero_si128(); int rb = row_info->rowbytes; - while (rb > 0) { + while (rb >= 4) { + a = d; d = load4(row); + d = _mm_add_epi8(d, a); + store3(row, d); + + row += 3; + rb -= 3; + } + if (rb > 0) { a = d; d = load3(row); d = _mm_add_epi8(d, a); store3(row, d); @@ -111,7 +131,23 @@ void png_read_filter_row_avg3_sse2(png_row_infop row_info, png_bytep row, __m128i a, d = zero; int rb = row_info->rowbytes; - while (rb > 0) { + while (rb >= 4) { + b = load4(prev); + a = d; d = load4(row ); + + /* PNG requires a truncating average, so we can't just use _mm_avg_epu8 */ + __m128i avg = _mm_avg_epu8(a,b); + /* ...but we can fix it up by subtracting off 1 if it rounded up. */ + avg = _mm_sub_epi8(avg, _mm_and_si128(_mm_xor_si128(a,b), + _mm_set1_epi8(1))); + d = _mm_add_epi8(d, avg); + store3(row, d); + + prev += 3; + row += 3; + rb -= 3; + } + if (rb > 0) { b = load3(prev); a = d; d = load3(row ); @@ -215,7 +251,42 @@ void png_read_filter_row_paeth3_sse2(png_row_infop row_info, png_bytep row, a, d = zero; int rb = row_info->rowbytes; - while (rb > 0) { + while (rb >= 4) { + /* It's easiest to do this math (particularly, deal with pc) with 16-bit + * intermediates. + */ + c = b; b = _mm_unpacklo_epi8(load4(prev), zero); + a = d; d = _mm_unpacklo_epi8(load4(row ), zero); + + /* (p-a) == (a+b-c - a) == (b-c) */ + __m128i pa = _mm_sub_epi16(b,c); + + /* (p-b) == (a+b-c - b) == (a-c) */ + __m128i pb = _mm_sub_epi16(a,c); + + /* (p-c) == (a+b-c - c) == (a+b-c-c) == (b-c)+(a-c) */ + __m128i pc = _mm_add_epi16(pa,pb); + + pa = abs_i16(pa); /* |p-a| */ + pb = abs_i16(pb); /* |p-b| */ + pc = abs_i16(pc); /* |p-c| */ + + __m128i smallest = _mm_min_epi16(pc, _mm_min_epi16(pa, pb)); + + /* Paeth breaks ties favoring a over b over c. */ + __m128i nearest = if_then_else(_mm_cmpeq_epi16(smallest, pa), a, + if_then_else(_mm_cmpeq_epi16(smallest, pb), b, + c)); + + /* Note `_epi8`: we need addition to wrap modulo 255. */ + d = _mm_add_epi8(d, nearest); + store3(row, _mm_packus_epi16(d,d)); + + prev += 3; + row += 3; + rb -= 3; + } + if (rb > 0) { /* It's easiest to do this math (particularly, deal with pc) with 16-bit * intermediates. */