SSE filter speed improvements for bpp=3.

- memcpy-free implementations of load3() / store3().
    These should have less variance compiler to compiler.

    - call load3() only when needed at the end of a scanline.
    In the middle, we can use the faster load4(), ignoring that byte.

contrib/intel/filter_sse2_intrinsics.c

@@ -29,29 +29,41 @@
  * whichever of a, b, or c is closest to p=a+b-c.
  */
 
-#ifndef PNG_NO_INTEL_SSE_3BPP
-static __m128i load3(const void* p) {
-   png_uint_32 packed;
-   memcpy(&packed, p, 3);
-   return _mm_cvtsi32_si128(packed);
-}
-#endif
-
 static __m128i load4(const void* p) {
    return _mm_cvtsi32_si128(*(const int*)p);
 }
 
-#ifndef PNG_NO_INTEL_SSE_3BPP
-static void store3(void* p, __m128i v) {
-   png_uint_32 packed = _mm_cvtsi128_si32(v);
-   memcpy(p, &packed, 3);
-}
-#endif
-
 static void store4(void* p, __m128i v) {
    *(int*)p = _mm_cvtsi128_si32(v);
 }
 
+#ifndef PNG_NO_INTEL_SSE_3BPP
+static __m128i load3(const void* p) {
+   /* We'll load 2 bytes, then 1 byte,
+    * then mask them together, and finally load into SSE.
+    */
+   const png_uint_16* p01 = p;
+   const png_byte*    p2  = (const png_byte*)(p01+1);
+
+   png_uint_32 v012 = (png_uint_32)(*p01)
+                    | (png_uint_32)(*p2) << 16;
+   return load4(&v012);
+}
+
+static void store3(void* p, __m128i v) {
+   /* We'll pull from SSE as a 32-bit int, then write
+    * its bottom two bytes, then its third byte.
+    */
+   png_uint_32 v012;
+   store4(&v012, v);
+
+   png_uint_16* p01 = p;
+   png_byte*    p2  = (png_byte*)(p01+1);
+   *p01 = v012;
+   *p2  = v012 >> 16;
+}
+#endif
+
 #ifndef PNG_NO_INTEL_SSE_3BPP
 void png_read_filter_row_sub3_sse2(png_row_infop row_info, png_bytep row,
    png_const_bytep prev)
@@ -64,7 +76,15 @@ void png_read_filter_row_sub3_sse2(png_row_infop row_info, png_bytep row,
    __m128i a, d = _mm_setzero_si128();
 
    int rb = row_info->rowbytes;
-   while (rb > 0) {
+   while (rb >= 4) {
+      a = d; d = load4(row);
+      d = _mm_add_epi8(d, a);
+      store3(row, d);
+
+      row += 3;
+      rb  -= 3;
+   }
+   if (rb > 0) {
       a = d; d = load3(row);
       d = _mm_add_epi8(d, a);
       store3(row, d);
@@ -111,7 +131,23 @@ void png_read_filter_row_avg3_sse2(png_row_infop row_info, png_bytep row,
    __m128i a, d = zero;
 
    int rb = row_info->rowbytes;
-   while (rb > 0) {
+   while (rb >= 4) {
+             b = load4(prev);
+      a = d; d = load4(row );
+
+      /* PNG requires a truncating average, so we can't just use _mm_avg_epu8 */
+      __m128i avg = _mm_avg_epu8(a,b);
+      /* ...but we can fix it up by subtracting off 1 if it rounded up. */
+      avg = _mm_sub_epi8(avg, _mm_and_si128(_mm_xor_si128(a,b),
+                                            _mm_set1_epi8(1)));
+      d = _mm_add_epi8(d, avg);
+      store3(row, d);
+
+      prev += 3;
+      row  += 3;
+      rb   -= 3;
+   }
+   if (rb > 0) {
              b = load3(prev);
       a = d; d = load3(row );
 
@@ -215,7 +251,42 @@ void png_read_filter_row_paeth3_sse2(png_row_infop row_info, png_bytep row,
            a, d = zero;
 
    int rb = row_info->rowbytes;
-   while (rb > 0) {
+   while (rb >= 4) {
+      /* It's easiest to do this math (particularly, deal with pc) with 16-bit
+       * intermediates.
+       */
+      c = b; b = _mm_unpacklo_epi8(load4(prev), zero);
+      a = d; d = _mm_unpacklo_epi8(load4(row ), zero);
+
+      /* (p-a) == (a+b-c - a) == (b-c) */
+      __m128i pa = _mm_sub_epi16(b,c);
+
+      /* (p-b) == (a+b-c - b) == (a-c) */
+      __m128i pb = _mm_sub_epi16(a,c);
+
+      /* (p-c) == (a+b-c - c) == (a+b-c-c) == (b-c)+(a-c) */
+      __m128i pc = _mm_add_epi16(pa,pb);
+
+      pa = abs_i16(pa);  /* |p-a| */
+      pb = abs_i16(pb);  /* |p-b| */
+      pc = abs_i16(pc);  /* |p-c| */
+
+      __m128i smallest = _mm_min_epi16(pc, _mm_min_epi16(pa, pb));
+
+      /* Paeth breaks ties favoring a over b over c. */
+      __m128i nearest  = if_then_else(_mm_cmpeq_epi16(smallest, pa), a,
+                         if_then_else(_mm_cmpeq_epi16(smallest, pb), b,
+                                                                     c));
+
+      /* Note `_epi8`: we need addition to wrap modulo 255. */
+      d = _mm_add_epi8(d, nearest);
+      store3(row, _mm_packus_epi16(d,d));
+
+      prev += 3;
+      row  += 3;
+      rb   -= 3;
+   }
+   if (rb > 0) {
       /* It's easiest to do this math (particularly, deal with pc) with 16-bit
        * intermediates.
        */