Add Power8 SHA extensions for SHA-384 and SHA-512

Attached and below is a patch for SHA-384 and SHA-512 using Power8 SHA extensions. Its another partial patch, and others will have to complete it.

SHA-384 and SHA-512 use the vshasigmad (double word version) of the instruction, and operate on vector unsigned long long.

The code is based on SHA-Intrinsics. It is public domain code without licensing requirements.

I think there is more performance to extract from the implementation, but we need to switch to a complete inline assembly version of the compression function. Based on PowerPC vec_xl_be replacement using inline assembly (which allows big-endian loads without the gratuitous endian swapping), performance drops when using the replacement VEC_XL_BE. It looks like xxswapd and xxlnand are removed, but the function is not optimized as well (overall).

Here are the relative numbers from GCC112, which is ppc64-le at 3.4 GHz. It also used the GCC 7.2.0 compiler located at /opt/cfarm/gcc-latest.

• Botan, C++

$ ./botan speed --msec=3000 --cpu-clock-speed=3400 MD5 SHA-1 SHA-384 SHA-512

MD5 hash buffer size 1024 bytes: 236.689 MiB/sec 13.70 cycles/byte (710.07 MiB in 3000.00 ms)
SHA-160 hash buffer size 1024 bytes: 355.444 MiB/sec 9.12 cycles/byte (1066.33 MiB in 3000.00 ms)
SHA-384 hash buffer size 1024 bytes: 239.836 MiB/sec 13.52 cycles/byte (719.51 MiB in 3000.00 ms)
SHA-512 hash buffer size 1024 bytes: 245.448 MiB/sec 13.21 cycles/byte (736.34 MiB in 3000.00 ms)

• Botan, Power8:

$ ./botan speed --msec=3000 --cpu-clock-speed=3400 SHA-384 SHA-512

SHA-384 hash buffer size 1024 bytes: 326.355 MiB/sec 9.94 cycles/byte (979.07 MiB in 3000.00 ms)
SHA-512 hash buffer size 1024 bytes: 327.679 MiB/sec 9.90 cycles/byte (983.04 MiB in 3000.00 ms)

---

$ git diff > sha2_32.diff
[noloader@gcc2-power8 botan]$ cat sha2_32.diff
diff --git a/src/lib/hash/sha2_32/sha2_32.cpp b/src/lib/hash/sha2_32/sha2_32.cpp
index 0710747..20614d4 100644
--- a/src/lib/hash/sha2_32/sha2_32.cpp
+++ b/src/lib/hash/sha2_32/sha2_32.cpp
@@ -9,6 +9,209 @@
 #include <botan/sha2_32.h>
 #include <botan/cpuid.h>

+#if defined(__ALTIVEC__)
+# include <altivec.h>
+# undef vector
+# undef pixel
+# undef bool
+#endif
+
+namespace {
+
+// ALIGN16 when the library controls alignment
+#define ALIGN16 __attribute__((aligned(16)))
+typedef __vector unsigned char uint8x16_p8;
+typedef __vector unsigned int  uint32x4_p8;
+
+// Indexes into the S[] array
+enum {A=0, B=1, C, D, E, F, G, H};
+
+static const ALIGN16 uint32_t SHA256_K[] =
+   {
+   0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
+   0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
+   0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
+   0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
+   0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
+   0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
+   0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
+   0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
+   0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
+   0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
+   0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
+   0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
+   0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
+   0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
+   0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
+   0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2
+   };
+
+// Aligned load
+template <class T> static inline
+uint32x4_p8 VectorLoad32x4(const T* data, int offset)
+   {
+   return (uint32x4_p8)vec_ld(offset, (uint8_t*)data);
+   }
+
+// Unaligned load
+template <class T> static inline
+uint32x4_p8 VectorLoad32x4u(const T* data, int offset)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   return (uint32x4_p8)vec_xl(offset, (uint8_t*)data);
+#else
+   return (uint32x4_p8)vec_vsx_ld(offset, (uint8_t*)data);
+#endif
+   }
+
+// Unaligned load of a user message. The load is big-endian,
+//   and then the message is permuted for 32-bit words.
+template <class T> static inline
+uint32x4_p8 VectorLoadMsg32x4(const T* data, int offset)
+   {
+#if __LITTLE_ENDIAN__
+   const uint8x16_p8 mask = {3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12};
+   const uint32x4_p8 r = VectorLoad32x4u(data, offset);
+   return (uint32x4_p8)vec_perm(r, r, mask);
+#else
+   return VectorLoad32x4u(data, offset);
+#endif
+   }
+
+// Aligned store
+template <class T> static inline
+void VectorStore32x4(const uint32x4_p8 val, T* data, int offset)
+   {
+   vec_st((uint8x16_p8)val, offset, (uint8_t*)data);
+   }
+
+// Unaligned store
+template <class T> static inline
+void VectorStore32x4u(const uint32x4_p8 val, T* data, int offset)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   vec_xst((uint8x16_p8)val, offset, (uint8_t*)data);
+#else
+   vec_vsx_st((uint8x16_p8)val, offset, (uint8_t*)data);
+#endif
+   }
+
+static inline
+uint32x4_p8 VectorCh(const uint32x4_p8 x, const uint32x4_p8 y, const uint32x4_p8 z)
+   {
+   // The trick below is due to Andy Polyakov and Jack Lloyd
+   return vec_sel(z,y,x);
+   }
+
+static inline
+uint32x4_p8 VectorMaj(const uint32x4_p8 x, const uint32x4_p8 y, const uint32x4_p8 z)
+   {
+   // The trick below is due to Andy Polyakov and Jack Lloyd
+   const uint32x4_p8 xy = vec_xor(x, y);
+   return vec_sel(y, z, xy);
+   }
+
+static inline
+uint32x4_p8 Vector_sigma0(const uint32x4_p8 val)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   return __vshasigmaw(val, 0, 0);
+#else
+   return __builtin_crypto_vshasigmaw(val, 0, 0);
+#endif
+   }
+
+static inline
+uint32x4_p8 Vector_sigma1(const uint32x4_p8 val)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   return __vshasigmaw(val, 0, 0xf);
+#else
+   return __builtin_crypto_vshasigmaw(val, 0, 0xf);
+#endif
+   }
+
+static inline
+uint32x4_p8 VectorSigma0(const uint32x4_p8 val)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   return __vshasigmaw(val, 1, 0);
+#else
+   return __builtin_crypto_vshasigmaw(val, 1, 0);
+#endif
+   }
+
+static inline
+uint32x4_p8 VectorSigma1(const uint32x4_p8 val)
+   {
+#if defined(__xlc__) || defined(__xlC__)
+   return __vshasigmaw(val, 1, 0xf);
+#else
+   return __builtin_crypto_vshasigmaw(val, 1, 0xf);
+#endif
+   }
+
+static inline
+uint32x4_p8 VectorPack(const uint32x4_p8 a, const uint32x4_p8 b,
+                       const uint32x4_p8 c, const uint32x4_p8 d)
+   {
+   const uint8x16_p8 m1 = {0,1,2,3, 16,17,18,19, 0,0,0,0, 0,0,0,0};
+   const uint8x16_p8 m2 = {0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
+   return vec_perm(vec_perm(a,b,m1),vec_perm(c,d,m1), m2);
+   }
+
+template <unsigned int L> static inline
+uint32x4_p8 VectorShiftLeft(const uint32x4_p8 val)
+   {
+#if __LITTLE_ENDIAN__
+   return (uint32x4_p8)vec_sld((uint8x16_p8)val, (uint8x16_p8)val, (16-L)&0xf);
+#else
+   return (uint32x4_p8)vec_sld((uint8x16_p8)val, (uint8x16_p8)val, L&0xf);
+#endif
+   }
+
+template <>
+uint32x4_p8 VectorShiftLeft<0>(const uint32x4_p8 val) { return val; }
+
+template <>
+uint32x4_p8 VectorShiftLeft<16>(const uint32x4_p8 val) { return val; }
+
+template <unsigned int R> static inline
+void SHA256_ROUND1(uint32x4_p8 W[16], uint32x4_p8 S[8], const uint32x4_p8 K, const uint32x4_p8 M)
+   {
+   uint32x4_p8 T1, T2;
+
+   W[R] = M;
+   T1 = S[H] + VectorSigma1(S[E]) + VectorCh(S[E],S[F],S[G]) + K + M;
+   T2 = VectorSigma0(S[A]) + VectorMaj(S[A],S[B],S[C]);
+
+   S[H] = S[G]; S[G] = S[F]; S[F] = S[E];
+   S[E] = S[D] + T1;
+   S[D] = S[C]; S[C] = S[B]; S[B] = S[A];
+   S[A] = T1 + T2;
+   }
+
+template <unsigned int R> static inline
+void SHA256_ROUND2(uint32x4_p8 W[16], uint32x4_p8 S[8], const uint32x4_p8 K)
+   {
+   // Indexes into the W[] array
+   enum {IDX0=(R+0)&0xf, IDX1=(R+1)&0xf, IDX9=(R+9)&0xf, IDX14=(R+14)&0xf};
+
+   const uint32x4_p8 s0 = Vector_sigma0(W[IDX1]);
+   const uint32x4_p8 s1 = Vector_sigma1(W[IDX14]);
+
+   uint32x4_p8 T1 = (W[IDX0] += s0 + s1 + W[IDX9]);
+   T1 += S[H] + VectorSigma1(S[E]) + VectorCh(S[E],S[F],S[G]) + K;
+   uint32x4_p8 T2 = VectorSigma0(S[A]) + VectorMaj(S[A],S[B],S[C]);
+
+   S[H] = S[G]; S[G] = S[F]; S[F] = S[E];
+   S[E] = S[D] + T1;
+   S[D] = S[C]; S[C] = S[B]; S[B] = S[A];
+   S[A] = T1 + T2;
+   }
+
+}
+
 namespace Botan {

 std::unique_ptr<HashFunction> SHA_224::copy_state() const
@@ -22,23 +225,6 @@ std::unique_ptr<HashFunction> SHA_256::copy_state() const
    }

 /*
-* SHA-256 F1 Function
-*
-* Use a macro as many compilers won't inline a function this big,
-* even though it is much faster if inlined.
-*/
-#define SHA2_32_F(A, B, C, D, E, F, G, H, M1, M2, M3, M4, magic) do {               \
-   uint32_t A_rho = rotr<2>(A) ^ rotr<13>(A) ^ rotr<22>(A); \
-   uint32_t E_rho = rotr<6>(E) ^ rotr<11>(E) ^ rotr<25>(E); \
-   uint32_t M2_sigma = rotr<17>(M2) ^ rotr<19>(M2) ^ (M2 >> 10);    \
-   uint32_t M4_sigma = rotr<7>(M4) ^ rotr<18>(M4) ^ (M4 >> 3);      \
-   H += magic + E_rho + ((E & F) ^ (~E & G)) + M1;                                  \
-   D += H;                                                                          \
-   H += A_rho + ((A & B) | ((A | B) & C));                                          \
-   M1 += M2_sigma + M3 + M4_sigma;                                                  \
-   } while(0);
-
-/*
 * SHA-224 / SHA-256 compression function
 */
 void SHA_256::compress_digest(secure_vector<uint32_t>& digest,
@@ -58,105 +244,138 @@ void SHA_256::compress_digest(secure_vector<uint32_t>& digest,
       }
 #endif

-   uint32_t A = digest[0], B = digest[1], C = digest[2],
-          D = digest[3], E = digest[4], F = digest[5],
-          G = digest[6], H = digest[7];
+   const uint32_t* k = reinterpret_cast<const uint32_t*>(SHA256_K);
+   const uint32_t* m = reinterpret_cast<const uint32_t*>(input);
+   const uint32_t* s = reinterpret_cast<const uint32_t*>(&digest[0]);
+
+   uint32x4_p8 abcd = VectorLoad32x4u(s+0, 0);
+   uint32x4_p8 efgh = VectorLoad32x4u(s+4, 0);
+   uint32x4_p8 W[16], S[8], vm, vk;

-   for(size_t i = 0; i != blocks; ++i)
+   while (blocks--)
       {
-      uint32_t W00 = load_be<uint32_t>(input,  0);
-      uint32_t W01 = load_be<uint32_t>(input,  1);
-      uint32_t W02 = load_be<uint32_t>(input,  2);
-      uint32_t W03 = load_be<uint32_t>(input,  3);
-      uint32_t W04 = load_be<uint32_t>(input,  4);
-      uint32_t W05 = load_be<uint32_t>(input,  5);
-      uint32_t W06 = load_be<uint32_t>(input,  6);
-      uint32_t W07 = load_be<uint32_t>(input,  7);
-      uint32_t W08 = load_be<uint32_t>(input,  8);
-      uint32_t W09 = load_be<uint32_t>(input,  9);
-      uint32_t W10 = load_be<uint32_t>(input, 10);
-      uint32_t W11 = load_be<uint32_t>(input, 11);
-      uint32_t W12 = load_be<uint32_t>(input, 12);
-      uint32_t W13 = load_be<uint32_t>(input, 13);
-      uint32_t W14 = load_be<uint32_t>(input, 14);
-      uint32_t W15 = load_be<uint32_t>(input, 15);
-
-      SHA2_32_F(A, B, C, D, E, F, G, H, W00, W14, W09, W01, 0x428A2F98);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W01, W15, W10, W02, 0x71374491);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W02, W00, W11, W03, 0xB5C0FBCF);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W03, W01, W12, W04, 0xE9B5DBA5);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W04, W02, W13, W05, 0x3956C25B);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W05, W03, W14, W06, 0x59F111F1);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W06, W04, W15, W07, 0x923F82A4);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W07, W05, W00, W08, 0xAB1C5ED5);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W08, W06, W01, W09, 0xD807AA98);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W09, W07, W02, W10, 0x12835B01);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W10, W08, W03, W11, 0x243185BE);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W11, W09, W04, W12, 0x550C7DC3);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W12, W10, W05, W13, 0x72BE5D74);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W13, W11, W06, W14, 0x80DEB1FE);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W14, W12, W07, W15, 0x9BDC06A7);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W15, W13, W08, W00, 0xC19BF174);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W00, W14, W09, W01, 0xE49B69C1);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W01, W15, W10, W02, 0xEFBE4786);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W02, W00, W11, W03, 0x0FC19DC6);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W03, W01, W12, W04, 0x240CA1CC);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W04, W02, W13, W05, 0x2DE92C6F);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W05, W03, W14, W06, 0x4A7484AA);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W06, W04, W15, W07, 0x5CB0A9DC);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W07, W05, W00, W08, 0x76F988DA);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W08, W06, W01, W09, 0x983E5152);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W09, W07, W02, W10, 0xA831C66D);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W10, W08, W03, W11, 0xB00327C8);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W11, W09, W04, W12, 0xBF597FC7);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W12, W10, W05, W13, 0xC6E00BF3);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W13, W11, W06, W14, 0xD5A79147);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W14, W12, W07, W15, 0x06CA6351);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W15, W13, W08, W00, 0x14292967);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W00, W14, W09, W01, 0x27B70A85);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W01, W15, W10, W02, 0x2E1B2138);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W02, W00, W11, W03, 0x4D2C6DFC);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W03, W01, W12, W04, 0x53380D13);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W04, W02, W13, W05, 0x650A7354);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W05, W03, W14, W06, 0x766A0ABB);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W06, W04, W15, W07, 0x81C2C92E);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W07, W05, W00, W08, 0x92722C85);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W08, W06, W01, W09, 0xA2BFE8A1);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W09, W07, W02, W10, 0xA81A664B);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W10, W08, W03, W11, 0xC24B8B70);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W11, W09, W04, W12, 0xC76C51A3);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W12, W10, W05, W13, 0xD192E819);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W13, W11, W06, W14, 0xD6990624);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W14, W12, W07, W15, 0xF40E3585);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W15, W13, W08, W00, 0x106AA070);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W00, W14, W09, W01, 0x19A4C116);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W01, W15, W10, W02, 0x1E376C08);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W02, W00, W11, W03, 0x2748774C);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W03, W01, W12, W04, 0x34B0BCB5);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W04, W02, W13, W05, 0x391C0CB3);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W05, W03, W14, W06, 0x4ED8AA4A);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W06, W04, W15, W07, 0x5B9CCA4F);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W07, W05, W00, W08, 0x682E6FF3);
-      SHA2_32_F(A, B, C, D, E, F, G, H, W08, W06, W01, W09, 0x748F82EE);
-      SHA2_32_F(H, A, B, C, D, E, F, G, W09, W07, W02, W10, 0x78A5636F);
-      SHA2_32_F(G, H, A, B, C, D, E, F, W10, W08, W03, W11, 0x84C87814);
-      SHA2_32_F(F, G, H, A, B, C, D, E, W11, W09, W04, W12, 0x8CC70208);
-      SHA2_32_F(E, F, G, H, A, B, C, D, W12, W10, W05, W13, 0x90BEFFFA);
-      SHA2_32_F(D, E, F, G, H, A, B, C, W13, W11, W06, W14, 0xA4506CEB);
-      SHA2_32_F(C, D, E, F, G, H, A, B, W14, W12, W07, W15, 0xBEF9A3F7);
-      SHA2_32_F(B, C, D, E, F, G, H, A, W15, W13, W08, W00, 0xC67178F2);
-
-      A = (digest[0] += A);
-      B = (digest[1] += B);
-      C = (digest[2] += C);
-      D = (digest[3] += D);
-      E = (digest[4] += E);
-      F = (digest[5] += F);
-      G = (digest[6] += G);
-      H = (digest[7] += H);
-
-      input += 64;
+      S[A] = abcd; S[E] = efgh;
+      S[B] = VectorShiftLeft<4>(S[A]);
+      S[F] = VectorShiftLeft<4>(S[E]);
+      S[C] = VectorShiftLeft<4>(S[B]);
+      S[G] = VectorShiftLeft<4>(S[F]);
+      S[D] = VectorShiftLeft<4>(S[C]);
+      S[H] = VectorShiftLeft<4>(S[G]);
+
+      unsigned int i, offset=0;
+
+      // Unroll the loop to provide the round number as a constexpr
+      // for (unsigned int i=0; i<16; ++i)
+         {
+         vk = VectorLoad32x4(k, offset);
+         vm = VectorLoadMsg32x4(m, offset);
+         SHA256_ROUND1<0>(W,S, vk,vm);
+         offset+=16;
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<1>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<2>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<3>(W,S, vk,vm);
+
+         vk = VectorLoad32x4(k, offset);
+         vm = VectorLoadMsg32x4(m, offset);
+         SHA256_ROUND1<4>(W,S, vk,vm);
+         offset+=16;
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<5>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<6>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<7>(W,S, vk,vm);
+
+         vk = VectorLoad32x4(k, offset);
+         vm = VectorLoadMsg32x4(m, offset);
+         SHA256_ROUND1<8>(W,S, vk,vm);
+         offset+=16;
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<9>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<10>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<11>(W,S, vk,vm);
+
+         vk = VectorLoad32x4(k, offset);
+         vm = VectorLoadMsg32x4(m, offset);
+         SHA256_ROUND1<12>(W,S, vk,vm);
+         offset+=16;
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<13>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<14>(W,S, vk,vm);
+
+         vk = VectorShiftLeft<4>(vk);
+         vm = VectorShiftLeft<4>(vm);
+         SHA256_ROUND1<15>(W,S, vk,vm);
+         }
+
+      // Number of 32-bit words, not bytes
+      m += 16;
+
+      for (i=16; i<64; i+=16)
+         {
+         vk = VectorLoad32x4(k, offset);
+         SHA256_ROUND2<0>(W,S, vk);
+         SHA256_ROUND2<1>(W,S, VectorShiftLeft<4>(vk));
+         SHA256_ROUND2<2>(W,S, VectorShiftLeft<8>(vk));
+         SHA256_ROUND2<3>(W,S, VectorShiftLeft<12>(vk));
+         offset+=16;
+
+         vk = VectorLoad32x4(k, offset);
+         SHA256_ROUND2<4>(W,S, vk);
+         SHA256_ROUND2<5>(W,S, VectorShiftLeft<4>(vk));
+         SHA256_ROUND2<6>(W,S, VectorShiftLeft<8>(vk));
+         SHA256_ROUND2<7>(W,S, VectorShiftLeft<12>(vk));
+         offset+=16;
+
+         vk = VectorLoad32x4(k, offset);
+         SHA256_ROUND2<8>(W,S, vk);
+         SHA256_ROUND2<9>(W,S, VectorShiftLeft<4>(vk));
+         SHA256_ROUND2<10>(W,S, VectorShiftLeft<8>(vk));
+         SHA256_ROUND2<11>(W,S, VectorShiftLeft<12>(vk));
+         offset+=16;
+
+         vk = VectorLoad32x4(k, offset);
+         SHA256_ROUND2<12>(W,S, vk);
+         SHA256_ROUND2<13>(W,S, VectorShiftLeft<4>(vk));
+         SHA256_ROUND2<14>(W,S, VectorShiftLeft<8>(vk));
+         SHA256_ROUND2<15>(W,S, VectorShiftLeft<12>(vk));
+         offset+=16;
+         }
+
+      abcd += VectorPack(S[A],S[B],S[C],S[D]);
+      efgh += VectorPack(S[E],S[F],S[G],S[H]);
       }
+
+   VectorStore32x4u(abcd, s+0, 0);
+   VectorStore32x4u(efgh, s+4, 0);
    }

 /*

Here is the updated sha2_64.cpp and the diff packaged as a ZIP file: sha2_64.zip.