CoCalc -- intel

GitHub Repository: freebsd/freebsd-src
Path: blob/main/sys/crypto/aesni/intel_sha256.c
³⁹⁴⁷⁸ views
1
/*******************************************************************************
2
* Copyright (c) 2013, Intel Corporation 
3
* 
4
* All rights reserved. 
5
* 
6
* Redistribution and use in source and binary forms, with or without
7
* modification, are permitted provided that the following conditions are
8
* met: 
9
* 
10
* * Redistributions of source code must retain the above copyright
11
*   notice, this list of conditions and the following disclaimer.  
12
* 
13
* * Redistributions in binary form must reproduce the above copyright
14
*   notice, this list of conditions and the following disclaimer in the
15
*   documentation and/or other materials provided with the
16
*   distribution. 
17
* 
18
* * Neither the name of the Intel Corporation nor the names of its
19
*   contributors may be used to endorse or promote products derived from
20
*   this software without specific prior written permission. 
21
* 
22
* 
23
* THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
24
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
27
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34
********************************************************************************
35
*
36
* Intel SHA Extensions optimized implementation of a SHA-256 update function 
37
*
38
* The function takes a pointer to the current hash values, a pointer to the 
39
* input data, and a number of 64 byte blocks to process.  Once all blocks have 
40
* been processed, the digest pointer is  updated with the resulting hash value.
41
* The function only processes complete blocks, there is no functionality to 
42
* store partial blocks.  All message padding and hash value initialization must
43
* be done outside the update function.  
44
*
45
* The indented lines in the loop are instructions related to rounds processing.
46
* The non-indented lines are instructions related to the message schedule.
47
*
48
* Author: Sean Gulley <[email protected]>
49
* Date:   July 2013
50
*
51
********************************************************************************
52
*
53
* Example complier command line:
54
* icc intel_sha_extensions_sha256_intrinsic.c
55
* gcc -msha -msse4 intel_sha_extensions_sha256_intrinsic.c
56
*
57
*******************************************************************************/
58

59
#include <sys/types.h>
60
#include <crypto/aesni/aesni_os.h>
61
#include <crypto/aesni/sha_sse.h>
62

63
#include <immintrin.h>
64

65
void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks) {
66
   __m128i state0, state1;
67
   __m128i msg;
68
   __m128i msgtmp0, msgtmp1, msgtmp2, msgtmp3;
69
   __m128i tmp;
70
   __m128i shuf_mask;
71
   __m128i abef_save, cdgh_save;
72

73
   // Load initial hash values
74
   // Need to reorder these appropriately
75
   // DCBA, HGFE -> ABEF, CDGH
76
   tmp    = _mm_loadu_si128((__m128i*) digest);
77
   state1 = _mm_loadu_si128((__m128i*) (digest+4));
78

79
   tmp    = _mm_shuffle_epi32(tmp, 0xB1);       // CDAB
80
   state1 = _mm_shuffle_epi32(state1, 0x1B);    // EFGH
81
   state0 = _mm_alignr_epi8(tmp, state1, 8);    // ABEF
82
   state1 = _mm_blend_epi16(state1, tmp, 0xF0); // CDGH
83

84
   shuf_mask = _mm_set_epi64x(0x0c0d0e0f08090a0bull, 0x0405060700010203ull);
85

86
   while (num_blks > 0) {
87
      // Save hash values for addition after rounds
88
      abef_save = state0;
89
      cdgh_save = state1;
90

91
      // Rounds 0-3
92
      msg     = _mm_loadu_si128((const __m128i*) data);
93
      msgtmp0 = _mm_shuffle_epi8(msg, shuf_mask);
94
         msg    = _mm_add_epi32(msgtmp0, 
95
                  _mm_set_epi64x(0xE9B5DBA5B5C0FBCFull, 0x71374491428A2F98ull));
96
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
97
         msg    = _mm_shuffle_epi32(msg, 0x0E);
98
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
99

100
      // Rounds 4-7
101
      msgtmp1 = _mm_loadu_si128((const __m128i*) (data+16));
102
      msgtmp1 = _mm_shuffle_epi8(msgtmp1, shuf_mask);
103
         msg    = _mm_add_epi32(msgtmp1, 
104
                  _mm_set_epi64x(0xAB1C5ED5923F82A4ull, 0x59F111F13956C25Bull));
105
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
106
         msg    = _mm_shuffle_epi32(msg, 0x0E);
107
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
108
      msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
109

110
      // Rounds 8-11
111
      msgtmp2 = _mm_loadu_si128((const __m128i*) (data+32));
112
      msgtmp2 = _mm_shuffle_epi8(msgtmp2, shuf_mask);
113
         msg    = _mm_add_epi32(msgtmp2, 
114
                  _mm_set_epi64x(0x550C7DC3243185BEull, 0x12835B01D807AA98ull));
115
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
116
         msg    = _mm_shuffle_epi32(msg, 0x0E);
117
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
118
      msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
119

120
      // Rounds 12-15
121
      msgtmp3 = _mm_loadu_si128((const __m128i*) (data+48));
122
      msgtmp3 = _mm_shuffle_epi8(msgtmp3, shuf_mask);
123
         msg    = _mm_add_epi32(msgtmp3, 
124
                  _mm_set_epi64x(0xC19BF1749BDC06A7ull, 0x80DEB1FE72BE5D74ull));
125
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
126
      tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
127
      msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
128
      msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
129
         msg    = _mm_shuffle_epi32(msg, 0x0E);
130
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
131
      msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
132

133
      // Rounds 16-19
134
         msg    = _mm_add_epi32(msgtmp0, 
135
                  _mm_set_epi64x(0x240CA1CC0FC19DC6ull, 0xEFBE4786E49B69C1ull));
136
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
137
      tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
138
      msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
139
      msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
140
         msg    = _mm_shuffle_epi32(msg, 0x0E);
141
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
142
      msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
143

144
      // Rounds 20-23
145
         msg    = _mm_add_epi32(msgtmp1, 
146
                  _mm_set_epi64x(0x76F988DA5CB0A9DCull, 0x4A7484AA2DE92C6Full));
147
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
148
      tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
149
      msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
150
      msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
151
         msg    = _mm_shuffle_epi32(msg, 0x0E);
152
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
153
      msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
154

155
      // Rounds 24-27
156
         msg    = _mm_add_epi32(msgtmp2, 
157
                  _mm_set_epi64x(0xBF597FC7B00327C8ull, 0xA831C66D983E5152ull));
158
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
159
      tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
160
      msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
161
      msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
162
         msg    = _mm_shuffle_epi32(msg, 0x0E);
163
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
164
      msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
165

166
      // Rounds 28-31
167
         msg    = _mm_add_epi32(msgtmp3, 
168
                  _mm_set_epi64x(0x1429296706CA6351ull, 0xD5A79147C6E00BF3ull));
169
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
170
      tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
171
      msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
172
      msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
173
         msg    = _mm_shuffle_epi32(msg, 0x0E);
174
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
175
      msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
176

177
      // Rounds 32-35
178
         msg    = _mm_add_epi32(msgtmp0, 
179
                  _mm_set_epi64x(0x53380D134D2C6DFCull, 0x2E1B213827B70A85ull));
180
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
181
      tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
182
      msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
183
      msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
184
         msg    = _mm_shuffle_epi32(msg, 0x0E);
185
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
186
      msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
187

188
      // Rounds 36-39
189
         msg    = _mm_add_epi32(msgtmp1, 
190
                  _mm_set_epi64x(0x92722C8581C2C92Eull, 0x766A0ABB650A7354ull));
191
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
192
      tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
193
      msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
194
      msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
195
         msg    = _mm_shuffle_epi32(msg, 0x0E);
196
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
197
      msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
198

199
      // Rounds 40-43
200
         msg    = _mm_add_epi32(msgtmp2, 
201
                  _mm_set_epi64x(0xC76C51A3C24B8B70ull, 0xA81A664BA2BFE8A1ull));
202
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
203
      tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
204
      msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
205
      msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
206
         msg    = _mm_shuffle_epi32(msg, 0x0E);
207
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
208
      msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
209

210
      // Rounds 44-47
211
         msg    = _mm_add_epi32(msgtmp3, 
212
                  _mm_set_epi64x(0x106AA070F40E3585ull, 0xD6990624D192E819ull));
213
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
214
      tmp     = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
215
      msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
216
      msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
217
         msg    = _mm_shuffle_epi32(msg, 0x0E);
218
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
219
      msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
220

221
      // Rounds 48-51
222
         msg    = _mm_add_epi32(msgtmp0, 
223
                  _mm_set_epi64x(0x34B0BCB52748774Cull, 0x1E376C0819A4C116ull));
224
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
225
      tmp     = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
226
      msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
227
      msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
228
         msg    = _mm_shuffle_epi32(msg, 0x0E);
229
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
230
      msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
231

232
      // Rounds 52-55
233
         msg    = _mm_add_epi32(msgtmp1, 
234
                  _mm_set_epi64x(0x682E6FF35B9CCA4Full, 0x4ED8AA4A391C0CB3ull));
235
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
236
      tmp     = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
237
      msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
238
      msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
239
         msg    = _mm_shuffle_epi32(msg, 0x0E);
240
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
241

242
      // Rounds 56-59
243
         msg    = _mm_add_epi32(msgtmp2, 
244
                  _mm_set_epi64x(0x8CC7020884C87814ull, 0x78A5636F748F82EEull));
245
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
246
      tmp     = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
247
      msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
248
      msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
249
         msg    = _mm_shuffle_epi32(msg, 0x0E);
250
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
251

252
      // Rounds 60-63
253
         msg    = _mm_add_epi32(msgtmp3, 
254
                  _mm_set_epi64x(0xC67178F2BEF9A3F7ull, 0xA4506CEB90BEFFFAull));
255
         state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
256
         msg    = _mm_shuffle_epi32(msg, 0x0E);
257
         state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
258

259
      // Add current hash values with previously saved
260
      state0 = _mm_add_epi32(state0, abef_save);
261
      state1 = _mm_add_epi32(state1, cdgh_save);
262

263
      data += 64;
264
      num_blks--;
265
   }
266

267
   // Write hash values back in the correct order
268
   tmp    = _mm_shuffle_epi32(state0, 0x1B);    // FEBA
269
   state1 = _mm_shuffle_epi32(state1, 0xB1);    // DCHG
270
   state0 = _mm_blend_epi16(tmp, state1, 0xF0); // DCBA
271
   state1 = _mm_alignr_epi8(state1, tmp, 8);    // ABEF
272

273
   _mm_store_si128((__m128i*) digest, state0);
274
   _mm_store_si128((__m128i*) (digest+4), state1);
275
}
276

277

278
Product

Resources

Company
