CoCalc -- blake3

GitHub Repository: freebsd/freebsd-src
Path: blob/main/contrib/llvm-project/llvm/lib/Support/BLAKE3/blake3_impl.h
³⁵²⁶⁹ views
1
#ifndef BLAKE3_IMPL_H
2
#define BLAKE3_IMPL_H
3

4
#include <assert.h>
5
#include <stdbool.h>
6
#include <stddef.h>
7
#include <stdint.h>
8
#include <string.h>
9

10
#include "llvm-c/blake3.h"
11
// For \p LLVM_LIBRARY_VISIBILITY
12
#include "llvm/Support/Compiler.h"
13

14
#include "llvm_blake3_prefix.h"
15

16
// internal flags
17
enum blake3_flags {
18
  CHUNK_START         = 1 << 0,
19
  CHUNK_END           = 1 << 1,
20
  PARENT              = 1 << 2,
21
  ROOT                = 1 << 3,
22
  KEYED_HASH          = 1 << 4,
23
  DERIVE_KEY_CONTEXT  = 1 << 5,
24
  DERIVE_KEY_MATERIAL = 1 << 6,
25
};
26

27
// This C implementation tries to support recent versions of GCC, Clang, and
28
// MSVC.
29
#if defined(_MSC_VER)
30
#define INLINE static __forceinline
31
#else
32
#define INLINE static inline __attribute__((always_inline))
33
#endif
34

35
#if defined(__x86_64__) || defined(_M_X64) 
36
#define IS_X86
37
#define IS_X86_64
38
#endif
39

40
#if defined(__i386__) || defined(_M_IX86)
41
#define IS_X86
42
#define IS_X86_32
43
#endif
44

45
#if defined(__aarch64__) || defined(_M_ARM64)
46
#define IS_AARCH64
47
#endif
48

49
#if defined(IS_X86)
50
#if defined(_MSC_VER)
51
#include <intrin.h>
52
#endif
53
#include <immintrin.h>
54
#endif
55

56
#if !defined(BLAKE3_USE_NEON) 
57
  // If BLAKE3_USE_NEON not manually set, autodetect based on
58
  // AArch64ness and endianness.
59
  #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)
60
    #define BLAKE3_USE_NEON 1
61
  #else
62
    #define BLAKE3_USE_NEON 0
63
  #endif
64
#endif
65

66
#if defined(IS_X86)
67
#define MAX_SIMD_DEGREE 16
68
#elif BLAKE3_USE_NEON == 1
69
#define MAX_SIMD_DEGREE 4
70
#else
71
#define MAX_SIMD_DEGREE 1
72
#endif
73

74
// There are some places where we want a static size that's equal to the
75
// MAX_SIMD_DEGREE, but also at least 2.
76
#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
77

78
static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
79
                               0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
80
                               0x1F83D9ABUL, 0x5BE0CD19UL};
81

82
static const uint8_t MSG_SCHEDULE[7][16] = {
83
    {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
84
    {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
85
    {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
86
    {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
87
    {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
88
    {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
89
    {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
90
};
91

92
/* Find index of the highest set bit */
93
/* x is assumed to be nonzero.       */
94
static unsigned int highest_one(uint64_t x) {
95
#if defined(__GNUC__) || defined(__clang__)
96
  return 63 ^ __builtin_clzll(x);
97
#elif defined(_MSC_VER) && defined(IS_X86_64)
98
  unsigned long index;
99
  _BitScanReverse64(&index, x);
100
  return index;
101
#elif defined(_MSC_VER) && defined(IS_X86_32)
102
  if(x >> 32) {
103
    unsigned long index;
104
    _BitScanReverse(&index, (unsigned long)(x >> 32));
105
    return 32 + index;
106
  } else {
107
    unsigned long index;
108
    _BitScanReverse(&index, (unsigned long)x);
109
    return index;
110
  }
111
#else
112
  unsigned int c = 0;
113
  if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
114
  if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
115
  if(x & 0x000000000000ff00ULL) { x >>=  8; c +=  8; }
116
  if(x & 0x00000000000000f0ULL) { x >>=  4; c +=  4; }
117
  if(x & 0x000000000000000cULL) { x >>=  2; c +=  2; }
118
  if(x & 0x0000000000000002ULL) {           c +=  1; }
119
  return c;
120
#endif
121
}
122

123
// Count the number of 1 bits.
124
INLINE unsigned int popcnt(uint64_t x) {
125
#if defined(__GNUC__) || defined(__clang__)
126
  return __builtin_popcountll(x);
127
#else
128
  unsigned int count = 0;
129
  while (x != 0) {
130
    count += 1;
131
    x &= x - 1;
132
  }
133
  return count;
134
#endif
135
}
136

137
// Largest power of two less than or equal to x. As a special case, returns 1
138
// when x is 0. 
139
INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
140
  return 1ULL << highest_one(x | 1);
141
}
142

143
INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
144

145
INLINE uint32_t counter_high(uint64_t counter) {
146
  return (uint32_t)(counter >> 32);
147
}
148

149
INLINE uint32_t load32(const void *src) {
150
  const uint8_t *p = (const uint8_t *)src;
151
  return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
152
         ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
153
}
154

155
INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
156
                           uint32_t key_words[8]) {
157
  key_words[0] = load32(&key[0 * 4]);
158
  key_words[1] = load32(&key[1 * 4]);
159
  key_words[2] = load32(&key[2 * 4]);
160
  key_words[3] = load32(&key[3 * 4]);
161
  key_words[4] = load32(&key[4 * 4]);
162
  key_words[5] = load32(&key[5 * 4]);
163
  key_words[6] = load32(&key[6 * 4]);
164
  key_words[7] = load32(&key[7 * 4]);
165
}
166

167
INLINE void store32(void *dst, uint32_t w) {
168
  uint8_t *p = (uint8_t *)dst;
169
  p[0] = (uint8_t)(w >> 0);
170
  p[1] = (uint8_t)(w >> 8);
171
  p[2] = (uint8_t)(w >> 16);
172
  p[3] = (uint8_t)(w >> 24);
173
}
174

175
INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
176
  store32(&bytes_out[0 * 4], cv_words[0]);
177
  store32(&bytes_out[1 * 4], cv_words[1]);
178
  store32(&bytes_out[2 * 4], cv_words[2]);
179
  store32(&bytes_out[3 * 4], cv_words[3]);
180
  store32(&bytes_out[4 * 4], cv_words[4]);
181
  store32(&bytes_out[5 * 4], cv_words[5]);
182
  store32(&bytes_out[6 * 4], cv_words[6]);
183
  store32(&bytes_out[7 * 4], cv_words[7]);
184
}
185

186
LLVM_LIBRARY_VISIBILITY
187
void blake3_compress_in_place(uint32_t cv[8],
188
                              const uint8_t block[BLAKE3_BLOCK_LEN],
189
                              uint8_t block_len, uint64_t counter,
190
                              uint8_t flags);
191

192
LLVM_LIBRARY_VISIBILITY
193
void blake3_compress_xof(const uint32_t cv[8],
194
                         const uint8_t block[BLAKE3_BLOCK_LEN],
195
                         uint8_t block_len, uint64_t counter, uint8_t flags,
196
                         uint8_t out[64]);
197

198
LLVM_LIBRARY_VISIBILITY
199
void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
200
                      size_t blocks, const uint32_t key[8], uint64_t counter,
201
                      bool increment_counter, uint8_t flags,
202
                      uint8_t flags_start, uint8_t flags_end, uint8_t *out);
203

204
LLVM_LIBRARY_VISIBILITY
205
size_t blake3_simd_degree(void);
206

207

208
// Declarations for implementation-specific functions.
209
LLVM_LIBRARY_VISIBILITY
210
void blake3_compress_in_place_portable(uint32_t cv[8],
211
                                       const uint8_t block[BLAKE3_BLOCK_LEN],
212
                                       uint8_t block_len, uint64_t counter,
213
                                       uint8_t flags);
214

215
LLVM_LIBRARY_VISIBILITY
216
void blake3_compress_xof_portable(const uint32_t cv[8],
217
                                  const uint8_t block[BLAKE3_BLOCK_LEN],
218
                                  uint8_t block_len, uint64_t counter,
219
                                  uint8_t flags, uint8_t out[64]);
220

221
LLVM_LIBRARY_VISIBILITY
222
void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
223
                               size_t blocks, const uint32_t key[8],
224
                               uint64_t counter, bool increment_counter,
225
                               uint8_t flags, uint8_t flags_start,
226
                               uint8_t flags_end, uint8_t *out);
227

228
#if defined(IS_X86)
229
#if !defined(BLAKE3_NO_SSE2)
230
LLVM_LIBRARY_VISIBILITY
231
void blake3_compress_in_place_sse2(uint32_t cv[8],
232
                                   const uint8_t block[BLAKE3_BLOCK_LEN],
233
                                   uint8_t block_len, uint64_t counter,
234
                                   uint8_t flags);
235
LLVM_LIBRARY_VISIBILITY
236
void blake3_compress_xof_sse2(const uint32_t cv[8],
237
                              const uint8_t block[BLAKE3_BLOCK_LEN],
238
                              uint8_t block_len, uint64_t counter,
239
                              uint8_t flags, uint8_t out[64]);
240
LLVM_LIBRARY_VISIBILITY
241
void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
242
                           size_t blocks, const uint32_t key[8],
243
                           uint64_t counter, bool increment_counter,
244
                           uint8_t flags, uint8_t flags_start,
245
                           uint8_t flags_end, uint8_t *out);
246
#endif
247
#if !defined(BLAKE3_NO_SSE41)
248
LLVM_LIBRARY_VISIBILITY
249
void blake3_compress_in_place_sse41(uint32_t cv[8],
250
                                    const uint8_t block[BLAKE3_BLOCK_LEN],
251
                                    uint8_t block_len, uint64_t counter,
252
                                    uint8_t flags);
253
LLVM_LIBRARY_VISIBILITY
254
void blake3_compress_xof_sse41(const uint32_t cv[8],
255
                               const uint8_t block[BLAKE3_BLOCK_LEN],
256
                               uint8_t block_len, uint64_t counter,
257
                               uint8_t flags, uint8_t out[64]);
258
LLVM_LIBRARY_VISIBILITY
259
void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
260
                            size_t blocks, const uint32_t key[8],
261
                            uint64_t counter, bool increment_counter,
262
                            uint8_t flags, uint8_t flags_start,
263
                            uint8_t flags_end, uint8_t *out);
264
#endif
265
#if !defined(BLAKE3_NO_AVX2)
266
LLVM_LIBRARY_VISIBILITY
267
void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
268
                           size_t blocks, const uint32_t key[8],
269
                           uint64_t counter, bool increment_counter,
270
                           uint8_t flags, uint8_t flags_start,
271
                           uint8_t flags_end, uint8_t *out);
272
#endif
273
#if !defined(BLAKE3_NO_AVX512)
274
LLVM_LIBRARY_VISIBILITY
275
void blake3_compress_in_place_avx512(uint32_t cv[8],
276
                                     const uint8_t block[BLAKE3_BLOCK_LEN],
277
                                     uint8_t block_len, uint64_t counter,
278
                                     uint8_t flags);
279

280
LLVM_LIBRARY_VISIBILITY
281
void blake3_compress_xof_avx512(const uint32_t cv[8],
282
                                const uint8_t block[BLAKE3_BLOCK_LEN],
283
                                uint8_t block_len, uint64_t counter,
284
                                uint8_t flags, uint8_t out[64]);
285

286
LLVM_LIBRARY_VISIBILITY
287
void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
288
                             size_t blocks, const uint32_t key[8],
289
                             uint64_t counter, bool increment_counter,
290
                             uint8_t flags, uint8_t flags_start,
291
                             uint8_t flags_end, uint8_t *out);
292
#endif
293
#endif
294

295
#if BLAKE3_USE_NEON == 1
296
LLVM_LIBRARY_VISIBILITY
297
void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
298
                           size_t blocks, const uint32_t key[8],
299
                           uint64_t counter, bool increment_counter,
300
                           uint8_t flags, uint8_t flags_start,
301
                           uint8_t flags_end, uint8_t *out);
302
#endif
303

304

305
#endif /* BLAKE3_IMPL_H */
306

307
Product

Resources

Company
