Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
godotengine
GitHub Repository: godotengine/godot
 Path: blob/master/thirdparty/embree/common/simd/vuint16_avx512.h
9912 views
1
// Copyright 2009-2021 Intel Corporation

2
// SPDX-License-Identifier: Apache-2.0

3


4
#pragma once

5


6
#define vboolf vboolf_impl

7
#define vboold vboold_impl

8
#define vint vint_impl

9
#define vuint vuint_impl

10
#define vllong vllong_impl

11
#define vfloat vfloat_impl

12
#define vdouble vdouble_impl

13


14
namespace embree

15
{ 

16
  /* 16-wide AVX-512 unsigned integer type */

17
  template<>

18
  struct vuint<16>

19
  {

20
    ALIGNED_STRUCT_(64);   

21


22
    typedef vboolf16 Bool;

23
    typedef vuint16  UInt;

24
    typedef vfloat16 Float;

25


26
    enum  { size = 16 }; // number of SIMD elements

27
    union {              // data

28
      __m512i v; 

29
      unsigned int i[16]; 

30
    };

31
    

32
    ////////////////////////////////////////////////////////////////////////////////

33
    /// Constructors, Assignment & Cast Operators

34
    ////////////////////////////////////////////////////////////////////////////////

35
       

36
    __forceinline vuint() {}

37
    __forceinline vuint(const vuint16& t) { v = t.v; }

38
    __forceinline vuint16& operator =(const vuint16& f) { v = f.v; return *this; }

39


40
    __forceinline vuint(const __m512i& t) { v = t; }

41
    __forceinline operator __m512i() const { return v; }

42
    __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); }

43


44
    __forceinline vuint(unsigned int i) {

45
      v = _mm512_set1_epi32(i);

46
    }

47


48
    __forceinline vuint(const vuint4& i) {

49
      v = _mm512_broadcast_i32x4(i);

50
    }

51


52
    __forceinline vuint(const vuint8& i) {

53
      v = _mm512_castps_si512(_mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castsi256_pd(i))));

54
    }

55
    

56
    __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) {

57
      v = _mm512_set4_epi32(d,c,b,a);      

58
    }

59


60
    __forceinline vuint(unsigned int a0 , unsigned int a1 , unsigned int a2 , unsigned int a3,

61
                        unsigned int a4 , unsigned int a5 , unsigned int a6 , unsigned int a7,

62
                        unsigned int a8 , unsigned int a9 , unsigned int a10, unsigned int a11,

63
                        unsigned int a12, unsigned int a13, unsigned int a14, unsigned int a15)

64
    {

65
      v = _mm512_set_epi32(a15,a14,a13,a12,a11,a10,a9,a8,a7,a6,a5,a4,a3,a2,a1,a0);

66
    }

67
   

68
    __forceinline explicit vuint(const __m512& f) {

69
      v = _mm512_cvtps_epu32(f);

70
    }

71
    

72
    ////////////////////////////////////////////////////////////////////////////////

73
    /// Constants

74
    ////////////////////////////////////////////////////////////////////////////////

75
    

76
    __forceinline vuint(ZeroTy) : v(_mm512_setzero_epi32()) {}

77
    __forceinline vuint(OneTy)  : v(_mm512_set1_epi32(1)) {}

78
    __forceinline vuint(StepTy) : v(_mm512_set_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}

79
    __forceinline vuint(ReverseStepTy) : v(_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}

80


81
    ////////////////////////////////////////////////////////////////////////////////

82
    /// Loads and Stores

83
    ////////////////////////////////////////////////////////////////////////////////

84


85
    static __forceinline void store_nt(void* __restrict__ ptr, const vuint16& a) {

86
      _mm512_stream_si512((__m512i*)ptr,a);

87
    }

88


89
    static __forceinline vuint16 loadu(const void* addr)

90
    {

91
      return _mm512_loadu_si512(addr);

92
    }

93


94
    static __forceinline vuint16 loadu(const unsigned char* ptr) { return _mm512_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)); }

95
    static __forceinline vuint16 loadu(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_loadu_si256((__m256i*)ptr)); }

96


97
    static __forceinline vuint16 load(const vuint16* addr) {

98
      return _mm512_load_si512(addr);

99
    }

100


101
    static __forceinline vuint16 load(const unsigned int* addr) {

102
      return _mm512_load_si512(addr);

103
    }

104


105
    static __forceinline vuint16 load(unsigned short* ptr) { return _mm512_cvtepu16_epi32(*(__m256i*)ptr); }

106


107


108
    static __forceinline void store(void* ptr, const vuint16& v) {

109
      _mm512_store_si512(ptr,v);

110
    }

111


112
    static __forceinline void storeu(void* ptr, const vuint16& v) {

113
      _mm512_storeu_si512(ptr,v);

114
    }

115


116
    static __forceinline void storeu(const vboolf16& mask, void* ptr, const vuint16& f) {

117
      _mm512_mask_storeu_epi32(ptr,mask,f);

118
    }

119


120
    static __forceinline void store(const vboolf16& mask, void* addr, const vuint16& v2) {

121
      _mm512_mask_store_epi32(addr,mask,v2);

122
    }

123


124
    static __forceinline vuint16 compact(const vboolf16& mask, vuint16& v) {

125
      return _mm512_mask_compress_epi32(v,mask,v);

126
    }

127


128
    static __forceinline vuint16 compact(const vboolf16& mask, const vuint16& a, vuint16& b) {

129
      return _mm512_mask_compress_epi32(a,mask,b);

130
    }

131


132
    static __forceinline vuint16 expand(const vboolf16& mask, const vuint16& a, vuint16& b) {

133
      return _mm512_mask_expand_epi32(b,mask,a);

134
    }

135


136
    template<int scale = 4>

137
    static __forceinline vuint16 gather(const unsigned int* ptr, const vint16& index) {

138
      return _mm512_i32gather_epi32(index,ptr,scale);

139
    }

140


141
    template<int scale = 4>

142
    static __forceinline vuint16 gather(const vboolf16& mask, const unsigned int* ptr, const vint16& index) {

143
      return _mm512_mask_i32gather_epi32(_mm512_undefined_epi32(),mask,index,ptr,scale);

144
    }

145


146
    template<int scale = 4>

147
    static __forceinline vuint16 gather(const vboolf16& mask, vuint16& dest, const unsigned int* ptr, const vint16& index) {

148
      return _mm512_mask_i32gather_epi32(dest,mask,index,ptr,scale);

149
    }

150


151
    template<int scale = 4>

152
    static __forceinline void scatter(unsigned int* ptr, const vint16& index, const vuint16& v) {

153
      _mm512_i32scatter_epi32((int*)ptr,index,v,scale);

154
    }

155


156
    template<int scale = 4>

157
    static __forceinline void scatter(const vboolf16& mask, unsigned int* ptr, const vint16& index, const vuint16& v) {

158
      _mm512_mask_i32scatter_epi32((int*)ptr,mask,index,v,scale);

159
    }

160


161
    ////////////////////////////////////////////////////////////////////////////////

162
    /// Array Access

163
    ////////////////////////////////////////////////////////////////////////////////

164
    

165
    __forceinline       unsigned int& operator [](size_t index)       { assert(index < 16); return i[index]; }

166
    __forceinline const unsigned int& operator [](size_t index) const { assert(index < 16); return i[index]; }

167


168
    __forceinline unsigned int uint    (size_t index) const { assert(index < 16); return ((unsigned int*)i)[index]; }

169
    __forceinline size_t&      uint64_t(size_t index) const { assert(index < 8);  return ((size_t*)i)[index]; }

170
  };

171
  

172
  ////////////////////////////////////////////////////////////////////////////////

173
  /// Unary Operators

174
  ////////////////////////////////////////////////////////////////////////////////

175


176
  __forceinline vboolf16 asBool(const vuint16& a) { return _mm512_movepi32_mask(a); }

177


178
  __forceinline vuint16 operator +(const vuint16& a) { return a; }

179
  __forceinline vuint16 operator -(const vuint16& a) { return _mm512_sub_epi32(_mm512_setzero_epi32(), a); }

180


181
  ////////////////////////////////////////////////////////////////////////////////

182
  /// Binary Operators

183
  ////////////////////////////////////////////////////////////////////////////////

184


185
  __forceinline vuint16 operator +(const vuint16& a, const vuint16& b) { return _mm512_add_epi32(a, b); }

186
  __forceinline vuint16 operator +(const vuint16& a, unsigned int   b) { return a + vuint16(b); }

187
  __forceinline vuint16 operator +(unsigned int   a, const vuint16& b) { return vuint16(a) + b; }

188


189
  __forceinline vuint16 operator -(const vuint16& a, const vuint16& b) { return _mm512_sub_epi32(a, b); }

190
  __forceinline vuint16 operator -(const vuint16& a, unsigned int   b) { return a - vuint16(b); }

191
  __forceinline vuint16 operator -(unsigned int   a, const vuint16& b) { return vuint16(a) - b; }

192


193
  __forceinline vuint16 operator *(const vuint16& a, const vuint16& b) { return _mm512_mul_epu32(a, b); }

194
  __forceinline vuint16 operator *(const vuint16& a, unsigned int   b) { return a * vuint16(b); }

195
  __forceinline vuint16 operator *(unsigned int   a, const vuint16& b) { return vuint16(a) * b; }

196


197
  __forceinline vuint16 operator &(const vuint16& a, const vuint16& b) { return _mm512_and_epi32(a, b); }

198
  __forceinline vuint16 operator &(const vuint16& a, unsigned int   b) { return a & vuint16(b); }

199
  __forceinline vuint16 operator &(unsigned int   a, const vuint16& b) { return vuint16(a) & b; }

200


201
  __forceinline vuint16 operator |(const vuint16& a, const vuint16& b) { return _mm512_or_epi32(a, b); }

202
  __forceinline vuint16 operator |(const vuint16& a, unsigned int   b) { return a | vuint16(b); }

203
  __forceinline vuint16 operator |(unsigned int   a, const vuint16& b) { return vuint16(a) | b; }

204


205
  __forceinline vuint16 operator ^(const vuint16& a, const vuint16& b) { return _mm512_xor_epi32(a, b); }

206
  __forceinline vuint16 operator ^(const vuint16& a, unsigned int   b) { return a ^ vuint16(b); }

207
  __forceinline vuint16 operator ^(unsigned int   a, const vuint16& b) { return vuint16(a) ^ b; }

208


209
  __forceinline vuint16 operator <<(const vuint16& a, unsigned int n) { return _mm512_slli_epi32(a, n); }

210
  __forceinline vuint16 operator >>(const vuint16& a, unsigned int n) { return _mm512_srli_epi32(a, n); }

211


212
  __forceinline vuint16 operator <<(const vuint16& a, const vuint16& n) { return _mm512_sllv_epi32(a, n); }

213
  __forceinline vuint16 operator >>(const vuint16& a, const vuint16& n) { return _mm512_srlv_epi32(a, n); }

214


215
  __forceinline vuint16 sll (const vuint16& a, unsigned int b) { return _mm512_slli_epi32(a, b); }

216
  __forceinline vuint16 sra (const vuint16& a, unsigned int b) { return _mm512_srai_epi32(a, b); }

217
  __forceinline vuint16 srl (const vuint16& a, unsigned int b) { return _mm512_srli_epi32(a, b); }

218
  

219
  __forceinline vuint16 min(const vuint16& a, const vuint16& b) { return _mm512_min_epu32(a, b); }

220
  __forceinline vuint16 min(const vuint16& a, unsigned int   b) { return min(a,vuint16(b)); }

221
  __forceinline vuint16 min(unsigned int   a, const vuint16& b) { return min(vuint16(a),b); }

222


223
  __forceinline vuint16 max(const vuint16& a, const vuint16& b) { return _mm512_max_epu32(a, b); }

224
  __forceinline vuint16 max(const vuint16& a, unsigned int   b) { return max(a,vuint16(b)); }

225
  __forceinline vuint16 max(unsigned int   a, const vuint16& b) { return max(vuint16(a),b); }

226
  

227
  __forceinline vuint16 mask_add(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_add_epi32(c,mask,a,b); }

228
  __forceinline vuint16 mask_sub(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_sub_epi32(c,mask,a,b); }

229


230
  __forceinline vuint16 mask_and(const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_and_epi32(c,m,a,b); }

231
  __forceinline vuint16 mask_or (const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_or_epi32(c,m,a,b); }

232


233
  ////////////////////////////////////////////////////////////////////////////////

234
  /// Assignment Operators

235
  ////////////////////////////////////////////////////////////////////////////////

236


237
  __forceinline vuint16& operator +=(vuint16& a, const vuint16& b) { return a = a + b; }

238
  __forceinline vuint16& operator +=(vuint16& a, unsigned int   b) { return a = a + b; }

239
  

240
  __forceinline vuint16& operator -=(vuint16& a, const vuint16& b) { return a = a - b; }

241
  __forceinline vuint16& operator -=(vuint16& a, unsigned int   b) { return a = a - b; }

242


243
  __forceinline vuint16& operator *=(vuint16& a, const vuint16& b) { return a = a * b; }

244
  __forceinline vuint16& operator *=(vuint16& a, unsigned int   b) { return a = a * b; }

245
  

246
  __forceinline vuint16& operator &=(vuint16& a, const vuint16& b) { return a = a & b; }

247
  __forceinline vuint16& operator &=(vuint16& a, unsigned int   b) { return a = a & b; }

248
  

249
  __forceinline vuint16& operator |=(vuint16& a, const vuint16& b) { return a = a | b; }

250
  __forceinline vuint16& operator |=(vuint16& a, unsigned int   b) { return a = a | b; }

251
  

252
  __forceinline vuint16& operator <<=(vuint16& a, unsigned int b) { return a = a << b; }

253
  __forceinline vuint16& operator >>=(vuint16& a, unsigned int b) { return a = a >> b; }

254


255


256
  ////////////////////////////////////////////////////////////////////////////////

257
  /// Comparison Operators + Select

258
  ////////////////////////////////////////////////////////////////////////////////

259


260
  __forceinline vboolf16 operator ==(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }

261
  __forceinline vboolf16 operator ==(const vuint16& a, unsigned int   b) { return a == vuint16(b); }

262
  __forceinline vboolf16 operator ==(unsigned int   a, const vuint16& b) { return vuint16(a) == b; }

263
  

264
  __forceinline vboolf16 operator !=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }

265
  __forceinline vboolf16 operator !=(const vuint16& a, unsigned int   b) { return a != vuint16(b); }

266
  __forceinline vboolf16 operator !=(unsigned int   a, const vuint16& b) { return vuint16(a) != b; }

267
  

268
  __forceinline vboolf16 operator < (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }

269
  __forceinline vboolf16 operator < (const vuint16& a, unsigned int   b) { return a <  vuint16(b); }

270
  __forceinline vboolf16 operator < (unsigned int   a, const vuint16& b) { return vuint16(a) <  b; }

271
  

272
  __forceinline vboolf16 operator >=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }

273
  __forceinline vboolf16 operator >=(const vuint16& a, unsigned int   b) { return a >= vuint16(b); }

274
  __forceinline vboolf16 operator >=(unsigned int   a, const vuint16& b) { return vuint16(a) >= b; }

275


276
  __forceinline vboolf16 operator > (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }

277
  __forceinline vboolf16 operator > (const vuint16& a, unsigned int   b) { return a >  vuint16(b); }

278
  __forceinline vboolf16 operator > (unsigned int   a, const vuint16& b) { return vuint16(a) >  b; }

279


280
  __forceinline vboolf16 operator <=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }

281
  __forceinline vboolf16 operator <=(const vuint16& a, unsigned int   b) { return a <= vuint16(b); }

282
  __forceinline vboolf16 operator <=(unsigned int   a, const vuint16& b) { return vuint16(a) <= b; }

283


284
  __forceinline vboolf16 eq(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }

285
  __forceinline vboolf16 ne(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }

286
  __forceinline vboolf16 lt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }

287
  __forceinline vboolf16 ge(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }

288
  __forceinline vboolf16 gt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }

289
  __forceinline vboolf16 le(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }

290


291
  __forceinline vboolf16 eq(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_EQ); }

292
  __forceinline vboolf16 ne(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_NE); }

293
  __forceinline vboolf16 lt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LT); }

294
  __forceinline vboolf16 ge(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GE); }

295
  __forceinline vboolf16 gt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GT); }

296
  __forceinline vboolf16 le(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LE); }

297
    

298
 

299
  __forceinline vuint16 select(const vboolf16& m, const vuint16& t, const vuint16& f) {

300
    return _mm512_mask_or_epi32(f,m,t,t); 

301
  }

302


303
  ////////////////////////////////////////////////////////////////////////////////

304
  // Movement/Shifting/Shuffling Functions

305
  ////////////////////////////////////////////////////////////////////////////////

306


307
  template<int i>

308
  __forceinline vuint16 shuffle(const vuint16& v) {

309
    return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i)));

310
  }

311


312
  template<int i0, int i1, int i2, int i3>

313
  __forceinline vuint16 shuffle(const vuint16& v) {

314
    return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));

315
  }

316


317
  template<int i>

318
  __forceinline vuint16 shuffle4(const vuint16& v) {

319
    return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v) ,_MM_SHUFFLE(i, i, i, i)));

320
  }

321


322
  template<int i0, int i1, int i2, int i3>

323
  __forceinline vuint16 shuffle4(const vuint16& v) {

324
    return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));

325
  }

326


327
  template<int i>

328
  __forceinline vuint16 align_shift_right(const vuint16& a, const vuint16& b) {

329
    return _mm512_alignr_epi32(a, b, i);

330
  };

331


332
  __forceinline unsigned int toScalar(const vuint16& v) {

333
    return _mm_cvtsi128_si32(_mm512_castsi512_si128(v));

334
  }

335


336
  ////////////////////////////////////////////////////////////////////////////////

337
  /// Reductions

338
  ////////////////////////////////////////////////////////////////////////////////

339


340
  __forceinline vuint16 vreduce_min2(vuint16 x) {                      return min(x, shuffle<1,0,3,2>(x)); }

341
  __forceinline vuint16 vreduce_min4(vuint16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }

342
  __forceinline vuint16 vreduce_min8(vuint16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); }

343
  __forceinline vuint16 vreduce_min (vuint16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); }

344


345
  __forceinline vuint16 vreduce_max2(vuint16 x) {                      return max(x, shuffle<1,0,3,2>(x)); }

346
  __forceinline vuint16 vreduce_max4(vuint16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }

347
  __forceinline vuint16 vreduce_max8(vuint16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); }

348
  __forceinline vuint16 vreduce_max (vuint16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); }

349


350
  __forceinline vuint16 vreduce_and2(vuint16 x) {                      return x & shuffle<1,0,3,2>(x); }

351
  __forceinline vuint16 vreduce_and4(vuint16 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); }

352
  __forceinline vuint16 vreduce_and8(vuint16 x) { x = vreduce_and4(x); return x & shuffle4<1,0,3,2>(x); }

353
  __forceinline vuint16 vreduce_and (vuint16 x) { x = vreduce_and8(x); return x & shuffle4<2,3,0,1>(x); }

354


355
  __forceinline vuint16 vreduce_or2(vuint16 x) {                     return x | shuffle<1,0,3,2>(x); }

356
  __forceinline vuint16 vreduce_or4(vuint16 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); }

357
  __forceinline vuint16 vreduce_or8(vuint16 x) { x = vreduce_or4(x); return x | shuffle4<1,0,3,2>(x); }

358
  __forceinline vuint16 vreduce_or (vuint16 x) { x = vreduce_or8(x); return x | shuffle4<2,3,0,1>(x); }

359


360
  __forceinline vuint16 vreduce_add2(vuint16 x) {                      return x + shuffle<1,0,3,2>(x); }

361
  __forceinline vuint16 vreduce_add4(vuint16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }

362
  __forceinline vuint16 vreduce_add8(vuint16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); }

363
  __forceinline vuint16 vreduce_add (vuint16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); }

364


365
  __forceinline unsigned int reduce_min(const vuint16& v) { return toScalar(vreduce_min(v)); }

366
  __forceinline unsigned int reduce_max(const vuint16& v) { return toScalar(vreduce_max(v)); }

367
  __forceinline unsigned int reduce_and(const vuint16& v) { return toScalar(vreduce_and(v)); }

368
  __forceinline unsigned int reduce_or (const vuint16& v) { return toScalar(vreduce_or (v)); }

369
  __forceinline unsigned int reduce_add(const vuint16& v) { return toScalar(vreduce_add(v)); }

370


371
  ////////////////////////////////////////////////////////////////////////////////

372
  /// Memory load and store operations

373
  ////////////////////////////////////////////////////////////////////////////////

374
  

375
  __forceinline vuint16 permute(vuint16 v, vuint16 index) {

376
    return _mm512_permutexvar_epi32(index,v);  

377
  }

378


379
  __forceinline vuint16 reverse(const vuint16& a) {

380
    return permute(a,vuint16(reverse_step));

381
  }

382


383
  __forceinline vuint16 prefix_sum(const vuint16& a) 

384
  {

385
    const vuint16 z(zero);

386
    vuint16 v = a;

387
    v = v + align_shift_right<16-1>(v,z);

388
    v = v + align_shift_right<16-2>(v,z);

389
    v = v + align_shift_right<16-4>(v,z);

390
    v = v + align_shift_right<16-8>(v,z);

391
    return v;  

392
  }

393


394
  __forceinline vuint16 reverse_prefix_sum(const vuint16& a) 

395
  {

396
    const vuint16 z(zero);

397
    vuint16 v = a;

398
    v = v + align_shift_right<1>(z,v);

399
    v = v + align_shift_right<2>(z,v);

400
    v = v + align_shift_right<4>(z,v);

401
    v = v + align_shift_right<8>(z,v);

402
    return v;  

403
  }

404


405
  ////////////////////////////////////////////////////////////////////////////////

406
  /// Output Operators

407
  ////////////////////////////////////////////////////////////////////////////////

408
  

409
  __forceinline embree_ostream operator <<(embree_ostream cout, const vuint16& v)

410
  {

411
    cout << "<" << v[0];

412
    for (int i=1; i<16; i++) cout << ", " << v[i];

413
    cout << ">";

414
    return cout;

415
  }

416
}

417


418
#undef vboolf

419
#undef vboold

420
#undef vint

421
#undef vuint

422
#undef vllong

423
#undef vfloat

424
#undef vdouble

425


426