Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
godotengine
GitHub Repository: godotengine/godot
 Path: blob/master/thirdparty/embree/common/simd/vuint4_sse2.h
9912 views
1
// Copyright 2009-2021 Intel Corporation

2
// SPDX-License-Identifier: Apache-2.0

3


4
#pragma once

5


6
#include "../math/emath.h"

7


8
#define vboolf vboolf_impl

9
#define vboold vboold_impl

10
#define vint vint_impl

11
#define vuint vuint_impl

12
#define vllong vllong_impl

13
#define vfloat vfloat_impl

14
#define vdouble vdouble_impl

15


16
namespace embree

17
{

18
  /* 4-wide SSE integer type */

19
  template<>

20
  struct vuint<4>

21
  {

22
    ALIGNED_STRUCT_(16);

23
    

24
    typedef vboolf4 Bool;

25
    typedef vuint4   Int;

26
    typedef vfloat4 Float;

27


28
    enum  { size = 4 }; // number of SIMD elements

29
    union { __m128i v; unsigned int i[4]; }; // data

30


31
    ////////////////////////////////////////////////////////////////////////////////

32
    /// Constructors, Assignment & Cast Operators

33
    ////////////////////////////////////////////////////////////////////////////////

34
    

35
    __forceinline vuint() {}

36
    __forceinline vuint(const vuint4& a) { v = a.v; }

37
    __forceinline vuint4& operator =(const vuint4& a) { v = a.v; return *this; }

38


39
    __forceinline vuint(const __m128i a) : v(a) {}

40
    __forceinline operator const __m128i&() const { return v; }

41
    __forceinline operator       __m128i&()       { return v; }

42


43


44
    __forceinline vuint(unsigned int a) : v(_mm_set1_epi32(a)) {}

45
    __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm_set_epi32(d, c, b, a)) {}

46


47
#if defined(__AVX512VL__)

48
    __forceinline explicit vuint(__m128 a) : v(_mm_cvtps_epu32(a)) {}

49
#endif

50


51
#if defined(__AVX512VL__)

52
    __forceinline explicit vuint(const vboolf4& a) : v(_mm_movm_epi32(a)) {}

53
#else

54
    __forceinline explicit vuint(const vboolf4& a) : v(_mm_castps_si128((__m128)a)) {}

55
#endif

56


57
    ////////////////////////////////////////////////////////////////////////////////

58
    /// Constants

59
    ////////////////////////////////////////////////////////////////////////////////

60


61
    __forceinline vuint(ZeroTy)   : v(_mm_setzero_si128()) {}

62
    __forceinline vuint(OneTy)    : v(_mm_set1_epi32(1)) {}

63
    __forceinline vuint(PosInfTy) : v(_mm_set1_epi32(unsigned(pos_inf))) {}

64
    __forceinline vuint(StepTy)   : v(_mm_set_epi32(3, 2, 1, 0)) {}

65
    __forceinline vuint(TrueTy)   { v = _mm_cmpeq_epi32(v,v); }

66
    __forceinline vuint(UndefinedTy) : v(_mm_castps_si128(_mm_undefined_ps())) {}

67


68
    ////////////////////////////////////////////////////////////////////////////////

69
    /// Loads and Stores

70
    ////////////////////////////////////////////////////////////////////////////////

71


72
    static __forceinline vuint4 load (const void* a) { return _mm_load_si128((__m128i*)a); }

73
    static __forceinline vuint4 loadu(const void* a) { return _mm_loadu_si128((__m128i*)a); }

74


75
    static __forceinline void store (void* ptr, const vuint4& v) { _mm_store_si128((__m128i*)ptr,v); }

76
    static __forceinline void storeu(void* ptr, const vuint4& v) { _mm_storeu_si128((__m128i*)ptr,v); }

77
    

78
#if defined(__AVX512VL__)

79
    static __forceinline vuint4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_epi32 (_mm_setzero_si128(),mask,ptr); }

80
    static __forceinline vuint4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_epi32(_mm_setzero_si128(),mask,ptr); }

81


82
    static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_store_epi32 (ptr,mask,v); }

83
    static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_storeu_epi32(ptr,mask,v); }

84
#elif defined(__AVX__)

85
    static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }

86
    static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }

87


88
    static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }

89
    static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }

90
#else

91
    static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_and_si128(_mm_load_si128 ((__m128i*)a),mask); }

92
    static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_and_si128(_mm_loadu_si128((__m128i*)a),mask); }

93


94
    static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { store (ptr,select(mask,i,load (ptr))); }

95
    static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { storeu(ptr,select(mask,i,loadu(ptr))); }

96
#endif

97


98
#if defined(__aarch64__)

99
    static __forceinline vuint4 load(const unsigned char* ptr) {

100
        return _mm_load4epu8_epi32(((__m128i*)ptr));

101
    }

102
    static __forceinline vuint4 loadu(const unsigned char* ptr) {

103
        return _mm_load4epu8_epi32(((__m128i*)ptr));

104
    }

105
#elif defined(__SSE4_1__)

106
    static __forceinline vuint4 load(const unsigned char* ptr) {

107
      return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));

108
    }

109


110
    static __forceinline vuint4 loadu(const unsigned char* ptr) {

111
      return  _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));

112
    }

113


114
#endif

115


116
    static __forceinline vuint4 load(const unsigned short* ptr) {

117
#if defined(__aarch64__)

118
      return _mm_load4epu16_epi32(((__m128i*)ptr));

119
#elif defined (__SSE4_1__)

120
      return _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr));

121
#else

122
      return vuint4(ptr[0],ptr[1],ptr[2],ptr[3]);

123
#endif

124
    } 

125


126
    static __forceinline vuint4 load_nt(void* ptr) {

127
#if (defined(__aarch64__)) || defined(__SSE4_1__)

128
      return _mm_stream_load_si128((__m128i*)ptr); 

129
#else

130
      return _mm_load_si128((__m128i*)ptr); 

131
#endif

132
    }

133
    

134
    static __forceinline void store_nt(void* ptr, const vuint4& v) {

135
#if !defined(__aarch64__) && defined(__SSE4_1__)

136
      _mm_stream_ps((float*)ptr, _mm_castsi128_ps(v));

137
#else

138
      _mm_store_si128((__m128i*)ptr,v);

139
#endif

140
    }

141


142
    template<int scale = 4>

143
    static __forceinline vuint4 gather(const unsigned int* ptr, const vint4& index) {

144
#if defined(__AVX2__) && !defined(__aarch64__)

145
      return _mm_i32gather_epi32((const int*)ptr, index, scale);

146
#else

147
      return vuint4(

148
          *(unsigned int*)(((char*)ptr)+scale*index[0]),

149
          *(unsigned int*)(((char*)ptr)+scale*index[1]),

150
          *(unsigned int*)(((char*)ptr)+scale*index[2]),

151
          *(unsigned int*)(((char*)ptr)+scale*index[3]));

152
#endif

153
    }

154


155
    template<int scale = 4>

156
    static __forceinline vuint4 gather(const vboolf4& mask, const unsigned int* ptr, const vint4& index) {

157
      vuint4 r = zero;

158
#if defined(__AVX512VL__)

159
      return _mm_mmask_i32gather_epi32(r, mask, index, ptr, scale);

160
#elif defined(__AVX2__) && !defined(__aarch64__)

161
      return _mm_mask_i32gather_epi32(r, (const int*)ptr, index, mask, scale);

162
#else

163
      if (likely(mask[0])) r[0] = *(unsigned int*)(((char*)ptr)+scale*index[0]);

164
      if (likely(mask[1])) r[1] = *(unsigned int*)(((char*)ptr)+scale*index[1]);

165
      if (likely(mask[2])) r[2] = *(unsigned int*)(((char*)ptr)+scale*index[2]);

166
      if (likely(mask[3])) r[3] = *(unsigned int*)(((char*)ptr)+scale*index[3]);

167
      return r;

168
#endif

169
    }

170


171
    ////////////////////////////////////////////////////////////////////////////////

172
    /// Array Access

173
    ////////////////////////////////////////////////////////////////////////////////

174


175
    __forceinline const unsigned int& operator [](size_t index) const { assert(index < 4); return i[index]; }

176
    __forceinline       unsigned int& operator [](size_t index)       { assert(index < 4); return i[index]; }

177


178
    friend __forceinline vuint4 select(const vboolf4& m, const vuint4& t, const vuint4& f) {

179
#if defined(__AVX512VL__)

180
      return _mm_mask_blend_epi32(m, (__m128i)f, (__m128i)t);

181
#elif defined(__SSE4_1__)

182
      return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), m)); 

183
#else

184
      return _mm_or_si128(_mm_and_si128(m, t), _mm_andnot_si128(m, f)); 

185
#endif

186
    }

187
  };

188


189
  ////////////////////////////////////////////////////////////////////////////////

190
  /// Unary Operators

191
  ////////////////////////////////////////////////////////////////////////////////

192


193
#if defined(__AVX512VL__)

194
  __forceinline vboolf4 asBool(const vuint4& a) { return _mm_movepi32_mask(a); }

195
#else

196
  __forceinline vboolf4 asBool(const vuint4& a) { return _mm_castsi128_ps(a); }

197
#endif

198


199
  __forceinline vuint4 operator +(const vuint4& a) { return a; }

200
  __forceinline vuint4 operator -(const vuint4& a) { return _mm_sub_epi32(_mm_setzero_si128(), a); }

201


202
  ////////////////////////////////////////////////////////////////////////////////

203
  /// Binary Operators

204
  ////////////////////////////////////////////////////////////////////////////////

205


206
  __forceinline vuint4 operator +(const vuint4& a, const vuint4& b) { return _mm_add_epi32(a, b); }

207
  __forceinline vuint4 operator +(const vuint4& a, unsigned int  b) { return a + vuint4(b); }

208
  __forceinline vuint4 operator +(unsigned int  a, const vuint4& b) { return vuint4(a) + b; }

209


210
  __forceinline vuint4 operator -(const vuint4& a, const vuint4& b) { return _mm_sub_epi32(a, b); }

211
  __forceinline vuint4 operator -(const vuint4& a, unsigned int  b) { return a - vuint4(b); }

212
  __forceinline vuint4 operator -(unsigned int  a, const vuint4& b) { return vuint4(a) - b; }

213


214
//#if defined(__SSE4_1__)

215
//  __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return _mm_mullo_epu32(a, b); }

216
//#else

217
//  __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return vuint4(a[0]*b[0],a[1]*b[1],a[2]*b[2],a[3]*b[3]); }

218
//#endif

219
//  __forceinline vuint4 operator *(const vuint4& a, unsigned int  b) { return a * vuint4(b); }

220
//  __forceinline vuint4 operator *(unsigned int  a, const vuint4& b) { return vuint4(a) * b; }

221


222
  __forceinline vuint4 operator &(const vuint4& a, const vuint4& b) { return _mm_and_si128(a, b); }

223
  __forceinline vuint4 operator &(const vuint4& a, unsigned int  b) { return a & vuint4(b); }

224
  __forceinline vuint4 operator &(unsigned int  a, const vuint4& b) { return vuint4(a) & b; }

225


226
  __forceinline vuint4 operator |(const vuint4& a, const vuint4& b) { return _mm_or_si128(a, b); }

227
  __forceinline vuint4 operator |(const vuint4& a, unsigned int  b) { return a | vuint4(b); }

228
  __forceinline vuint4 operator |(unsigned int  a, const vuint4& b) { return vuint4(a) | b; }

229


230
  __forceinline vuint4 operator ^(const vuint4& a, const vuint4& b) { return _mm_xor_si128(a, b); }

231
  __forceinline vuint4 operator ^(const vuint4& a, unsigned int  b) { return a ^ vuint4(b); }

232
  __forceinline vuint4 operator ^(unsigned int  a, const vuint4& b) { return vuint4(a) ^ b; }

233


234
  __forceinline vuint4 operator <<(const vuint4& a, unsigned int n) { return _mm_slli_epi32(a, n); }

235
  __forceinline vuint4 operator >>(const vuint4& a, unsigned int n) { return _mm_srli_epi32(a, n); }

236


237
  __forceinline vuint4 sll (const vuint4& a, unsigned int b) { return _mm_slli_epi32(a, b); }

238
  __forceinline vuint4 sra (const vuint4& a, unsigned int b) { return _mm_srai_epi32(a, b); }

239
  __forceinline vuint4 srl (const vuint4& a, unsigned int b) { return _mm_srli_epi32(a, b); }

240
  

241
  ////////////////////////////////////////////////////////////////////////////////

242
  /// Assignment Operators

243
  ////////////////////////////////////////////////////////////////////////////////

244


245
  __forceinline vuint4& operator +=(vuint4& a, const vuint4& b) { return a = a + b; }

246
  __forceinline vuint4& operator +=(vuint4& a, unsigned int  b) { return a = a + b; }

247
  

248
  __forceinline vuint4& operator -=(vuint4& a, const vuint4& b) { return a = a - b; }

249
  __forceinline vuint4& operator -=(vuint4& a, unsigned int  b) { return a = a - b; }

250


251
//#if defined(__SSE4_1__)

252
//  __forceinline vuint4& operator *=(vuint4& a, const vuint4& b) { return a = a * b; }

253
//  __forceinline vuint4& operator *=(vuint4& a, unsigned int  b) { return a = a * b; }

254
//#endif

255
  

256
  __forceinline vuint4& operator &=(vuint4& a, const vuint4& b) { return a = a & b; }

257
  __forceinline vuint4& operator &=(vuint4& a, unsigned int  b) { return a = a & b; }

258
  

259
  __forceinline vuint4& operator |=(vuint4& a, const vuint4& b) { return a = a | b; }

260
  __forceinline vuint4& operator |=(vuint4& a, unsigned int  b) { return a = a | b; }

261
  

262
  __forceinline vuint4& operator <<=(vuint4& a, unsigned int  b) { return a = a << b; }

263
  __forceinline vuint4& operator >>=(vuint4& a, unsigned int  b) { return a = a >> b; }

264


265
  ////////////////////////////////////////////////////////////////////////////////

266
  /// Comparison Operators + Select

267
  ////////////////////////////////////////////////////////////////////////////////

268


269
#if defined(__AVX512VL__)

270
  __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }

271
  __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }

272
  //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }

273
  //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }

274
  //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }

275
  //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }

276
#else

277
  __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); }

278
  __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return !(a == b); }

279
  //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmplt_epu32(a, b)); }

280
  //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return !(a <  b); }

281
  //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpgt_epu32(a, b)); }

282
  //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return !(a >  b); }

283
#endif

284


285
  __forceinline vboolf4 operator ==(const vuint4& a, unsigned int  b) { return a == vuint4(b); }

286
  __forceinline vboolf4 operator ==(unsigned int  a, const vuint4& b) { return vuint4(a) == b; }

287


288
  __forceinline vboolf4 operator !=(const vuint4& a, unsigned int  b) { return a != vuint4(b); }

289
  __forceinline vboolf4 operator !=(unsigned int  a, const vuint4& b) { return vuint4(a) != b; }

290


291
  //__forceinline vboolf4 operator < (const vuint4& a, unsigned int  b) { return a <  vuint4(b); }

292
  //__forceinline vboolf4 operator < (unsigned int  a, const vuint4& b) { return vuint4(a) <  b; }

293


294
  //__forceinline vboolf4 operator >=(const vuint4& a, unsigned int  b) { return a >= vuint4(b); }

295
  //__forceinline vboolf4 operator >=(unsigned int  a, const vuint4& b) { return vuint4(a) >= b; }

296


297
  //__forceinline vboolf4 operator > (const vuint4& a, unsigned int  b) { return a >  vuint4(b); }

298
  //__forceinline vboolf4 operator > (unsigned int  a, const vuint4& b) { return vuint4(a) >  b; }

299


300
  //__forceinline vboolf4 operator <=(const vuint4& a, unsigned int  b) { return a <= vuint4(b); }

301
  //__forceinline vboolf4 operator <=(unsigned int  a, const vuint4& b) { return vuint4(a) <= b; }

302


303
  __forceinline vboolf4 eq(const vuint4& a, const vuint4& b) { return a == b; }

304
  __forceinline vboolf4 ne(const vuint4& a, const vuint4& b) { return a != b; }

305
  //__forceinline vboolf4 lt(const vuint4& a, const vuint4& b) { return a <  b; }

306
  //__forceinline vboolf4 ge(const vuint4& a, const vuint4& b) { return a >= b; }

307
  //__forceinline vboolf4 gt(const vuint4& a, const vuint4& b) { return a >  b; }

308
  //__forceinline vboolf4 le(const vuint4& a, const vuint4& b) { return a <= b; }

309


310
#if defined(__AVX512VL__)

311
  __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_EQ); }

312
  __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_NE); }

313
  //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); }

314
  //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GE); }

315
  //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GT); }

316
  //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LE); }

317
#else

318
  __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a == b); }

319
  __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a != b); }

320
  //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a <  b); }

321
  //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a >= b); }

322
  //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a >  b); }

323
  //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a <= b); }

324
#endif

325


326
  template<int mask>

327
  __forceinline vuint4 select(const vuint4& t, const vuint4& f) {

328
#if defined(__SSE4_1__) 

329
    return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), mask));

330
#else

331
    return select(vboolf4(mask), t, f);

332
#endif    

333
  }

334


335
/*#if defined(__SSE4_1__)

336
  __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return _mm_min_epu32(a, b); }

337
  __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return _mm_max_epu32(a, b); }

338


339
#else

340
  __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return select(a < b,a,b); }

341
  __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return select(a < b,b,a); }

342
#endif

343


344
  __forceinline vuint4 min(const vuint4& a, unsigned int  b) { return min(a,vuint4(b)); }

345
  __forceinline vuint4 min(unsigned int  a, const vuint4& b) { return min(vuint4(a),b); }

346
  __forceinline vuint4 max(const vuint4& a, unsigned int  b) { return max(a,vuint4(b)); }

347
  __forceinline vuint4 max(unsigned int  a, const vuint4& b) { return max(vuint4(a),b); }*/

348


349
  ////////////////////////////////////////////////////////////////////////////////

350
  // Movement/Shifting/Shuffling Functions

351
  ////////////////////////////////////////////////////////////////////////////////

352


353
  __forceinline vuint4 unpacklo(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpacklo_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }

354
  __forceinline vuint4 unpackhi(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpackhi_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }

355


356
#if defined(__aarch64__)

357
  template<int i0, int i1, int i2, int i3>

358
  __forceinline vuint4 shuffle(const vuint4& v) {

359
    return vreinterpretq_s32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3)));

360
  }

361
  template<int i0, int i1, int i2, int i3>

362
  __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) {

363
    return vreinterpretq_s32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3)));

364
  }

365
#else

366
  template<int i0, int i1, int i2, int i3>

367
  __forceinline vuint4 shuffle(const vuint4& v) {

368
    return _mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0));

369
  }

370


371
  template<int i0, int i1, int i2, int i3>

372
  __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) {

373
    return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));

374
  }

375
#endif

376
#if defined(__SSE3__)

377
  template<> __forceinline vuint4 shuffle<0, 0, 2, 2>(const vuint4& v) { return _mm_castps_si128(_mm_moveldup_ps(_mm_castsi128_ps(v))); }

378
  template<> __forceinline vuint4 shuffle<1, 1, 3, 3>(const vuint4& v) { return _mm_castps_si128(_mm_movehdup_ps(_mm_castsi128_ps(v))); }

379
  template<> __forceinline vuint4 shuffle<0, 1, 0, 1>(const vuint4& v) { return _mm_castpd_si128(_mm_movedup_pd (_mm_castsi128_pd(v))); }

380
#endif

381


382
  template<int i>

383
  __forceinline vuint4 shuffle(const vuint4& v) {

384
    return shuffle<i,i,i,i>(v);

385
  }

386


387
#if defined(__SSE4_1__) && !defined(__aarch64__)

388
  template<int src> __forceinline unsigned int extract(const vuint4& b) { return _mm_extract_epi32(b, src); }

389
  template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { return _mm_insert_epi32(a, b, dst); }

390
#else

391
  template<int src> __forceinline unsigned int extract(const vuint4& b) { return b[src&3]; }

392
  template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { vuint4 c = a; c[dst&3] = b; return c; }

393
#endif

394


395
  template<> __forceinline unsigned int extract<0>(const vuint4& b) { return _mm_cvtsi128_si32(b); }

396


397
  __forceinline unsigned int toScalar(const vuint4& v) { return _mm_cvtsi128_si32(v); }

398


399
  ////////////////////////////////////////////////////////////////////////////////

400
  /// Reductions

401
  ////////////////////////////////////////////////////////////////////////////////

402


403
#if 0

404
#if defined(__SSE4_1__)

405


406
  __forceinline vuint4 vreduce_min(const vuint4& v) { vuint4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); }

407
  __forceinline vuint4 vreduce_max(const vuint4& v) { vuint4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); }

408
  __forceinline vuint4 vreduce_add(const vuint4& v) { vuint4 h = shuffle<1,0,3,2>(v)   + v ; return shuffle<2,3,0,1>(h)   + h ; }

409


410
  __forceinline unsigned int reduce_min(const vuint4& v) { return toScalar(vreduce_min(v)); }

411
  __forceinline unsigned int reduce_max(const vuint4& v) { return toScalar(vreduce_max(v)); }

412
  __forceinline unsigned int reduce_add(const vuint4& v) { return toScalar(vreduce_add(v)); }

413


414
  __forceinline size_t select_min(const vuint4& v) { return bsf(movemask(v == vreduce_min(v))); }

415
  __forceinline size_t select_max(const vuint4& v) { return bsf(movemask(v == vreduce_max(v))); }

416


417
  //__forceinline size_t select_min(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }

418
  //__forceinline size_t select_max(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }

419


420
#else

421


422
  __forceinline unsigned int reduce_min(const vuint4& v) { return min(v[0],v[1],v[2],v[3]); }

423
  __forceinline unsigned int reduce_max(const vuint4& v) { return max(v[0],v[1],v[2],v[3]); }

424
  __forceinline unsigned int reduce_add(const vuint4& v) { return v[0]+v[1]+v[2]+v[3]; }

425


426
#endif

427
#endif

428


429
  ////////////////////////////////////////////////////////////////////////////////

430
  /// Output Operators

431
  ////////////////////////////////////////////////////////////////////////////////

432


433
  __forceinline embree_ostream operator <<(embree_ostream cout, const vuint4& a) {

434
    return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">";

435
  }

436
}

437


438
#undef vboolf

439
#undef vboold

440
#undef vint

441
#undef vuint

442
#undef vllong

443
#undef vfloat

444
#undef vdouble

445


446