Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
godotengine
GitHub Repository: godotengine/godot
 Path: blob/master/thirdparty/embree/common/simd/vdouble4_avx.h
9912 views
1
// Copyright 2009-2021 Intel Corporation

2
// SPDX-License-Identifier: Apache-2.0

3


4
#pragma once

5


6
#define vboolf vboolf_impl

7
#define vboold vboold_impl

8
#define vint vint_impl

9
#define vuint vuint_impl

10
#define vllong vllong_impl

11
#define vfloat vfloat_impl

12
#define vdouble vdouble_impl

13


14
namespace embree

15
{ 

16
  /* 4-wide AVX 64-bit double type */

17
  template<>

18
  struct vdouble<4>

19
  {

20
    ALIGNED_STRUCT_(32);

21
            

22
    typedef vboold4 Bool;

23


24
    enum  { size = 4 }; // number of SIMD elements

25
    union {             // data

26
      __m256d v; 

27
      double i[4]; 

28
    };

29
    

30
    ////////////////////////////////////////////////////////////////////////////////

31
    /// Constructors, Assignment & Cast Operators

32
    ////////////////////////////////////////////////////////////////////////////////

33
       

34
    __forceinline vdouble() {}

35
    __forceinline vdouble(const vdouble4& t) { v = t.v; }

36
    __forceinline vdouble4& operator =(const vdouble4& f) { v = f.v; return *this; }

37


38
    __forceinline vdouble(const __m256d& t) { v = t; }

39
    __forceinline operator __m256d() const { return v; }

40


41
    __forceinline vdouble(double i) {

42
      v = _mm256_set1_pd(i);

43
    }

44
    

45
    __forceinline vdouble(double a, double b, double c, double d) {

46
      v = _mm256_set_pd(d,c,b,a);      

47
    }

48
   

49
    

50
    ////////////////////////////////////////////////////////////////////////////////

51
    /// Constants

52
    ////////////////////////////////////////////////////////////////////////////////

53
    

54
    __forceinline vdouble(ZeroTy) : v(_mm256_setzero_pd()) {}

55
    __forceinline vdouble(OneTy)  : v(_mm256_set1_pd(1)) {}

56
    __forceinline vdouble(StepTy) : v(_mm256_set_pd(3.0,2.0,1.0,0.0)) {}

57
    __forceinline vdouble(ReverseStepTy) : v(_mm256_setr_pd(3.0,2.0,1.0,0.0)) {}

58


59
    ////////////////////////////////////////////////////////////////////////////////

60
    /// Loads and Stores

61
    ////////////////////////////////////////////////////////////////////////////////

62


63
    static __forceinline void store_nt(double *__restrict__ ptr, const vdouble4& a) {

64
      _mm256_stream_pd(ptr, a);

65
    }

66


67
    static __forceinline vdouble4 loadu(const double* addr) {

68
      return _mm256_loadu_pd(addr);

69
    }

70


71
    static __forceinline vdouble4 load(const vdouble4* addr) {

72
      return _mm256_load_pd((double*)addr);

73
    }

74


75
    static __forceinline vdouble4 load(const double* addr) {

76
      return _mm256_load_pd(addr);

77
    }

78


79
    static __forceinline void store(double* ptr, const vdouble4& v) {

80
      _mm256_store_pd(ptr, v);

81
    }

82


83
    static __forceinline void storeu(double* ptr, const vdouble4& v) {

84
      _mm256_storeu_pd(ptr, v);

85
    }

86


87
    static __forceinline vdouble4 broadcast(const void* a) { return _mm256_set1_pd(*(double*)a); }

88


89
    ////////////////////////////////////////////////////////////////////////////////

90
    /// Array Access

91
    ////////////////////////////////////////////////////////////////////////////////

92
    

93
    __forceinline       double& operator [](size_t index)       { assert(index < 4); return i[index]; }

94
    __forceinline const double& operator [](size_t index) const { assert(index < 4); return i[index]; }

95
  };

96
  

97
  ////////////////////////////////////////////////////////////////////////////////

98
  /// Unary Operators

99
  ////////////////////////////////////////////////////////////////////////////////

100


101
#if defined(__AVX2__)

102
  __forceinline vdouble4 asDouble(const vllong4&  a) { return _mm256_castsi256_pd(a); }

103
  __forceinline vllong4  asLLong (const vdouble4& a) { return _mm256_castpd_si256(a); }

104
#endif

105


106
  __forceinline vdouble4 operator +(const vdouble4& a) { return a; }

107
  __forceinline vdouble4 operator -(const vdouble4& a) { return _mm256_sub_pd(_mm256_setzero_pd(), a); }

108


109
  ////////////////////////////////////////////////////////////////////////////////

110
  /// Binary Operators

111
  ////////////////////////////////////////////////////////////////////////////////

112


113
  __forceinline vdouble4 operator +(const vdouble4& a, const vdouble4& b) { return _mm256_add_pd(a, b); }

114
  __forceinline vdouble4 operator +(const vdouble4& a, double          b) { return a + vdouble4(b); }

115
  __forceinline vdouble4 operator +(double          a, const vdouble4& b) { return vdouble4(a) + b; }

116


117
  __forceinline vdouble4 operator -(const vdouble4& a, const vdouble4& b) { return _mm256_sub_pd(a, b); }

118
  __forceinline vdouble4 operator -(const vdouble4& a, double          b) { return a - vdouble4(b); }

119
  __forceinline vdouble4 operator -(double          a, const vdouble4& b) { return vdouble4(a) - b; }

120


121
  __forceinline vdouble4 operator *(const vdouble4& a, const vdouble4& b) { return _mm256_mul_pd(a, b); }

122
  __forceinline vdouble4 operator *(const vdouble4& a, double          b) { return a * vdouble4(b); }

123
  __forceinline vdouble4 operator *(double          a, const vdouble4& b) { return vdouble4(a) * b; }

124


125
  __forceinline vdouble4 operator &(const vdouble4& a, const vdouble4& b) { return _mm256_and_pd(a, b); }

126
  __forceinline vdouble4 operator &(const vdouble4& a, double          b) { return a & vdouble4(b); }

127
  __forceinline vdouble4 operator &(double          a, const vdouble4& b) { return vdouble4(a) & b; }

128


129
  __forceinline vdouble4 operator |(const vdouble4& a, const vdouble4& b) { return _mm256_or_pd(a, b); }

130
  __forceinline vdouble4 operator |(const vdouble4& a, double          b) { return a | vdouble4(b); }

131
  __forceinline vdouble4 operator |(double          a, const vdouble4& b) { return vdouble4(a) | b; }

132


133
  __forceinline vdouble4 operator ^(const vdouble4& a, const vdouble4& b) { return _mm256_xor_pd(a, b); }

134
  __forceinline vdouble4 operator ^(const vdouble4& a, double          b) { return a ^ vdouble4(b); }

135
  __forceinline vdouble4 operator ^(double          a, const vdouble4& b) { return vdouble4(a) ^ b; }

136
  

137
  __forceinline vdouble4 min(const vdouble4& a, const vdouble4& b) { return _mm256_min_pd(a, b); }

138
  __forceinline vdouble4 min(const vdouble4& a, double          b) { return min(a,vdouble4(b)); }

139
  __forceinline vdouble4 min(double          a, const vdouble4& b) { return min(vdouble4(a),b); }

140


141
  __forceinline vdouble4 max(const vdouble4& a, const vdouble4& b) { return _mm256_max_pd(a, b); }

142
  __forceinline vdouble4 max(const vdouble4& a, double          b) { return max(a,vdouble4(b)); }

143
  __forceinline vdouble4 max(double          a, const vdouble4& b) { return max(vdouble4(a),b); }

144
  

145
  ////////////////////////////////////////////////////////////////////////////////

146
  /// Ternary Operators

147
  ////////////////////////////////////////////////////////////////////////////////

148


149
#if defined(__FMA__)

150
  __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmadd_pd(a,b,c); }

151
  __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmsub_pd(a,b,c); }

152
  __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmadd_pd(a,b,c); }

153
  __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmsub_pd(a,b,c); }

154
#else

155
  __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b+c; }

156
  __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b-c; }

157
  __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b+c;}

158
  __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b-c; }

159
#endif

160


161
  ////////////////////////////////////////////////////////////////////////////////

162
  /// Assignment Operators

163
  ////////////////////////////////////////////////////////////////////////////////

164


165
  __forceinline vdouble4& operator +=(vdouble4& a, const vdouble4& b) { return a = a + b; }

166
  __forceinline vdouble4& operator +=(vdouble4& a, double          b) { return a = a + b; }

167
  

168
  __forceinline vdouble4& operator -=(vdouble4& a, const vdouble4& b) { return a = a - b; }

169
  __forceinline vdouble4& operator -=(vdouble4& a, double          b) { return a = a - b; }

170


171
  __forceinline vdouble4& operator *=(vdouble4& a, const vdouble4& b) { return a = a * b; }

172
  __forceinline vdouble4& operator *=(vdouble4& a, double          b) { return a = a * b; }

173
  

174
  __forceinline vdouble4& operator &=(vdouble4& a, const vdouble4& b) { return a = a & b; }

175
  __forceinline vdouble4& operator &=(vdouble4& a, double          b) { return a = a & b; }

176
  

177
  __forceinline vdouble4& operator |=(vdouble4& a, const vdouble4& b) { return a = a | b; }

178
  __forceinline vdouble4& operator |=(vdouble4& a, double          b) { return a = a | b; }

179
  

180


181
  ////////////////////////////////////////////////////////////////////////////////

182
  /// Comparison Operators + Select

183
  ////////////////////////////////////////////////////////////////////////////////

184


185
#if defined(__AVX512VL__)

186
  __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_EQ); }

187
  __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_NE); }

188
  __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LT); }

189
  __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GE); }

190
  __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GT); }

191
  __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LE); }

192
#elif !defined(__aarch64__)

193
  __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_EQ_OQ);  }

194
  __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NEQ_UQ); }

195
  __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LT_OS);  }

196
  __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLT_US); }

197
  __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLE_US); }

198
  __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LE_OS);  }

199
#else

200
  __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmpeq_pd(a, b);  }

201
  __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpneq_pd(a, b); }

202
  __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmplt_pd(a, b);  }

203
  __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpnlt_pd(a, b); }

204
  __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmpnle_pd(a, b); }

205
  __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmple_pd(a, b);  }

206
#endif

207


208
  __forceinline vboold4 operator ==(const vdouble4& a, double          b) { return a == vdouble4(b); }

209
  __forceinline vboold4 operator ==(double          a, const vdouble4& b) { return vdouble4(a) == b; }

210


211
  __forceinline vboold4 operator !=(const vdouble4& a, double          b) { return a != vdouble4(b); }

212
  __forceinline vboold4 operator !=(double          a, const vdouble4& b) { return vdouble4(a) != b; }

213


214
  __forceinline vboold4 operator < (const vdouble4& a, double          b) { return a <  vdouble4(b); }

215
  __forceinline vboold4 operator < (double          a, const vdouble4& b) { return vdouble4(a) <  b; }

216


217
  __forceinline vboold4 operator >=(const vdouble4& a, double          b) { return a >= vdouble4(b); }

218
  __forceinline vboold4 operator >=(double          a, const vdouble4& b) { return vdouble4(a) >= b; }

219


220
  __forceinline vboold4 operator > (const vdouble4& a, double          b) { return a >  vdouble4(b); }

221
  __forceinline vboold4 operator > (double          a, const vdouble4& b) { return vdouble4(a) >  b; }

222


223
  __forceinline vboold4 operator <=(const vdouble4& a, double          b) { return a <= vdouble4(b); }

224
  __forceinline vboold4 operator <=(double          a, const vdouble4& b) { return vdouble4(a) <= b; }

225


226
  __forceinline vboold4 eq(const vdouble4& a, const vdouble4& b) { return a == b; }

227
  __forceinline vboold4 ne(const vdouble4& a, const vdouble4& b) { return a != b; }

228
  __forceinline vboold4 lt(const vdouble4& a, const vdouble4& b) { return a <  b; }

229
  __forceinline vboold4 ge(const vdouble4& a, const vdouble4& b) { return a >= b; }

230
  __forceinline vboold4 gt(const vdouble4& a, const vdouble4& b) { return a >  b; }

231
  __forceinline vboold4 le(const vdouble4& a, const vdouble4& b) { return a <= b; }

232


233
#if defined(__AVX512VL__)

234
  __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_EQ); }

235
  __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_NE); }

236
  __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LT); }

237
  __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GE); }

238
  __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GT); }

239
  __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LE); }

240
#else

241
  __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a == b); }

242
  __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a != b); }

243
  __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a <  b); }

244
  __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a >= b); }

245
  __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a >  b); }

246
  __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a <= b); }

247
#endif

248
 

249
  __forceinline vdouble4 select(const vboold4& m, const vdouble4& t, const vdouble4& f) {

250
#if defined(__AVX512VL__)

251
    return _mm256_mask_blend_pd(m, f, t);

252
#else

253
    return _mm256_blendv_pd(f, t, m);

254
#endif

255
  }

256


257
  ////////////////////////////////////////////////////////////////////////////////

258
  // Movement/Shifting/Shuffling Functions

259
  ////////////////////////////////////////////////////////////////////////////////

260


261
  template<int i0, int i1>

262
  __forceinline vdouble4 shuffle(const vdouble4& v) {

263
    return _mm256_permute_pd(v, (i1 << 3) | (i0 << 2) | (i1 << 1) | i0);

264
  }

265


266
  template<int i>

267
  __forceinline vdouble4 shuffle(const vdouble4& v) {

268
    return shuffle<i, i>(v);

269
  }

270


271
  template<int i0, int i1>

272
  __forceinline vdouble4 shuffle2(const vdouble4& v) {

273
    return _mm256_permute2f128_pd(v, v, (i1 << 4) | i0);

274
  }

275


276
  __forceinline double toScalar(const vdouble4& v) {

277
    return _mm_cvtsd_f64(_mm256_castpd256_pd128(v));

278
  }

279


280
  ////////////////////////////////////////////////////////////////////////////////

281
  /// Reductions

282
  ////////////////////////////////////////////////////////////////////////////////

283


284
  __forceinline vdouble4 vreduce_min2(const vdouble4& x) { return min(x, shuffle<1,0>(x)); }

285
  __forceinline vdouble4 vreduce_min (const vdouble4& y) { const vdouble4 x = vreduce_min2(y); return min(x, shuffle2<1,0>(x)); }

286


287
  __forceinline vdouble4 vreduce_max2(const vdouble4& x) { return max(x,shuffle<1,0>(x)); }

288
  __forceinline vdouble4 vreduce_max (const vdouble4& y) { const vdouble4 x = vreduce_max2(y); return max(x, shuffle2<1,0>(x)); }

289


290
  __forceinline vdouble4 vreduce_and2(const vdouble4& x) { return x & shuffle<1,0>(x); }

291
  __forceinline vdouble4 vreduce_and (const vdouble4& y) { const vdouble4 x = vreduce_and2(y); return x & shuffle2<1,0>(x); }

292


293
  __forceinline vdouble4 vreduce_or2(const vdouble4& x) { return x | shuffle<1,0>(x); }

294
  __forceinline vdouble4 vreduce_or (const vdouble4& y) { const vdouble4 x = vreduce_or2(y); return x | shuffle2<1,0>(x); }

295


296
  __forceinline vdouble4 vreduce_add2(const vdouble4& x) { return x + shuffle<1,0>(x); }

297
  __forceinline vdouble4 vreduce_add (const vdouble4& y) { const vdouble4 x = vreduce_add2(y); return x + shuffle2<1,0>(x); }

298


299
  __forceinline double reduce_add(const vdouble4& a) { return toScalar(vreduce_add(a)); }

300
  __forceinline double reduce_min(const vdouble4& a) { return toScalar(vreduce_min(a)); }

301
  __forceinline double reduce_max(const vdouble4& a) { return toScalar(vreduce_max(a)); }

302


303
  ////////////////////////////////////////////////////////////////////////////////

304
  /// Memory load and store operations

305
  ////////////////////////////////////////////////////////////////////////////////

306


307


308
  

309
  ////////////////////////////////////////////////////////////////////////////////

310
  /// Output Operators

311
  ////////////////////////////////////////////////////////////////////////////////

312
  

313
  __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble4& v)

314
  {

315
    cout << "<" << v[0];

316
    for (size_t i=1; i<4; i++) cout << ", " << v[i];

317
    cout << ">";

318
    return cout;

319
  }

320
}

321


322
#undef vboolf

323
#undef vboold

324
#undef vint

325
#undef vuint

326
#undef vllong

327
#undef vfloat

328
#undef vdouble

329


330