Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
godotengine
GitHub Repository: godotengine/godot
 Path: blob/master/thirdparty/embree/common/simd/vdouble8_avx512.h
9912 views
1
// Copyright 2009-2021 Intel Corporation

2
// SPDX-License-Identifier: Apache-2.0

3


4
#pragma once

5


6
#define vboolf vboolf_impl

7
#define vboold vboold_impl

8
#define vint vint_impl

9
#define vuint vuint_impl

10
#define vllong vllong_impl

11
#define vfloat vfloat_impl

12
#define vdouble vdouble_impl

13


14
namespace embree

15
{

16
  /* 8-wide AVX-512 64-bit double type */

17
  template<>

18
  struct vdouble<8>

19
  {

20
    ALIGNED_STRUCT_(64);

21
    

22
    typedef vboold8 Bool;

23


24
    enum  { size = 8 }; // number of SIMD elements

25
    union {              // data

26
      __m512d v;

27
      double i[8];

28
    };

29


30
    ////////////////////////////////////////////////////////////////////////////////

31
    /// Constructors, Assignment & Cast Operators

32
    ////////////////////////////////////////////////////////////////////////////////

33


34
    __forceinline vdouble() {}

35
    __forceinline vdouble(const vdouble8& t) { v = t.v; }

36
    __forceinline vdouble8& operator =(const vdouble8& f) { v = f.v; return *this; }

37


38
    __forceinline vdouble(const __m512d& t) { v = t; }

39
    __forceinline operator __m512d() const { return v; }

40
    __forceinline operator __m256d() const { return _mm512_castpd512_pd256(v); }

41


42
    __forceinline vdouble(double i) {

43
      v = _mm512_set1_pd(i);

44
    }

45


46
    __forceinline vdouble(double a, double b, double c, double d) {

47
      v = _mm512_set4_pd(d,c,b,a);

48
    }

49


50
    __forceinline vdouble(double a0, double a1, double a2, double a3,

51
                          double a4, double a5, double a6, double a7)

52
    {

53
      v = _mm512_set_pd(a7,a6,a5,a4,a3,a2,a1,a0);

54
    }

55


56


57
    ////////////////////////////////////////////////////////////////////////////////

58
    /// Constants

59
    ////////////////////////////////////////////////////////////////////////////////

60


61
    __forceinline vdouble(ZeroTy) : v(_mm512_setzero_pd()) {}

62
    __forceinline vdouble(OneTy)  : v(_mm512_set1_pd(1)) {}

63
    __forceinline vdouble(StepTy) : v(_mm512_set_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {}

64
    __forceinline vdouble(ReverseStepTy) : v(_mm512_setr_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {}

65


66
    ////////////////////////////////////////////////////////////////////////////////

67
    /// Loads and Stores

68
    ////////////////////////////////////////////////////////////////////////////////

69


70
    static __forceinline void store_nt(void *__restrict__ ptr, const vdouble8& a) {

71
      _mm512_stream_pd((double*)ptr, a);

72
    }

73


74
    static __forceinline vdouble8 loadu(const void* addr) {

75
      return _mm512_loadu_pd((double*)addr);

76
    }

77


78
    static __forceinline vdouble8 load(const vdouble8* addr) {

79
      return _mm512_load_pd((double*)addr);

80
    }

81


82
    static __forceinline vdouble8 load(const double* addr) {

83
      return _mm512_load_pd(addr);

84
    }

85


86
    static __forceinline void store(void* ptr, const vdouble8& v) {

87
      _mm512_store_pd(ptr, v);

88
    }

89


90
    static __forceinline void storeu(void* ptr, const vdouble8& v) {

91
      _mm512_storeu_pd(ptr, v);

92
    }

93


94
    static __forceinline void storeu(const vboold8& mask, double* ptr, const vdouble8& f) {

95
      _mm512_mask_storeu_pd(ptr, mask, f);

96
    }

97


98
    static __forceinline void store(const vboold8& mask, void* addr, const vdouble8& v2) {

99
      _mm512_mask_store_pd(addr, mask, v2);

100
    }

101


102
    static __forceinline vdouble8 compact(const vboold8& mask, vdouble8& v) {

103
      return _mm512_mask_compress_pd(v, mask, v);

104
    }

105


106
    static __forceinline vdouble8 compact(const vboold8& mask, const vdouble8& a, vdouble8& b) {

107
      return _mm512_mask_compress_pd(a, mask, b);

108
    }

109


110
    static __forceinline vdouble8 broadcast(const void* a) { return _mm512_set1_pd(*(double*)a); }

111


112
    ////////////////////////////////////////////////////////////////////////////////

113
    /// Array Access

114
    ////////////////////////////////////////////////////////////////////////////////

115


116
    __forceinline       double& operator [](size_t index)       { assert(index < 8); return i[index]; }

117
    __forceinline const double& operator [](size_t index) const { assert(index < 8); return i[index]; }

118


119
  };

120


121
  ////////////////////////////////////////////////////////////////////////////////

122
  /// Unary Operators

123
  ////////////////////////////////////////////////////////////////////////////////

124


125
  __forceinline vdouble8 asDouble(const vllong8&  a) { return _mm512_castsi512_pd(a); }

126
  __forceinline vllong8  asLLong (const vdouble8& a) { return _mm512_castpd_si512(a); }

127


128
  __forceinline vdouble8 operator +(const vdouble8& a) { return a; }

129
  __forceinline vdouble8 operator -(const vdouble8& a) { return _mm512_sub_pd(_mm512_setzero_pd(), a); }

130


131
  ////////////////////////////////////////////////////////////////////////////////

132
  /// Binary Operators

133
  ////////////////////////////////////////////////////////////////////////////////

134


135
  __forceinline vdouble8 operator +(const vdouble8& a, const vdouble8& b) { return _mm512_add_pd(a, b); }

136
  __forceinline vdouble8 operator +(const vdouble8& a, double          b) { return a + vdouble8(b); }

137
  __forceinline vdouble8 operator +(double          a, const vdouble8& b) { return vdouble8(a) + b; }

138


139
  __forceinline vdouble8 operator -(const vdouble8& a, const vdouble8& b) { return _mm512_sub_pd(a, b); }

140
  __forceinline vdouble8 operator -(const vdouble8& a, double          b) { return a - vdouble8(b); }

141
  __forceinline vdouble8 operator -(double          a, const vdouble8& b) { return vdouble8(a) - b; }

142


143
  __forceinline vdouble8 operator *(const vdouble8& a, const vdouble8& b) { return _mm512_mul_pd(a, b); }

144
  __forceinline vdouble8 operator *(const vdouble8& a, double          b) { return a * vdouble8(b); }

145
  __forceinline vdouble8 operator *(double          a, const vdouble8& b) { return vdouble8(a) * b; }

146


147
  __forceinline vdouble8 operator &(const vdouble8& a, const vdouble8& b) { return _mm512_and_pd(a, b); }

148
  __forceinline vdouble8 operator &(const vdouble8& a, double          b) { return a & vdouble8(b); }

149
  __forceinline vdouble8 operator &(double          a, const vdouble8& b) { return vdouble8(a) & b; }

150


151
  __forceinline vdouble8 operator |(const vdouble8& a, const vdouble8& b) { return _mm512_or_pd(a, b); }

152
  __forceinline vdouble8 operator |(const vdouble8& a, double          b) { return a | vdouble8(b); }

153
  __forceinline vdouble8 operator |(double          a, const vdouble8& b) { return vdouble8(a) | b; }

154


155
  __forceinline vdouble8 operator ^(const vdouble8& a, const vdouble8& b) { return _mm512_xor_pd(a, b); }

156
  __forceinline vdouble8 operator ^(const vdouble8& a, double          b) { return a ^ vdouble8(b); }

157
  __forceinline vdouble8 operator ^(double          a, const vdouble8& b) { return vdouble8(a) ^ b; }

158


159
  __forceinline vdouble8 operator <<(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), n)); }

160
  __forceinline vdouble8 operator >>(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), n)); }

161


162
  __forceinline vdouble8 operator <<(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_sllv_epi64(_mm512_castpd_si512(a), n)); }

163
  __forceinline vdouble8 operator >>(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_srav_epi64(_mm512_castpd_si512(a), n)); }

164


165
  __forceinline vdouble8 sll (const vdouble8& a, const unsigned int b) { return  _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), b)); }

166
  __forceinline vdouble8 sra (const vdouble8& a, const unsigned int b) { return  _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), b)); }

167
  __forceinline vdouble8 srl (const vdouble8& a, const unsigned int b) { return  _mm512_castsi512_pd(_mm512_srli_epi64(_mm512_castpd_si512(a), b)); }

168


169
  __forceinline vdouble8 min(const vdouble8& a, const vdouble8& b) { return _mm512_min_pd(a, b); }

170
  __forceinline vdouble8 min(const vdouble8& a, double          b) { return min(a,vdouble8(b)); }

171
  __forceinline vdouble8 min(double          a, const vdouble8& b) { return min(vdouble8(a),b); }

172


173
  __forceinline vdouble8 max(const vdouble8& a, const vdouble8& b) { return _mm512_max_pd(a, b); }

174
  __forceinline vdouble8 max(const vdouble8& a, double          b) { return max(a,vdouble8(b)); }

175
  __forceinline vdouble8 max(double          a, const vdouble8& b) { return max(vdouble8(a),b); }

176


177
  __forceinline vdouble8 mask_add(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_add_pd(c,mask,a,b); }

178
  __forceinline vdouble8 mask_sub(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_sub_pd(c,mask,a,b); }

179


180
  __forceinline vdouble8 mask_and(const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_and_pd(c,m,a,b); }

181
  __forceinline vdouble8 mask_or (const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_or_pd(c,m,a,b); }

182


183
  ////////////////////////////////////////////////////////////////////////////////

184
  /// Ternary Operators

185
  ////////////////////////////////////////////////////////////////////////////////

186


187
  __forceinline vdouble8 madd (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmadd_pd(a,b,c); }

188
  __forceinline vdouble8 msub (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmsub_pd(a,b,c); }

189
  __forceinline vdouble8 nmadd(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmadd_pd(a,b,c); }

190
  __forceinline vdouble8 nmsub(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmsub_pd(a,b,c); }

191


192


193
  ////////////////////////////////////////////////////////////////////////////////

194
  /// Assignment Operators

195
  ////////////////////////////////////////////////////////////////////////////////

196


197
  __forceinline vdouble8& operator +=(vdouble8& a, const vdouble8& b) { return a = a + b; }

198
  __forceinline vdouble8& operator +=(vdouble8& a, double          b) { return a = a + b; }

199


200
  __forceinline vdouble8& operator -=(vdouble8& a, const vdouble8& b) { return a = a - b; }

201
  __forceinline vdouble8& operator -=(vdouble8& a, double          b) { return a = a - b; }

202


203
  __forceinline vdouble8& operator *=(vdouble8& a, const vdouble8& b) { return a = a * b; }

204
  __forceinline vdouble8& operator *=(vdouble8& a, double          b) { return a = a * b; }

205


206
  __forceinline vdouble8& operator &=(vdouble8& a, const vdouble8& b) { return a = a & b; }

207
  __forceinline vdouble8& operator &=(vdouble8& a, double          b) { return a = a & b; }

208


209
  __forceinline vdouble8& operator |=(vdouble8& a, const vdouble8& b) { return a = a | b; }

210
  __forceinline vdouble8& operator |=(vdouble8& a, double          b) { return a = a | b; }

211


212
  __forceinline vdouble8& operator <<=(vdouble8& a, const double b) { return a = a << b; }

213
  __forceinline vdouble8& operator >>=(vdouble8& a, const double b) { return a = a >> b; }

214


215


216
  ////////////////////////////////////////////////////////////////////////////////

217
  /// Comparison Operators + Select

218
  ////////////////////////////////////////////////////////////////////////////////

219


220
  __forceinline vboold8 operator ==(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); }

221
  __forceinline vboold8 operator ==(const vdouble8& a, double          b) { return a == vdouble8(b); }

222
  __forceinline vboold8 operator ==(double          a, const vdouble8& b) { return vdouble8(a) == b; }

223


224
  __forceinline vboold8 operator !=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); }

225
  __forceinline vboold8 operator !=(const vdouble8& a, double          b) { return a != vdouble8(b); }

226
  __forceinline vboold8 operator !=(double          a, const vdouble8& b) { return vdouble8(a) != b; }

227


228
  __forceinline vboold8 operator < (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); }

229
  __forceinline vboold8 operator < (const vdouble8& a, double          b) { return a <  vdouble8(b); }

230
  __forceinline vboold8 operator < (double          a, const vdouble8& b) { return vdouble8(a) <  b; }

231


232
  __forceinline vboold8 operator >=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); }

233
  __forceinline vboold8 operator >=(const vdouble8& a, double          b) { return a >= vdouble8(b); }

234
  __forceinline vboold8 operator >=(double          a, const vdouble8& b) { return vdouble8(a) >= b; }

235


236
  __forceinline vboold8 operator > (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); }

237
  __forceinline vboold8 operator > (const vdouble8& a, double          b) { return a >  vdouble8(b); }

238
  __forceinline vboold8 operator > (double          a, const vdouble8& b) { return vdouble8(a) >  b; }

239


240
  __forceinline vboold8 operator <=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); }

241
  __forceinline vboold8 operator <=(const vdouble8& a, double          b) { return a <= vdouble8(b); }

242
  __forceinline vboold8 operator <=(double          a, const vdouble8& b) { return vdouble8(a) <= b; }

243


244
  __forceinline vboold8 eq(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); }

245
  __forceinline vboold8 ne(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); }

246
  __forceinline vboold8 lt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); }

247
  __forceinline vboold8 ge(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); }

248
  __forceinline vboold8 gt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); }

249
  __forceinline vboold8 le(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); }

250


251
  __forceinline vboold8 eq(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_EQ); }

252
  __forceinline vboold8 ne(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_NE); }

253
  __forceinline vboold8 lt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LT); }

254
  __forceinline vboold8 ge(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GE); }

255
  __forceinline vboold8 gt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GT); }

256
  __forceinline vboold8 le(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LE); }

257


258
  __forceinline vdouble8 select(const vboold8& m, const vdouble8& t, const vdouble8& f) {

259
    return _mm512_mask_or_pd(f,m,t,t);

260
  }

261


262
  ////////////////////////////////////////////////////////////////////////////////

263
  // Movement/Shifting/Shuffling Functions

264
  ////////////////////////////////////////////////////////////////////////////////

265


266
  template<int i0, int i1>

267
  __forceinline vdouble8 shuffle(const vdouble8& v) {

268
    return _mm512_permute_pd(v, (i1 << 7) | (i0 << 6) | (i1 << 5) | (i0 << 4) | (i1 << 3) | (i0 << 2) | (i1 << 1) | i0);

269
  }

270


271
  template<int i>

272
  __forceinline vdouble8 shuffle(const vdouble8& v) {

273
    return shuffle<i, i>(v);

274
  }

275


276
  template<int i0, int i1, int i2, int i3>

277
  __forceinline vdouble8 shuffle(const vdouble8& v) {

278
    return _mm512_permutex_pd(v, _MM_SHUFFLE(i3, i2, i1, i0));

279
  }

280


281
  template<int i0, int i1>

282
  __forceinline vdouble8 shuffle4(const vdouble8& v) {

283
    return _mm512_shuffle_f64x2(v, v, _MM_SHUFFLE(i1*2+1, i1*2, i0*2+1, i0*2));

284
  }

285


286
  template<int i>

287
  __forceinline vdouble8 shuffle4(const vdouble8& v) {

288
    return shuffle4<i, i>(v);

289
  }

290
  

291
  template<int i>

292
  __forceinline vdouble8 align_shift_right(const vdouble8& a, const vdouble8& b) {

293
    return _mm512_castsi512_pd(_mm512_alignr_epi64(_mm512_castpd_si512(a), _mm512_castpd_si512(b), i));

294
  }

295


296
  __forceinline double toScalar(const vdouble8& v) {

297
    return _mm_cvtsd_f64(_mm512_castpd512_pd128(v));

298
  }

299


300
  ////////////////////////////////////////////////////////////////////////////////

301
  /// Reductions

302
  ////////////////////////////////////////////////////////////////////////////////

303


304
  __forceinline vdouble8 vreduce_add2(vdouble8 x) {                      return x + shuffle<1,0,3,2>(x); }

305
  __forceinline vdouble8 vreduce_add4(vdouble8 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }

306
  __forceinline vdouble8 vreduce_add (vdouble8 x) { x = vreduce_add4(x); return x + shuffle4<1,0>(x); }

307


308
  __forceinline vdouble8 vreduce_min2(vdouble8 x) {                      return min(x, shuffle<1,0,3,2>(x)); }

309
  __forceinline vdouble8 vreduce_min4(vdouble8 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }

310
  __forceinline vdouble8 vreduce_min (vdouble8 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0>(x)); }

311


312
  __forceinline vdouble8 vreduce_max2(vdouble8 x) {                      return max(x, shuffle<1,0,3,2>(x)); }

313
  __forceinline vdouble8 vreduce_max4(vdouble8 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }

314
  __forceinline vdouble8 vreduce_max (vdouble8 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0>(x)); }

315


316
  __forceinline double reduce_add(const vdouble8& v) { return toScalar(vreduce_add(v)); }

317
  __forceinline double reduce_min(const vdouble8& v) { return toScalar(vreduce_min(v)); }

318
  __forceinline double reduce_max(const vdouble8& v) { return toScalar(vreduce_max(v)); }

319


320
  ////////////////////////////////////////////////////////////////////////////////

321
  /// Memory load and store operations

322
  ////////////////////////////////////////////////////////////////////////////////

323


324
  __forceinline vdouble8 permute(const vdouble8& v, const vllong8& index) {

325
    return _mm512_permutexvar_pd(index, v);

326
  }

327


328
  __forceinline vdouble8 reverse(const vdouble8& a) {

329
    return permute(a, vllong8(reverse_step));

330
  }

331


332
  ////////////////////////////////////////////////////////////////////////////////

333
  /// Output Operators

334
  ////////////////////////////////////////////////////////////////////////////////

335


336
  __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble8& v)

337
  {

338
    cout << "<" << v[0];

339
    for (size_t i=1; i<8; i++) cout << ", " << v[i];

340
    cout << ">";

341
    return cout;

342
  }

343
}

344


345
#undef vboolf

346
#undef vboold

347
#undef vint

348
#undef vuint

349
#undef vllong

350
#undef vfloat

351
#undef vdouble

352


353