CoCalc -- StoreTile.h

GitHub Repository: PojavLauncherTeam/mesa
Path: blob/21.2-virgl/src/gallium/drivers/swr/rasterizer/memory/StoreTile.h
⁴⁵⁷⁴ views
1
/****************************************************************************
2
* Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
3
*
4
* Permission is hereby granted, free of charge, to any person obtaining a
5
* copy of this software and associated documentation files (the "Software"),
6
* to deal in the Software without restriction, including without limitation
7
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
* and/or sell copies of the Software, and to permit persons to whom the
9
* Software is furnished to do so, subject to the following conditions:
10
*
11
* The above copyright notice and this permission notice (including the next
12
* paragraph) shall be included in all copies or substantial portions of the
13
* Software.
14
*
15
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21
* IN THE SOFTWARE.
22
*
23
* @file StoreTile.h
24
*
25
* @brief Functionality for Store.
26
*
27
******************************************************************************/
28
#pragma once
29

30
#include "common/os.h"
31
#include "common/formats.h"
32
#include "core/context.h"
33
#include "core/rdtsc_core.h"
34
#include "core/format_conversion.h"
35

36
#include "memory/TilingFunctions.h"
37
#include "memory/Convert.h"
38
#include "memory/SurfaceState.h"
39
#include "core/multisample.h"
40

41
#include <array>
42
#include <sstream>
43

44
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
45

46
// Function pointer to different storing functions for color, depth, and stencil based on incoming formats.
47
typedef void(*PFN_STORE_TILES)(uint8_t*, SWR_SURFACE_STATE*, uint32_t, uint32_t, uint32_t);
48

49
//////////////////////////////////////////////////////////////////////////
50
/// Store Raster Tile Function Tables.
51
//////////////////////////////////////////////////////////////////////////
52
extern PFN_STORE_TILES sStoreTilesTableColor[SWR_TILE_MODE_COUNT][NUM_SWR_FORMATS];
53
extern PFN_STORE_TILES sStoreTilesTableDepth[SWR_TILE_MODE_COUNT][NUM_SWR_FORMATS];
54
extern PFN_STORE_TILES sStoreTilesTableStencil[SWR_TILE_MODE_COUNT][NUM_SWR_FORMATS];
55

56
void InitStoreTilesTable_Linear_1();
57
void InitStoreTilesTable_Linear_2();
58
void InitStoreTilesTable_TileX_1();
59
void InitStoreTilesTable_TileX_2();
60
void InitStoreTilesTable_TileY_1();
61
void InitStoreTilesTable_TileY_2();
62
void InitStoreTilesTable_TileW();
63
void InitStoreTilesTable();
64

65
//////////////////////////////////////////////////////////////////////////
66
/// StorePixels
67
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
68
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
69
/// @param ppDsts   - Array of destination pointers.  Each pointer is
70
///                   to a single row of at most 16B.
71
/// @tparam NumDests - Number of destination pointers.  Each pair of
72
///                    pointers is for a 16-byte column of two rows.
73
//////////////////////////////////////////////////////////////////////////
74
template <size_t PixelSize, size_t NumDests>
75
struct StorePixels
76
{
77
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests]) = delete;
78
};
79

80
//////////////////////////////////////////////////////////////////////////
81
/// StorePixels (32-bit pixel specialization)
82
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
83
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
84
/// @param ppDsts   - Array of destination pointers.  Each pointer is
85
///                   to a single row of at most 16B.
86
/// @tparam NumDests - Number of destination pointers.  Each pair of
87
///                    pointers is for a 16-byte column of two rows.
88
//////////////////////////////////////////////////////////////////////////
89
template <>
90
struct StorePixels<8, 2>
91
{
92
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[2])
93
    {
94
        // Each 4-pixel row is 4 bytes.
95
        const uint16_t* pPixSrc = (const uint16_t*)pSrc;
96

97
        // Unswizzle from SWR-Z order
98
        uint16_t* pRow = (uint16_t*)ppDsts[0];
99
        pRow[0] = pPixSrc[0];
100
        pRow[1] = pPixSrc[2];
101

102
        pRow = (uint16_t*)ppDsts[1];
103
        pRow[0] = pPixSrc[1];
104
        pRow[1] = pPixSrc[3];
105
    }
106
};
107

108
template <>
109
struct StorePixels<8, 4>
110
{
111
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[4])
112
    {
113
        // 8 x 2 bytes = 16 bytes, 16 pixels
114
        const uint16_t *pSrc16 = reinterpret_cast<const uint16_t *>(pSrc);
115

116
        uint16_t **ppDsts16 = reinterpret_cast<uint16_t **>(ppDsts);
117

118
        // Unswizzle from SWR-Z order
119
        ppDsts16[0][0] = pSrc16[0];     // 0 1
120
        ppDsts16[0][1] = pSrc16[2];     // 4 5
121

122
        ppDsts16[1][0] = pSrc16[1];     // 2 3
123
        ppDsts16[1][1] = pSrc16[3];     // 6 7
124

125
        ppDsts16[2][0] = pSrc16[4];     // 8 9
126
        ppDsts16[2][1] = pSrc16[6];     // C D
127

128
        ppDsts16[3][0] = pSrc16[5];     // A B
129
        ppDsts16[3][1] = pSrc16[7];     // E F
130
    }
131
};
132

133
//////////////////////////////////////////////////////////////////////////
134
/// StorePixels (32-bit pixel specialization)
135
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
136
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
137
/// @param ppDsts   - Array of destination pointers.  Each pointer is
138
///                   to a single row of at most 16B.
139
/// @tparam NumDests - Number of destination pointers.  Each pair of
140
///                    pointers is for a 16-byte column of two rows.
141
//////////////////////////////////////////////////////////////////////////
142
template <>
143
struct StorePixels<16, 2>
144
{
145
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[2])
146
    {
147
        // Each 4-pixel row is 8 bytes.
148
        const uint32_t* pPixSrc = (const uint32_t*)pSrc;
149

150
        // Unswizzle from SWR-Z order
151
        uint32_t* pRow = (uint32_t*)ppDsts[0];
152
        pRow[0] = pPixSrc[0];
153
        pRow[1] = pPixSrc[2];
154

155
        pRow = (uint32_t*)ppDsts[1];
156
        pRow[0] = pPixSrc[1];
157
        pRow[1] = pPixSrc[3];
158
    }
159
};
160

161
template <>
162
struct StorePixels<16, 4>
163
{
164
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[4])
165
    {
166
        // 8 x 4 bytes = 32 bytes, 16 pixels
167
        const uint32_t *pSrc32 = reinterpret_cast<const uint32_t *>(pSrc);
168

169
        uint32_t **ppDsts32 = reinterpret_cast<uint32_t **>(ppDsts);
170

171
        // Unswizzle from SWR-Z order
172
        ppDsts32[0][0] = pSrc32[0];     // 0 1
173
        ppDsts32[0][1] = pSrc32[2];     // 4 5
174

175
        ppDsts32[1][0] = pSrc32[1];     // 2 3
176
        ppDsts32[1][1] = pSrc32[3];     // 6 7
177

178
        ppDsts32[2][0] = pSrc32[4];     // 8 9
179
        ppDsts32[2][1] = pSrc32[6];     // C D
180

181
        ppDsts32[3][0] = pSrc32[5];     // A B
182
        ppDsts32[3][1] = pSrc32[7];     // E F
183
    }
184
};
185

186
//////////////////////////////////////////////////////////////////////////
187
/// StorePixels (32-bit pixel specialization)
188
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
189
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
190
/// @param ppDsts   - Array of destination pointers.  Each pointer is
191
///                   to a single row of at most 16B.
192
/// @tparam NumDests - Number of destination pointers.  Each pair of
193
///                    pointers is for a 16-byte column of two rows.
194
//////////////////////////////////////////////////////////////////////////
195
template <>
196
struct StorePixels<32, 2>
197
{
198
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[2])
199
    {
200
        // Each 4-pixel row is 16-bytes
201
        simd4scalari *pZRow01 = (simd4scalari*)pSrc;
202
        simd4scalari vQuad00 = SIMD128::load_si(pZRow01);
203
        simd4scalari vQuad01 = SIMD128::load_si(pZRow01 + 1);
204

205
        simd4scalari vRow00 = SIMD128::unpacklo_epi64(vQuad00, vQuad01);
206
        simd4scalari vRow10 = SIMD128::unpackhi_epi64(vQuad00, vQuad01);
207

208
        SIMD128::storeu_si((simd4scalari*)ppDsts[0], vRow00);
209
        SIMD128::storeu_si((simd4scalari*)ppDsts[1], vRow10);
210
    }
211
};
212

213
template <>
214
struct StorePixels<32, 4>
215
{
216
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[4])
217
    {
218
        // 4 x 16 bytes = 64 bytes, 16 pixels
219
        const simd4scalari *pSrc128 = reinterpret_cast<const simd4scalari *>(pSrc);
220

221
        simd4scalari **ppDsts128 = reinterpret_cast<simd4scalari **>(ppDsts);
222

223
        // Unswizzle from SWR-Z order
224
        simd4scalari quad0 = SIMD128::load_si(&pSrc128[0]);                        // 0 1 2 3
225
        simd4scalari quad1 = SIMD128::load_si(&pSrc128[1]);                        // 4 5 6 7
226
        simd4scalari quad2 = SIMD128::load_si(&pSrc128[2]);                        // 8 9 A B
227
        simd4scalari quad3 = SIMD128::load_si(&pSrc128[3]);                        // C D E F
228

229
        SIMD128::storeu_si(ppDsts128[0], SIMD128::unpacklo_epi64(quad0, quad1));   // 0 1 4 5
230
        SIMD128::storeu_si(ppDsts128[1], SIMD128::unpackhi_epi64(quad0, quad1));   // 2 3 6 7
231
        SIMD128::storeu_si(ppDsts128[2], SIMD128::unpacklo_epi64(quad2, quad3));   // 8 9 C D
232
        SIMD128::storeu_si(ppDsts128[3], SIMD128::unpackhi_epi64(quad2, quad3));   // A B E F
233
    }
234
};
235

236
//////////////////////////////////////////////////////////////////////////
237
/// StorePixels (32-bit pixel specialization)
238
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
239
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
240
/// @param ppDsts   - Array of destination pointers.  Each pointer is
241
///                   to a single row of at most 16B.
242
/// @tparam NumDests - Number of destination pointers.  Each pair of
243
///                    pointers is for a 16-byte column of two rows.
244
//////////////////////////////////////////////////////////////////////////
245
template <>
246
struct StorePixels<64, 4>
247
{
248
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[4])
249
    {
250
        // Each 4-pixel row is 32 bytes.
251
        const simd4scalari* pPixSrc = (const simd4scalari*)pSrc;
252

253
        // order of pointers match SWR-Z layout
254
        simd4scalari** pvDsts = (simd4scalari**)&ppDsts[0];
255
        *pvDsts[0] = pPixSrc[0];
256
        *pvDsts[1] = pPixSrc[1];
257
        *pvDsts[2] = pPixSrc[2];
258
        *pvDsts[3] = pPixSrc[3];
259
    }
260
};
261

262
template <>
263
struct StorePixels<64, 8>
264
{
265
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[8])
266
    {
267
        // 8 x 16 bytes = 128 bytes, 16 pixels
268
        const simd4scalari *pSrc128 = reinterpret_cast<const simd4scalari *>(pSrc);
269

270
        simd4scalari **ppDsts128 = reinterpret_cast<simd4scalari **>(ppDsts);
271

272
        // order of pointers match SWR-Z layout
273
        *ppDsts128[0] = pSrc128[0];     // 0 1
274
        *ppDsts128[1] = pSrc128[1];     // 2 3
275
        *ppDsts128[2] = pSrc128[2];     // 4 5
276
        *ppDsts128[3] = pSrc128[3];     // 6 7
277
        *ppDsts128[4] = pSrc128[4];     // 8 9
278
        *ppDsts128[5] = pSrc128[5];     // A B
279
        *ppDsts128[6] = pSrc128[6];     // C D
280
        *ppDsts128[7] = pSrc128[7];     // E F
281
    }
282
};
283

284
//////////////////////////////////////////////////////////////////////////
285
/// StorePixels (32-bit pixel specialization)
286
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
287
/// @param pSrc     - Pointer to source raster tile in SWRZ pixel order
288
/// @param ppDsts   - Array of destination pointers.  Each pointer is
289
///                   to a single row of at most 16B.
290
/// @tparam NumDests - Number of destination pointers.  Each pair of
291
///                    pointers is for a 16-byte column of two rows.
292
//////////////////////////////////////////////////////////////////////////
293
template <>
294
struct StorePixels<128, 8>
295
{
296
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[8])
297
    {
298
        // Each 4-pixel row is 64 bytes.
299
        const simd4scalari* pPixSrc = (const simd4scalari*)pSrc;
300

301
        // Unswizzle from SWR-Z order
302
        simd4scalari** pvDsts = (simd4scalari**)&ppDsts[0];
303
        *pvDsts[0] = pPixSrc[0];
304
        *pvDsts[1] = pPixSrc[2];
305
        *pvDsts[2] = pPixSrc[1];
306
        *pvDsts[3] = pPixSrc[3];
307
        *pvDsts[4] = pPixSrc[4];
308
        *pvDsts[5] = pPixSrc[6];
309
        *pvDsts[6] = pPixSrc[5];
310
        *pvDsts[7] = pPixSrc[7];
311
    }
312
};
313

314
template <>
315
struct StorePixels<128, 16>
316
{
317
    static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[16])
318
    {
319
        // 16 x 16 bytes = 256 bytes, 16 pixels
320
        const simd4scalari *pSrc128 = reinterpret_cast<const simd4scalari *>(pSrc);
321

322
        simd4scalari **ppDsts128 = reinterpret_cast<simd4scalari **>(ppDsts);
323

324
        for (uint32_t i = 0; i < 16; i += 4)
325
        {
326
            *ppDsts128[i + 0] = pSrc128[i + 0];
327
            *ppDsts128[i + 1] = pSrc128[i + 2];
328
            *ppDsts128[i + 2] = pSrc128[i + 1];
329
            *ppDsts128[i + 3] = pSrc128[i + 3];
330
        }
331
    }
332
};
333

334
//////////////////////////////////////////////////////////////////////////
335
/// ConvertPixelsSOAtoAOS - Conversion for SIMD pixel (4x2 or 2x2)
336
//////////////////////////////////////////////////////////////////////////
337
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
338
struct ConvertPixelsSOAtoAOS
339
{
340
    //////////////////////////////////////////////////////////////////////////
341
    /// @brief Converts a SIMD from the Hot Tile to the destination format
342
    ///        and converts from SOA to AOS.
343
    /// @param pSrc - Pointer to raster tile.
344
    /// @param pDst - Pointer to destination surface or deswizzling buffer.
345
    template <size_t NumDests>
346
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
347
    {
348
        static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 16; // 16 pixels * 16 bytes per pixel
349

350
        OSALIGNSIMD16(uint8_t) soaTile[MAX_RASTER_TILE_BYTES] = {0};
351
        OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES] = {0};
352

353
        // Convert from SrcFormat --> DstFormat
354
        simd16vector src;
355
        LoadSOA<SrcFormat>(pSrc, src);
356
        StoreSOA<DstFormat>(src, soaTile);
357

358
        // Convert from SOA --> AOS
359
        FormatTraits<DstFormat>::TransposeT::Transpose_simd16(soaTile, aosTile);
360

361
        // Store data into destination
362
        StorePixels<FormatTraits<DstFormat>::bpp, NumDests>::Store(aosTile, ppDsts);
363
    }
364
};
365

366
//////////////////////////////////////////////////////////////////////////
367
/// ConvertPixelsSOAtoAOS - Conversion for SIMD pixel (4x2 or 2x2)
368
/// Specialization for no format conversion
369
//////////////////////////////////////////////////////////////////////////
370
template<SWR_FORMAT Format>
371
struct ConvertPixelsSOAtoAOS<Format, Format>
372
{
373
    //////////////////////////////////////////////////////////////////////////
374
    /// @brief Converts a SIMD from the Hot Tile to the destination format
375
    ///        and converts from SOA to AOS.
376
    /// @param pSrc - Pointer to raster tile.
377
    /// @param pDst - Pointer to destination surface or deswizzling buffer.
378
    template <size_t NumDests>
379
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
380
    {
381
        static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 16; // 16 pixels * 16 bytes per pixel
382

383
        OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
384

385
        // Convert from SOA --> AOS
386
        FormatTraits<Format>::TransposeT::Transpose_simd16(pSrc, aosTile);
387

388
        // Store data into destination
389
        StorePixels<FormatTraits<Format>::bpp, NumDests>::Store(aosTile, ppDsts);
390
    }
391
};
392

393
//////////////////////////////////////////////////////////////////////////
394
/// ConvertPixelsSOAtoAOS - Specialization conversion for B5G6R6_UNORM
395
//////////////////////////////////////////////////////////////////////////
396
template<>
397
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, B5G6R5_UNORM >
398
{
399
    //////////////////////////////////////////////////////////////////////////
400
    /// @brief Converts a SIMD from the Hot Tile to the destination format
401
    ///        and converts from SOA to AOS.
402
    /// @param pSrc - Pointer to raster tile.
403
    /// @param pDst - Pointer to destination surface or deswizzling buffer.
404
    template <size_t NumDests>
405
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
406
    {
407
        static const SWR_FORMAT SrcFormat = R32G32B32A32_FLOAT;
408
        static const SWR_FORMAT DstFormat = B5G6R5_UNORM;
409

410
        static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 16; // 16 pixels * 16 bytes per pixel
411

412
        OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
413

414
        // Load hot-tile
415
        simd16vector src, dst;
416
        LoadSOA<SrcFormat>(pSrc, src);
417

418
        // deswizzle
419
        dst.x = src[FormatTraits<DstFormat>::swizzle(0)];
420
        dst.y = src[FormatTraits<DstFormat>::swizzle(1)];
421
        dst.z = src[FormatTraits<DstFormat>::swizzle(2)];
422

423
        // clamp
424
        dst.x = Clamp<DstFormat>(dst.x, 0);
425
        dst.y = Clamp<DstFormat>(dst.y, 1);
426
        dst.z = Clamp<DstFormat>(dst.z, 2);
427

428
        // normalize
429
        dst.x = Normalize<DstFormat>(dst.x, 0);
430
        dst.y = Normalize<DstFormat>(dst.y, 1);
431
        dst.z = Normalize<DstFormat>(dst.z, 2);
432

433
        // pack
434
        simd16scalari packed = _simd16_castps_si(dst.x);
435

436
        SWR_ASSERT(FormatTraits<DstFormat>::GetBPC(0) == 5);
437
        SWR_ASSERT(FormatTraits<DstFormat>::GetBPC(1) == 6);
438

439
        packed = _simd16_or_si(packed, _simd16_slli_epi32(_simd16_castps_si(dst.y), 5));
440
        packed = _simd16_or_si(packed, _simd16_slli_epi32(_simd16_castps_si(dst.z), 5 + 6));
441

442
        // pack low 16 bits of each 32 bit lane to low 128 bits of dst
443
        uint32_t *pPacked = (uint32_t*)&packed;
444
        uint16_t *pAosTile = (uint16_t*)&aosTile[0];
445
        for (uint32_t t = 0; t < KNOB_SIMD16_WIDTH; ++t)
446
        {
447
            *pAosTile++ = *pPacked++;
448
        }
449

450
        // Store data into destination
451
        StorePixels<FormatTraits<DstFormat>::bpp, NumDests>::Store(aosTile, ppDsts);
452
    }
453
};
454

455
//////////////////////////////////////////////////////////////////////////
456
/// ConvertPixelsSOAtoAOS - Conversion for SIMD pixel (4x2 or 2x2)
457
//////////////////////////////////////////////////////////////////////////
458
template<>
459
struct ConvertPixelsSOAtoAOS<R32_FLOAT, R24_UNORM_X8_TYPELESS>
460
{
461
    static const SWR_FORMAT SrcFormat = R32_FLOAT;
462
    static const SWR_FORMAT DstFormat = R24_UNORM_X8_TYPELESS;
463

464
    //////////////////////////////////////////////////////////////////////////
465
    /// @brief Converts a SIMD from the Hot Tile to the destination format
466
    ///        and converts from SOA to AOS.
467
    /// @param pSrc - Pointer to raster tile.
468
    /// @param pDst - Pointer to destination surface or deswizzling buffer.
469
    template <size_t NumDests>
470
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
471
    {
472
        simd16scalar comp = _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
473

474
        // clamp
475
        const simd16scalar zero = _simd16_setzero_ps();
476
        const simd16scalar ones = _simd16_set1_ps(1.0f);
477

478
        comp = _simd16_max_ps(comp, zero);
479
        comp = _simd16_min_ps(comp, ones);
480

481
        // normalize
482
        comp = _simd16_mul_ps(comp, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
483

484
        simd16scalari temp = _simd16_cvtps_epi32(comp);
485

486
        // swizzle
487
        temp = _simd16_permute_epi32(temp, _simd16_set_epi32(15, 14, 11, 10, 13, 12, 9, 8, 7, 6, 3, 2, 5, 4, 1, 0));
488

489
        // merge/store data into destination but don't overwrite the X8 bits
490
        simdscalari destlo = _simd_loadu2_si(reinterpret_cast<simd4scalari *>(ppDsts[1]), reinterpret_cast<simd4scalari *>(ppDsts[0]));
491
        simdscalari desthi = _simd_loadu2_si(reinterpret_cast<simd4scalari *>(ppDsts[3]), reinterpret_cast<simd4scalari *>(ppDsts[2]));
492

493
        simd16scalari dest = _simd16_setzero_si();
494

495
        dest = _simd16_insert_si(dest, destlo, 0);
496
        dest = _simd16_insert_si(dest, desthi, 1);
497

498
        simd16scalari mask = _simd16_set1_epi32(0x00FFFFFF);
499

500
        dest = _simd16_or_si(_simd16_andnot_si(mask, dest), _simd16_and_si(mask, temp));
501

502
        _simd_storeu2_si(reinterpret_cast<simd4scalari *>(ppDsts[1]), reinterpret_cast<simd4scalari *>(ppDsts[0]), _simd16_extract_si(dest, 0));
503
        _simd_storeu2_si(reinterpret_cast<simd4scalari *>(ppDsts[3]), reinterpret_cast<simd4scalari *>(ppDsts[2]), _simd16_extract_si(dest, 1));
504
    }
505
};
506

507
template<SWR_FORMAT DstFormat>
508
INLINE static void FlatConvert(const uint8_t* pSrc, uint8_t* pDst0, uint8_t* pDst1, uint8_t* pDst2, uint8_t* pDst3)
509
{
510
    // swizzle rgba -> bgra while we load
511
    simd16scalar comp0 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(0) * sizeof(simd16scalar))); // float32 rrrrrrrrrrrrrrrr
512
    simd16scalar comp1 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(1) * sizeof(simd16scalar))); // float32 gggggggggggggggg
513
    simd16scalar comp2 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(2) * sizeof(simd16scalar))); // float32 bbbbbbbbbbbbbbbb
514
    simd16scalar comp3 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(3) * sizeof(simd16scalar))); // float32 aaaaaaaaaaaaaaaa
515

516
    // clamp
517
    const simd16scalar zero = _simd16_setzero_ps();
518
    const simd16scalar ones = _simd16_set1_ps(1.0f);
519

520
    comp0 = _simd16_max_ps(comp0, zero);
521
    comp0 = _simd16_min_ps(comp0, ones);
522

523
    comp1 = _simd16_max_ps(comp1, zero);
524
    comp1 = _simd16_min_ps(comp1, ones);
525

526
    comp2 = _simd16_max_ps(comp2, zero);
527
    comp2 = _simd16_min_ps(comp2, ones);
528

529
    comp3 = _simd16_max_ps(comp3, zero);
530
    comp3 = _simd16_min_ps(comp3, ones);
531

532
    // gamma-correct only rgb
533
    if (FormatTraits<DstFormat>::isSRGB)
534
    {
535
        comp0 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(0, comp0);
536
        comp1 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(1, comp1);
537
        comp2 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(2, comp2);
538
    }
539

540
    // convert float components from 0.0f..1.0f to correct scale for 0..255 dest format
541
    comp0 = _simd16_mul_ps(comp0, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
542
    comp1 = _simd16_mul_ps(comp1, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(1)));
543
    comp2 = _simd16_mul_ps(comp2, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(2)));
544
    comp3 = _simd16_mul_ps(comp3, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(3)));
545

546
    // moving to 16 wide integer vector types
547
    simd16scalari src0 = _simd16_cvtps_epi32(comp0); // padded byte rrrrrrrrrrrrrrrr
548
    simd16scalari src1 = _simd16_cvtps_epi32(comp1); // padded byte gggggggggggggggg
549
    simd16scalari src2 = _simd16_cvtps_epi32(comp2); // padded byte bbbbbbbbbbbbbbbb
550
    simd16scalari src3 = _simd16_cvtps_epi32(comp3); // padded byte aaaaaaaaaaaaaaaa
551

552
    // SOA to AOS conversion
553
    src1 = _simd16_slli_epi32(src1,  8);
554
    src2 = _simd16_slli_epi32(src2, 16);
555
    src3 = _simd16_slli_epi32(src3, 24);
556

557
    simd16scalari final = _simd16_or_si(_simd16_or_si(src0, src1), _simd16_or_si(src2, src3));  // 0 1 2 3 4 5 6 7 8 9 A B C D E F
558

559
    // de-swizzle conversion
560
#if 1
561
    simd16scalari final0 = _simd16_permute2f128_si(final, final, 0xA0); // (2, 2, 0, 0)         // 0 1 2 3 0 1 2 3 8 9 A B 8 9 A B
562
    simd16scalari final1 = _simd16_permute2f128_si(final, final, 0xF5); // (3, 3, 1, 1)         // 4 5 6 7 4 5 6 7 C D E F C D E F
563

564
    final = _simd16_shuffle_epi64(final0, final1, 0xCC); // (1 1 0 0 1 1 0 0)                   // 0 1 4 5 2 3 6 7 8 9 C D A B E F
565

566
#else
567
    final = _simd16_permute_epi32(final, _simd16_set_epi32(15, 14, 11, 10, 13, 12, 9, 8, 7, 6, 3, 2, 5, 4, 1, 0));
568

569
#endif
570
    // store 8x2 memory order:
571
    //  row0: [ pDst0, pDst2 ] = { 0 1 4 5 }, { 8 9 C D }
572
    //  row1: [ pDst1, pDst3 ] = { 2 3 6 7 }, { A B E F }
573
    _simd_storeu2_si(reinterpret_cast<simd4scalari *>(pDst1), reinterpret_cast<simd4scalari *>(pDst0), _simd16_extract_si(final, 0));
574
    _simd_storeu2_si(reinterpret_cast<simd4scalari *>(pDst3), reinterpret_cast<simd4scalari *>(pDst2), _simd16_extract_si(final, 1));
575
}
576

577
template<SWR_FORMAT DstFormat>
578
INLINE static void FlatConvert(const uint8_t* pSrc, uint8_t* pDst, uint8_t* pDst1)
579
{
580
    static const uint32_t offset = sizeof(simdscalar);
581

582
    // swizzle rgba -> bgra while we load
583
    simdscalar vComp0 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(0))*offset)); // float32 rrrrrrrr
584
    simdscalar vComp1 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(1))*offset)); // float32 gggggggg
585
    simdscalar vComp2 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(2))*offset)); // float32 bbbbbbbb
586
    simdscalar vComp3 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(3))*offset)); // float32 aaaaaaaa
587

588
    // clamp
589
    vComp0 = _simd_max_ps(vComp0, _simd_setzero_ps());
590
    vComp0 = _simd_min_ps(vComp0, _simd_set1_ps(1.0f));
591

592
    vComp1 = _simd_max_ps(vComp1, _simd_setzero_ps());
593
    vComp1 = _simd_min_ps(vComp1, _simd_set1_ps(1.0f));
594

595
    vComp2 = _simd_max_ps(vComp2, _simd_setzero_ps());
596
    vComp2 = _simd_min_ps(vComp2, _simd_set1_ps(1.0f));
597

598
    vComp3 = _simd_max_ps(vComp3, _simd_setzero_ps());
599
    vComp3 = _simd_min_ps(vComp3, _simd_set1_ps(1.0f));
600

601
    if (FormatTraits<DstFormat>::isSRGB)
602
    {
603
        // Gamma-correct only rgb
604
        vComp0 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(0, vComp0);
605
        vComp1 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(1, vComp1);
606
        vComp2 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(2, vComp2);
607
    }
608

609
    // convert float components from 0.0f .. 1.0f to correct scale for 0 .. 255 dest format
610
    vComp0 = _simd_mul_ps(vComp0, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
611
    vComp1 = _simd_mul_ps(vComp1, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(1)));
612
    vComp2 = _simd_mul_ps(vComp2, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(2)));
613
    vComp3 = _simd_mul_ps(vComp3, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(3)));
614

615
    // moving to 8 wide integer vector types
616
    simdscalari src0 = _simd_cvtps_epi32(vComp0); // padded byte rrrrrrrr
617
    simdscalari src1 = _simd_cvtps_epi32(vComp1); // padded byte gggggggg
618
    simdscalari src2 = _simd_cvtps_epi32(vComp2); // padded byte bbbbbbbb
619
    simdscalari src3 = _simd_cvtps_epi32(vComp3); // padded byte aaaaaaaa
620

621
#if KNOB_ARCH <= KNOB_ARCH_AVX
622

623
    // splitting into two sets of 4 wide integer vector types
624
    // because AVX doesn't have instructions to support this operation at 8 wide
625
    simd4scalari srcLo0 = _mm256_castsi256_si128(src0); // 000r000r000r000r
626
    simd4scalari srcLo1 = _mm256_castsi256_si128(src1); // 000g000g000g000g
627
    simd4scalari srcLo2 = _mm256_castsi256_si128(src2); // 000b000b000b000b
628
    simd4scalari srcLo3 = _mm256_castsi256_si128(src3); // 000a000a000a000a
629

630
    simd4scalari srcHi0 = _mm256_extractf128_si256(src0, 1); // 000r000r000r000r
631
    simd4scalari srcHi1 = _mm256_extractf128_si256(src1, 1); // 000g000g000g000g
632
    simd4scalari srcHi2 = _mm256_extractf128_si256(src2, 1); // 000b000b000b000b
633
    simd4scalari srcHi3 = _mm256_extractf128_si256(src3, 1); // 000a000a000a000a
634

635
    srcLo1 = _mm_slli_si128(srcLo1, 1); // 00g000g000g000g0
636
    srcHi1 = _mm_slli_si128(srcHi1, 1); // 00g000g000g000g0
637
    srcLo2 = _mm_slli_si128(srcLo2, 2); // 0b000b000b000b00
638
    srcHi2 = _mm_slli_si128(srcHi2, 2); // 0b000b000b000b00
639
    srcLo3 = _mm_slli_si128(srcLo3, 3); // a000a000a000a000
640
    srcHi3 = _mm_slli_si128(srcHi3, 3); // a000a000a000a000
641

642
    srcLo0 = SIMD128::or_si(srcLo0, srcLo1); // 00gr00gr00gr00gr
643
    srcLo2 = SIMD128::or_si(srcLo2, srcLo3); // ab00ab00ab00ab00
644

645
    srcHi0 = SIMD128::or_si(srcHi0, srcHi1); // 00gr00gr00gr00gr
646
    srcHi2 = SIMD128::or_si(srcHi2, srcHi3); // ab00ab00ab00ab00
647

648
    srcLo0 = SIMD128::or_si(srcLo0, srcLo2); // abgrabgrabgrabgr
649
    srcHi0 = SIMD128::or_si(srcHi0, srcHi2); // abgrabgrabgrabgr
650

651
    // unpack into rows that get the tiling order correct
652
    simd4scalari vRow00 = SIMD128::unpacklo_epi64(srcLo0, srcHi0);  // abgrabgrabgrabgrabgrabgrabgrabgr
653
    simd4scalari vRow10 = SIMD128::unpackhi_epi64(srcLo0, srcHi0);
654

655
    simdscalari final = _mm256_castsi128_si256(vRow00);
656
    final = _mm256_insertf128_si256(final, vRow10, 1);
657

658
#else
659

660
    // logic is as above, only wider
661
    src1 = _mm256_slli_si256(src1, 1);
662
    src2 = _mm256_slli_si256(src2, 2);
663
    src3 = _mm256_slli_si256(src3, 3);
664

665
    src0 = _mm256_or_si256(src0, src1);
666
    src2 = _mm256_or_si256(src2, src3);
667

668
    simdscalari final = _mm256_or_si256(src0, src2);
669

670
    // adjust the data to get the tiling order correct 0 1 2 3 -> 0 2 1 3
671
    final = _mm256_permute4x64_epi64(final, 0xD8);
672
#endif
673

674
    _simd_storeu2_si((simd4scalari*)pDst1, (simd4scalari*)pDst, final);
675
}
676

677
template<SWR_FORMAT DstFormat>
678
INLINE static void FlatConvertNoAlpha(const uint8_t* pSrc, uint8_t* pDst0, uint8_t* pDst1, uint8_t* pDst2, uint8_t* pDst3)
679
{
680
    // swizzle rgba -> bgra while we load
681
    simd16scalar comp0 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(0) * sizeof(simd16scalar))); // float32 rrrrrrrrrrrrrrrr
682
    simd16scalar comp1 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(1) * sizeof(simd16scalar))); // float32 gggggggggggggggg
683
    simd16scalar comp2 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(2) * sizeof(simd16scalar))); // float32 bbbbbbbbbbbbbbbb
684

685
    // clamp
686
    const simd16scalar zero = _simd16_setzero_ps();
687
    const simd16scalar ones = _simd16_set1_ps(1.0f);
688

689
    comp0 = _simd16_max_ps(comp0, zero);
690
    comp0 = _simd16_min_ps(comp0, ones);
691

692
    comp1 = _simd16_max_ps(comp1, zero);
693
    comp1 = _simd16_min_ps(comp1, ones);
694

695
    comp2 = _simd16_max_ps(comp2, zero);
696
    comp2 = _simd16_min_ps(comp2, ones);
697

698
    // gamma-correct only rgb
699
    if (FormatTraits<DstFormat>::isSRGB)
700
    {
701
        comp0 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(0, comp0);
702
        comp1 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(1, comp1);
703
        comp2 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(2, comp2);
704
    }
705

706
    // convert float components from 0.0f..1.0f to correct scale for 0..255 dest format
707
    comp0 = _simd16_mul_ps(comp0, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
708
    comp1 = _simd16_mul_ps(comp1, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(1)));
709
    comp2 = _simd16_mul_ps(comp2, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(2)));
710

711
    // moving to 16 wide integer vector types
712
    simd16scalari src0 = _simd16_cvtps_epi32(comp0); // padded byte rrrrrrrrrrrrrrrr
713
    simd16scalari src1 = _simd16_cvtps_epi32(comp1); // padded byte gggggggggggggggg
714
    simd16scalari src2 = _simd16_cvtps_epi32(comp2); // padded byte bbbbbbbbbbbbbbbb
715

716
    // SOA to AOS conversion
717
    src1 = _simd16_slli_epi32(src1,  8);
718
    src2 = _simd16_slli_epi32(src2, 16);
719

720
    simd16scalari final = _simd16_or_si(_simd16_or_si(src0, src1), src2);                       // 0 1 2 3 4 5 6 7 8 9 A B C D E F
721

722
    // de-swizzle conversion
723
#if 1
724
    simd16scalari final0 = _simd16_permute2f128_si(final, final, 0xA0); // (2, 2, 0, 0)         // 0 1 2 3 0 1 2 3 8 9 A B 8 9 A B
725
    simd16scalari final1 = _simd16_permute2f128_si(final, final, 0xF5); // (3, 3, 1, 1)         // 4 5 6 7 4 5 6 7 C D E F C D E F
726

727
    final = _simd16_shuffle_epi64(final0, final1, 0xCC); // (1 1 0 0 1 1 0 0)                   // 0 1 4 5 2 3 6 7 8 9 C D A B E F
728

729
#else
730
    final = _simd16_permute_epi32(final, _simd16_set_epi32(15, 14, 11, 10, 13, 12, 9, 8, 7, 6, 3, 2, 5, 4, 1, 0));
731

732
#endif
733
    // store 8x2 memory order:
734
    //  row0: [ pDst0, pDst2 ] = { 0 1 4 5 }, { 8 9 C D }
735
    //  row1: [ pDst1, pDst3 ] = { 2 3 6 7 }, { A B E F }
736
    _simd_storeu2_si(reinterpret_cast<simd4scalari *>(pDst1), reinterpret_cast<simd4scalari *>(pDst0), _simd16_extract_si(final, 0));
737
    _simd_storeu2_si(reinterpret_cast<simd4scalari *>(pDst3), reinterpret_cast<simd4scalari *>(pDst2), _simd16_extract_si(final, 1));
738
}
739

740
template<SWR_FORMAT DstFormat>
741
INLINE static void FlatConvertNoAlpha(const uint8_t* pSrc, uint8_t* pDst, uint8_t* pDst1)
742
{
743
    static const uint32_t offset = sizeof(simdscalar);
744

745
    // swizzle rgba -> bgra while we load
746
    simdscalar vComp0 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(0))*offset)); // float32 rrrrrrrr
747
    simdscalar vComp1 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(1))*offset)); // float32 gggggggg
748
    simdscalar vComp2 = _simd_load_ps((const float*)(pSrc + (FormatTraits<DstFormat>::swizzle(2))*offset)); // float32 bbbbbbbb
749
                                                                                                            // clamp
750
    vComp0 = _simd_max_ps(vComp0, _simd_setzero_ps());
751
    vComp0 = _simd_min_ps(vComp0, _simd_set1_ps(1.0f));
752

753
    vComp1 = _simd_max_ps(vComp1, _simd_setzero_ps());
754
    vComp1 = _simd_min_ps(vComp1, _simd_set1_ps(1.0f));
755

756
    vComp2 = _simd_max_ps(vComp2, _simd_setzero_ps());
757
    vComp2 = _simd_min_ps(vComp2, _simd_set1_ps(1.0f));
758

759
    if (FormatTraits<DstFormat>::isSRGB)
760
    {
761
        // Gamma-correct only rgb
762
        vComp0 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(0, vComp0);
763
        vComp1 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(1, vComp1);
764
        vComp2 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(2, vComp2);
765
    }
766

767
    // convert float components from 0.0f .. 1.0f to correct scale for 0 .. 255 dest format
768
    vComp0 = _simd_mul_ps(vComp0, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
769
    vComp1 = _simd_mul_ps(vComp1, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(1)));
770
    vComp2 = _simd_mul_ps(vComp2, _simd_set1_ps(FormatTraits<DstFormat>::fromFloat(2)));
771

772
    // moving to 8 wide integer vector types
773
    simdscalari src0 = _simd_cvtps_epi32(vComp0); // padded byte rrrrrrrr
774
    simdscalari src1 = _simd_cvtps_epi32(vComp1); // padded byte gggggggg
775
    simdscalari src2 = _simd_cvtps_epi32(vComp2); // padded byte bbbbbbbb
776

777
#if KNOB_ARCH <= KNOB_ARCH_AVX
778

779
    // splitting into two sets of 4 wide integer vector types
780
    // because AVX doesn't have instructions to support this operation at 8 wide
781
    simd4scalari srcLo0 = _mm256_castsi256_si128(src0); // 000r000r000r000r
782
    simd4scalari srcLo1 = _mm256_castsi256_si128(src1); // 000g000g000g000g
783
    simd4scalari srcLo2 = _mm256_castsi256_si128(src2); // 000b000b000b000b
784

785
    simd4scalari srcHi0 = _mm256_extractf128_si256(src0, 1); // 000r000r000r000r
786
    simd4scalari srcHi1 = _mm256_extractf128_si256(src1, 1); // 000g000g000g000g
787
    simd4scalari srcHi2 = _mm256_extractf128_si256(src2, 1); // 000b000b000b000b
788

789
    srcLo1 = _mm_slli_si128(srcLo1, 1); // 00g000g000g000g0
790
    srcHi1 = _mm_slli_si128(srcHi1, 1); // 00g000g000g000g0
791
    srcLo2 = _mm_slli_si128(srcLo2, 2); // 0b000b000b000b00
792
    srcHi2 = _mm_slli_si128(srcHi2, 2); // 0b000b000b000b00
793

794
    srcLo0 = SIMD128::or_si(srcLo0, srcLo1); // 00gr00gr00gr00gr
795

796
    srcHi0 = SIMD128::or_si(srcHi0, srcHi1); // 00gr00gr00gr00gr
797

798
    srcLo0 = SIMD128::or_si(srcLo0, srcLo2); // 0bgr0bgr0bgr0bgr
799
    srcHi0 = SIMD128::or_si(srcHi0, srcHi2); // 0bgr0bgr0bgr0bgr
800

801
    // unpack into rows that get the tiling order correct
802
    simd4scalari vRow00 = SIMD128::unpacklo_epi64(srcLo0, srcHi0);  // 0bgr0bgr0bgr0bgr0bgr0bgr0bgr0bgr
803
    simd4scalari vRow10 = SIMD128::unpackhi_epi64(srcLo0, srcHi0);
804

805
    simdscalari final = _mm256_castsi128_si256(vRow00);
806
    final = _mm256_insertf128_si256(final, vRow10, 1);
807

808
#else
809

810
                                              // logic is as above, only wider
811
    src1 = _mm256_slli_si256(src1, 1);
812
    src2 = _mm256_slli_si256(src2, 2);
813

814
    src0 = _mm256_or_si256(src0, src1);
815

816
    simdscalari final = _mm256_or_si256(src0, src2);
817

818
    // adjust the data to get the tiling order correct 0 1 2 3 -> 0 2 1 3
819
    final = _mm256_permute4x64_epi64(final, 0xD8);
820

821
#endif
822

823
    _simd_storeu2_si((simd4scalari*)pDst1, (simd4scalari*)pDst, final);
824
}
825

826
template<>
827
struct ConvertPixelsSOAtoAOS<R32G32B32A32_FLOAT, B8G8R8A8_UNORM>
828
{
829
    template <size_t NumDests>
830
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
831
    {
832
        FlatConvert<B8G8R8A8_UNORM>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
833
    }
834
};
835

836
template<>
837
struct ConvertPixelsSOAtoAOS<R32G32B32A32_FLOAT, B8G8R8X8_UNORM>
838
{
839
    template <size_t NumDests>
840
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
841
    {
842
        FlatConvertNoAlpha<B8G8R8X8_UNORM>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
843
    }
844
};
845

846
template<>
847
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, B8G8R8A8_UNORM_SRGB >
848
{
849
    template <size_t NumDests>
850
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
851
    {
852
        FlatConvert<B8G8R8A8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
853
    }
854
};
855

856
template<>
857
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, B8G8R8X8_UNORM_SRGB >
858
{
859
    template <size_t NumDests>
860
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
861
    {
862
        FlatConvertNoAlpha<B8G8R8X8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
863
    }
864
};
865

866
template<>
867
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, R8G8B8A8_UNORM >
868
{
869
    template <size_t NumDests>
870
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
871
    {
872
        FlatConvert<R8G8B8A8_UNORM>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
873
    }
874
};
875

876
template<>
877
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, R8G8B8X8_UNORM >
878
{
879
    template <size_t NumDests>
880
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
881
    {
882
        FlatConvertNoAlpha<R8G8B8X8_UNORM>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
883
    }
884
};
885

886
template<>
887
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, R8G8B8A8_UNORM_SRGB >
888
{
889
    template <size_t NumDests>
890
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
891
    {
892
        FlatConvert<R8G8B8A8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
893
    }
894
};
895

896
template<>
897
struct ConvertPixelsSOAtoAOS < R32G32B32A32_FLOAT, R8G8B8X8_UNORM_SRGB >
898
{
899
    template <size_t NumDests>
900
    INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
901
    {
902
        FlatConvertNoAlpha<R8G8B8X8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
903
    }
904
};
905

906
//////////////////////////////////////////////////////////////////////////
907
/// StoreRasterTile
908
//////////////////////////////////////////////////////////////////////////
909
template<typename TTraits, SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
910
struct StoreRasterTile
911
{
912
    //////////////////////////////////////////////////////////////////////////
913
    /// @brief Retrieve color from hot tile source which is always float.
914
    /// @param pSrc - Pointer to raster tile.
915
    /// @param x, y - Coordinates to raster tile.
916
    /// @param output - output color
917
    INLINE static void GetSwizzledSrcColor(
918
        uint8_t* pSrc,
919
        uint32_t x, uint32_t y,
920
        float outputColor[4])
921
    {
922
        typedef SimdTile_16<SrcFormat, DstFormat> SimdT;
923

924
        SimdT *pSrcSimdTiles = reinterpret_cast<SimdT *>(pSrc);
925

926
        // Compute which simd tile we're accessing within 8x8 tile.
927
        //   i.e. Compute linear simd tile coordinate given (x, y) in pixel coordinates.
928
        uint32_t simdIndex = (y / SIMD16_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD16_TILE_X_DIM) + (x / SIMD16_TILE_X_DIM);
929

930
        SimdT *pSimdTile = &pSrcSimdTiles[simdIndex];
931

932
        uint32_t simdOffset = (y % SIMD16_TILE_Y_DIM) * SIMD16_TILE_X_DIM + (x % SIMD16_TILE_X_DIM);
933

934
        pSimdTile->GetSwizzledColor(simdOffset, outputColor);
935
    }
936

937
    //////////////////////////////////////////////////////////////////////////
938
    /// @brief Stores an 8x8 raster tile to the destination surface.
939
    /// @param pSrc - Pointer to raster tile.
940
    /// @param pDstSurface - Destination surface state
941
    /// @param x, y - Coordinates to raster tile.
942
    INLINE static void Store(
943
        uint8_t *pSrc,
944
        SWR_SURFACE_STATE* pDstSurface,
945
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex) // (x, y) pixel coordinate to start of raster tile.
946
    {
947
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
948
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
949

950
        // For each raster tile pixel (rx, ry)
951
        for (uint32_t ry = 0; ry < KNOB_TILE_Y_DIM; ++ry)
952
        {
953
            for (uint32_t rx = 0; rx < KNOB_TILE_X_DIM; ++rx)
954
            {
955
                // Perform bounds checking.
956
                if (((x + rx) < lodWidth) &&
957
                    ((y + ry) < lodHeight))
958
                {
959
                    float srcColor[4];
960
                    GetSwizzledSrcColor(pSrc, rx, ry, srcColor);
961

962
                    uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>((x + rx), (y + ry),
963
                        pDstSurface->arrayIndex + renderTargetArrayIndex, pDstSurface->arrayIndex + renderTargetArrayIndex,
964
                        sampleNum, pDstSurface->lod, pDstSurface);
965
                    {
966
                        ConvertPixelFromFloat<DstFormat>(pDst, srcColor);
967
                    }
968
                }
969
            }
970
        }
971
    }
972

973
    //////////////////////////////////////////////////////////////////////////
974
    /// @brief Resolves an 8x8 raster tile to the resolve destination surface.
975
    /// @param pSrc - Pointer to raster tile.
976
    /// @param pDstSurface - Destination surface state
977
    /// @param x, y - Coordinates to raster tile.
978
    /// @param sampleOffset - Offset between adjacent multisamples
979
    INLINE static void Resolve(
980
        uint8_t *pSrc,
981
        SWR_SURFACE_STATE* pDstSurface,
982
        uint32_t x, uint32_t y, uint32_t sampleOffset, uint32_t renderTargetArrayIndex) // (x, y) pixel coordinate to start of raster tile.
983
    {
984
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
985
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
986

987
        float oneOverNumSamples = 1.0f / pDstSurface->numSamples;
988

989
        // For each raster tile pixel (rx, ry)
990
        for (uint32_t ry = 0; ry < KNOB_TILE_Y_DIM; ++ry)
991
        {
992
            for (uint32_t rx = 0; rx < KNOB_TILE_X_DIM; ++rx)
993
            {
994
                // Perform bounds checking.
995
                if (((x + rx) < lodWidth) &&
996
                        ((y + ry) < lodHeight))
997
                {
998
                    // Sum across samples
999
                    float resolveColor[4] = {0};
1000
                    for (uint32_t sampleNum = 0; sampleNum < pDstSurface->numSamples; sampleNum++)
1001
                    {
1002
                        float sampleColor[4] = {0};
1003
                        uint8_t *pSampleSrc = pSrc + sampleOffset * sampleNum;
1004
                        GetSwizzledSrcColor(pSampleSrc, rx, ry, sampleColor);
1005
                        resolveColor[0] += sampleColor[0];
1006
                        resolveColor[1] += sampleColor[1];
1007
                        resolveColor[2] += sampleColor[2];
1008
                        resolveColor[3] += sampleColor[3];
1009
                    }
1010

1011
                    // Divide by numSamples to average
1012
                    resolveColor[0] *= oneOverNumSamples;
1013
                    resolveColor[1] *= oneOverNumSamples;
1014
                    resolveColor[2] *= oneOverNumSamples;
1015
                    resolveColor[3] *= oneOverNumSamples;
1016

1017
                    // Use the resolve surface state
1018
                    SWR_SURFACE_STATE* pResolveSurface = (SWR_SURFACE_STATE*)pDstSurface->xpAuxBaseAddress;
1019
                    uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>((x + rx), (y + ry),
1020
                        pResolveSurface->arrayIndex + renderTargetArrayIndex, pResolveSurface->arrayIndex + renderTargetArrayIndex,
1021
                        0, pResolveSurface->lod, pResolveSurface);
1022
                    {
1023
                        ConvertPixelFromFloat<DstFormat>(pDst, resolveColor);
1024
                    }
1025
                }
1026
            }
1027
        }
1028
    }
1029

1030
};
1031

1032
template<typename TTraits, SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1033
struct OptStoreRasterTile : StoreRasterTile<TTraits, SrcFormat, DstFormat>
1034
{};
1035

1036
//////////////////////////////////////////////////////////////////////////
1037
/// OptStoreRasterTile - SWR_TILE_MODE_NONE specialization for 8bpp
1038
//////////////////////////////////////////////////////////////////////////
1039
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1040
struct OptStoreRasterTile< TilingTraits<SWR_TILE_NONE, 8>, SrcFormat, DstFormat>
1041
{
1042
    typedef StoreRasterTile<TilingTraits<SWR_TILE_NONE, 8>, SrcFormat, DstFormat> GenericStoreTile;
1043
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1044
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1045

1046
    //////////////////////////////////////////////////////////////////////////
1047
    /// @brief Stores an 8x8 raster tile to the destination surface.
1048
    /// @param pSrc - Pointer to raster tile.
1049
    /// @param pDstSurface - Destination surface state
1050
    /// @param x, y - Coordinates to raster tile.
1051
    INLINE static void Store(
1052
        uint8_t *pSrc,
1053
        SWR_SURFACE_STATE* pDstSurface,
1054
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1055
    {
1056
        // Punt non-full tiles to generic store
1057
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1058
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1059

1060
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1061
        {
1062
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1063
        }
1064

1065
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1066
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1067

1068
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1069
        const uint32_t dy = SIMD16_TILE_Y_DIM * pDstSurface->pitch - KNOB_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1070

1071
        uint8_t* ppDsts[] =
1072
        {
1073
            pDst,                                           // row 0, col 0
1074
            pDst + pDstSurface->pitch,                      // row 1, col 0
1075
            pDst + dx / 2,                                  // row 0, col 1
1076
            pDst + pDstSurface->pitch + dx / 2              // row 1, col 1
1077
        };
1078

1079
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1080
        {
1081
            for (uint32_t xx = 0; xx < KNOB_TILE_X_DIM; xx += SIMD16_TILE_X_DIM)
1082
            {
1083
                ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1084

1085
                pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1086

1087
                ppDsts[0] += dx;
1088
                ppDsts[1] += dx;
1089
                ppDsts[2] += dx;
1090
                ppDsts[3] += dx;
1091
            }
1092

1093
            ppDsts[0] += dy;
1094
            ppDsts[1] += dy;
1095
            ppDsts[2] += dy;
1096
            ppDsts[3] += dy;
1097
        }
1098
    }
1099
};
1100

1101
//////////////////////////////////////////////////////////////////////////
1102
/// OptStoreRasterTile - SWR_TILE_MODE_NONE specialization for 16bpp
1103
//////////////////////////////////////////////////////////////////////////
1104
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1105
struct OptStoreRasterTile< TilingTraits<SWR_TILE_NONE, 16>, SrcFormat, DstFormat>
1106
{
1107
    typedef StoreRasterTile<TilingTraits<SWR_TILE_NONE, 16>, SrcFormat, DstFormat> GenericStoreTile;
1108
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1109
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1110

1111
    //////////////////////////////////////////////////////////////////////////
1112
    /// @brief Stores an 8x8 raster tile to the destination surface.
1113
    /// @param pSrc - Pointer to raster tile.
1114
    /// @param pDstSurface - Destination surface state
1115
    /// @param x, y - Coordinates to raster tile.
1116
    INLINE static void Store(
1117
        uint8_t *pSrc,
1118
        SWR_SURFACE_STATE* pDstSurface,
1119
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1120
    {
1121
        // Punt non-full tiles to generic store
1122
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1123
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1124

1125
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1126
        {
1127
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1128
        }
1129

1130
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1131
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1132

1133
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1134
        const uint32_t dy = SIMD16_TILE_Y_DIM * pDstSurface->pitch - KNOB_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1135

1136
        uint8_t* ppDsts[] =
1137
        {
1138
            pDst,                                           // row 0, col 0
1139
            pDst + pDstSurface->pitch,                      // row 1, col 0
1140
            pDst + dx / 2,                                  // row 0, col 1
1141
            pDst + pDstSurface->pitch + dx / 2              // row 1, col 1
1142
        };
1143

1144
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1145
        {
1146
            for (uint32_t xx = 0; xx < KNOB_TILE_X_DIM; xx += SIMD16_TILE_X_DIM)
1147
            {
1148
                ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1149

1150
                pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1151

1152
                ppDsts[0] += dx;
1153
                ppDsts[1] += dx;
1154
                ppDsts[2] += dx;
1155
                ppDsts[3] += dx;
1156
            }
1157

1158
            ppDsts[0] += dy;
1159
            ppDsts[1] += dy;
1160
            ppDsts[2] += dy;
1161
            ppDsts[3] += dy;
1162
        }
1163
    }
1164
};
1165

1166
//////////////////////////////////////////////////////////////////////////
1167
/// OptStoreRasterTile - SWR_TILE_MODE_NONE specialization for 32bpp
1168
//////////////////////////////////////////////////////////////////////////
1169
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1170
struct OptStoreRasterTile< TilingTraits<SWR_TILE_NONE, 32>, SrcFormat, DstFormat>
1171
{
1172
    typedef StoreRasterTile<TilingTraits<SWR_TILE_NONE, 32>, SrcFormat, DstFormat> GenericStoreTile;
1173
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1174
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1175

1176
    //////////////////////////////////////////////////////////////////////////
1177
    /// @brief Stores an 8x8 raster tile to the destination surface.
1178
    /// @param pSrc - Pointer to raster tile.
1179
    /// @param pDstSurface - Destination surface state
1180
    /// @param x, y - Coordinates to raster tile.
1181
    INLINE static void Store(
1182
        uint8_t *pSrc,
1183
        SWR_SURFACE_STATE* pDstSurface,
1184
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1185
    {
1186
        // Punt non-full tiles to generic store
1187
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1188
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1189

1190
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1191
        {
1192
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1193
        }
1194

1195
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1196
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1197

1198
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1199
        const uint32_t dy = SIMD16_TILE_Y_DIM * pDstSurface->pitch - KNOB_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1200

1201
        uint8_t* ppDsts[] =
1202
        {
1203
            pDst,                                           // row 0, col 0
1204
            pDst + pDstSurface->pitch,                      // row 1, col 0
1205
            pDst + dx / 2,                                  // row 0, col 1
1206
            pDst + pDstSurface->pitch + dx / 2              // row 1, col 1
1207
        };
1208

1209
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1210
        {
1211
            for (uint32_t xx = 0; xx < KNOB_TILE_X_DIM; xx += SIMD16_TILE_X_DIM)
1212
            {
1213
                ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1214

1215
                pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1216

1217
                ppDsts[0] += dx;
1218
                ppDsts[1] += dx;
1219
                ppDsts[2] += dx;
1220
                ppDsts[3] += dx;
1221
            }
1222

1223
            ppDsts[0] += dy;
1224
            ppDsts[1] += dy;
1225
            ppDsts[2] += dy;
1226
            ppDsts[3] += dy;
1227
        }
1228
    }
1229
};
1230

1231
//////////////////////////////////////////////////////////////////////////
1232
/// OptStoreRasterTile - SWR_TILE_MODE_NONE specialization for 64bpp
1233
//////////////////////////////////////////////////////////////////////////
1234
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1235
struct OptStoreRasterTile< TilingTraits<SWR_TILE_NONE, 64>, SrcFormat, DstFormat>
1236
{
1237
    typedef StoreRasterTile<TilingTraits<SWR_TILE_NONE, 64>, SrcFormat, DstFormat> GenericStoreTile;
1238
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1239
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1240
    static const size_t MAX_DST_COLUMN_BYTES = 16;
1241

1242
    //////////////////////////////////////////////////////////////////////////
1243
    /// @brief Stores an 8x8 raster tile to the destination surface.
1244
    /// @param pSrc - Pointer to raster tile.
1245
    /// @param pDstSurface - Destination surface state
1246
    /// @param x, y - Coordinates to raster tile.
1247
    INLINE static void Store(
1248
        uint8_t *pSrc,
1249
        SWR_SURFACE_STATE* pDstSurface,
1250
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1251
    {
1252
        // Punt non-full tiles to generic store
1253
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1254
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1255

1256
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1257
        {
1258
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1259
        }
1260

1261
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1262
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1263

1264
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1265
        const uint32_t dy = SIMD16_TILE_Y_DIM * pDstSurface->pitch;
1266

1267
        // we have to break these large spans up, since ConvertPixelsSOAtoAOS() can only work on max 16B spans (a TileY limitation)
1268
        static_assert(dx == MAX_DST_COLUMN_BYTES * 4, "Invalid column offsets");
1269

1270
        uint8_t *ppDsts[] =
1271
        {
1272
            pDst,                                                               // row 0, col 0
1273
            pDst + pDstSurface->pitch,                                          // row 1, col 0
1274
            pDst + MAX_DST_COLUMN_BYTES,                                        // row 0, col 1
1275
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES,                   // row 1, col 1
1276
            pDst + MAX_DST_COLUMN_BYTES * 2,                                    // row 0, col 2
1277
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 2,               // row 1, col 2
1278
            pDst + MAX_DST_COLUMN_BYTES * 3,                                    // row 0, col 3
1279
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 3                // row 1, col 3
1280
        };
1281

1282
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1283
        {
1284
            // Raster tile width is same as simd16 tile width
1285
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1286

1287
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1288

1289
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1290

1291
            for (uint32_t i = 0; i < ARRAY_SIZE(ppDsts); i += 1)
1292
            {
1293
                ppDsts[i] += dy;
1294
            }
1295
        }
1296
    }
1297
};
1298

1299
//////////////////////////////////////////////////////////////////////////
1300
/// OptStoreRasterTile - SWR_TILE_MODE_NONE specialization for 128bpp
1301
//////////////////////////////////////////////////////////////////////////
1302
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1303
struct OptStoreRasterTile< TilingTraits<SWR_TILE_NONE, 128>, SrcFormat, DstFormat>
1304
{
1305
    typedef StoreRasterTile<TilingTraits<SWR_TILE_NONE, 128>, SrcFormat, DstFormat> GenericStoreTile;
1306
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1307
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1308
    static const size_t MAX_DST_COLUMN_BYTES = 16;
1309

1310
    //////////////////////////////////////////////////////////////////////////
1311
    /// @brief Stores an 8x8 raster tile to the destination surface.
1312
    /// @param pSrc - Pointer to raster tile.
1313
    /// @param pDstSurface - Destination surface state
1314
    /// @param x, y - Coordinates to raster tile.
1315
    INLINE static void Store(
1316
        uint8_t *pSrc,
1317
        SWR_SURFACE_STATE* pDstSurface,
1318
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1319
    {
1320
        // Punt non-full tiles to generic store
1321
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1322
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1323

1324
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1325
        {
1326
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1327
        }
1328

1329
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1330
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1331

1332
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1333
        const uint32_t dy = SIMD16_TILE_Y_DIM * pDstSurface->pitch;
1334

1335
        // we have to break these large spans up, since ConvertPixelsSOAtoAOS() can only work on max 16B spans (a TileY limitation)
1336
        static_assert(dx == MAX_DST_COLUMN_BYTES * 8, "Invalid column offsets");
1337

1338
        uint8_t* ppDsts[] =
1339
        {
1340
            pDst,                                                               // row 0, col 0
1341
            pDst + pDstSurface->pitch,                                          // row 1, col 0
1342
            pDst + MAX_DST_COLUMN_BYTES,                                        // row 0, col 1
1343
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES,                   // row 1, col 1
1344
            pDst + MAX_DST_COLUMN_BYTES * 2,                                    // row 0, col 2
1345
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 2,               // row 1, col 2
1346
            pDst + MAX_DST_COLUMN_BYTES * 3,                                    // row 0, col 3
1347
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 3,               // row 1, col 3
1348
            pDst + MAX_DST_COLUMN_BYTES * 4,                                    // row 0, col 4
1349
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 4,               // row 1, col 4
1350
            pDst + MAX_DST_COLUMN_BYTES * 5,                                    // row 0, col 5
1351
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 5,               // row 1, col 5
1352
            pDst + MAX_DST_COLUMN_BYTES * 6,                                    // row 0, col 6
1353
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 6,               // row 1, col 6
1354
            pDst + MAX_DST_COLUMN_BYTES * 7,                                    // row 0, col 7
1355
            pDst + pDstSurface->pitch + MAX_DST_COLUMN_BYTES * 7,               // row 1, col 7
1356
        };
1357

1358
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1359
        {
1360
            // Raster tile width is same as simd16 tile width
1361
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1362

1363
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1364

1365
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1366

1367
            for (uint32_t i = 0; i < ARRAY_SIZE(ppDsts); i += 1)
1368
            {
1369
                ppDsts[i] += dy;
1370
            }
1371
        }
1372
    }
1373
};
1374

1375
//////////////////////////////////////////////////////////////////////////
1376
/// OptStoreRasterTile - TILE_MODE_YMAJOR specialization for 8bpp
1377
//////////////////////////////////////////////////////////////////////////
1378
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1379
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 8>, SrcFormat, DstFormat>
1380
{
1381
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 8>, SrcFormat, DstFormat> GenericStoreTile;
1382
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1383

1384
    //////////////////////////////////////////////////////////////////////////
1385
    /// @brief Stores an 8x8 raster tile to the destination surface.
1386
    /// @param pSrc - Pointer to raster tile.
1387
    /// @param pDstSurface - Destination surface state
1388
    /// @param x, y - Coordinates to raster tile.
1389
    INLINE static void Store(
1390
        uint8_t *pSrc,
1391
        SWR_SURFACE_STATE* pDstSurface,
1392
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1393
    {
1394
        static const uint32_t DestRowWidthBytes = 16;                    // 16B rows
1395

1396
        // Punt non-full tiles to generic store
1397
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1398
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1399

1400
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1401
        {
1402
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1403
        }
1404

1405
        // TileY is a column-major tiling mode where each 4KB tile consist of 8 columns of 32 x 16B rows.
1406
        // We can compute the offsets to each column within the raster tile once and increment from these.
1407
        // There will be 4 8x2 simd tiles in an 8x8 raster tile.
1408
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1409
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1410

1411
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes;
1412

1413
        // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
1414
        uint8_t *ppDsts[] =
1415
        {
1416
            pDst,
1417
            pDst + DestRowWidthBytes,
1418
            pDst + DestRowWidthBytes / 4,
1419
            pDst + DestRowWidthBytes + DestRowWidthBytes / 4
1420
        };
1421

1422
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1423
        {
1424
            // Raster tile width is same as simd16 tile width
1425
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1426

1427
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1428

1429
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1430

1431
            ppDsts[0] += dy;
1432
            ppDsts[1] += dy;
1433
            ppDsts[2] += dy;
1434
            ppDsts[3] += dy;
1435
        }
1436
    }
1437
};
1438

1439
//////////////////////////////////////////////////////////////////////////
1440
/// OptStoreRasterTile - TILE_MODE_YMAJOR specialization for 16bpp
1441
//////////////////////////////////////////////////////////////////////////
1442
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1443
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 16>, SrcFormat, DstFormat>
1444
{
1445
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 16>, SrcFormat, DstFormat> GenericStoreTile;
1446
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1447

1448
    //////////////////////////////////////////////////////////////////////////
1449
    /// @brief Stores an 8x8 raster tile to the destination surface.
1450
    /// @param pSrc - Pointer to raster tile.
1451
    /// @param pDstSurface - Destination surface state
1452
    /// @param x, y - Coordinates to raster tile.
1453
    INLINE static void Store(
1454
        uint8_t *pSrc,
1455
        SWR_SURFACE_STATE* pDstSurface,
1456
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1457
    {
1458
        static const uint32_t DestRowWidthBytes = 16;                    // 16B rows
1459

1460
        // Punt non-full tiles to generic store
1461
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1462
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1463

1464
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1465
        {
1466
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1467
        }
1468

1469
        // TileY is a column-major tiling mode where each 4KB tile consist of 8 columns of 32 x 16B rows.
1470
        // We can compute the offsets to each column within the raster tile once and increment from these.
1471
        // There will be 4 8x2 simd tiles in an 8x8 raster tile.
1472
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1473
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1474

1475
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes;
1476

1477
        // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
1478
        uint8_t *ppDsts[] =
1479
        {
1480
            pDst,
1481
            pDst + DestRowWidthBytes,
1482
            pDst + DestRowWidthBytes / 2,
1483
            pDst + DestRowWidthBytes + DestRowWidthBytes / 2
1484
        };
1485

1486
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1487
        {
1488
            // Raster tile width is same as simd16 tile width
1489
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1490

1491
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1492

1493
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1494

1495
            ppDsts[0] += dy;
1496
            ppDsts[1] += dy;
1497
            ppDsts[2] += dy;
1498
            ppDsts[3] += dy;
1499
        }
1500
    }
1501
};
1502

1503
//////////////////////////////////////////////////////////////////////////
1504
/// OptStoreRasterTile - TILE_MODE_XMAJOR specialization for 32bpp
1505
//////////////////////////////////////////////////////////////////////////
1506
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1507
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_XMAJOR, 32>, SrcFormat, DstFormat>
1508
{
1509
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_XMAJOR, 32>, SrcFormat, DstFormat> GenericStoreTile;
1510
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1511
    static const size_t DST_BYTES_PER_PIXEL = FormatTraits<DstFormat>::bpp / 8;
1512

1513
    //////////////////////////////////////////////////////////////////////////
1514
    /// @brief Stores an 8x8 raster tile to the destination surface.
1515
    /// @param pSrc - Pointer to raster tile.
1516
    /// @param pDstSurface - Destination surface state
1517
    /// @param x, y - Coordinates to raster tile.
1518
    INLINE static void Store(
1519
        uint8_t *pSrc,
1520
        SWR_SURFACE_STATE* pDstSurface,
1521
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1522
    {
1523
        static const uint32_t DestRowWidthBytes = 512;                   // 512B rows
1524

1525
        // Punt non-full tiles to generic store
1526
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1527
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1528

1529
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1530
        {
1531
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1532
        }
1533

1534
        // TileX is a row-major tiling mode where each 4KB tile consist of 8 x 512B rows.
1535
        // We can compute the offsets to each column within the raster tile once and increment from these.
1536
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1537
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1538

1539
        const uint32_t dx = SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1540
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes - KNOB_TILE_X_DIM * DST_BYTES_PER_PIXEL;
1541

1542
        uint8_t* ppDsts[] =
1543
        {
1544
            pDst,                                           // row 0, col 0
1545
            pDst + DestRowWidthBytes,                       // row 1, col 0
1546
            pDst + dx / 2,                                  // row 0, col 1
1547
            pDst + DestRowWidthBytes + dx / 2               // row 1, col 1
1548
        };
1549

1550
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1551
        {
1552
            for (uint32_t xx = 0; xx < KNOB_TILE_X_DIM; xx += SIMD16_TILE_X_DIM)
1553
            {
1554
                ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1555

1556
                pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1557

1558
                ppDsts[0] += dx;
1559
                ppDsts[1] += dx;
1560
                ppDsts[2] += dx;
1561
                ppDsts[3] += dx;
1562
            }
1563

1564
            ppDsts[0] += dy;
1565
            ppDsts[1] += dy;
1566
            ppDsts[2] += dy;
1567
            ppDsts[3] += dy;
1568
        }
1569
    }
1570
};
1571

1572
//////////////////////////////////////////////////////////////////////////
1573
/// OptStoreRasterTile - TILE_MODE_YMAJOR specialization for 32bpp
1574
//////////////////////////////////////////////////////////////////////////
1575
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1576
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 32>, SrcFormat, DstFormat>
1577
{
1578
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 32>, SrcFormat, DstFormat> GenericStoreTile;
1579
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1580

1581
    //////////////////////////////////////////////////////////////////////////
1582
    /// @brief Stores an 8x8 raster tile to the destination surface.
1583
    /// @param pSrc - Pointer to raster tile.
1584
    /// @param pDstSurface - Destination surface state
1585
    /// @param x, y - Coordinates to raster tile.
1586
    INLINE static void Store(
1587
        uint8_t *pSrc,
1588
        SWR_SURFACE_STATE* pDstSurface,
1589
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1590
    {
1591
        static const uint32_t DestRowWidthBytes = 16;                    // 16B rows
1592
        static const uint32_t DestColumnBytes = DestRowWidthBytes * 32;  // 16B x 32 rows.
1593

1594
        // Punt non-full tiles to generic store
1595
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1596
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1597

1598
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1599
        {
1600
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1601
        }
1602

1603
        // TileY is a column-major tiling mode where each 4KB tile consist of 8 columns of 32 x 16B rows.
1604
        // We can compute the offsets to each column within the raster tile once and increment from these.
1605
        // There will be 4 8x2 simd tiles in an 8x8 raster tile.
1606
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1607
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1608

1609
        // we have to break these large spans up, since ConvertPixelsSOAtoAOS() can only work on max 16B spans (a TileY limitation)
1610
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes;
1611

1612
        // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
1613
        uint8_t *ppDsts[] =
1614
        {
1615
            pDst,                                           // row 0, col 0
1616
            pDst + DestRowWidthBytes,                       // row 1, col 0
1617
            pDst + DestColumnBytes,                         // row 0, col 1
1618
            pDst + DestRowWidthBytes + DestColumnBytes      // row 1, col 1
1619
        };
1620

1621
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1622
        {
1623
            // Raster tile width is same as simd16 tile width
1624
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1625

1626
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1627

1628
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1629

1630
            ppDsts[0] += dy;
1631
            ppDsts[1] += dy;
1632
            ppDsts[2] += dy;
1633
            ppDsts[3] += dy;
1634
        }
1635
    }
1636
};
1637

1638
//////////////////////////////////////////////////////////////////////////
1639
/// OptStoreRasterTile - TILE_MODE_YMAJOR specialization for 64bpp
1640
//////////////////////////////////////////////////////////////////////////
1641
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1642
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 64>, SrcFormat, DstFormat>
1643
{
1644
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 64>, SrcFormat, DstFormat> GenericStoreTile;
1645
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1646

1647
    //////////////////////////////////////////////////////////////////////////
1648
    /// @brief Stores an 8x8 raster tile to the destination surface.
1649
    /// @param pSrc - Pointer to raster tile.
1650
    /// @param pDstSurface - Destination surface state
1651
    /// @param x, y - Coordinates to raster tile.
1652
    INLINE static void Store(
1653
        uint8_t *pSrc,
1654
        SWR_SURFACE_STATE* pDstSurface,
1655
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1656
    {
1657
        static const uint32_t DestRowWidthBytes = 16;                    // 16B rows
1658
        static const uint32_t DestColumnBytes = DestRowWidthBytes * 32;  // 16B x 32 rows.
1659

1660
        // Punt non-full tiles to generic store
1661
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1662
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1663

1664
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1665
        {
1666
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1667
        }
1668

1669
        // TileY is a column-major tiling mode where each 4KB tile consist of 8 columns of 32 x 16B rows.
1670
        // We can compute the offsets to each column within the raster tile once and increment from these.
1671
        // There will be 4 8x2 simd tiles in an 8x8 raster tile.
1672
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1673
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1674

1675
        // we have to break these large spans up, since ConvertPixelsSOAtoAOS() can only work on max 16B spans (a TileY limitation)
1676
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes;
1677

1678
        // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
1679
        uint8_t *ppDsts[] =
1680
        {
1681
            pDst,                                           // row 0, col 0
1682
            pDst + DestRowWidthBytes,                       // row 1, col 0
1683
            pDst + DestColumnBytes,                         // row 0, col 1
1684
            pDst + DestRowWidthBytes + DestColumnBytes,     // row 1, col 1
1685
            pDst + DestColumnBytes * 2,                     // row 0, col 2
1686
            pDst + DestRowWidthBytes + DestColumnBytes * 2, // row 1, col 2
1687
            pDst + DestColumnBytes * 3,                     // row 0, col 3
1688
            pDst + DestRowWidthBytes + DestColumnBytes * 3  // row 1, col 3
1689
        };
1690

1691
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1692
        {
1693
            // Raster tile width is same as simd16 tile width
1694
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1695

1696
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1697

1698
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1699

1700
            for (uint32_t i = 0; i < ARRAY_SIZE(ppDsts); i += 1)
1701
            {
1702
                ppDsts[i] += dy;
1703
            }
1704
        }
1705
    }
1706
};
1707

1708
//////////////////////////////////////////////////////////////////////////
1709
/// OptStoreRasterTile - SWR_TILE_MODE_YMAJOR specialization for 128bpp
1710
//////////////////////////////////////////////////////////////////////////
1711
template<SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1712
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 128>, SrcFormat, DstFormat>
1713
{
1714
    typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 128>, SrcFormat, DstFormat> GenericStoreTile;
1715
    static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
1716

1717
    //////////////////////////////////////////////////////////////////////////
1718
    /// @brief Stores an 8x8 raster tile to the destination surface.
1719
    /// @param pSrc - Pointer to raster tile.
1720
    /// @param pDstSurface - Destination surface state
1721
    /// @param x, y - Coordinates to raster tile.
1722
    INLINE static void Store(
1723
        uint8_t *pSrc,
1724
        SWR_SURFACE_STATE* pDstSurface,
1725
        uint32_t x, uint32_t y, uint32_t sampleNum, uint32_t renderTargetArrayIndex)
1726
    {
1727
        static const uint32_t DestRowWidthBytes = 16;                    // 16B rows
1728
        static const uint32_t DestColumnBytes = DestRowWidthBytes * 32;  // 16B x 32 rows.
1729

1730
        // Punt non-full tiles to generic store
1731
        uint32_t lodWidth = std::max(pDstSurface->width >> pDstSurface->lod, 1U);
1732
        uint32_t lodHeight = std::max(pDstSurface->height >> pDstSurface->lod, 1U);
1733

1734
        if (x + KNOB_TILE_X_DIM > lodWidth || y + KNOB_TILE_Y_DIM > lodHeight)
1735
        {
1736
            return GenericStoreTile::Store(pSrc, pDstSurface, x, y, sampleNum, renderTargetArrayIndex);
1737
        }
1738

1739
        // TileY is a column-major tiling mode where each 4KB tile consist of 8 columns of 32 x 16B rows.
1740
        // We can compute the offsets to each column within the raster tile once and increment from these.
1741
        // There will be 4 8x2 simd tiles in an 8x8 raster tile.
1742
        uint8_t *pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
1743
            pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
1744

1745
        // we have to break these large spans up, since ConvertPixelsSOAtoAOS() can only work on max 16B spans (a TileY limitation)
1746
        const uint32_t dy = SIMD16_TILE_Y_DIM * DestRowWidthBytes;
1747

1748
        // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
1749
        uint8_t *ppDsts[] =
1750
        {
1751
            pDst,                                           // row 0, col 0
1752
            pDst + DestRowWidthBytes,                       // row 1, col 0
1753
            pDst + DestColumnBytes,                         // row 0, col 1
1754
            pDst + DestRowWidthBytes + DestColumnBytes,     // row 1, col 1
1755
            pDst + DestColumnBytes * 2,                     // row 0, col 2
1756
            pDst + DestRowWidthBytes + DestColumnBytes * 2, // row 1, col 2
1757
            pDst + DestColumnBytes * 3,                     // row 0, col 3
1758
            pDst + DestRowWidthBytes + DestColumnBytes * 3, // row 1, col 3
1759
            pDst + DestColumnBytes * 4,                     // row 0, col 4
1760
            pDst + DestRowWidthBytes + DestColumnBytes * 4, // row 1, col 4
1761
            pDst + DestColumnBytes * 5,                     // row 0, col 5
1762
            pDst + DestRowWidthBytes + DestColumnBytes * 5, // row 1, col 5
1763
            pDst + DestColumnBytes * 6,                     // row 0, col 6
1764
            pDst + DestRowWidthBytes + DestColumnBytes * 6, // row 1, col 6
1765
            pDst + DestColumnBytes * 7,                     // row 0, col 7
1766
            pDst + DestRowWidthBytes + DestColumnBytes * 7  // row 1, col 7
1767
        };
1768

1769
        for (uint32_t yy = 0; yy < KNOB_TILE_Y_DIM; yy += SIMD16_TILE_Y_DIM)
1770
        {
1771
            // Raster tile width is same as simd16 tile width
1772
            static_assert(KNOB_TILE_X_DIM == SIMD16_TILE_X_DIM, "Invalid tile x dim");
1773

1774
            ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
1775

1776
            pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
1777

1778
            for (uint32_t i = 0; i < ARRAY_SIZE(ppDsts); i += 1)
1779
            {
1780
                ppDsts[i] += dy;
1781
            }
1782
        }
1783
    }
1784
};
1785

1786
//////////////////////////////////////////////////////////////////////////
1787
/// StoreMacroTile - Stores a macro tile which consists of raster tiles.
1788
//////////////////////////////////////////////////////////////////////////
1789
template<typename TTraits, SWR_FORMAT SrcFormat, SWR_FORMAT DstFormat>
1790
struct StoreMacroTile
1791
{
1792
    //////////////////////////////////////////////////////////////////////////
1793
    /// @brief Stores a macrotile to the destination surface using safe implementation.
1794
    /// @param pSrc - Pointer to macro tile.
1795
    /// @param pDstSurface - Destination surface state
1796
    /// @param x, y - Coordinates to macro tile
1797
    static void StoreGeneric(
1798
        uint8_t *pSrcHotTile,
1799
        SWR_SURFACE_STATE* pDstSurface,
1800
        uint32_t x, uint32_t y, uint32_t renderTargetArrayIndex)
1801
    {
1802
        PFN_STORE_TILES_INTERNAL pfnStore;
1803
        pfnStore = StoreRasterTile<TTraits, SrcFormat, DstFormat>::Store;
1804

1805
        // Store each raster tile from the hot tile to the destination surface.
1806
        for (uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
1807
        {
1808
            for (uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
1809
            {
1810
                for (uint32_t sampleNum = 0; sampleNum < pDstSurface->numSamples; sampleNum++)
1811
                {
1812
                    pfnStore(pSrcHotTile, pDstSurface, (x + col), (y + row), sampleNum, renderTargetArrayIndex);
1813
                    pSrcHotTile += KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<SrcFormat>::bpp / 8);
1814
                }
1815
            }
1816
        }
1817

1818
    }
1819

1820
    typedef void(*PFN_STORE_TILES_INTERNAL)(uint8_t*, SWR_SURFACE_STATE*, uint32_t, uint32_t, uint32_t, uint32_t);
1821
    //////////////////////////////////////////////////////////////////////////
1822
    /// @brief Stores a macrotile to the destination surface.
1823
    /// @param pSrc - Pointer to macro tile.
1824
    /// @param pDstSurface - Destination surface state
1825
    /// @param x, y - Coordinates to macro tile
1826
    static void Store(
1827
        uint8_t *pSrcHotTile,
1828
        SWR_SURFACE_STATE* pDstSurface,
1829
        uint32_t x, uint32_t y, uint32_t renderTargetArrayIndex)
1830
    {
1831
        PFN_STORE_TILES_INTERNAL pfnStore[SWR_MAX_NUM_MULTISAMPLES];
1832

1833
        for (uint32_t sampleNum = 0; sampleNum < pDstSurface->numSamples; sampleNum++)
1834
        {
1835
            size_t dstSurfAddress = (size_t)ComputeSurfaceAddress<false, false>(
1836
                0,
1837
                0,
1838
                pDstSurface->arrayIndex + renderTargetArrayIndex, // z for 3D surfaces
1839
                pDstSurface->arrayIndex + renderTargetArrayIndex, // array index for 2D arrays
1840
                sampleNum,
1841
                pDstSurface->lod,
1842
                pDstSurface);
1843

1844
            // Only support generic store-tile if lod surface doesn't start on a page boundary and is non-linear
1845
            bool bForceGeneric = ((pDstSurface->tileMode != SWR_TILE_NONE) && (0 != (dstSurfAddress & 0xfff))) ||
1846
                (pDstSurface->bInterleavedSamples);
1847

1848
            pfnStore[sampleNum] = (bForceGeneric || KNOB_USE_GENERIC_STORETILE) ? StoreRasterTile<TTraits, SrcFormat, DstFormat>::Store : OptStoreRasterTile<TTraits, SrcFormat, DstFormat>::Store;
1849
        }
1850

1851
        // Save original for pSrcHotTile resolve.
1852
        uint8_t *pResolveSrcHotTile = pSrcHotTile;
1853

1854
        // Store each raster tile from the hot tile to the destination surface.
1855
        for(uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
1856
        {
1857
            for(uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
1858
            {
1859
                for(uint32_t sampleNum = 0; sampleNum < pDstSurface->numSamples; sampleNum++)
1860
                {
1861
                    pfnStore[sampleNum](pSrcHotTile, pDstSurface, (x + col), (y + row), sampleNum, renderTargetArrayIndex);
1862
                    pSrcHotTile += KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<SrcFormat>::bpp / 8);
1863
                }
1864
            }
1865
        }
1866

1867
        if (pDstSurface->xpAuxBaseAddress)
1868
        {
1869
            uint32_t sampleOffset = KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<SrcFormat>::bpp / 8);
1870
            // Store each raster tile from the hot tile to the destination surface.
1871
            for(uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
1872
            {
1873
                for(uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
1874
                {
1875
                    StoreRasterTile<TTraits, SrcFormat, DstFormat>::Resolve(pResolveSrcHotTile, pDstSurface, (x + col), (y + row), sampleOffset, renderTargetArrayIndex);
1876
                    pResolveSrcHotTile += sampleOffset * pDstSurface->numSamples;
1877
                }
1878
            }
1879
        }
1880
    }
1881
};
1882

1883
//////////////////////////////////////////////////////////////////////////
1884
/// InitStoreTilesTable - Helper for setting up the tables.
1885
template <SWR_TILE_MODE TTileMode, size_t NumTileModesT, size_t ArraySizeT>
1886
void InitStoreTilesTableColor_Half1(
1887
    PFN_STORE_TILES (&table)[NumTileModesT][ArraySizeT])
1888
{
1889
    table[TTileMode][R32G32B32A32_FLOAT]            = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32A32_FLOAT>::Store;
1890
    table[TTileMode][R32G32B32A32_SINT]             = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32A32_SINT>::Store;
1891
    table[TTileMode][R32G32B32A32_UINT]             = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32A32_UINT>::Store;
1892
    table[TTileMode][R32G32B32X32_FLOAT]            = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32X32_FLOAT>::Store;
1893
    table[TTileMode][R32G32B32A32_SSCALED]          = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32A32_SSCALED>::Store;
1894
    table[TTileMode][R32G32B32A32_USCALED]          = StoreMacroTile<TilingTraits<TTileMode, 128>, R32G32B32A32_FLOAT, R32G32B32A32_USCALED>::Store;
1895
    table[TTileMode][R32G32B32_FLOAT]               = StoreMacroTile<TilingTraits<TTileMode, 96>, R32G32B32A32_FLOAT, R32G32B32_FLOAT>::Store;
1896
    table[TTileMode][R32G32B32_SINT]                = StoreMacroTile<TilingTraits<TTileMode, 96>, R32G32B32A32_FLOAT, R32G32B32_SINT>::Store;
1897
    table[TTileMode][R32G32B32_UINT]                = StoreMacroTile<TilingTraits<TTileMode, 96>, R32G32B32A32_FLOAT, R32G32B32_UINT>::Store;
1898
    table[TTileMode][R32G32B32_SSCALED]             = StoreMacroTile<TilingTraits<TTileMode, 96>, R32G32B32A32_FLOAT, R32G32B32_SSCALED>::Store;
1899
    table[TTileMode][R32G32B32_USCALED]             = StoreMacroTile<TilingTraits<TTileMode, 96>, R32G32B32A32_FLOAT, R32G32B32_USCALED>::Store;
1900
    table[TTileMode][R16G16B16A16_UNORM]            = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_UNORM>::Store;
1901
    table[TTileMode][R16G16B16A16_SNORM]            = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_SNORM>::Store;
1902
    table[TTileMode][R16G16B16A16_SINT]             = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_SINT>::Store;
1903
    table[TTileMode][R16G16B16A16_UINT]             = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_UINT>::Store;
1904
    table[TTileMode][R16G16B16A16_FLOAT]            = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_FLOAT>::Store;
1905
    table[TTileMode][R32G32_FLOAT]                  = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32G32_FLOAT>::Store;
1906
    table[TTileMode][R32G32_SINT]                   = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32G32_SINT>::Store;
1907
    table[TTileMode][R32G32_UINT]                   = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32G32_UINT>::Store;
1908
    table[TTileMode][R32_FLOAT_X8X24_TYPELESS]      = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32_FLOAT_X8X24_TYPELESS>::Store;
1909
    table[TTileMode][X32_TYPELESS_G8X24_UINT]       = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, X32_TYPELESS_G8X24_UINT>::Store;
1910
    table[TTileMode][R16G16B16X16_UNORM]            = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16X16_UNORM>::Store;
1911
    table[TTileMode][R16G16B16X16_FLOAT]            = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16X16_FLOAT>::Store;
1912
    table[TTileMode][R16G16B16A16_SSCALED]          = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_SSCALED>::Store;
1913
    table[TTileMode][R16G16B16A16_USCALED]          = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R16G16B16A16_USCALED>::Store;
1914
    table[TTileMode][R32G32_SSCALED]                = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32G32_SSCALED>::Store;
1915
    table[TTileMode][R32G32_USCALED]                = StoreMacroTile<TilingTraits<TTileMode, 64>, R32G32B32A32_FLOAT, R32G32_USCALED>::Store;
1916
    table[TTileMode][B8G8R8A8_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B8G8R8A8_UNORM>::Store;
1917
    table[TTileMode][B8G8R8A8_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B8G8R8A8_UNORM_SRGB>::Store;
1918
    table[TTileMode][R10G10B10A2_UNORM]             = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_UNORM>::StoreGeneric;
1919
    table[TTileMode][R10G10B10A2_UNORM_SRGB]        = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_UNORM_SRGB>::StoreGeneric;
1920
    table[TTileMode][R10G10B10A2_UINT]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_UINT>::StoreGeneric;
1921
    table[TTileMode][R8G8B8A8_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_UNORM>::Store;
1922
    table[TTileMode][R8G8B8A8_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_UNORM_SRGB>::Store;
1923
    table[TTileMode][R8G8B8A8_SNORM]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_SNORM>::Store;
1924
    table[TTileMode][R8G8B8A8_SINT]                 = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_SINT>::Store;
1925
    table[TTileMode][R8G8B8A8_UINT]                 = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_UINT>::Store;
1926
    table[TTileMode][R16G16_UNORM]                  = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_UNORM>::Store;
1927
    table[TTileMode][R16G16_SNORM]                  = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_SNORM>::Store;
1928
    table[TTileMode][R16G16_SINT]                   = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_SINT>::Store;
1929
    table[TTileMode][R16G16_UINT]                   = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_UINT>::Store;
1930
    table[TTileMode][R16G16_FLOAT]                  = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_FLOAT>::Store;
1931
    table[TTileMode][B10G10R10A2_UNORM]             = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_UNORM>::StoreGeneric;
1932
    table[TTileMode][B10G10R10A2_UNORM_SRGB]        = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_UNORM_SRGB>::StoreGeneric;
1933
    table[TTileMode][R11G11B10_FLOAT]               = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R11G11B10_FLOAT>::StoreGeneric;
1934
    table[TTileMode][R10G10B10_FLOAT_A2_UNORM]      = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10_FLOAT_A2_UNORM>::StoreGeneric;
1935
    table[TTileMode][R32_SINT]                      = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R32_SINT>::Store;
1936
    table[TTileMode][R32_UINT]                      = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R32_UINT>::Store;
1937
    table[TTileMode][R32_FLOAT]                     = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R32_FLOAT>::Store;
1938
    table[TTileMode][R24_UNORM_X8_TYPELESS]         = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R24_UNORM_X8_TYPELESS>::StoreGeneric;
1939
    table[TTileMode][X24_TYPELESS_G8_UINT]          = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, X24_TYPELESS_G8_UINT>::StoreGeneric;
1940
    table[TTileMode][A32_FLOAT]                     = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, A32_FLOAT>::Store;
1941
    table[TTileMode][B8G8R8X8_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B8G8R8X8_UNORM>::Store;
1942
    table[TTileMode][B8G8R8X8_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B8G8R8X8_UNORM_SRGB>::Store;
1943
    table[TTileMode][R8G8B8X8_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8X8_UNORM>::Store;
1944
    table[TTileMode][R8G8B8X8_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8X8_UNORM_SRGB>::Store;
1945
}
1946

1947
template <SWR_TILE_MODE TTileMode, size_t NumTileModesT, size_t ArraySizeT>
1948
void InitStoreTilesTableColor_Half2(
1949
    PFN_STORE_TILES(&table)[NumTileModesT][ArraySizeT])
1950
{
1951
    table[TTileMode][R9G9B9E5_SHAREDEXP]            = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R9G9B9E5_SHAREDEXP>::StoreGeneric;
1952
    table[TTileMode][B10G10R10X2_UNORM]             = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10X2_UNORM>::StoreGeneric;
1953
    table[TTileMode][R10G10B10X2_USCALED]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10X2_USCALED>::StoreGeneric;
1954
    table[TTileMode][R8G8B8A8_SSCALED]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_SSCALED>::Store;
1955
    table[TTileMode][R8G8B8A8_USCALED]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R8G8B8A8_USCALED>::Store;
1956
    table[TTileMode][R16G16_SSCALED]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_SSCALED>::Store;
1957
    table[TTileMode][R16G16_USCALED]                = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R16G16_USCALED>::Store;
1958
    table[TTileMode][R32_SSCALED]                   = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R32_SSCALED>::Store;
1959
    table[TTileMode][R32_USCALED]                   = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R32_USCALED>::Store;
1960
    table[TTileMode][B5G6R5_UNORM]                  = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G6R5_UNORM>::Store;
1961
    table[TTileMode][B5G6R5_UNORM_SRGB]             = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G6R5_UNORM_SRGB>::StoreGeneric;
1962
    table[TTileMode][B5G5R5A1_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G5R5A1_UNORM>::StoreGeneric;
1963
    table[TTileMode][B5G5R5A1_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G5R5A1_UNORM_SRGB>::StoreGeneric;
1964
    table[TTileMode][B4G4R4A4_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B4G4R4A4_UNORM>::StoreGeneric;
1965
    table[TTileMode][B4G4R4A4_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B4G4R4A4_UNORM_SRGB>::StoreGeneric;
1966
    table[TTileMode][R8G8_UNORM]                    = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_UNORM>::Store;
1967
    table[TTileMode][R8G8_SNORM]                    = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_SNORM>::Store;
1968
    table[TTileMode][R8G8_SINT]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_SINT>::Store;
1969
    table[TTileMode][R8G8_UINT]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_UINT>::Store;
1970
    table[TTileMode][R16_UNORM]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_UNORM>::Store;
1971
    table[TTileMode][R16_SNORM]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_SNORM>::Store;
1972
    table[TTileMode][R16_SINT]                      = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_SINT>::Store;
1973
    table[TTileMode][R16_UINT]                      = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_UINT>::Store;
1974
    table[TTileMode][R16_FLOAT]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_FLOAT>::Store;
1975
    table[TTileMode][A16_UNORM]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, A16_UNORM>::Store;
1976
    table[TTileMode][A16_FLOAT]                     = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, A16_FLOAT>::Store;
1977
    table[TTileMode][B5G5R5X1_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G5R5X1_UNORM>::StoreGeneric;
1978
    table[TTileMode][B5G5R5X1_UNORM_SRGB]           = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, B5G5R5X1_UNORM_SRGB>::StoreGeneric;
1979
    table[TTileMode][R8G8_SSCALED]                  = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_SSCALED>::Store;
1980
    table[TTileMode][R8G8_USCALED]                  = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R8G8_USCALED>::Store;
1981
    table[TTileMode][R16_SSCALED]                   = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_SSCALED>::Store;
1982
    table[TTileMode][R16_USCALED]                   = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, R16_USCALED>::Store;
1983
    table[TTileMode][A1B5G5R5_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, A1B5G5R5_UNORM>::StoreGeneric;
1984
    table[TTileMode][A4B4G4R4_UNORM]                = StoreMacroTile<TilingTraits<TTileMode, 16>, R32G32B32A32_FLOAT, A4B4G4R4_UNORM>::StoreGeneric;
1985
    table[TTileMode][R8_UNORM]                      = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_UNORM>::Store;
1986
    table[TTileMode][R8_SNORM]                      = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_SNORM>::Store;
1987
    table[TTileMode][R8_SINT]                       = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_SINT>::Store;
1988
    table[TTileMode][R8_UINT]                       = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_UINT>::Store;
1989
    table[TTileMode][A8_UNORM]                      = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, A8_UNORM>::Store;
1990
    table[TTileMode][R8_SSCALED]                    = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_SSCALED>::Store;
1991
    table[TTileMode][R8_USCALED]                    = StoreMacroTile<TilingTraits<TTileMode, 8>, R32G32B32A32_FLOAT, R8_USCALED>::Store;
1992
    table[TTileMode][R8G8B8_UNORM]                  = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_UNORM>::Store;
1993
    table[TTileMode][R8G8B8_SNORM]                  = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_SNORM>::Store;
1994
    table[TTileMode][R8G8B8_SSCALED]                = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_SSCALED>::Store;
1995
    table[TTileMode][R8G8B8_USCALED]                = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_USCALED>::Store;
1996
    table[TTileMode][R16G16B16_FLOAT]               = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_FLOAT>::Store;
1997
    table[TTileMode][R16G16B16_UNORM]               = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_UNORM>::Store;
1998
    table[TTileMode][R16G16B16_SNORM]               = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_SNORM>::Store;
1999
    table[TTileMode][R16G16B16_SSCALED]             = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_SSCALED>::Store;
2000
    table[TTileMode][R16G16B16_USCALED]             = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_USCALED>::Store;
2001
    table[TTileMode][R8G8B8_UNORM_SRGB]             = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_UNORM_SRGB>::Store;
2002
    table[TTileMode][R16G16B16_UINT]                = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_UINT>::Store;
2003
    table[TTileMode][R16G16B16_SINT]                = StoreMacroTile<TilingTraits<TTileMode, 48>, R32G32B32A32_FLOAT, R16G16B16_SINT>::Store;
2004
    table[TTileMode][R10G10B10A2_SNORM]             = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_SNORM>::StoreGeneric;
2005
    table[TTileMode][R10G10B10A2_USCALED]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_USCALED>::StoreGeneric;
2006
    table[TTileMode][R10G10B10A2_SSCALED]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_SSCALED>::StoreGeneric;
2007
    table[TTileMode][R10G10B10A2_SINT]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, R10G10B10A2_SINT>::StoreGeneric;
2008
    table[TTileMode][B10G10R10A2_SNORM]             = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_SNORM>::StoreGeneric;
2009
    table[TTileMode][B10G10R10A2_USCALED]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_USCALED>::StoreGeneric;
2010
    table[TTileMode][B10G10R10A2_SSCALED]           = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_SSCALED>::StoreGeneric;
2011
    table[TTileMode][B10G10R10A2_UINT]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_UINT>::StoreGeneric;
2012
    table[TTileMode][B10G10R10A2_SINT]              = StoreMacroTile<TilingTraits<TTileMode, 32>, R32G32B32A32_FLOAT, B10G10R10A2_SINT>::StoreGeneric;
2013
    table[TTileMode][R8G8B8_UINT]                   = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_UINT>::Store;
2014
    table[TTileMode][R8G8B8_SINT]                   = StoreMacroTile<TilingTraits<TTileMode, 24>, R32G32B32A32_FLOAT, R8G8B8_SINT>::Store;
2015
}
2016

2017
//////////////////////////////////////////////////////////////////////////
2018
/// INIT_STORE_TILES_TABLE - Helper macro for setting up the tables.
2019
template <SWR_TILE_MODE TTileMode, size_t NumTileModes, size_t ArraySizeT>
2020
void InitStoreTilesTableDepth(
2021
    PFN_STORE_TILES(&table)[NumTileModes][ArraySizeT])
2022
{
2023
   table[TTileMode][R32_FLOAT]                      = StoreMacroTile<TilingTraits<TTileMode, 32>, R32_FLOAT, R32_FLOAT>::Store;
2024
   table[TTileMode][R32_FLOAT_X8X24_TYPELESS]       = StoreMacroTile<TilingTraits<TTileMode, 64>, R32_FLOAT, R32_FLOAT_X8X24_TYPELESS>::Store;
2025
   table[TTileMode][R24_UNORM_X8_TYPELESS]          = StoreMacroTile<TilingTraits<TTileMode, 32>, R32_FLOAT, R24_UNORM_X8_TYPELESS>::Store;
2026
   table[TTileMode][R16_UNORM]                      = StoreMacroTile<TilingTraits<TTileMode, 16>, R32_FLOAT, R16_UNORM>::Store;
2027
}
2028

2029
template <SWR_TILE_MODE TTileMode, size_t NumTileModes, size_t ArraySizeT>
2030
void InitStoreTilesTableStencil(
2031
    PFN_STORE_TILES(&table)[NumTileModes][ArraySizeT])
2032
{
2033
    table[TTileMode][R8_UINT]                       = StoreMacroTile<TilingTraits<TTileMode, 8>, R8_UINT, R8_UINT>::Store;
2034
}
2035

2036

2037
//////////////////////////////////////////////////////////////////////////
2038
/// @brief Deswizzles and stores a full hottile to a render surface
2039
/// @param hPrivateContext - Handle to private DC
2040
/// @param srcFormat - Format for hot tile.
2041
/// @param renderTargetIndex - Index to destination render target
2042
/// @param x, y - Coordinates to raster tile.
2043
/// @param pSrcHotTile - Pointer to Hot Tile
2044
void SwrStoreHotTileToSurface(
2045
        HANDLE hWorkerPrivateData,
2046
        SWR_SURFACE_STATE *pDstSurface,
2047
	 BucketManager* pBucketMgr,
2048
        SWR_FORMAT srcFormat,
2049
        SWR_RENDERTARGET_ATTACHMENT renderTargetIndex,
2050
        uint32_t x, uint32_t y, uint32_t renderTargetArrayIndex,
2051
        uint8_t *pSrcHotTile);
2052

2053
Product

Resources

Company
