CoCalc -- jpgd

CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

GitHub Repository: hrydgard/ppsspp
Path: blob/master/ext/jpge/jpgd_idct.h
Views: ¹⁴⁰¹
1
// Copyright 2009 Intel Corporation
2
// All Rights Reserved
3
//
4
// Permission is granted to use, copy, distribute and prepare derivative works of this
5
// software for any purpose and without fee, provided, that the above copyright notice
6
// and this statement appear in all copies.  Intel makes no representations about the
7
// suitability of this software for any purpose.  THIS SOFTWARE IS PROVIDED "AS IS."
8
// INTEL SPECIFICALLY DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, AND ALL LIABILITY,
9
// INCLUDING CONSEQUENTIAL AND OTHER INDIRECT DAMAGES, FOR THE USE OF THIS SOFTWARE,
10
// INCLUDING LIABILITY FOR INFRINGEMENT OF ANY PROPRIETARY RIGHTS, AND INCLUDING THE
11
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  Intel does not
12
// assume any responsibility for any errors which may appear in this software nor any
13
// responsibility to update it.
14
//
15
// From:
16
// https://software.intel.com/sites/default/files/m/d/4/1/d/8/UsingIntelAVXToImplementIDCT-r1_5.pdf
17
// https://software.intel.com/file/29048
18
//
19
// Requires SSE
20
//
21
#ifdef _MSC_VER
22
#include <intrin.h>
23
#endif
24
#include <immintrin.h>
25

26
#ifdef _MSC_VER
27
	#define JPGD_SIMD_ALIGN(type, name) __declspec(align(16)) type name
28
#else
29
	#define JPGD_SIMD_ALIGN(type, name) type name __attribute__((aligned(16)))
30
#endif
31

32
#define BITS_INV_ACC 4
33
#define SHIFT_INV_ROW 16 - BITS_INV_ACC
34
#define SHIFT_INV_COL 1 + BITS_INV_ACC
35
const short IRND_INV_ROW = 1024 * (6 - BITS_INV_ACC);	//1 << (SHIFT_INV_ROW-1)
36
const short IRND_INV_COL = 16 * (BITS_INV_ACC - 3);		// 1 << (SHIFT_INV_COL-1)
37
const short IRND_INV_CORR = IRND_INV_COL - 1;			// correction -1.0 and round
38

39
JPGD_SIMD_ALIGN(short, shortM128_one_corr[8]) = {1, 1, 1, 1, 1, 1, 1, 1};
40
JPGD_SIMD_ALIGN(short, shortM128_round_inv_row[8]) = {IRND_INV_ROW, 0, IRND_INV_ROW, 0, IRND_INV_ROW, 0, IRND_INV_ROW, 0};
41
JPGD_SIMD_ALIGN(short, shortM128_round_inv_col[8]) = {IRND_INV_COL, IRND_INV_COL, IRND_INV_COL, IRND_INV_COL, IRND_INV_COL, IRND_INV_COL, IRND_INV_COL, IRND_INV_COL};
42
JPGD_SIMD_ALIGN(short, shortM128_round_inv_corr[8])= {IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR, IRND_INV_CORR};
43
JPGD_SIMD_ALIGN(short, shortM128_tg_1_16[8]) = {13036, 13036, 13036, 13036, 13036, 13036, 13036, 13036}; // tg * (2<<16) + 0.5
44
JPGD_SIMD_ALIGN(short, shortM128_tg_2_16[8]) = {27146, 27146, 27146, 27146, 27146, 27146, 27146, 27146}; // tg * (2<<16) + 0.5
45
JPGD_SIMD_ALIGN(short, shortM128_tg_3_16[8]) = {-21746, -21746, -21746, -21746, -21746, -21746, -21746, -21746}; // tg * (2<<16) + 0.5
46
JPGD_SIMD_ALIGN(short, shortM128_cos_4_16[8]) = {-19195, -19195, -19195, -19195, -19195, -19195, -19195, -19195};// cos * (2<<16) + 0.5
47

48
//-----------------------------------------------------------------------------
49
// Table for rows 0,4 - constants are multiplied on cos_4_16
50
// w15 w14 w11 w10 w07 w06 w03 w02
51
// w29 w28 w25 w24 w21 w20 w17 w16
52
// w31 w30 w27 w26 w23 w22 w19 w18
53
//movq -> w05 w04 w01 w00
54
JPGD_SIMD_ALIGN(short, shortM128_tab_i_04[]) = {
55
	16384, 21407, 16384, 8867,
56
	16384, -8867, 16384, -21407, // w13 w12 w09 w08
57
	16384, 8867, -16384, -21407, // w07 w06 w03 w02
58
	-16384, 21407, 16384, -8867, // w15 w14 w11 w10
59
	22725, 19266, 19266, -4520, // w21 w20 w17 w16
60
	12873, -22725, 4520, -12873, // w29 w28 w25 w24
61
	12873, 4520, -22725, -12873, // w23 w22 w19 w18
62
	4520, 19266, 19266, -22725}; // w31 w30 w27 w26
63

64
	// Table for rows 1,7 - constants are multiplied on cos_1_16
65
//movq -> w05 w04 w01 w00
66
JPGD_SIMD_ALIGN(short, shortM128_tab_i_17[]) = {
67
	22725, 29692, 22725, 12299,
68
	22725, -12299, 22725, -29692, // w13 w12 w09 w08
69
	22725, 12299, -22725, -29692, // w07 w06 w03 w02
70
	-22725, 29692, 22725, -12299, // w15 w14 w11 w10
71
	31521, 26722, 26722, -6270, // w21 w20 w17 w16
72
	17855, -31521, 6270, -17855, // w29 w28 w25 w24
73
	17855, 6270, -31521, -17855, // w23 w22 w19 w18
74
	6270, 26722, 26722, -31521}; // w31 w30 w27 w26
75

76
// Table for rows 2,6 - constants are multiplied on cos_2_16
77
//movq -> w05 w04 w01 w00
78
JPGD_SIMD_ALIGN(short, shortM128_tab_i_26[]) = {
79
	21407, 27969, 21407, 11585,
80
	21407, -11585, 21407, -27969, // w13 w12 w09 w08
81
	21407, 11585, -21407, -27969, // w07 w06 w03 w02
82
	-21407, 27969, 21407, -11585, // w15 w14 w11 w10
83
	29692, 25172, 25172, -5906,	// w21 w20 w17 w16
84
	16819, -29692, 5906, -16819, // w29 w28 w25 w24
85
	16819, 5906, -29692, -16819, // w23 w22 w19 w18
86
	5906, 25172, 25172, -29692}; // w31 w30 w27 w26
87
// Table for rows 3,5 - constants are multiplied on cos_3_16
88
//movq -> w05 w04 w01 w00
89
JPGD_SIMD_ALIGN(short, shortM128_tab_i_35[]) = {
90
	19266, 25172, 19266, 10426,
91
	19266, -10426, 19266, -25172, // w13 w12 w09 w08
92
	19266, 10426, -19266, -25172, // w07 w06 w03 w02
93
	-19266, 25172, 19266, -10426, // w15 w14 w11 w10
94
	26722, 22654, 22654, -5315, // w21 w20 w17 w16
95
	15137, -26722, 5315, -15137, // w29 w28 w25 w24
96
	15137, 5315, -26722, -15137, // w23 w22 w19 w18
97
	5315, 22654, 22654, -26722}; // w31 w30 w27 w26
98

99
JPGD_SIMD_ALIGN(short, shortM128_128[8]) = { 128, 128, 128, 128, 128, 128, 128, 128 };
100

101
void idctSSEShortU8(const short *pInput, uint8_t * pOutputUB)
102
{
103
	__m128i r_xmm0, r_xmm4;
104
	__m128i r_xmm1, r_xmm2, r_xmm3, r_xmm5, r_xmm6, r_xmm7;
105
	__m128i row0, row1, row2, row3, row4, row5, row6, row7;
106
	short * pTab_i_04 = shortM128_tab_i_04;
107
	short * pTab_i_26 = shortM128_tab_i_26;
108

109
	//Get pointers for this input and output
110
	pTab_i_04 = shortM128_tab_i_04;
111
	pTab_i_26 = shortM128_tab_i_26;
112

113
	//Row 1 and Row 3
114
	r_xmm0 = _mm_load_si128((__m128i *) pInput);
115
	r_xmm4 = _mm_load_si128((__m128i *) (&pInput[2*8]));
116

117
	// *** Work on the data in xmm0
118
	//low shuffle mask = 0xd8 = 11 01 10 00
119
	//get short 2 and short 0 into ls 32-bits
120
	r_xmm0 = _mm_shufflelo_epi16(r_xmm0, 0xd8);
121

122
	// copy short 2 and short 0 to all locations
123
	r_xmm1 = _mm_shuffle_epi32(r_xmm0, 0);
124
		
125
	// add to those copies
126
	r_xmm1 = _mm_madd_epi16(r_xmm1, *((__m128i *) pTab_i_04));
127

128
	// shuffle mask = 0x55 = 01 01 01 01
129
	// copy short 3 and short 1 to all locations
130
	r_xmm3 = _mm_shuffle_epi32(r_xmm0, 0x55);
131
		
132
	// high shuffle mask = 0xd8 = 11 01 10 00
133
	// get short 6 and short 4 into bit positions 64-95
134
	// get short 7 and short 5 into bit positions 96-127
135
	r_xmm0 = _mm_shufflehi_epi16(r_xmm0, 0xd8);
136
		
137
	// add to short 3 and short 1
138
	r_xmm3 = _mm_madd_epi16(r_xmm3, *((__m128i *) &pTab_i_04[16]));
139
		
140
	// shuffle mask = 0xaa = 10 10 10 10
141
	// copy short 6 and short 4 to all locations
142
	r_xmm2 = _mm_shuffle_epi32(r_xmm0, 0xaa);
143
		
144
	// shuffle mask = 0xaa = 11 11 11 11
145
	// copy short 7 and short 5 to all locations
146
	r_xmm0 = _mm_shuffle_epi32(r_xmm0, 0xff);
147
		
148
	// add to short 6 and short 4
149
	r_xmm2 = _mm_madd_epi16(r_xmm2, *((__m128i *) &pTab_i_04[8])); 
150
		
151
	// *** Work on the data in xmm4
152
	// high shuffle mask = 0xd8 11 01 10 00
153
	// get short 6 and short 4 into bit positions 64-95
154
	// get short 7 and short 5 into bit positions 96-127
155
	r_xmm4 = _mm_shufflehi_epi16(r_xmm4, 0xd8);
156
		
157
	// (xmm0 short 2 and short 0 plus pSi) + some constants
158
	r_xmm1 = _mm_add_epi32(r_xmm1, *((__m128i *) shortM128_round_inv_row));
159
	r_xmm4 = _mm_shufflelo_epi16(r_xmm4, 0xd8);
160
	r_xmm0 = _mm_madd_epi16(r_xmm0, *((__m128i *) &pTab_i_04[24]));
161
	r_xmm5 = _mm_shuffle_epi32(r_xmm4, 0);
162
	r_xmm6 = _mm_shuffle_epi32(r_xmm4, 0xaa);
163
	r_xmm5 = _mm_madd_epi16(r_xmm5, *((__m128i *) &shortM128_tab_i_26[0]));
164
	r_xmm1 = _mm_add_epi32(r_xmm1, r_xmm2);
165
	r_xmm2 = r_xmm1;
166
	r_xmm7 = _mm_shuffle_epi32(r_xmm4, 0x55);
167
	r_xmm6 = _mm_madd_epi16(r_xmm6, *((__m128i *) &shortM128_tab_i_26[8])); 
168
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm3);
169
	r_xmm4 = _mm_shuffle_epi32(r_xmm4, 0xff);
170
	r_xmm2 = _mm_sub_epi32(r_xmm2, r_xmm0);
171
	r_xmm7 = _mm_madd_epi16(r_xmm7, *((__m128i *) &shortM128_tab_i_26[16])); 
172
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm1);
173
	r_xmm2 = _mm_srai_epi32(r_xmm2, 12);
174
	r_xmm5 = _mm_add_epi32(r_xmm5, *((__m128i *) shortM128_round_inv_row));
175
	r_xmm4 = _mm_madd_epi16(r_xmm4, *((__m128i *) &shortM128_tab_i_26[24]));
176
	r_xmm5 = _mm_add_epi32(r_xmm5, r_xmm6);
177
	r_xmm6 = r_xmm5;
178
	r_xmm0 = _mm_srai_epi32(r_xmm0, 12);
179
	r_xmm2 = _mm_shuffle_epi32(r_xmm2, 0x1b);
180
	row0 = _mm_packs_epi32(r_xmm0, r_xmm2);
181
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm7);
182
	r_xmm6 = _mm_sub_epi32(r_xmm6, r_xmm4);
183
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm5);
184
	r_xmm6 = _mm_srai_epi32(r_xmm6, 12);
185
	r_xmm4 = _mm_srai_epi32(r_xmm4, 12);
186
	r_xmm6 = _mm_shuffle_epi32(r_xmm6, 0x1b);
187
	row2 = _mm_packs_epi32(r_xmm4, r_xmm6);
188

189
	//Row 5 and row 7
190
	r_xmm0 = _mm_load_si128((__m128i *) (&pInput[4*8]));
191
	r_xmm4 = _mm_load_si128((__m128i *) (&pInput[6*8]));
192

193
	r_xmm0 = _mm_shufflelo_epi16(r_xmm0, 0xd8);
194
	r_xmm1 = _mm_shuffle_epi32(r_xmm0, 0);
195
	r_xmm1 = _mm_madd_epi16(r_xmm1, *((__m128i *) pTab_i_04));
196
	r_xmm3 = _mm_shuffle_epi32(r_xmm0, 0x55);
197
	r_xmm0 = _mm_shufflehi_epi16(r_xmm0, 0xd8);
198
	r_xmm3 = _mm_madd_epi16(r_xmm3, *((__m128i *) &pTab_i_04[16]));
199
	r_xmm2 = _mm_shuffle_epi32(r_xmm0, 0xaa);
200
	r_xmm0 = _mm_shuffle_epi32(r_xmm0, 0xff);
201
	r_xmm2 = _mm_madd_epi16(r_xmm2, *((__m128i *) &pTab_i_04[8])); 
202
	r_xmm4 = _mm_shufflehi_epi16(r_xmm4, 0xd8);
203
	r_xmm1 = _mm_add_epi32(r_xmm1, *((__m128i *) shortM128_round_inv_row));
204
	r_xmm4 = _mm_shufflelo_epi16(r_xmm4, 0xd8);
205
	r_xmm0 = _mm_madd_epi16(r_xmm0, *((__m128i *) &pTab_i_04[24]));
206
	r_xmm5 = _mm_shuffle_epi32(r_xmm4, 0);
207
	r_xmm6 = _mm_shuffle_epi32(r_xmm4, 0xaa);
208
	r_xmm5 = _mm_madd_epi16(r_xmm5, *((__m128i *) &shortM128_tab_i_26[0]));
209
	r_xmm1 = _mm_add_epi32(r_xmm1, r_xmm2);
210
	r_xmm2 = r_xmm1;
211
	r_xmm7 = _mm_shuffle_epi32(r_xmm4, 0x55);
212
	r_xmm6 = _mm_madd_epi16(r_xmm6, *((__m128i *) &shortM128_tab_i_26[8])); 
213
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm3);
214
	r_xmm4 = _mm_shuffle_epi32(r_xmm4, 0xff);
215
	r_xmm2 = _mm_sub_epi32(r_xmm2, r_xmm0);
216
	r_xmm7 = _mm_madd_epi16(r_xmm7, *((__m128i *) &shortM128_tab_i_26[16])); 
217
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm1);
218
	r_xmm2 = _mm_srai_epi32(r_xmm2, 12);
219
	r_xmm5 = _mm_add_epi32(r_xmm5, *((__m128i *) shortM128_round_inv_row));
220
	r_xmm4 = _mm_madd_epi16(r_xmm4, *((__m128i *) &shortM128_tab_i_26[24]));
221
	r_xmm5 = _mm_add_epi32(r_xmm5, r_xmm6);
222
	r_xmm6 = r_xmm5;
223
	r_xmm0 = _mm_srai_epi32(r_xmm0, 12);
224
	r_xmm2 = _mm_shuffle_epi32(r_xmm2, 0x1b);
225
	row4 = _mm_packs_epi32(r_xmm0, r_xmm2);
226
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm7);
227
	r_xmm6 = _mm_sub_epi32(r_xmm6, r_xmm4);
228
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm5);
229
	r_xmm6 = _mm_srai_epi32(r_xmm6, 12);
230
	r_xmm4 = _mm_srai_epi32(r_xmm4, 12);
231
	r_xmm6 = _mm_shuffle_epi32(r_xmm6, 0x1b);
232
	row6 = _mm_packs_epi32(r_xmm4, r_xmm6);
233

234
	//Row 4 and row 2
235
	pTab_i_04 = shortM128_tab_i_35;
236
	pTab_i_26 = shortM128_tab_i_17;
237
	r_xmm0 = _mm_load_si128((__m128i *) (&pInput[3*8]));
238
	r_xmm4 = _mm_load_si128((__m128i *) (&pInput[1*8]));
239

240
	r_xmm0 = _mm_shufflelo_epi16(r_xmm0, 0xd8);
241
	r_xmm1 = _mm_shuffle_epi32(r_xmm0, 0);
242
	r_xmm1 = _mm_madd_epi16(r_xmm1, *((__m128i *) pTab_i_04));
243
	r_xmm3 = _mm_shuffle_epi32(r_xmm0, 0x55);
244
	r_xmm0 = _mm_shufflehi_epi16(r_xmm0, 0xd8);
245
	r_xmm3 = _mm_madd_epi16(r_xmm3, *((__m128i *) &pTab_i_04[16]));
246
	r_xmm2 = _mm_shuffle_epi32(r_xmm0, 0xaa);
247
	r_xmm0 = _mm_shuffle_epi32(r_xmm0, 0xff);
248
	r_xmm2 = _mm_madd_epi16(r_xmm2, *((__m128i *) &pTab_i_04[8])); 
249
	r_xmm4 = _mm_shufflehi_epi16(r_xmm4, 0xd8);
250
	r_xmm1 = _mm_add_epi32(r_xmm1, *((__m128i *) shortM128_round_inv_row));
251
	r_xmm4 = _mm_shufflelo_epi16(r_xmm4, 0xd8);
252
	r_xmm0 = _mm_madd_epi16(r_xmm0, *((__m128i *) &pTab_i_04[24]));
253
	r_xmm5 = _mm_shuffle_epi32(r_xmm4, 0);
254
	r_xmm6 = _mm_shuffle_epi32(r_xmm4, 0xaa);
255
	r_xmm5 = _mm_madd_epi16(r_xmm5, *((__m128i *) &pTab_i_26[0]));
256
	r_xmm1 = _mm_add_epi32(r_xmm1, r_xmm2);
257
	r_xmm2 = r_xmm1;
258
	r_xmm7 = _mm_shuffle_epi32(r_xmm4, 0x55);
259
	r_xmm6 = _mm_madd_epi16(r_xmm6, *((__m128i *) &pTab_i_26[8])); 
260
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm3);
261
	r_xmm4 = _mm_shuffle_epi32(r_xmm4, 0xff);
262
	r_xmm2 = _mm_sub_epi32(r_xmm2, r_xmm0);
263
	r_xmm7 = _mm_madd_epi16(r_xmm7, *((__m128i *) &pTab_i_26[16])); 
264
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm1);
265
	r_xmm2 = _mm_srai_epi32(r_xmm2, 12);
266
	r_xmm5 = _mm_add_epi32(r_xmm5, *((__m128i *) shortM128_round_inv_row));
267
	r_xmm4 = _mm_madd_epi16(r_xmm4, *((__m128i *) &pTab_i_26[24]));
268
	r_xmm5 = _mm_add_epi32(r_xmm5, r_xmm6);
269
	r_xmm6 = r_xmm5;
270
	r_xmm0 = _mm_srai_epi32(r_xmm0, 12);
271
	r_xmm2 = _mm_shuffle_epi32(r_xmm2, 0x1b);
272
	row3 = _mm_packs_epi32(r_xmm0, r_xmm2);
273
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm7);
274
	r_xmm6 = _mm_sub_epi32(r_xmm6, r_xmm4);
275
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm5);
276
	r_xmm6 = _mm_srai_epi32(r_xmm6, 12);
277
	r_xmm4 = _mm_srai_epi32(r_xmm4, 12);
278
	r_xmm6 = _mm_shuffle_epi32(r_xmm6, 0x1b);
279
	row1 = _mm_packs_epi32(r_xmm4, r_xmm6);
280

281
	//Row 6 and row 8
282
	r_xmm0 = _mm_load_si128((__m128i *) (&pInput[5*8]));
283
	r_xmm4 = _mm_load_si128((__m128i *) (&pInput[7*8]));
284

285
	r_xmm0 = _mm_shufflelo_epi16(r_xmm0, 0xd8);
286
	r_xmm1 = _mm_shuffle_epi32(r_xmm0, 0);
287
	r_xmm1 = _mm_madd_epi16(r_xmm1, *((__m128i *) pTab_i_04));
288
	r_xmm3 = _mm_shuffle_epi32(r_xmm0, 0x55);
289
	r_xmm0 = _mm_shufflehi_epi16(r_xmm0, 0xd8);
290
	r_xmm3 = _mm_madd_epi16(r_xmm3, *((__m128i *) &pTab_i_04[16]));
291
	r_xmm2 = _mm_shuffle_epi32(r_xmm0, 0xaa);
292
	r_xmm0 = _mm_shuffle_epi32(r_xmm0, 0xff);
293
	r_xmm2 = _mm_madd_epi16(r_xmm2, *((__m128i *) &pTab_i_04[8])); 
294
	r_xmm4 = _mm_shufflehi_epi16(r_xmm4, 0xd8);
295
	r_xmm1 = _mm_add_epi32(r_xmm1, *((__m128i *) shortM128_round_inv_row));
296
	r_xmm4 = _mm_shufflelo_epi16(r_xmm4, 0xd8);
297
	r_xmm0 = _mm_madd_epi16(r_xmm0, *((__m128i *) &pTab_i_04[24]));
298
	r_xmm5 = _mm_shuffle_epi32(r_xmm4, 0);
299
	r_xmm6 = _mm_shuffle_epi32(r_xmm4, 0xaa);
300
	r_xmm5 = _mm_madd_epi16(r_xmm5, *((__m128i *) &pTab_i_26[0]));
301
	r_xmm1 = _mm_add_epi32(r_xmm1, r_xmm2);
302
	r_xmm2 = r_xmm1;
303
	r_xmm7 = _mm_shuffle_epi32(r_xmm4, 0x55);
304
	r_xmm6 = _mm_madd_epi16(r_xmm6, *((__m128i *) &pTab_i_26[8])); 
305
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm3);
306
	r_xmm4 = _mm_shuffle_epi32(r_xmm4, 0xff);
307
	r_xmm2 = _mm_sub_epi32(r_xmm2, r_xmm0);
308
	r_xmm7 = _mm_madd_epi16(r_xmm7, *((__m128i *) &pTab_i_26[16])); 
309
	r_xmm0 = _mm_add_epi32(r_xmm0, r_xmm1);
310
	r_xmm2 = _mm_srai_epi32(r_xmm2, 12);
311
	r_xmm5 = _mm_add_epi32(r_xmm5, *((__m128i *) shortM128_round_inv_row));
312
	r_xmm4 = _mm_madd_epi16(r_xmm4, *((__m128i *) &pTab_i_26[24]));
313
	r_xmm5 = _mm_add_epi32(r_xmm5, r_xmm6);
314
	r_xmm6 = r_xmm5;
315
	r_xmm0 = _mm_srai_epi32(r_xmm0, 12);
316
	r_xmm2 = _mm_shuffle_epi32(r_xmm2, 0x1b);
317
	row5 = _mm_packs_epi32(r_xmm0, r_xmm2);
318
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm7);
319
	r_xmm6 = _mm_sub_epi32(r_xmm6, r_xmm4);
320
	r_xmm4 = _mm_add_epi32(r_xmm4, r_xmm5);
321
	r_xmm6 = _mm_srai_epi32(r_xmm6, 12);
322
	r_xmm4 = _mm_srai_epi32(r_xmm4, 12);
323
	r_xmm6 = _mm_shuffle_epi32(r_xmm6, 0x1b);
324
	row7 = _mm_packs_epi32(r_xmm4, r_xmm6);
325

326
	r_xmm1 = _mm_load_si128((__m128i *) shortM128_tg_3_16);
327
	r_xmm2 = row5;
328
	r_xmm3 = row3;
329
	r_xmm0 = _mm_mulhi_epi16(row5, r_xmm1);
330

331
	r_xmm1 = _mm_mulhi_epi16(r_xmm1, r_xmm3);
332
	r_xmm5 = _mm_load_si128((__m128i *) shortM128_tg_1_16);
333
	r_xmm6 = row7;
334
	r_xmm4 = _mm_mulhi_epi16(row7, r_xmm5);
335

336
	r_xmm0 = _mm_adds_epi16(r_xmm0, r_xmm2);
337
	r_xmm5 = _mm_mulhi_epi16(r_xmm5, row1);
338
	r_xmm1 = _mm_adds_epi16(r_xmm1, r_xmm3);
339
	r_xmm7 = row6;
340

341
	r_xmm0 = _mm_adds_epi16(r_xmm0, r_xmm3);
342
	r_xmm3 = _mm_load_si128((__m128i *) shortM128_tg_2_16);
343
	r_xmm2 = _mm_subs_epi16(r_xmm2, r_xmm1);
344
	r_xmm7 = _mm_mulhi_epi16(r_xmm7, r_xmm3);
345
	r_xmm1 = r_xmm0;
346
	r_xmm3 = _mm_mulhi_epi16(r_xmm3, row2);
347
	r_xmm5 = _mm_subs_epi16(r_xmm5, r_xmm6);
348
	r_xmm4 = _mm_adds_epi16(r_xmm4, row1);
349
	r_xmm0 = _mm_adds_epi16(r_xmm0, r_xmm4);
350
	r_xmm0 = _mm_adds_epi16(r_xmm0, *((__m128i *) shortM128_one_corr));
351
	r_xmm4 = _mm_subs_epi16(r_xmm4, r_xmm1);
352
	r_xmm6 = r_xmm5;
353
	r_xmm5 = _mm_subs_epi16(r_xmm5, r_xmm2);
354
	r_xmm5 = _mm_adds_epi16(r_xmm5, *((__m128i *) shortM128_one_corr));
355
	r_xmm6 = _mm_adds_epi16(r_xmm6, r_xmm2);
356

357
	//Intermediate results, needed later
358
	__m128i temp3, temp7;
359
	temp7 = r_xmm0;
360

361
	r_xmm1 = r_xmm4;
362
	r_xmm0 = _mm_load_si128((__m128i *) shortM128_cos_4_16);
363
	r_xmm4 = _mm_adds_epi16(r_xmm4, r_xmm5);
364
	r_xmm2 = _mm_load_si128((__m128i *) shortM128_cos_4_16);
365
	r_xmm2 = _mm_mulhi_epi16(r_xmm2, r_xmm4);
366

367
	//Intermediate results, needed later
368
	temp3 = r_xmm6;
369

370
	r_xmm1 = _mm_subs_epi16(r_xmm1, r_xmm5);
371
	r_xmm7 = _mm_adds_epi16(r_xmm7, row2);
372
	r_xmm3 = _mm_subs_epi16(r_xmm3, row6);
373
	r_xmm6 = row0;
374
	r_xmm0 = _mm_mulhi_epi16(r_xmm0, r_xmm1);
375
	r_xmm5 = row4;
376
	r_xmm5 = _mm_adds_epi16(r_xmm5, r_xmm6);
377
	r_xmm6 = _mm_subs_epi16(r_xmm6, row4);
378
	r_xmm4 = _mm_adds_epi16(r_xmm4, r_xmm2);
379

380
	r_xmm4 = _mm_or_si128(r_xmm4, *((__m128i *) shortM128_one_corr));
381
	r_xmm0 = _mm_adds_epi16(r_xmm0, r_xmm1);
382
	r_xmm0 = _mm_or_si128(r_xmm0, *((__m128i *) shortM128_one_corr));
383

384
	r_xmm2 = r_xmm5;
385
	r_xmm5 = _mm_adds_epi16(r_xmm5, r_xmm7);
386
	r_xmm1 = r_xmm6;
387
	r_xmm5 = _mm_adds_epi16(r_xmm5, *((__m128i *) shortM128_round_inv_col));
388
	r_xmm2 = _mm_subs_epi16(r_xmm2, r_xmm7);
389
	r_xmm7 = temp7;
390
	r_xmm6 = _mm_adds_epi16(r_xmm6, r_xmm3);
391
	r_xmm6 = _mm_adds_epi16(r_xmm6, *((__m128i *) shortM128_round_inv_col));
392
	r_xmm7 = _mm_adds_epi16(r_xmm7, r_xmm5);
393
	r_xmm7 = _mm_srai_epi16(r_xmm7, SHIFT_INV_COL);
394
	r_xmm1 = _mm_subs_epi16(r_xmm1, r_xmm3);
395
	r_xmm1 = _mm_adds_epi16(r_xmm1, *((__m128i *) shortM128_round_inv_corr));
396
	r_xmm3 = r_xmm6;
397
	r_xmm2 = _mm_adds_epi16(r_xmm2, *((__m128i *) shortM128_round_inv_corr));
398
	r_xmm6 = _mm_adds_epi16(r_xmm6, r_xmm4);
399

400
	//Store results for row 0
401
	//_mm_store_si128((__m128i *) pOutput, r_xmm7);
402
	__m128i r0 = r_xmm7;
403

404
	r_xmm6 = _mm_srai_epi16(r_xmm6, SHIFT_INV_COL);
405
	r_xmm7 = r_xmm1;
406
	r_xmm1 = _mm_adds_epi16(r_xmm1, r_xmm0);
407

408
	//Store results for row 1
409
	//_mm_store_si128((__m128i *) (&pOutput[1*8]), r_xmm6); 
410
	__m128i r1 = r_xmm6;
411

412
	r_xmm1 = _mm_srai_epi16(r_xmm1, SHIFT_INV_COL);
413
	r_xmm6 = temp3;
414
	r_xmm7 = _mm_subs_epi16(r_xmm7, r_xmm0);
415
	r_xmm7 = _mm_srai_epi16(r_xmm7, SHIFT_INV_COL);
416

417
	//Store results for row 2
418
	//_mm_store_si128((__m128i *) (&pOutput[2*8]), r_xmm1); 
419
	__m128i r2 = r_xmm1;
420

421
	r_xmm5 = _mm_subs_epi16(r_xmm5, temp7); 
422
	r_xmm5 = _mm_srai_epi16(r_xmm5, SHIFT_INV_COL);
423

424
	//Store results for row 7
425
	//_mm_store_si128((__m128i *) (&pOutput[7*8]), r_xmm5); 
426
	__m128i r7 = r_xmm5;
427

428
	r_xmm3 = _mm_subs_epi16(r_xmm3, r_xmm4);
429
	r_xmm6 = _mm_adds_epi16(r_xmm6, r_xmm2);
430
	r_xmm2 = _mm_subs_epi16(r_xmm2, temp3); 
431
	r_xmm6 = _mm_srai_epi16(r_xmm6, SHIFT_INV_COL);
432
	r_xmm2 = _mm_srai_epi16(r_xmm2, SHIFT_INV_COL);
433

434
	//Store results for row 3
435
	//_mm_store_si128((__m128i *) (&pOutput[3*8]), r_xmm6); 
436
	__m128i r3 = r_xmm6;
437

438
	r_xmm3 = _mm_srai_epi16(r_xmm3, SHIFT_INV_COL);
439

440
	//Store results for rows 4, 5, and 6
441
	//_mm_store_si128((__m128i *) (&pOutput[4*8]), r_xmm2);
442
	//_mm_store_si128((__m128i *) (&pOutput[5*8]), r_xmm7);
443
	//_mm_store_si128((__m128i *) (&pOutput[6*8]), r_xmm3);
444

445
	__m128i r4 = r_xmm2;
446
	__m128i r5 = r_xmm7;
447
	__m128i r6 = r_xmm3;
448

449
	r0 = _mm_add_epi16(*(const __m128i *)shortM128_128, r0);
450
	r1 = _mm_add_epi16(*(const __m128i *)shortM128_128, r1);
451
	r2 = _mm_add_epi16(*(const __m128i *)shortM128_128, r2);
452
	r3 = _mm_add_epi16(*(const __m128i *)shortM128_128, r3);
453
	r4 = _mm_add_epi16(*(const __m128i *)shortM128_128, r4);
454
	r5 = _mm_add_epi16(*(const __m128i *)shortM128_128, r5);
455
	r6 = _mm_add_epi16(*(const __m128i *)shortM128_128, r6);
456
	r7 = _mm_add_epi16(*(const __m128i *)shortM128_128, r7);
457

458
	((__m128i *)pOutputUB)[0] = _mm_packus_epi16(r0, r1);
459
	((__m128i *)pOutputUB)[1] = _mm_packus_epi16(r2, r3);
460
	((__m128i *)pOutputUB)[2] = _mm_packus_epi16(r4, r5);
461
	((__m128i *)pOutputUB)[3] = _mm_packus_epi16(r6, r7);
462
}
463

464
CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

Product

Resources

Company
