CoCalc -- sljitNativeX86

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitNativeX86_common.c
²²⁴⁶⁷ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28
{
29
	return "x86" SLJIT_CPUINFO;
30
}
31

32
/*
33
   32b register indexes:
34
     0 - EAX
35
     1 - ECX
36
     2 - EDX
37
     3 - EBX
38
     4 - ESP
39
     5 - EBP
40
     6 - ESI
41
     7 - EDI
42
*/
43

44
/*
45
   64b register indexes:
46
     0 - RAX
47
     1 - RCX
48
     2 - RDX
49
     3 - RBX
50
     4 - RSP
51
     5 - RBP
52
     6 - RSI
53
     7 - RDI
54
     8 - R8   - From now on REX prefix is required
55
     9 - R9
56
    10 - R10
57
    11 - R11
58
    12 - R12
59
    13 - R13
60
    14 - R14
61
    15 - R15
62
*/
63

64
#define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
65
#define TMP_FREG	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
66

67
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
68

69
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 3] = {
70
	0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 5, 7, 6, 4, 3
71
};
72

73
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
74
	0, 1, 2, 3, 4, 5, 6, 7, 0
75
};
76

77
#define CHECK_EXTRA_REGS(p, w, do) \
78
	if (p >= SLJIT_R3 && p <= SLJIT_S3) { \
79
		w = (2 * SSIZE_OF(sw)) + ((p) - SLJIT_R3) * SSIZE_OF(sw); \
80
		p = SLJIT_MEM1(SLJIT_SP); \
81
		do; \
82
	}
83

84
#else /* SLJIT_CONFIG_X86_32 */
85

86
#define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
87

88
/* Note: r12 & 0x7 == 0b100, which decoded as SIB byte present
89
   Note: avoid to use r12 and r13 for memory addressing
90
   therefore r12 is better to be a higher saved register. */
91
#ifndef _WIN64
92
/* Args: rdi(=7), rsi(=6), rdx(=2), rcx(=1), r8, r9. Scratches: rax(=0), r10, r11 */
93
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
94
	0, 0, 6, 7, 1, 8, 11, 10, 12, 5, 13, 14, 15, 3, 4, 2, 9
95
};
96
/* low-map. reg_map & 0x7. */
97
static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
98
	0, 0, 6, 7, 1, 0,  3,  2,  4, 5,  5,  6,  7, 3, 4, 2, 1
99
};
100
#else
101
/* Args: rcx(=1), rdx(=2), r8, r9. Scratches: rax(=0), r10, r11 */
102
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
103
	0, 0, 2, 8, 1, 11, 12, 5, 13, 14, 15, 7, 6, 3, 4, 9, 10
104
};
105
/* low-map. reg_map & 0x7. */
106
static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
107
	0, 0, 2, 0, 1,  3,  4, 5,  5,  6,  7, 7, 6, 3, 4, 1,  2
108
};
109
#endif
110

111
/* Args: xmm0-xmm3 */
112
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
113
	0, 0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4
114
};
115
/* low-map. freg_map & 0x7. */
116
static const sljit_u8 freg_lmap[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
117
	0, 0, 1, 2, 3, 5, 6, 7, 0, 1,  2,  3,  4,  5,  6,  7, 4
118
};
119

120
#define REX_W		0x48
121
#define REX_R		0x44
122
#define REX_X		0x42
123
#define REX_B		0x41
124
#define REX		0x40
125

126
#ifndef _WIN64
127
#define HALFWORD_MAX 0x7fffffffl
128
#define HALFWORD_MIN -0x80000000l
129
#else
130
#define HALFWORD_MAX 0x7fffffffll
131
#define HALFWORD_MIN -0x80000000ll
132
#endif
133

134
#define IS_HALFWORD(x)		((x) <= HALFWORD_MAX && (x) >= HALFWORD_MIN)
135
#define NOT_HALFWORD(x)		((x) > HALFWORD_MAX || (x) < HALFWORD_MIN)
136

137
#define CHECK_EXTRA_REGS(p, w, do)
138

139
#endif /* SLJIT_CONFIG_X86_32 */
140

141
#define U8(v)			((sljit_u8)(v))
142

143
/* Size flags for emit_x86_instruction: */
144
#define EX86_BIN_INS		((sljit_uw)0x000010)
145
#define EX86_SHIFT_INS		((sljit_uw)0x000020)
146
#define EX86_BYTE_ARG		((sljit_uw)0x000040)
147
#define EX86_HALF_ARG		((sljit_uw)0x000080)
148
/* Size flags for both emit_x86_instruction and emit_vex_instruction: */
149
#define EX86_REX		((sljit_uw)0x000100)
150
#define EX86_NO_REXW		((sljit_uw)0x000200)
151
#define EX86_PREF_66		((sljit_uw)0x000400)
152
#define EX86_PREF_F2		((sljit_uw)0x000800)
153
#define EX86_PREF_F3		((sljit_uw)0x001000)
154
#define EX86_SSE2_OP1		((sljit_uw)0x002000)
155
#define EX86_SSE2_OP2		((sljit_uw)0x004000)
156
#define EX86_SSE2		(EX86_SSE2_OP1 | EX86_SSE2_OP2)
157
#define EX86_VEX_EXT		((sljit_uw)0x008000)
158
/* Op flags for emit_vex_instruction: */
159
#define VEX_OP_0F38		((sljit_uw)0x010000)
160
#define VEX_OP_0F3A		((sljit_uw)0x020000)
161
#define VEX_SSE2_OPV		((sljit_uw)0x040000)
162
#define VEX_AUTO_W		((sljit_uw)0x080000)
163
#define VEX_W			((sljit_uw)0x100000)
164
#define VEX_256			((sljit_uw)0x200000)
165

166
#define EX86_SELECT_66(op)	(((op) & SLJIT_32) ? 0 : EX86_PREF_66)
167
#define EX86_SELECT_F2_F3(op)	(((op) & SLJIT_32) ? EX86_PREF_F3 : EX86_PREF_F2)
168

169
/* --------------------------------------------------------------------- */
170
/*  Instruction forms                                                    */
171
/* --------------------------------------------------------------------- */
172

173
#define ADD			(/* BINARY */ 0 << 3)
174
#define ADD_EAX_i32		0x05
175
#define ADD_r_rm		0x03
176
#define ADD_rm_r		0x01
177
#define ADDSD_x_xm		0x58
178
#define ADC			(/* BINARY */ 2 << 3)
179
#define ADC_EAX_i32		0x15
180
#define ADC_r_rm		0x13
181
#define ADC_rm_r		0x11
182
#define AND			(/* BINARY */ 4 << 3)
183
#define AND_EAX_i32		0x25
184
#define AND_r_rm		0x23
185
#define AND_rm_r		0x21
186
#define ANDPD_x_xm		0x54
187
#define BSR_r_rm		(/* GROUP_0F */ 0xbd)
188
#define BSF_r_rm		(/* GROUP_0F */ 0xbc)
189
#define BSWAP_r			(/* GROUP_0F */ 0xc8)
190
#define CALL_i32		0xe8
191
#define CALL_rm			(/* GROUP_FF */ 2 << 3)
192
#define CDQ			0x99
193
#define CMOVE_r_rm		(/* GROUP_0F */ 0x44)
194
#define CMP			(/* BINARY */ 7 << 3)
195
#define CMP_EAX_i32		0x3d
196
#define CMP_r_rm		0x3b
197
#define CMP_rm_r		0x39
198
#define CMPS_x_xm		0xc2
199
#define CMPXCHG_rm_r		0xb1
200
#define CMPXCHG_rm8_r		0xb0
201
#define CVTPD2PS_x_xm		0x5a
202
#define CVTPS2PD_x_xm		0x5a
203
#define CVTSI2SD_x_rm		0x2a
204
#define CVTTSD2SI_r_xm		0x2c
205
#define DIV			(/* GROUP_F7 */ 6 << 3)
206
#define DIVSD_x_xm		0x5e
207
#define EXTRACTPS_x_xm		0x17
208
#define FLDS			0xd9
209
#define FLDL			0xdd
210
#define FSTPS			0xd9
211
#define FSTPD			0xdd
212
#define INSERTPS_x_xm		0x21
213
#define INT3			0xcc
214
#define IDIV			(/* GROUP_F7 */ 7 << 3)
215
#define IMUL			(/* GROUP_F7 */ 5 << 3)
216
#define IMUL_r_rm		(/* GROUP_0F */ 0xaf)
217
#define IMUL_r_rm_i8		0x6b
218
#define IMUL_r_rm_i32		0x69
219
#define JL_i8			0x7c
220
#define JE_i8			0x74
221
#define JNC_i8			0x73
222
#define JNE_i8			0x75
223
#define JMP_i8			0xeb
224
#define JMP_i32			0xe9
225
#define JMP_rm			(/* GROUP_FF */ 4 << 3)
226
#define LEA_r_m			0x8d
227
#define LOOP_i8			0xe2
228
#define LZCNT_r_rm		(/* GROUP_F3 */ /* GROUP_0F */ 0xbd)
229
#define MOV_r_rm		0x8b
230
#define MOV_r_i32		0xb8
231
#define MOV_rm_r		0x89
232
#define MOV_rm_i32		0xc7
233
#define MOV_rm8_i8		0xc6
234
#define MOV_rm8_r8		0x88
235
#define MOVAPS_x_xm		0x28
236
#define MOVAPS_xm_x		0x29
237
#define MOVD_x_rm		0x6e
238
#define MOVD_rm_x		0x7e
239
#define MOVDDUP_x_xm		0x12
240
#define MOVDQA_x_xm		0x6f
241
#define MOVDQA_xm_x		0x7f
242
#define MOVDQU_x_xm		0x6f
243
#define MOVHLPS_x_x		0x12
244
#define MOVHPD_m_x		0x17
245
#define MOVHPD_x_m		0x16
246
#define MOVLHPS_x_x		0x16
247
#define MOVLPD_m_x		0x13
248
#define MOVLPD_x_m		0x12
249
#define MOVMSKPS_r_x		(/* GROUP_0F */ 0x50)
250
#define MOVQ_x_xm		(/* GROUP_0F */ 0x7e)
251
#define MOVSD_x_xm		0x10
252
#define MOVSD_xm_x		0x11
253
#define MOVSHDUP_x_xm		0x16
254
#define MOVSXD_r_rm		0x63
255
#define MOVSX_r_rm8		(/* GROUP_0F */ 0xbe)
256
#define MOVSX_r_rm16		(/* GROUP_0F */ 0xbf)
257
#define MOVUPS_x_xm		0x10
258
#define MOVZX_r_rm8		(/* GROUP_0F */ 0xb6)
259
#define MOVZX_r_rm16		(/* GROUP_0F */ 0xb7)
260
#define MUL			(/* GROUP_F7 */ 4 << 3)
261
#define MULSD_x_xm		0x59
262
#define NEG_rm			(/* GROUP_F7 */ 3 << 3)
263
#define NOP			0x90
264
#define NOT_rm			(/* GROUP_F7 */ 2 << 3)
265
#define OR			(/* BINARY */ 1 << 3)
266
#define OR_r_rm			0x0b
267
#define OR_EAX_i32		0x0d
268
#define OR_rm_r			0x09
269
#define OR_rm8_r8		0x08
270
#define ORPD_x_xm		0x56
271
#define PACKSSWB_x_xm		(/* GROUP_0F */ 0x63)
272
#define PAND_x_xm		0xdb
273
#define PCMPEQD_x_xm		0x76
274
#define PINSRB_x_rm_i8		0x20
275
#define PINSRW_x_rm_i8		0xc4
276
#define PINSRD_x_rm_i8		0x22
277
#define PEXTRB_rm_x_i8		0x14
278
#define PEXTRW_rm_x_i8		0x15
279
#define PEXTRD_rm_x_i8		0x16
280
#define PMOVMSKB_r_x		(/* GROUP_0F */ 0xd7)
281
#define PMOVSXBD_x_xm		0x21
282
#define PMOVSXBQ_x_xm		0x22
283
#define PMOVSXBW_x_xm		0x20
284
#define PMOVSXDQ_x_xm		0x25
285
#define PMOVSXWD_x_xm		0x23
286
#define PMOVSXWQ_x_xm		0x24
287
#define PMOVZXBD_x_xm		0x31
288
#define PMOVZXBQ_x_xm		0x32
289
#define PMOVZXBW_x_xm		0x30
290
#define PMOVZXDQ_x_xm		0x35
291
#define PMOVZXWD_x_xm		0x33
292
#define PMOVZXWQ_x_xm		0x34
293
#define POP_r			0x58
294
#define POP_rm			0x8f
295
#define POPF			0x9d
296
#define POR_x_xm		0xeb
297
#define PREFETCH		0x18
298
#define PSHUFB_x_xm		0x00
299
#define PSHUFD_x_xm		0x70
300
#define PSHUFLW_x_xm		0x70
301
#define PSRLDQ_x		0x73
302
#define PSLLD_x_i8		0x72
303
#define PSLLQ_x_i8		0x73
304
#define PUSH_i32		0x68
305
#define PUSH_r			0x50
306
#define PUSH_rm			(/* GROUP_FF */ 6 << 3)
307
#define PUSHF			0x9c
308
#define PXOR_x_xm		0xef
309
#define ROL			(/* SHIFT */ 0 << 3)
310
#define ROR			(/* SHIFT */ 1 << 3)
311
#define RET_near		0xc3
312
#define RET_i16			0xc2
313
#define SBB			(/* BINARY */ 3 << 3)
314
#define SBB_EAX_i32		0x1d
315
#define SBB_r_rm		0x1b
316
#define SBB_rm_r		0x19
317
#define SAR			(/* SHIFT */ 7 << 3)
318
#define SHL			(/* SHIFT */ 4 << 3)
319
#define SHLD			(/* GROUP_0F */ 0xa5)
320
#define SHRD			(/* GROUP_0F */ 0xad)
321
#define SHR			(/* SHIFT */ 5 << 3)
322
#define SHUFPS_x_xm		0xc6
323
#define SUB			(/* BINARY */ 5 << 3)
324
#define SUB_EAX_i32		0x2d
325
#define SUB_r_rm		0x2b
326
#define SUB_rm_r		0x29
327
#define SUBSD_x_xm		0x5c
328
#define TEST_EAX_i32		0xa9
329
#define TEST_rm_r		0x85
330
#define TZCNT_r_rm		(/* GROUP_F3 */ /* GROUP_0F */ 0xbc)
331
#define UCOMISD_x_xm		0x2e
332
#define UNPCKLPD_x_xm		0x14
333
#define UNPCKLPS_x_xm		0x14
334
#define VBROADCASTSD_x_xm	0x19
335
#define VBROADCASTSS_x_xm	0x18
336
#define VEXTRACTF128_x_ym	0x19
337
#define VEXTRACTI128_x_ym	0x39
338
#define VINSERTF128_y_y_xm	0x18
339
#define VINSERTI128_y_y_xm	0x38
340
#define VPBROADCASTB_x_xm	0x78
341
#define VPBROADCASTD_x_xm	0x58
342
#define VPBROADCASTQ_x_xm	0x59
343
#define VPBROADCASTW_x_xm	0x79
344
#define VPERMPD_y_ym		0x01
345
#define VPERMQ_y_ym		0x00
346
#define XCHG_EAX_r		0x90
347
#define XCHG_r_rm		0x87
348
#define XOR			(/* BINARY */ 6 << 3)
349
#define XOR_EAX_i32		0x35
350
#define XOR_r_rm		0x33
351
#define XOR_rm_r		0x31
352
#define XORPD_x_xm		0x57
353

354
#define GROUP_0F		0x0f
355
#define GROUP_66		0x66
356
#define GROUP_F3		0xf3
357
#define GROUP_F7		0xf7
358
#define GROUP_FF		0xff
359
#define GROUP_BINARY_81		0x81
360
#define GROUP_BINARY_83		0x83
361
#define GROUP_SHIFT_1		0xd1
362
#define GROUP_SHIFT_N		0xc1
363
#define GROUP_SHIFT_CL		0xd3
364
#define GROUP_LOCK		0xf0
365

366
#define MOD_REG			0xc0
367
#define MOD_DISP8		0x40
368

369
#define INC_SIZE(s)		(*inst++ = U8(s), compiler->size += (s))
370

371
#define PUSH_REG(r)		(*inst++ = U8(PUSH_r + (r)))
372
#define POP_REG(r)		(*inst++ = U8(POP_r + (r)))
373
#define RET()			(*inst++ = RET_near)
374
#define RET_I16(n)		(*inst++ = RET_i16, *inst++ = U8(n), *inst++ = 0)
375

376
#define SLJIT_INST_LABEL	255
377
#define SLJIT_INST_JUMP		254
378
#define SLJIT_INST_MOV_ADDR	253
379
#define SLJIT_INST_CONST	252
380

381
/* Multithreading does not affect these static variables, since they store
382
   built-in CPU features. Therefore they can be overwritten by different threads
383
   if they detect the CPU features in the same time. */
384
#define CPU_FEATURE_DETECTED		0x001
385
#if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
386
#define CPU_FEATURE_SSE2		0x002
387
#endif
388
#define CPU_FEATURE_SSE41		0x004
389
#define CPU_FEATURE_LZCNT		0x008
390
#define CPU_FEATURE_TZCNT		0x010
391
#define CPU_FEATURE_CMOV		0x020
392
#define CPU_FEATURE_AVX			0x040
393
#define CPU_FEATURE_AVX2		0x080
394
#define CPU_FEATURE_OSXSAVE		0x100
395

396
static sljit_u32 cpu_feature_list = 0;
397

398
#ifdef _WIN32_WCE
399
#include <cmnintrin.h>
400
#elif defined(_MSC_VER) && _MSC_VER >= 1400
401
#include <intrin.h>
402
#elif defined(__INTEL_COMPILER)
403
#include <cpuid.h>
404
#endif
405

406
#if (defined(_MSC_VER) && _MSC_VER >= 1400) || defined(__INTEL_COMPILER) \
407
	|| (defined(__INTEL_LLVM_COMPILER) && defined(__XSAVE__))
408
#include <immintrin.h>
409
#endif
410

411
/******************************************************/
412
/*    Unaligned-store functions                       */
413
/******************************************************/
414

415
static SLJIT_INLINE void sljit_unaligned_store_s16(void *addr, sljit_s16 value)
416
{
417
	SLJIT_MEMCPY(addr, &value, sizeof(value));
418
}
419

420
static SLJIT_INLINE void sljit_unaligned_store_s32(void *addr, sljit_s32 value)
421
{
422
	SLJIT_MEMCPY(addr, &value, sizeof(value));
423
}
424

425
static SLJIT_INLINE void sljit_unaligned_store_sw(void *addr, sljit_sw value)
426
{
427
	SLJIT_MEMCPY(addr, &value, sizeof(value));
428
}
429

430
/******************************************************/
431
/*    Utility functions                               */
432
/******************************************************/
433

434
static void execute_cpu_id(sljit_u32 info[4])
435
{
436
#if (defined(_MSC_VER) && _MSC_VER >= 1400) \
437
	|| (defined(__INTEL_COMPILER) && __INTEL_COMPILER == 2021 && __INTEL_COMPILER_UPDATE >= 7)
438

439
	__cpuidex((int*)info, (int)info[0], (int)info[2]);
440

441
#elif (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900)
442

443
	__get_cpuid_count(info[0], info[2], info, info + 1, info + 2, info + 3);
444

445
#elif (defined(_MSC_VER) || defined(__INTEL_COMPILER)) \
446
	&& (defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32)
447

448
	/* Intel syntax. */
449
	__asm {
450
		mov esi, info
451
		mov eax, [esi]
452
		mov ecx, [esi + 8]
453
		cpuid
454
		mov [esi], eax
455
		mov [esi + 4], ebx
456
		mov [esi + 8], ecx
457
		mov [esi + 12], edx
458
	}
459

460
#else
461

462
	__asm__ __volatile__ (
463
		"cpuid\n"
464
		: "=a" (info[0]), "=b" (info[1]), "=c" (info[2]), "=d" (info[3])
465
		: "0" (info[0]), "2" (info[2])
466
	);
467

468
#endif
469
}
470

471
static sljit_u32 execute_get_xcr0_low(void)
472
{
473
	sljit_u32 xcr0;
474

475
#if (defined(_MSC_VER) && _MSC_VER >= 1400) || defined(__INTEL_COMPILER) \
476
	|| (defined(__INTEL_LLVM_COMPILER) && defined(__XSAVE__))
477

478
	xcr0 = (sljit_u32)_xgetbv(0);
479

480
#elif defined(__TINYC__)
481

482
	__asm__ (
483
		"xorl %%ecx, %%ecx\n"
484
		".byte 0x0f\n"
485
		".byte 0x01\n"
486
		".byte 0xd0\n"
487
		: "=a" (xcr0)
488
		:
489
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
490
		: "ecx", "edx"
491
#else /* !SLJIT_CONFIG_X86_32 */
492
		: "rcx", "rdx"
493
#endif /* SLJIT_CONFIG_X86_32 */
494
	);
495

496
#elif (defined(__INTEL_LLVM_COMPILER) && __INTEL_LLVM_COMPILER < 20220100) \
497
	|| (defined(__clang__) && __clang_major__ < 14) \
498
	|| (defined(__GNUC__) && __GNUC__ < 3) \
499
	|| defined(__SUNPRO_C) || defined(__SUNPRO_CC)
500

501
	/* AT&T syntax. */
502
	__asm__ (
503
		"xorl %%ecx, %%ecx\n"
504
		"xgetbv\n"
505
		: "=a" (xcr0)
506
		:
507
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
508
		: "ecx", "edx"
509
#else /* !SLJIT_CONFIG_X86_32 */
510
		: "rcx", "rdx"
511
#endif /* SLJIT_CONFIG_X86_32 */
512
	);
513

514
#elif defined(_MSC_VER)
515

516
	/* Intel syntax. */
517
	__asm {
518
		xor ecx, ecx
519
		xgetbv
520
		mov xcr0, eax
521
	}
522

523
#else
524

525
	__asm__ (
526
		"xor{l %%ecx, %%ecx | ecx, ecx}\n"
527
		"xgetbv\n"
528
		: "=a" (xcr0)
529
		:
530
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
531
		: "ecx", "edx"
532
#else /* !SLJIT_CONFIG_X86_32 */
533
		: "rcx", "rdx"
534
#endif /* SLJIT_CONFIG_X86_32 */
535
	);
536

537
#endif
538
	return xcr0;
539
}
540

541
static void get_cpu_features(void)
542
{
543
	sljit_u32 feature_list = CPU_FEATURE_DETECTED;
544
	sljit_u32 info[4] = {0};
545
	sljit_u32 max_id;
546

547
	execute_cpu_id(info);
548
	max_id = info[0];
549

550
	if (max_id >= 7) {
551
		info[0] = 7;
552
		info[2] = 0;
553
		execute_cpu_id(info);
554

555
		if (info[1] & 0x8)
556
			feature_list |= CPU_FEATURE_TZCNT;
557
		if (info[1] & 0x20)
558
			feature_list |= CPU_FEATURE_AVX2;
559
	}
560

561
	if (max_id >= 1) {
562
		info[0] = 1;
563
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
564
		/* Winchip 2 and Cyrix MII bugs */
565
		info[1] = info[2] = 0;
566
#endif
567
		execute_cpu_id(info);
568

569
		if (info[2] & 0x80000)
570
			feature_list |= CPU_FEATURE_SSE41;
571
		if (info[2] & 0x8000000)
572
			feature_list |= CPU_FEATURE_OSXSAVE;
573
		if (info[2] & 0x10000000)
574
			feature_list |= CPU_FEATURE_AVX;
575
#if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
576
		if (info[3] & 0x4000000)
577
			feature_list |= CPU_FEATURE_SSE2;
578
#endif
579
		if (info[3] & 0x8000)
580
			feature_list |= CPU_FEATURE_CMOV;
581
	}
582

583
	info[0] = 0x80000000;
584
	execute_cpu_id(info);
585
	max_id = info[0];
586

587
	if (max_id >= 0x80000001) {
588
		info[0] = 0x80000001;
589
		execute_cpu_id(info);
590

591
		if (info[2] & 0x20)
592
			feature_list |= CPU_FEATURE_LZCNT;
593
	}
594

595
	if ((feature_list & CPU_FEATURE_OSXSAVE) && (execute_get_xcr0_low() & 0x4) == 0)
596
		feature_list &= ~(sljit_u32)(CPU_FEATURE_AVX | CPU_FEATURE_AVX2);
597

598
	cpu_feature_list = feature_list;
599
}
600

601
static sljit_u8 get_jump_code(sljit_uw type)
602
{
603
	switch (type) {
604
	case SLJIT_EQUAL:
605
	case SLJIT_ATOMIC_STORED:
606
	case SLJIT_F_EQUAL:
607
	case SLJIT_UNORDERED_OR_EQUAL:
608
		return 0x84 /* je */;
609

610
	case SLJIT_NOT_EQUAL:
611
	case SLJIT_ATOMIC_NOT_STORED:
612
	case SLJIT_F_NOT_EQUAL:
613
	case SLJIT_ORDERED_NOT_EQUAL:
614
		return 0x85 /* jne */;
615

616
	case SLJIT_LESS:
617
	case SLJIT_CARRY:
618
	case SLJIT_F_LESS:
619
	case SLJIT_UNORDERED_OR_LESS:
620
	case SLJIT_UNORDERED_OR_GREATER:
621
		return 0x82 /* jc */;
622

623
	case SLJIT_GREATER_EQUAL:
624
	case SLJIT_NOT_CARRY:
625
	case SLJIT_F_GREATER_EQUAL:
626
	case SLJIT_ORDERED_GREATER_EQUAL:
627
	case SLJIT_ORDERED_LESS_EQUAL:
628
		return 0x83 /* jae */;
629

630
	case SLJIT_GREATER:
631
	case SLJIT_F_GREATER:
632
	case SLJIT_ORDERED_LESS:
633
	case SLJIT_ORDERED_GREATER:
634
		return 0x87 /* jnbe */;
635

636
	case SLJIT_LESS_EQUAL:
637
	case SLJIT_F_LESS_EQUAL:
638
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
639
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
640
		return 0x86 /* jbe */;
641

642
	case SLJIT_SIG_LESS:
643
		return 0x8c /* jl */;
644

645
	case SLJIT_SIG_GREATER_EQUAL:
646
		return 0x8d /* jnl */;
647

648
	case SLJIT_SIG_GREATER:
649
		return 0x8f /* jnle */;
650

651
	case SLJIT_SIG_LESS_EQUAL:
652
		return 0x8e /* jle */;
653

654
	case SLJIT_OVERFLOW:
655
		return 0x80 /* jo */;
656

657
	case SLJIT_NOT_OVERFLOW:
658
		return 0x81 /* jno */;
659

660
	case SLJIT_UNORDERED:
661
	case SLJIT_ORDERED_EQUAL: /* NaN. */
662
		return 0x8a /* jp */;
663

664
	case SLJIT_ORDERED:
665
	case SLJIT_UNORDERED_OR_NOT_EQUAL: /* Not NaN. */
666
		return 0x8b /* jpo */;
667
	}
668
	return 0;
669
}
670

671
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
672
static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset);
673
#else /* !SLJIT_CONFIG_X86_32 */
674
static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr);
675
static sljit_u8* generate_mov_addr_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset);
676
#endif /* SLJIT_CONFIG_X86_32 */
677

678
static sljit_u8* detect_near_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset)
679
{
680
	sljit_uw type = jump->flags >> TYPE_SHIFT;
681
	sljit_s32 short_jump;
682
	sljit_uw label_addr;
683
	sljit_uw jump_addr;
684

685
	jump_addr = (sljit_uw)code_ptr;
686
	if (!(jump->flags & JUMP_ADDR)) {
687
		label_addr = (sljit_uw)(code + jump->u.label->size);
688

689
		if (jump->u.label->size > jump->addr)
690
			jump_addr = (sljit_uw)(code + jump->addr);
691
	} else
692
		label_addr = jump->u.target - (sljit_uw)executable_offset;
693

694
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
695
	if ((sljit_sw)(label_addr - (jump_addr + 6)) > HALFWORD_MAX || (sljit_sw)(label_addr - (jump_addr + 5)) < HALFWORD_MIN)
696
		return detect_far_jump_type(jump, code_ptr);
697
#endif /* SLJIT_CONFIG_X86_64 */
698

699
	short_jump = (sljit_sw)(label_addr - (jump_addr + 2)) >= -0x80 && (sljit_sw)(label_addr - (jump_addr + 2)) <= 0x7f;
700

701
	if (type == SLJIT_JUMP) {
702
		if (short_jump)
703
			*code_ptr++ = JMP_i8;
704
		else
705
			*code_ptr++ = JMP_i32;
706
	} else if (type > SLJIT_JUMP) {
707
		short_jump = 0;
708
		*code_ptr++ = CALL_i32;
709
	} else if (short_jump) {
710
		*code_ptr++ = U8(get_jump_code(type) - 0x10);
711
	} else {
712
		*code_ptr++ = GROUP_0F;
713
		*code_ptr++ = get_jump_code(type);
714
	}
715

716
	jump->addr = (sljit_uw)code_ptr;
717

718
	if (short_jump) {
719
		jump->flags |= PATCH_MB;
720
		code_ptr += sizeof(sljit_s8);
721
	} else {
722
		jump->flags |= PATCH_MW;
723
		code_ptr += sizeof(sljit_s32);
724
	}
725

726
	return code_ptr;
727
}
728

729
static void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
730
{
731
	sljit_uw flags = jump->flags;
732
	sljit_uw addr = (flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
733
	sljit_uw jump_addr = jump->addr;
734
	SLJIT_UNUSED_ARG(executable_offset);
735

736
	if (SLJIT_UNLIKELY(flags & JUMP_MOV_ADDR)) {
737
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
738
		sljit_unaligned_store_sw((void*)(jump_addr - sizeof(sljit_sw)), (sljit_sw)addr);
739
#else /* SLJIT_CONFIG_X86_32 */
740
		if (flags & PATCH_MD) {
741
			SLJIT_ASSERT(addr > HALFWORD_MAX);
742
			sljit_unaligned_store_sw((void*)(jump_addr - sizeof(sljit_sw)), (sljit_sw)addr);
743
			return;
744
		}
745

746
		if (flags & PATCH_MW) {
747
			addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET((sljit_u8*)jump_addr, executable_offset);
748
			SLJIT_ASSERT((sljit_sw)addr <= HALFWORD_MAX && (sljit_sw)addr >= HALFWORD_MIN);
749
		} else {
750
			SLJIT_ASSERT(addr <= HALFWORD_MAX);
751
		}
752
		sljit_unaligned_store_s32((void*)(jump_addr - sizeof(sljit_s32)), (sljit_s32)addr);
753
#endif /* !SLJIT_CONFIG_X86_32 */
754
		return;
755
	}
756

757
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
758
	if (SLJIT_UNLIKELY(flags & PATCH_MD)) {
759
		SLJIT_ASSERT(!(flags & JUMP_ADDR));
760
		sljit_unaligned_store_sw((void*)jump_addr, (sljit_sw)addr);
761
		return;
762
	}
763
#endif /* SLJIT_CONFIG_X86_64 */
764

765
	addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET((sljit_u8*)jump_addr, executable_offset);
766

767
	if (flags & PATCH_MB) {
768
		addr -= sizeof(sljit_s8);
769
		SLJIT_ASSERT((sljit_sw)addr <= 0x7f && (sljit_sw)addr >= -0x80);
770
		*(sljit_u8*)jump_addr = U8(addr);
771
		return;
772
	} else if (flags & PATCH_MW) {
773
		addr -= sizeof(sljit_s32);
774
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
775
		sljit_unaligned_store_sw((void*)jump_addr, (sljit_sw)addr);
776
#else /* !SLJIT_CONFIG_X86_32 */
777
		SLJIT_ASSERT((sljit_sw)addr <= HALFWORD_MAX && (sljit_sw)addr >= HALFWORD_MIN);
778
		sljit_unaligned_store_s32((void*)jump_addr, (sljit_s32)addr);
779
#endif /* SLJIT_CONFIG_X86_32 */
780
	}
781
}
782

783
static sljit_u8 *process_extended_label(sljit_u8 *code_ptr, struct sljit_extended_label *ext_label)
784
{
785
	sljit_uw mask;
786
	sljit_u8 *ptr = code_ptr;
787

788
	SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
789
	mask = ext_label->data;
790

791
	code_ptr = (sljit_u8*)(((sljit_uw)code_ptr + mask) & ~mask);
792

793
	while (ptr < code_ptr)
794
		*ptr++ = NOP;
795

796
	return code_ptr;
797
}
798

799
static void reduce_code_size(struct sljit_compiler *compiler)
800
{
801
	struct sljit_label *label;
802
	struct sljit_jump *jump;
803
	sljit_uw next_label_size;
804
	sljit_uw next_jump_addr;
805
	sljit_uw next_min_addr;
806
	sljit_uw size_reduce = 0;
807
	sljit_sw diff;
808
	sljit_uw type;
809
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
810
	sljit_uw size_reduce_max;
811
#endif /* SLJIT_DEBUG */
812

813
	label = compiler->labels;
814
	jump = compiler->jumps;
815

816
	next_label_size = SLJIT_GET_NEXT_SIZE(label);
817
	next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
818

819
	while (1) {
820
		next_min_addr = next_label_size;
821
		if (next_jump_addr < next_min_addr)
822
			next_min_addr = next_jump_addr;
823

824
		if (next_min_addr == SLJIT_MAX_ADDRESS)
825
			break;
826

827
		if (next_min_addr == next_label_size) {
828
			label->size -= size_reduce;
829

830
			label = label->next;
831
			next_label_size = SLJIT_GET_NEXT_SIZE(label);
832
		}
833

834
		if (next_min_addr != next_jump_addr)
835
			continue;
836

837
		jump->addr -= size_reduce;
838
		if (!(jump->flags & JUMP_MOV_ADDR)) {
839
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
840
			size_reduce_max = size_reduce + (((jump->flags >> TYPE_SHIFT) < SLJIT_JUMP) ? CJUMP_MAX_SIZE : JUMP_MAX_SIZE);
841
#endif /* SLJIT_DEBUG */
842

843
			if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
844
				if (jump->flags & JUMP_ADDR) {
845
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
846
					if (jump->u.target <= 0xffffffffl)
847
						size_reduce += sizeof(sljit_s32);
848
#endif /* SLJIT_CONFIG_X86_64 */
849
				} else {
850
					/* Unit size: instruction. */
851
					diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
852
					if (jump->u.label->size > jump->addr) {
853
						SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
854
						diff -= (sljit_sw)size_reduce;
855
					}
856
					type = jump->flags >> TYPE_SHIFT;
857

858
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
859
					if (type == SLJIT_JUMP) {
860
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
861
							size_reduce += JUMP_MAX_SIZE - 2;
862
						else if (diff <= HALFWORD_MAX + 5 && diff >= HALFWORD_MIN + 5)
863
							size_reduce += JUMP_MAX_SIZE - 5;
864
					} else if (type < SLJIT_JUMP) {
865
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
866
							size_reduce += CJUMP_MAX_SIZE - 2;
867
						else if (diff <= HALFWORD_MAX + 6 && diff >= HALFWORD_MIN + 6)
868
							size_reduce += CJUMP_MAX_SIZE - 6;
869
					} else  {
870
						if (diff <= HALFWORD_MAX + 5 && diff >= HALFWORD_MIN + 5)
871
							size_reduce += JUMP_MAX_SIZE - 5;
872
					}
873
#else /* !SLJIT_CONFIG_X86_64 */
874
					if (type == SLJIT_JUMP) {
875
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
876
							size_reduce += JUMP_MAX_SIZE - 2;
877
					} else if (type < SLJIT_JUMP) {
878
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
879
							size_reduce += CJUMP_MAX_SIZE - 2;
880
					}
881
#endif /* SLJIT_CONFIG_X86_64 */
882
				}
883
			}
884

885
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
886
			jump->flags |= (size_reduce_max - size_reduce) << JUMP_SIZE_SHIFT;
887
#endif /* SLJIT_DEBUG */
888
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
889
		} else {
890
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
891
			size_reduce_max = size_reduce + 10;
892
#endif /* SLJIT_DEBUG */
893

894
			if (!(jump->flags & JUMP_ADDR)) {
895
				diff = (sljit_sw)jump->u.label->size - (sljit_sw)(jump->addr - 3);
896

897
				if (diff <= HALFWORD_MAX && diff >= HALFWORD_MIN)
898
					size_reduce += 3;
899
			} else if (jump->u.target <= 0xffffffffl)
900
				size_reduce += (jump->flags & MOV_ADDR_HI) ? 4 : 5;
901

902
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
903
			jump->flags |= (size_reduce_max - size_reduce) << JUMP_SIZE_SHIFT;
904
#endif /* SLJIT_DEBUG */
905
#endif /* SLJIT_CONFIG_X86_64 */
906
		}
907

908
		jump = jump->next;
909
		next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
910
	}
911

912
	compiler->size -= size_reduce;
913
}
914

915
SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
916
{
917
	struct sljit_memory_fragment *buf;
918
	sljit_u8 *code;
919
	sljit_u8 *code_ptr;
920
	sljit_u8 *buf_ptr;
921
	sljit_u8 *buf_end;
922
	sljit_u8 len;
923
	sljit_sw executable_offset;
924
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
925
	sljit_uw addr;
926
#endif /* SLJIT_DEBUG */
927

928
	struct sljit_label *label;
929
	struct sljit_jump *jump;
930
	struct sljit_const *const_;
931

932
	CHECK_ERROR_PTR();
933
	CHECK_PTR(check_sljit_generate_code(compiler, options));
934

935
	reduce_code_size(compiler);
936

937
	/* Second code generation pass. */
938
	code = (sljit_u8*)allocate_executable_memory(compiler->size, options, exec_allocator_data, &executable_offset);
939
	PTR_FAIL_WITH_EXEC_IF(code);
940

941
	reverse_buf(compiler);
942
	buf = compiler->buf;
943

944
	code_ptr = code;
945
	label = compiler->labels;
946
	jump = compiler->jumps;
947
	const_ = compiler->consts;
948

949
	do {
950
		buf_ptr = buf->memory;
951
		buf_end = buf_ptr + buf->used_size;
952
		do {
953
			len = *buf_ptr++;
954
			SLJIT_ASSERT(len > 0);
955
			if (len < SLJIT_INST_CONST) {
956
				/* The code is already generated. */
957
				SLJIT_MEMCPY(code_ptr, buf_ptr, len);
958
				code_ptr += len;
959
				buf_ptr += len;
960
			} else {
961
				switch (len) {
962
				case SLJIT_INST_LABEL:
963
					if (label->u.index >= SLJIT_LABEL_ALIGNED)
964
						code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
965

966
					label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
967
					label->size = (sljit_uw)(code_ptr - code);
968
					label = label->next;
969
					break;
970
				case SLJIT_INST_JUMP:
971
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
972
					addr = (sljit_uw)code_ptr;
973
#endif /* SLJIT_DEBUG */
974
					if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
975
						code_ptr = detect_near_jump_type(jump, code_ptr, code, executable_offset);
976
					else {
977
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
978
						code_ptr = detect_far_jump_type(jump, code_ptr, executable_offset);
979
#else /* !SLJIT_CONFIG_X86_32 */
980
						code_ptr = detect_far_jump_type(jump, code_ptr);
981
#endif /* SLJIT_CONFIG_X86_32 */
982
					}
983

984
					SLJIT_ASSERT((sljit_uw)code_ptr - addr <= ((jump->flags >> JUMP_SIZE_SHIFT) & 0xff));
985
					jump = jump->next;
986
					break;
987
				case SLJIT_INST_MOV_ADDR:
988
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
989
					code_ptr = generate_mov_addr_code(jump, code_ptr, code, executable_offset);
990
#endif /* SLJIT_CONFIG_X86_64 */
991
					jump->addr = (sljit_uw)code_ptr;
992
					jump = jump->next;
993
					break;
994
				default:
995
					SLJIT_ASSERT(len == SLJIT_INST_CONST);
996
					const_->addr = (sljit_uw)code_ptr;
997
					const_ = const_->next;
998
					break;
999
				}
1000
			}
1001
		} while (buf_ptr < buf_end);
1002

1003
		SLJIT_ASSERT(buf_ptr == buf_end);
1004
		buf = buf->next;
1005
	} while (buf);
1006

1007
	SLJIT_ASSERT(!label);
1008
	SLJIT_ASSERT(!jump);
1009
	SLJIT_ASSERT(!const_);
1010
	SLJIT_ASSERT(code_ptr <= code + compiler->size);
1011

1012
	jump = compiler->jumps;
1013
	while (jump) {
1014
		generate_jump_or_mov_addr(jump, executable_offset);
1015
		jump = jump->next;
1016
	}
1017

1018
	compiler->error = SLJIT_ERR_COMPILED;
1019
	compiler->executable_offset = executable_offset;
1020
	compiler->executable_size = (sljit_uw)(code_ptr - code);
1021

1022
	code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
1023

1024
	SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1);
1025
	return (void*)code;
1026
}
1027

1028
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
1029
{
1030
	switch (feature_type) {
1031
	case SLJIT_HAS_FPU:
1032
#ifdef SLJIT_IS_FPU_AVAILABLE
1033
		return (SLJIT_IS_FPU_AVAILABLE) != 0;
1034
#elif (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
1035
		if (cpu_feature_list == 0)
1036
			get_cpu_features();
1037
		return (cpu_feature_list & CPU_FEATURE_SSE2) != 0;
1038
#else /* SLJIT_DETECT_SSE2 */
1039
		return 1;
1040
#endif /* SLJIT_DETECT_SSE2 */
1041

1042
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1043
	case SLJIT_HAS_VIRTUAL_REGISTERS:
1044
		return 1;
1045
#endif /* SLJIT_CONFIG_X86_32 */
1046

1047
	case SLJIT_HAS_CLZ:
1048
		if (cpu_feature_list == 0)
1049
			get_cpu_features();
1050

1051
		return (cpu_feature_list & CPU_FEATURE_LZCNT) ? 1 : 2;
1052

1053
	case SLJIT_HAS_CTZ:
1054
		if (cpu_feature_list == 0)
1055
			get_cpu_features();
1056

1057
		return (cpu_feature_list & CPU_FEATURE_TZCNT) ? 1 : 2;
1058

1059
	case SLJIT_HAS_CMOV:
1060
		if (cpu_feature_list == 0)
1061
			get_cpu_features();
1062
		return (cpu_feature_list & CPU_FEATURE_CMOV) != 0;
1063

1064
	case SLJIT_HAS_REV:
1065
	case SLJIT_HAS_ROT:
1066
	case SLJIT_HAS_PREFETCH:
1067
	case SLJIT_HAS_COPY_F32:
1068
	case SLJIT_HAS_COPY_F64:
1069
	case SLJIT_HAS_ATOMIC:
1070
	case SLJIT_HAS_MEMORY_BARRIER:
1071
		return 1;
1072

1073
#if !(defined SLJIT_IS_FPU_AVAILABLE) || SLJIT_IS_FPU_AVAILABLE
1074
	case SLJIT_HAS_AVX:
1075
		if (cpu_feature_list == 0)
1076
			get_cpu_features();
1077
		return (cpu_feature_list & CPU_FEATURE_AVX) != 0;
1078
	case SLJIT_HAS_AVX2:
1079
		if (cpu_feature_list == 0)
1080
			get_cpu_features();
1081
		return (cpu_feature_list & CPU_FEATURE_AVX2) != 0;
1082
	case SLJIT_HAS_SIMD:
1083
		if (cpu_feature_list == 0)
1084
			get_cpu_features();
1085
		return (cpu_feature_list & CPU_FEATURE_SSE41) != 0;
1086
#endif /* SLJIT_IS_FPU_AVAILABLE */
1087
	default:
1088
		return 0;
1089
	}
1090
}
1091

1092
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
1093
{
1094
	switch (type) {
1095
	case SLJIT_ORDERED_EQUAL:
1096
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
1097
		return 2;
1098
	}
1099

1100
	return 0;
1101
}
1102

1103
/* --------------------------------------------------------------------- */
1104
/*  Operators                                                            */
1105
/* --------------------------------------------------------------------- */
1106

1107
#define BINARY_OPCODE(opcode) (((opcode ## _EAX_i32) << 24) | ((opcode ## _r_rm) << 16) | ((opcode ## _rm_r) << 8) | (opcode))
1108

1109
#define BINARY_IMM32(op_imm, immw, arg, argw) \
1110
	do { \
1111
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1112
		FAIL_IF(!inst); \
1113
		*(inst + 1) |= (op_imm); \
1114
	} while (0)
1115

1116
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1117

1118
#define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1119
	do { \
1120
		if (IS_HALFWORD(immw) || compiler->mode32) { \
1121
			BINARY_IMM32(op_imm, immw, arg, argw); \
1122
		} \
1123
		else { \
1124
			FAIL_IF(emit_load_imm64(compiler, FAST_IS_REG(arg) ? TMP_REG2 : TMP_REG1, immw)); \
1125
			inst = emit_x86_instruction(compiler, 1, FAST_IS_REG(arg) ? TMP_REG2 : TMP_REG1, 0, arg, argw); \
1126
			FAIL_IF(!inst); \
1127
			*inst = (op_mr); \
1128
		} \
1129
	} while (0)
1130

1131
#define BINARY_EAX_IMM(op_eax_imm, immw) \
1132
	FAIL_IF(emit_do_imm32(compiler, (!compiler->mode32) ? REX_W : 0, (op_eax_imm), immw))
1133

1134
#else /* !SLJIT_CONFIG_X86_64 */
1135

1136
#define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1137
	BINARY_IMM32(op_imm, immw, arg, argw)
1138

1139
#define BINARY_EAX_IMM(op_eax_imm, immw) \
1140
	FAIL_IF(emit_do_imm(compiler, (op_eax_imm), immw))
1141

1142
#endif /* SLJIT_CONFIG_X86_64 */
1143

1144
static sljit_s32 emit_byte(struct sljit_compiler *compiler, sljit_u8 byte)
1145
{
1146
	sljit_u8 *inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1147
	FAIL_IF(!inst);
1148
	INC_SIZE(1);
1149
	*inst = byte;
1150
	return SLJIT_SUCCESS;
1151
}
1152

1153
static sljit_s32 emit_mov(struct sljit_compiler *compiler,
1154
	sljit_s32 dst, sljit_sw dstw,
1155
	sljit_s32 src, sljit_sw srcw);
1156

1157
#define EMIT_MOV(compiler, dst, dstw, src, srcw) \
1158
	FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
1159

1160
static sljit_s32 emit_groupf(struct sljit_compiler *compiler,
1161
	sljit_uw op,
1162
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1163

1164
static sljit_s32 emit_groupf_ext(struct sljit_compiler *compiler,
1165
	sljit_uw op,
1166
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1167

1168
static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
1169
	sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src);
1170

1171
static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
1172
	sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1173

1174
static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
1175
	sljit_s32 src1, sljit_sw src1w,
1176
	sljit_s32 src2, sljit_sw src2w);
1177

1178
static sljit_s32 emit_cmov_generic(struct sljit_compiler *compiler, sljit_s32 type,
1179
	sljit_s32 dst_reg,
1180
	sljit_s32 src, sljit_sw srcw);
1181

1182
static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler)
1183
{
1184
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET)
1185
	/* Emit endbr32/endbr64 when CET is enabled.  */
1186
	sljit_u8 *inst;
1187
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1188
	FAIL_IF(!inst);
1189
	INC_SIZE(4);
1190
	inst[0] = GROUP_F3;
1191
	inst[1] = GROUP_0F;
1192
	inst[2] = 0x1e;
1193
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1194
	inst[3] = 0xfb;
1195
#else /* !SLJIT_CONFIG_X86_32 */
1196
	inst[3] = 0xfa;
1197
#endif /* SLJIT_CONFIG_X86_32 */
1198
#else /* !SLJIT_CONFIG_X86_CET */
1199
	SLJIT_UNUSED_ARG(compiler);
1200
#endif /* SLJIT_CONFIG_X86_CET */
1201
	return SLJIT_SUCCESS;
1202
}
1203

1204
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1205

1206
static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg)
1207
{
1208
	sljit_u8 *inst;
1209
	sljit_s32 size;
1210

1211
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1212
	size = 5;
1213
#else
1214
	size = 4;
1215
#endif
1216

1217
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1218
	FAIL_IF(!inst);
1219
	INC_SIZE(size);
1220
	*inst++ = GROUP_F3;
1221
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1222
	*inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
1223
#endif
1224
	inst[0] = GROUP_0F;
1225
	inst[1] = 0x1e;
1226
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1227
	inst[2] = U8(MOD_REG | (0x1 << 3) | reg_lmap[reg]);
1228
#else
1229
	inst[2] = U8(MOD_REG | (0x1 << 3) | reg_map[reg]);
1230
#endif
1231
	return SLJIT_SUCCESS;
1232
}
1233

1234
static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg)
1235
{
1236
	sljit_u8 *inst;
1237
	sljit_s32 size;
1238

1239
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1240
	size = 5;
1241
#else
1242
	size = 4;
1243
#endif
1244

1245
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1246
	FAIL_IF(!inst);
1247
	INC_SIZE(size);
1248
	*inst++ = GROUP_F3;
1249
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1250
	*inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
1251
#endif
1252
	inst[0] = GROUP_0F;
1253
	inst[1] = 0xae;
1254
	inst[2] = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7);
1255
	return SLJIT_SUCCESS;
1256
}
1257

1258
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1259

1260
static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void)
1261
{
1262
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1263
	return _get_ssp() != 0;
1264
#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
1265
	return 0;
1266
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1267
}
1268

1269
static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler,
1270
	sljit_s32 src, sljit_sw srcw)
1271
{
1272
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1273
	sljit_u8 *inst, *jz_after_cmp_inst;
1274
	sljit_uw size_jz_after_cmp_inst;
1275

1276
	sljit_uw size_before_rdssp_inst = compiler->size;
1277

1278
	/* Generate "RDSSP TMP_REG1". */
1279
	FAIL_IF(emit_rdssp(compiler, TMP_REG1));
1280

1281
	/* Load return address on shadow stack into TMP_REG1. */
1282
	EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), 0);
1283

1284
	/* Compare return address against TMP_REG1. */
1285
	FAIL_IF(emit_cmp_binary (compiler, TMP_REG1, 0, src, srcw));
1286

1287
	/* Generate JZ to skip shadow stack ajdustment when shadow
1288
	   stack matches normal stack. */
1289
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1290
	FAIL_IF(!inst);
1291
	INC_SIZE(2);
1292
	*inst++ = get_jump_code(SLJIT_EQUAL) - 0x10;
1293
	size_jz_after_cmp_inst = compiler->size;
1294
	jz_after_cmp_inst = inst;
1295

1296
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1297
	/* REX_W is not necessary. */
1298
	compiler->mode32 = 1;
1299
#endif
1300
	/* Load 1 into TMP_REG1. */
1301
	EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
1302

1303
	/* Generate "INCSSP TMP_REG1". */
1304
	FAIL_IF(emit_incssp(compiler, TMP_REG1));
1305

1306
	/* Jump back to "RDSSP TMP_REG1" to check shadow stack again. */
1307
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1308
	FAIL_IF(!inst);
1309
	INC_SIZE(2);
1310
	inst[0] = JMP_i8;
1311
	inst[1] = size_before_rdssp_inst - compiler->size;
1312

1313
	*jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst;
1314
#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
1315
	SLJIT_UNUSED_ARG(compiler);
1316
	SLJIT_UNUSED_ARG(src);
1317
	SLJIT_UNUSED_ARG(srcw);
1318
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1319
	return SLJIT_SUCCESS;
1320
}
1321

1322
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1323
#include "sljitNativeX86_32.c"
1324
#else
1325
#include "sljitNativeX86_64.c"
1326
#endif
1327

1328
static sljit_s32 emit_mov(struct sljit_compiler *compiler,
1329
	sljit_s32 dst, sljit_sw dstw,
1330
	sljit_s32 src, sljit_sw srcw)
1331
{
1332
	sljit_u8* inst;
1333

1334
	if (FAST_IS_REG(src)) {
1335
		inst = emit_x86_instruction(compiler, 1, src, 0, dst, dstw);
1336
		FAIL_IF(!inst);
1337
		*inst = MOV_rm_r;
1338
		return SLJIT_SUCCESS;
1339
	}
1340

1341
	if (src == SLJIT_IMM) {
1342
		if (FAST_IS_REG(dst)) {
1343
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1344
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1345
#else
1346
			if (!compiler->mode32) {
1347
				if (NOT_HALFWORD(srcw))
1348
					return emit_load_imm64(compiler, dst, srcw);
1349
			}
1350
			else
1351
				return emit_do_imm32(compiler, (reg_map[dst] >= 8) ? REX_B : 0, U8(MOV_r_i32 | reg_lmap[dst]), srcw);
1352
#endif
1353
		}
1354
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1355
		if (!compiler->mode32 && NOT_HALFWORD(srcw)) {
1356
			/* Immediate to memory move. Only SLJIT_MOV operation copies
1357
			   an immediate directly into memory so TMP_REG1 can be used. */
1358
			FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
1359
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1360
			FAIL_IF(!inst);
1361
			*inst = MOV_rm_r;
1362
			return SLJIT_SUCCESS;
1363
		}
1364
#endif
1365
		inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, dstw);
1366
		FAIL_IF(!inst);
1367
		*inst = MOV_rm_i32;
1368
		return SLJIT_SUCCESS;
1369
	}
1370
	if (FAST_IS_REG(dst)) {
1371
		inst = emit_x86_instruction(compiler, 1, dst, 0, src, srcw);
1372
		FAIL_IF(!inst);
1373
		*inst = MOV_r_rm;
1374
		return SLJIT_SUCCESS;
1375
	}
1376

1377
	/* Memory to memory move. Only SLJIT_MOV operation copies
1378
	   data from memory to memory so TMP_REG1 can be used. */
1379
	inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src, srcw);
1380
	FAIL_IF(!inst);
1381
	*inst = MOV_r_rm;
1382
	inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1383
	FAIL_IF(!inst);
1384
	*inst = MOV_rm_r;
1385
	return SLJIT_SUCCESS;
1386
}
1387

1388
static sljit_s32 emit_cmov_generic(struct sljit_compiler *compiler, sljit_s32 type,
1389
	sljit_s32 dst_reg,
1390
	sljit_s32 src, sljit_sw srcw)
1391
{
1392
	sljit_u8* inst;
1393
	sljit_uw size;
1394

1395
	SLJIT_ASSERT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
1396

1397
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1398
	FAIL_IF(!inst);
1399
	INC_SIZE(2);
1400
	inst[0] = U8(get_jump_code((sljit_uw)type ^ 0x1) - 0x10);
1401

1402
	size = compiler->size;
1403
	EMIT_MOV(compiler, dst_reg, 0, src, srcw);
1404

1405
	inst[1] = U8(compiler->size - size);
1406
	return SLJIT_SUCCESS;
1407
}
1408

1409
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1410
{
1411
	sljit_u8 *inst;
1412
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1413
	sljit_uw size;
1414
#endif
1415

1416
	CHECK_ERROR();
1417
	CHECK(check_sljit_emit_op0(compiler, op));
1418

1419
	switch (GET_OPCODE(op)) {
1420
	case SLJIT_BREAKPOINT:
1421
		return emit_byte(compiler, INT3);
1422
	case SLJIT_NOP:
1423
		return emit_byte(compiler, NOP);
1424
	case SLJIT_LMUL_UW:
1425
	case SLJIT_LMUL_SW:
1426
	case SLJIT_DIVMOD_UW:
1427
	case SLJIT_DIVMOD_SW:
1428
	case SLJIT_DIV_UW:
1429
	case SLJIT_DIV_SW:
1430
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1431
#ifdef _WIN64
1432
		SLJIT_ASSERT(
1433
			reg_map[SLJIT_R0] == 0
1434
			&& reg_map[SLJIT_R1] == 2
1435
			&& reg_map[TMP_REG1] > 7);
1436
#else
1437
		SLJIT_ASSERT(
1438
			reg_map[SLJIT_R0] == 0
1439
			&& reg_map[SLJIT_R1] < 7
1440
			&& reg_map[TMP_REG1] == 2);
1441
#endif
1442
		compiler->mode32 = op & SLJIT_32;
1443
#endif
1444
		SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1445

1446
		op = GET_OPCODE(op);
1447
		if ((op | 0x2) == SLJIT_DIV_UW) {
1448
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1449
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1450
			inst = emit_x86_instruction(compiler, 1, SLJIT_R1, 0, SLJIT_R1, 0);
1451
#else
1452
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
1453
#endif
1454
			FAIL_IF(!inst);
1455
			*inst = XOR_r_rm;
1456
		}
1457

1458
		if ((op | 0x2) == SLJIT_DIV_SW) {
1459
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1460
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1461
#endif
1462

1463
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1464
			FAIL_IF(emit_byte(compiler, CDQ));
1465
#else
1466
			if (!compiler->mode32) {
1467
				inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1468
				FAIL_IF(!inst);
1469
				INC_SIZE(2);
1470
				inst[0] = REX_W;
1471
				inst[1] = CDQ;
1472
			} else
1473
				FAIL_IF(emit_byte(compiler, CDQ));
1474
#endif
1475
		}
1476

1477
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1478
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1479
		FAIL_IF(!inst);
1480
		INC_SIZE(2);
1481
		inst[0] = GROUP_F7;
1482
		inst[1] = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_map[TMP_REG1] : reg_map[SLJIT_R1]);
1483
#else /* !SLJIT_CONFIG_X86_32 */
1484
#ifdef _WIN64
1485
		size = (!compiler->mode32 || op >= SLJIT_DIVMOD_UW) ? 3 : 2;
1486
#else /* !_WIN64 */
1487
		size = (!compiler->mode32) ? 3 : 2;
1488
#endif /* _WIN64 */
1489
		inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1490
		FAIL_IF(!inst);
1491
		INC_SIZE(size);
1492
#ifdef _WIN64
1493
		if (!compiler->mode32)
1494
			*inst++ = REX_W | ((op >= SLJIT_DIVMOD_UW) ? REX_B : 0);
1495
		else if (op >= SLJIT_DIVMOD_UW)
1496
			*inst++ = REX_B;
1497
		inst[0] = GROUP_F7;
1498
		inst[1] = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_lmap[TMP_REG1] : reg_lmap[SLJIT_R1]);
1499
#else /* !_WIN64 */
1500
		if (!compiler->mode32)
1501
			*inst++ = REX_W;
1502
		inst[0] = GROUP_F7;
1503
		inst[1] = MOD_REG | reg_map[SLJIT_R1];
1504
#endif /* _WIN64 */
1505
#endif /* SLJIT_CONFIG_X86_32 */
1506
		switch (op) {
1507
		case SLJIT_LMUL_UW:
1508
			inst[1] |= MUL;
1509
			break;
1510
		case SLJIT_LMUL_SW:
1511
			inst[1] |= IMUL;
1512
			break;
1513
		case SLJIT_DIVMOD_UW:
1514
		case SLJIT_DIV_UW:
1515
			inst[1] |= DIV;
1516
			break;
1517
		case SLJIT_DIVMOD_SW:
1518
		case SLJIT_DIV_SW:
1519
			inst[1] |= IDIV;
1520
			break;
1521
		}
1522
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && !defined(_WIN64)
1523
		if (op <= SLJIT_DIVMOD_SW)
1524
			EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1525
#else
1526
		if (op >= SLJIT_DIV_UW)
1527
			EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1528
#endif
1529
		break;
1530
	case SLJIT_MEMORY_BARRIER:
1531
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1532
		FAIL_IF(!inst);
1533
		INC_SIZE(3);
1534
		inst[0] = GROUP_0F;
1535
		inst[1] = 0xae;
1536
		inst[2] = 0xf0;
1537
		return SLJIT_SUCCESS;
1538
	case SLJIT_ENDBR:
1539
		return emit_endbranch(compiler);
1540
	case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1541
		return skip_frames_before_return(compiler);
1542
	}
1543

1544
	return SLJIT_SUCCESS;
1545
}
1546

1547
static sljit_s32 emit_mov_byte(struct sljit_compiler *compiler, sljit_s32 sign,
1548
	sljit_s32 dst, sljit_sw dstw,
1549
	sljit_s32 src, sljit_sw srcw)
1550
{
1551
	sljit_u8* inst;
1552
	sljit_s32 dst_r;
1553

1554
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1555
	compiler->mode32 = 0;
1556
#endif
1557

1558
	if (src == SLJIT_IMM) {
1559
		if (FAST_IS_REG(dst)) {
1560
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1561
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1562
#else
1563
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1564
			FAIL_IF(!inst);
1565
			*inst = MOV_rm_i32;
1566
			return SLJIT_SUCCESS;
1567
#endif
1568
		}
1569
		inst = emit_x86_instruction(compiler, 1 | EX86_BYTE_ARG | EX86_NO_REXW, SLJIT_IMM, srcw, dst, dstw);
1570
		FAIL_IF(!inst);
1571
		*inst = MOV_rm8_i8;
1572
		return SLJIT_SUCCESS;
1573
	}
1574

1575
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1576

1577
	if ((dst & SLJIT_MEM) && FAST_IS_REG(src)) {
1578
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1579
		if (reg_map[src] >= 4) {
1580
			SLJIT_ASSERT(dst_r == TMP_REG1);
1581
			EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1582
		} else
1583
			dst_r = src;
1584
#else
1585
		dst_r = src;
1586
#endif
1587
	} else {
1588
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1589
		if (FAST_IS_REG(src) && reg_map[src] >= 4) {
1590
			/* Both src and dst are registers. */
1591
			SLJIT_ASSERT(FAST_IS_REG(dst));
1592

1593
			if (src == dst && !sign) {
1594
				inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0xff, dst, 0);
1595
				FAIL_IF(!inst);
1596
				*(inst + 1) |= AND;
1597
				return SLJIT_SUCCESS;
1598
			}
1599

1600
			EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1601
			src = TMP_REG1;
1602
			srcw = 0;
1603
		}
1604
#endif /* !SLJIT_CONFIG_X86_32 */
1605

1606
		/* src can be memory addr or reg_map[src] < 4 on x86_32 architectures. */
1607
		FAIL_IF(emit_groupf(compiler, sign ? MOVSX_r_rm8 : MOVZX_r_rm8, dst_r, src, srcw));
1608
	}
1609

1610
	if (dst & SLJIT_MEM) {
1611
		inst = emit_x86_instruction(compiler, 1 | EX86_REX | EX86_NO_REXW, dst_r, 0, dst, dstw);
1612
		FAIL_IF(!inst);
1613
		*inst = MOV_rm8_r8;
1614
	}
1615

1616
	return SLJIT_SUCCESS;
1617
}
1618

1619
static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 op,
1620
	sljit_s32 src, sljit_sw srcw)
1621
{
1622
	sljit_u8* inst;
1623

1624
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1625
	compiler->mode32 = 1;
1626
#endif
1627

1628
	inst = emit_x86_instruction(compiler, 2, 0, 0, src, srcw);
1629
	FAIL_IF(!inst);
1630
	inst[0] = GROUP_0F;
1631
	inst[1] = PREFETCH;
1632

1633
	if (op == SLJIT_PREFETCH_L1)
1634
		inst[2] |= (1 << 3);
1635
	else if (op == SLJIT_PREFETCH_L2)
1636
		inst[2] |= (2 << 3);
1637
	else if (op == SLJIT_PREFETCH_L3)
1638
		inst[2] |= (3 << 3);
1639

1640
	return SLJIT_SUCCESS;
1641
}
1642

1643
static sljit_s32 emit_mov_half(struct sljit_compiler *compiler, sljit_s32 sign,
1644
	sljit_s32 dst, sljit_sw dstw,
1645
	sljit_s32 src, sljit_sw srcw)
1646
{
1647
	sljit_u8* inst;
1648
	sljit_s32 dst_r;
1649

1650
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1651
	compiler->mode32 = 0;
1652
#endif
1653

1654
	if (src == SLJIT_IMM) {
1655
		if (FAST_IS_REG(dst)) {
1656
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1657
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1658
#else
1659
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1660
			FAIL_IF(!inst);
1661
			*inst = MOV_rm_i32;
1662
			return SLJIT_SUCCESS;
1663
#endif
1664
		}
1665
		inst = emit_x86_instruction(compiler, 1 | EX86_HALF_ARG | EX86_NO_REXW | EX86_PREF_66, SLJIT_IMM, srcw, dst, dstw);
1666
		FAIL_IF(!inst);
1667
		*inst = MOV_rm_i32;
1668
		return SLJIT_SUCCESS;
1669
	}
1670

1671
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1672

1673
	if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
1674
		dst_r = src;
1675
	else
1676
		FAIL_IF(emit_groupf(compiler, sign ? MOVSX_r_rm16 : MOVZX_r_rm16, dst_r, src, srcw));
1677

1678
	if (dst & SLJIT_MEM) {
1679
		inst = emit_x86_instruction(compiler, 1 | EX86_NO_REXW | EX86_PREF_66, dst_r, 0, dst, dstw);
1680
		FAIL_IF(!inst);
1681
		*inst = MOV_rm_r;
1682
	}
1683

1684
	return SLJIT_SUCCESS;
1685
}
1686

1687
static sljit_s32 emit_unary(struct sljit_compiler *compiler, sljit_u8 opcode,
1688
	sljit_s32 dst, sljit_sw dstw,
1689
	sljit_s32 src, sljit_sw srcw)
1690
{
1691
	sljit_u8* inst;
1692

1693
	if (dst == src && dstw == srcw) {
1694
		/* Same input and output */
1695
		inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1696
		FAIL_IF(!inst);
1697
		inst[0] = GROUP_F7;
1698
		inst[1] |= opcode;
1699
		return SLJIT_SUCCESS;
1700
	}
1701

1702
	if (FAST_IS_REG(dst)) {
1703
		EMIT_MOV(compiler, dst, 0, src, srcw);
1704
		inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1705
		FAIL_IF(!inst);
1706
		inst[0] = GROUP_F7;
1707
		inst[1] |= opcode;
1708
		return SLJIT_SUCCESS;
1709
	}
1710

1711
	EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1712
	inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1713
	FAIL_IF(!inst);
1714
	inst[0] = GROUP_F7;
1715
	inst[1] |= opcode;
1716
	EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1717
	return SLJIT_SUCCESS;
1718
}
1719

1720
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1721
static const sljit_sw emit_clz_arg = 32 + 31;
1722
static const sljit_sw emit_ctz_arg = 32;
1723
#endif
1724

1725
static sljit_s32 emit_clz_ctz(struct sljit_compiler *compiler, sljit_s32 is_clz,
1726
	sljit_s32 dst, sljit_sw dstw,
1727
	sljit_s32 src, sljit_sw srcw)
1728
{
1729
	sljit_u8* inst;
1730
	sljit_s32 dst_r;
1731
	sljit_sw max;
1732

1733
	SLJIT_ASSERT(cpu_feature_list != 0);
1734

1735
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1736

1737
	if (is_clz ? (cpu_feature_list & CPU_FEATURE_LZCNT) : (cpu_feature_list & CPU_FEATURE_TZCNT)) {
1738
		FAIL_IF(emit_groupf(compiler, (is_clz ? LZCNT_r_rm : TZCNT_r_rm) | EX86_PREF_F3, dst_r, src, srcw));
1739

1740
		if (dst & SLJIT_MEM)
1741
			EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1742
		return SLJIT_SUCCESS;
1743
	}
1744

1745
	FAIL_IF(emit_groupf(compiler, is_clz ? BSR_r_rm : BSF_r_rm, dst_r, src, srcw));
1746

1747
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1748
	max = is_clz ? (32 + 31) : 32;
1749

1750
	if (cpu_feature_list & CPU_FEATURE_CMOV) {
1751
		if (dst_r != TMP_REG1) {
1752
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, max);
1753
			inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG1, 0);
1754
		}
1755
		else
1756
			inst = emit_x86_instruction(compiler, 2, dst_r, 0, SLJIT_MEM0(), is_clz ? (sljit_sw)&emit_clz_arg : (sljit_sw)&emit_ctz_arg);
1757

1758
		FAIL_IF(!inst);
1759
		inst[0] = GROUP_0F;
1760
		inst[1] = CMOVE_r_rm;
1761
	}
1762
	else
1763
		FAIL_IF(emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, max));
1764

1765
	if (is_clz) {
1766
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 31, dst_r, 0);
1767
		FAIL_IF(!inst);
1768
		*(inst + 1) |= XOR;
1769
	}
1770
#else
1771
	if (is_clz)
1772
		max = compiler->mode32 ? (32 + 31) : (64 + 63);
1773
	else
1774
		max = compiler->mode32 ? 32 : 64;
1775

1776
	if (cpu_feature_list & CPU_FEATURE_CMOV) {
1777
		EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, max);
1778
		FAIL_IF(emit_groupf(compiler, CMOVE_r_rm, dst_r, TMP_REG2, 0));
1779
	} else
1780
		FAIL_IF(emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, max));
1781

1782
	if (is_clz) {
1783
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, max >> 1, dst_r, 0);
1784
		FAIL_IF(!inst);
1785
		*(inst + 1) |= XOR;
1786
	}
1787
#endif
1788

1789
	if (dst & SLJIT_MEM)
1790
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1791
	return SLJIT_SUCCESS;
1792
}
1793

1794
static sljit_s32 emit_bswap(struct sljit_compiler *compiler,
1795
	sljit_s32 op,
1796
	sljit_s32 dst, sljit_sw dstw,
1797
	sljit_s32 src, sljit_sw srcw)
1798
{
1799
	sljit_u8 *inst;
1800
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1801
	sljit_uw size;
1802
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1803
	sljit_u8 rex = 0;
1804
#else /* !SLJIT_CONFIG_X86_64 */
1805
	sljit_s32 dst_is_ereg = op & SLJIT_32;
1806
#endif /* SLJIT_CONFIG_X86_64 */
1807

1808
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1809
	if (op == SLJIT_REV_U32 || op == SLJIT_REV_S32)
1810
		compiler->mode32 = 1;
1811
#else /* !SLJIT_CONFIG_X86_64 */
1812
	op &= ~SLJIT_32;
1813
#endif /* SLJIT_CONFIG_X86_64 */
1814

1815
	if (src != dst_r) {
1816
		/* Only the lower 16 bit is read for eregs. */
1817
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16)
1818
			FAIL_IF(emit_mov_half(compiler, 0, dst_r, 0, src, srcw));
1819
		else
1820
			EMIT_MOV(compiler, dst_r, 0, src, srcw);
1821
	}
1822

1823
	size = 2;
1824
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1825
	if (!compiler->mode32)
1826
		rex = REX_W;
1827

1828
	if (reg_map[dst_r] >= 8)
1829
		rex |= REX_B;
1830

1831
	if (rex != 0)
1832
		size++;
1833
#endif /* SLJIT_CONFIG_X86_64 */
1834

1835
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1836
	FAIL_IF(!inst);
1837
	INC_SIZE(size);
1838

1839
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1840
	if (rex != 0)
1841
		*inst++ = rex;
1842

1843
	inst[0] = GROUP_0F;
1844
	inst[1] = BSWAP_r | reg_lmap[dst_r];
1845
#else /* !SLJIT_CONFIG_X86_64 */
1846
	inst[0] = GROUP_0F;
1847
	inst[1] = BSWAP_r | reg_map[dst_r];
1848
#endif /* SLJIT_CONFIG_X86_64 */
1849

1850
	if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1851
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1852
		size = compiler->mode32 ? 16 : 48;
1853
#else /* !SLJIT_CONFIG_X86_64 */
1854
		size = 16;
1855
#endif /* SLJIT_CONFIG_X86_64 */
1856

1857
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, (sljit_sw)size, dst_r, 0);
1858
		FAIL_IF(!inst);
1859
		if (op == SLJIT_REV_U16)
1860
			inst[1] |= SHR;
1861
		else
1862
			inst[1] |= SAR;
1863
	}
1864

1865
	if (dst & SLJIT_MEM) {
1866
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1867
		if (dst_is_ereg)
1868
			op = SLJIT_REV;
1869
#endif /* SLJIT_CONFIG_X86_32 */
1870
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16)
1871
			return emit_mov_half(compiler, 0, dst, dstw, TMP_REG1, 0);
1872

1873
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1874
	}
1875

1876
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1877
	if (op == SLJIT_REV_S32) {
1878
		compiler->mode32 = 0;
1879
		inst = emit_x86_instruction(compiler, 1, dst, 0, dst, 0);
1880
		FAIL_IF(!inst);
1881
		*inst = MOVSXD_r_rm;
1882
	}
1883
#endif /* SLJIT_CONFIG_X86_64 */
1884

1885
	return SLJIT_SUCCESS;
1886
}
1887

1888
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1889
	sljit_s32 dst, sljit_sw dstw,
1890
	sljit_s32 src, sljit_sw srcw)
1891
{
1892
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1893
	sljit_s32 dst_is_ereg = 0;
1894
#else /* !SLJIT_CONFIG_X86_32 */
1895
	sljit_s32 op_flags = GET_ALL_FLAGS(op);
1896
#endif /* SLJIT_CONFIG_X86_32 */
1897

1898
	CHECK_ERROR();
1899
	CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1900
	ADJUST_LOCAL_OFFSET(dst, dstw);
1901
	ADJUST_LOCAL_OFFSET(src, srcw);
1902

1903
	CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
1904
	CHECK_EXTRA_REGS(src, srcw, (void)0);
1905
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1906
	compiler->mode32 = op_flags & SLJIT_32;
1907
#endif /* SLJIT_CONFIG_X86_64 */
1908

1909
	op = GET_OPCODE(op);
1910

1911
	if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1912
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1913
		compiler->mode32 = 0;
1914
#endif /* SLJIT_CONFIG_X86_64 */
1915

1916
		if (FAST_IS_REG(src) && src == dst) {
1917
			if (!TYPE_CAST_NEEDED(op))
1918
				return SLJIT_SUCCESS;
1919
		}
1920

1921
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1922
		if (op_flags & SLJIT_32) {
1923
			if (src & SLJIT_MEM) {
1924
				if (op == SLJIT_MOV_S32)
1925
					op = SLJIT_MOV_U32;
1926
			}
1927
			else if (src == SLJIT_IMM) {
1928
				if (op == SLJIT_MOV_U32)
1929
					op = SLJIT_MOV_S32;
1930
			}
1931
		}
1932
#endif /* SLJIT_CONFIG_X86_64 */
1933

1934
		if (src == SLJIT_IMM) {
1935
			switch (op) {
1936
			case SLJIT_MOV_U8:
1937
				srcw = (sljit_u8)srcw;
1938
				break;
1939
			case SLJIT_MOV_S8:
1940
				srcw = (sljit_s8)srcw;
1941
				break;
1942
			case SLJIT_MOV_U16:
1943
				srcw = (sljit_u16)srcw;
1944
				break;
1945
			case SLJIT_MOV_S16:
1946
				srcw = (sljit_s16)srcw;
1947
				break;
1948
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1949
			case SLJIT_MOV_U32:
1950
				srcw = (sljit_u32)srcw;
1951
				break;
1952
			case SLJIT_MOV_S32:
1953
				srcw = (sljit_s32)srcw;
1954
				break;
1955
#endif /* SLJIT_CONFIG_X86_64 */
1956
			}
1957
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1958
			if (SLJIT_UNLIKELY(dst_is_ereg))
1959
				return emit_mov(compiler, dst, dstw, src, srcw);
1960
#endif /* SLJIT_CONFIG_X86_32 */
1961
		}
1962

1963
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1964
		if (SLJIT_UNLIKELY(dst_is_ereg) && (!(op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P) || (src & SLJIT_MEM))) {
1965
			SLJIT_ASSERT(dst == SLJIT_MEM1(SLJIT_SP));
1966
			dst = TMP_REG1;
1967
		}
1968
#endif /* SLJIT_CONFIG_X86_32 */
1969

1970
		switch (op) {
1971
		case SLJIT_MOV:
1972
		case SLJIT_MOV_P:
1973
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1974
		case SLJIT_MOV_U32:
1975
		case SLJIT_MOV_S32:
1976
		case SLJIT_MOV32:
1977
#endif /* SLJIT_CONFIG_X86_32 */
1978
			EMIT_MOV(compiler, dst, dstw, src, srcw);
1979
			break;
1980
		case SLJIT_MOV_U8:
1981
			FAIL_IF(emit_mov_byte(compiler, 0, dst, dstw, src, srcw));
1982
			break;
1983
		case SLJIT_MOV_S8:
1984
			FAIL_IF(emit_mov_byte(compiler, 1, dst, dstw, src, srcw));
1985
			break;
1986
		case SLJIT_MOV_U16:
1987
			FAIL_IF(emit_mov_half(compiler, 0, dst, dstw, src, srcw));
1988
			break;
1989
		case SLJIT_MOV_S16:
1990
			FAIL_IF(emit_mov_half(compiler, 1, dst, dstw, src, srcw));
1991
			break;
1992
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1993
		case SLJIT_MOV_U32:
1994
			FAIL_IF(emit_mov_int(compiler, 0, dst, dstw, src, srcw));
1995
			break;
1996
		case SLJIT_MOV_S32:
1997
			FAIL_IF(emit_mov_int(compiler, 1, dst, dstw, src, srcw));
1998
			break;
1999
		case SLJIT_MOV32:
2000
			compiler->mode32 = 1;
2001
			EMIT_MOV(compiler, dst, dstw, src, srcw);
2002
			compiler->mode32 = 0;
2003
			break;
2004
#endif /* SLJIT_CONFIG_X86_64 */
2005
		}
2006

2007
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2008
		if (SLJIT_UNLIKELY(dst_is_ereg) && dst == TMP_REG1)
2009
			return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), dstw, TMP_REG1, 0);
2010
#endif /* SLJIT_CONFIG_X86_32 */
2011
		return SLJIT_SUCCESS;
2012
	}
2013

2014
	switch (op) {
2015
	case SLJIT_CLZ:
2016
	case SLJIT_CTZ:
2017
		return emit_clz_ctz(compiler, (op == SLJIT_CLZ), dst, dstw, src, srcw);
2018
	case SLJIT_REV:
2019
	case SLJIT_REV_U16:
2020
	case SLJIT_REV_S16:
2021
	case SLJIT_REV_U32:
2022
	case SLJIT_REV_S32:
2023
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2024
		if (dst_is_ereg)
2025
			op |= SLJIT_32;
2026
#endif /* SLJIT_CONFIG_X86_32 */
2027
		return emit_bswap(compiler, op, dst, dstw, src, srcw);
2028
	}
2029

2030
	return SLJIT_SUCCESS;
2031
}
2032

2033
static sljit_s32 emit_cum_binary(struct sljit_compiler *compiler,
2034
	sljit_u32 op_types,
2035
	sljit_s32 dst, sljit_sw dstw,
2036
	sljit_s32 src1, sljit_sw src1w,
2037
	sljit_s32 src2, sljit_sw src2w)
2038
{
2039
	sljit_u8* inst;
2040
	sljit_u8 op_eax_imm = U8(op_types >> 24);
2041
	sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
2042
	sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
2043
	sljit_u8 op_imm = U8(op_types & 0xff);
2044

2045
	if (dst == src1 && dstw == src1w) {
2046
		if (src2 == SLJIT_IMM) {
2047
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2048
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2049
#else
2050
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
2051
#endif
2052
				BINARY_EAX_IMM(op_eax_imm, src2w);
2053
			}
2054
			else {
2055
				BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
2056
			}
2057
		}
2058
		else if (FAST_IS_REG(dst)) {
2059
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
2060
			FAIL_IF(!inst);
2061
			*inst = op_rm;
2062
		}
2063
		else if (FAST_IS_REG(src2)) {
2064
			/* Special exception for sljit_emit_op_flags. */
2065
			inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
2066
			FAIL_IF(!inst);
2067
			*inst = op_mr;
2068
		}
2069
		else {
2070
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
2071
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2072
			FAIL_IF(!inst);
2073
			*inst = op_mr;
2074
		}
2075
		return SLJIT_SUCCESS;
2076
	}
2077

2078
	/* Only for cumulative operations. */
2079
	if (dst == src2 && dstw == src2w) {
2080
		if (src1 == SLJIT_IMM) {
2081
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2082
			if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2083
#else
2084
			if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128)) {
2085
#endif
2086
				BINARY_EAX_IMM(op_eax_imm, src1w);
2087
			}
2088
			else {
2089
				BINARY_IMM(op_imm, op_mr, src1w, dst, dstw);
2090
			}
2091
		}
2092
		else if (FAST_IS_REG(dst)) {
2093
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src1, src1w);
2094
			FAIL_IF(!inst);
2095
			*inst = op_rm;
2096
		}
2097
		else if (FAST_IS_REG(src1)) {
2098
			inst = emit_x86_instruction(compiler, 1, src1, src1w, dst, dstw);
2099
			FAIL_IF(!inst);
2100
			*inst = op_mr;
2101
		}
2102
		else {
2103
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2104
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2105
			FAIL_IF(!inst);
2106
			*inst = op_mr;
2107
		}
2108
		return SLJIT_SUCCESS;
2109
	}
2110

2111
	/* General version. */
2112
	if (FAST_IS_REG(dst)) {
2113
		EMIT_MOV(compiler, dst, 0, src1, src1w);
2114
		if (src2 == SLJIT_IMM) {
2115
			BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
2116
		}
2117
		else {
2118
			inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
2119
			FAIL_IF(!inst);
2120
			*inst = op_rm;
2121
		}
2122
	}
2123
	else {
2124
		/* This version requires less memory writing. */
2125
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2126
		if (src2 == SLJIT_IMM) {
2127
			BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
2128
		}
2129
		else {
2130
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2131
			FAIL_IF(!inst);
2132
			*inst = op_rm;
2133
		}
2134
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2135
	}
2136

2137
	return SLJIT_SUCCESS;
2138
}
2139

2140
static sljit_s32 emit_non_cum_binary(struct sljit_compiler *compiler,
2141
	sljit_u32 op_types,
2142
	sljit_s32 dst, sljit_sw dstw,
2143
	sljit_s32 src1, sljit_sw src1w,
2144
	sljit_s32 src2, sljit_sw src2w)
2145
{
2146
	sljit_u8* inst;
2147
	sljit_u8 op_eax_imm = U8(op_types >> 24);
2148
	sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
2149
	sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
2150
	sljit_u8 op_imm = U8(op_types & 0xff);
2151

2152
	if (dst == src1 && dstw == src1w) {
2153
		if (src2 == SLJIT_IMM) {
2154
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2155
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2156
#else
2157
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
2158
#endif
2159
				BINARY_EAX_IMM(op_eax_imm, src2w);
2160
			}
2161
			else {
2162
				BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
2163
			}
2164
		}
2165
		else if (FAST_IS_REG(dst)) {
2166
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
2167
			FAIL_IF(!inst);
2168
			*inst = op_rm;
2169
		}
2170
		else if (FAST_IS_REG(src2)) {
2171
			inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
2172
			FAIL_IF(!inst);
2173
			*inst = op_mr;
2174
		}
2175
		else {
2176
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
2177
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2178
			FAIL_IF(!inst);
2179
			*inst = op_mr;
2180
		}
2181
		return SLJIT_SUCCESS;
2182
	}
2183

2184
	/* General version. */
2185
	if (FAST_IS_REG(dst) && dst != src2) {
2186
		EMIT_MOV(compiler, dst, 0, src1, src1w);
2187
		if (src2 == SLJIT_IMM) {
2188
			BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
2189
		}
2190
		else {
2191
			inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
2192
			FAIL_IF(!inst);
2193
			*inst = op_rm;
2194
		}
2195
	}
2196
	else {
2197
		/* This version requires less memory writing. */
2198
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2199
		if (src2 == SLJIT_IMM) {
2200
			BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
2201
		}
2202
		else {
2203
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2204
			FAIL_IF(!inst);
2205
			*inst = op_rm;
2206
		}
2207
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2208
	}
2209

2210
	return SLJIT_SUCCESS;
2211
}
2212

2213
static sljit_s32 emit_mul(struct sljit_compiler *compiler,
2214
	sljit_s32 dst, sljit_sw dstw,
2215
	sljit_s32 src1, sljit_sw src1w,
2216
	sljit_s32 src2, sljit_sw src2w)
2217
{
2218
	sljit_u8* inst;
2219
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2220

2221
	/* Register destination. */
2222
	if (dst_r == src1 && src2 != SLJIT_IMM) {
2223
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src2, src2w));
2224
	} else if (dst_r == src2 && src1 != SLJIT_IMM) {
2225
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src1, src1w));
2226
	} else if (src1 == SLJIT_IMM) {
2227
		if (src2 == SLJIT_IMM) {
2228
			EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, src2w);
2229
			src2 = dst_r;
2230
			src2w = 0;
2231
		}
2232

2233
		if (src1w <= 127 && src1w >= -128) {
2234
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2235
			FAIL_IF(!inst);
2236
			*inst = IMUL_r_rm_i8;
2237

2238
			FAIL_IF(emit_byte(compiler, U8(src1w)));
2239
		}
2240
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2241
		else {
2242
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2243
			FAIL_IF(!inst);
2244
			*inst = IMUL_r_rm_i32;
2245
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2246
			FAIL_IF(!inst);
2247
			INC_SIZE(4);
2248
			sljit_unaligned_store_sw(inst, src1w);
2249
		}
2250
#else
2251
		else if (IS_HALFWORD(src1w)) {
2252
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2253
			FAIL_IF(!inst);
2254
			*inst = IMUL_r_rm_i32;
2255
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2256
			FAIL_IF(!inst);
2257
			INC_SIZE(4);
2258
			sljit_unaligned_store_s32(inst, (sljit_s32)src1w);
2259
		}
2260
		else {
2261
			if (dst_r != src2)
2262
				EMIT_MOV(compiler, dst_r, 0, src2, src2w);
2263
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src1w));
2264
			FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, TMP_REG2, 0));
2265
		}
2266
#endif
2267
	}
2268
	else if (src2 == SLJIT_IMM) {
2269
		/* Note: src1 is NOT immediate. */
2270

2271
		if (src2w <= 127 && src2w >= -128) {
2272
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2273
			FAIL_IF(!inst);
2274
			*inst = IMUL_r_rm_i8;
2275

2276
			FAIL_IF(emit_byte(compiler, U8(src2w)));
2277
		}
2278
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2279
		else {
2280
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2281
			FAIL_IF(!inst);
2282
			*inst = IMUL_r_rm_i32;
2283

2284
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2285
			FAIL_IF(!inst);
2286
			INC_SIZE(4);
2287
			sljit_unaligned_store_sw(inst, src2w);
2288
		}
2289
#else
2290
		else if (IS_HALFWORD(src2w)) {
2291
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2292
			FAIL_IF(!inst);
2293
			*inst = IMUL_r_rm_i32;
2294

2295
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2296
			FAIL_IF(!inst);
2297
			INC_SIZE(4);
2298
			sljit_unaligned_store_s32(inst, (sljit_s32)src2w);
2299
		} else {
2300
			if (dst_r != src1)
2301
				EMIT_MOV(compiler, dst_r, 0, src1, src1w);
2302
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2303
			FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, TMP_REG2, 0));
2304
		}
2305
#endif
2306
	} else {
2307
		/* Neither argument is immediate. */
2308
		if (ADDRESSING_DEPENDS_ON(src2, dst_r))
2309
			dst_r = TMP_REG1;
2310
		EMIT_MOV(compiler, dst_r, 0, src1, src1w);
2311
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src2, src2w));
2312
	}
2313

2314
	if (dst & SLJIT_MEM)
2315
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2316

2317
	return SLJIT_SUCCESS;
2318
}
2319

2320
static sljit_s32 emit_lea_binary(struct sljit_compiler *compiler,
2321
	sljit_s32 dst, sljit_sw dstw,
2322
	sljit_s32 src1, sljit_sw src1w,
2323
	sljit_s32 src2, sljit_sw src2w)
2324
{
2325
	sljit_u8* inst;
2326
	sljit_s32 dst_r, done = 0;
2327

2328
	/* These cases better be left to handled by normal way. */
2329
	if (dst == src1 && dstw == src1w)
2330
		return SLJIT_ERR_UNSUPPORTED;
2331
	if (dst == src2 && dstw == src2w)
2332
		return SLJIT_ERR_UNSUPPORTED;
2333

2334
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2335

2336
	if (FAST_IS_REG(src1)) {
2337
		if (FAST_IS_REG(src2)) {
2338
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), 0);
2339
			FAIL_IF(!inst);
2340
			*inst = LEA_r_m;
2341
			done = 1;
2342
		}
2343
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2344
		if (src2 == SLJIT_IMM && (compiler->mode32 || IS_HALFWORD(src2w))) {
2345
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), (sljit_s32)src2w);
2346
#else
2347
		if (src2 == SLJIT_IMM) {
2348
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), src2w);
2349
#endif
2350
			FAIL_IF(!inst);
2351
			*inst = LEA_r_m;
2352
			done = 1;
2353
		}
2354
	}
2355
	else if (FAST_IS_REG(src2)) {
2356
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2357
		if (src1 == SLJIT_IMM && (compiler->mode32 || IS_HALFWORD(src1w))) {
2358
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), (sljit_s32)src1w);
2359
#else
2360
		if (src1 == SLJIT_IMM) {
2361
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), src1w);
2362
#endif
2363
			FAIL_IF(!inst);
2364
			*inst = LEA_r_m;
2365
			done = 1;
2366
		}
2367
	}
2368

2369
	if (done) {
2370
		if (dst_r == TMP_REG1)
2371
			return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2372
		return SLJIT_SUCCESS;
2373
	}
2374
	return SLJIT_ERR_UNSUPPORTED;
2375
}
2376

2377
static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
2378
	sljit_s32 src1, sljit_sw src1w,
2379
	sljit_s32 src2, sljit_sw src2w)
2380
{
2381
	sljit_u8* inst;
2382

2383
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2384
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2385
#else
2386
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128)) {
2387
#endif
2388
		BINARY_EAX_IMM(CMP_EAX_i32, src2w);
2389
		return SLJIT_SUCCESS;
2390
	}
2391

2392
	if (FAST_IS_REG(src1)) {
2393
		if (src2 == SLJIT_IMM) {
2394
			BINARY_IMM(CMP, CMP_rm_r, src2w, src1, 0);
2395
		}
2396
		else {
2397
			inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2398
			FAIL_IF(!inst);
2399
			*inst = CMP_r_rm;
2400
		}
2401
		return SLJIT_SUCCESS;
2402
	}
2403

2404
	if (FAST_IS_REG(src2) && src1 != SLJIT_IMM) {
2405
		inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2406
		FAIL_IF(!inst);
2407
		*inst = CMP_rm_r;
2408
		return SLJIT_SUCCESS;
2409
	}
2410

2411
	if (src2 == SLJIT_IMM) {
2412
		if (src1 == SLJIT_IMM) {
2413
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2414
			src1 = TMP_REG1;
2415
			src1w = 0;
2416
		}
2417
		BINARY_IMM(CMP, CMP_rm_r, src2w, src1, src1w);
2418
	}
2419
	else {
2420
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2421
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2422
		FAIL_IF(!inst);
2423
		*inst = CMP_r_rm;
2424
	}
2425
	return SLJIT_SUCCESS;
2426
}
2427

2428
static sljit_s32 emit_test_binary(struct sljit_compiler *compiler,
2429
	sljit_s32 src1, sljit_sw src1w,
2430
	sljit_s32 src2, sljit_sw src2w)
2431
{
2432
	sljit_u8* inst;
2433

2434
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2435
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2436
#else
2437
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128)) {
2438
#endif
2439
		BINARY_EAX_IMM(TEST_EAX_i32, src2w);
2440
		return SLJIT_SUCCESS;
2441
	}
2442

2443
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2444
	if (src2 == SLJIT_R0 && src1 == SLJIT_IMM && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2445
#else
2446
	if (src2 == SLJIT_R0 && src1 == SLJIT_IMM && (src1w > 127 || src1w < -128)) {
2447
#endif
2448
		BINARY_EAX_IMM(TEST_EAX_i32, src1w);
2449
		return SLJIT_SUCCESS;
2450
	}
2451

2452
	if (src1 != SLJIT_IMM) {
2453
		if (src2 == SLJIT_IMM) {
2454
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2455
			if (IS_HALFWORD(src2w) || compiler->mode32) {
2456
				inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2457
				FAIL_IF(!inst);
2458
				*inst = GROUP_F7;
2459
			} else {
2460
				FAIL_IF(emit_load_imm64(compiler, FAST_IS_REG(src1) ? TMP_REG2 : TMP_REG1, src2w));
2461
				inst = emit_x86_instruction(compiler, 1, FAST_IS_REG(src1) ? TMP_REG2 : TMP_REG1, 0, src1, src1w);
2462
				FAIL_IF(!inst);
2463
				*inst = TEST_rm_r;
2464
			}
2465
#else
2466
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2467
			FAIL_IF(!inst);
2468
			*inst = GROUP_F7;
2469
#endif
2470
			return SLJIT_SUCCESS;
2471
		}
2472
		else if (FAST_IS_REG(src1)) {
2473
			inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2474
			FAIL_IF(!inst);
2475
			*inst = TEST_rm_r;
2476
			return SLJIT_SUCCESS;
2477
		}
2478
	}
2479

2480
	if (src2 != SLJIT_IMM) {
2481
		if (src1 == SLJIT_IMM) {
2482
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2483
			if (IS_HALFWORD(src1w) || compiler->mode32) {
2484
				inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src1w, src2, src2w);
2485
				FAIL_IF(!inst);
2486
				*inst = GROUP_F7;
2487
			}
2488
			else {
2489
				FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src1w));
2490
				inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2491
				FAIL_IF(!inst);
2492
				*inst = TEST_rm_r;
2493
			}
2494
#else
2495
			inst = emit_x86_instruction(compiler, 1, src1, src1w, src2, src2w);
2496
			FAIL_IF(!inst);
2497
			*inst = GROUP_F7;
2498
#endif
2499
			return SLJIT_SUCCESS;
2500
		}
2501
		else if (FAST_IS_REG(src2)) {
2502
			inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2503
			FAIL_IF(!inst);
2504
			*inst = TEST_rm_r;
2505
			return SLJIT_SUCCESS;
2506
		}
2507
	}
2508

2509
	EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2510
	if (src2 == SLJIT_IMM) {
2511
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2512
		if (IS_HALFWORD(src2w) || compiler->mode32) {
2513
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2514
			FAIL_IF(!inst);
2515
			*inst = GROUP_F7;
2516
		}
2517
		else {
2518
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2519
			inst = emit_x86_instruction(compiler, 1, TMP_REG2, 0, TMP_REG1, 0);
2520
			FAIL_IF(!inst);
2521
			*inst = TEST_rm_r;
2522
		}
2523
#else
2524
		inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2525
		FAIL_IF(!inst);
2526
		*inst = GROUP_F7;
2527
#endif
2528
	}
2529
	else {
2530
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2531
		FAIL_IF(!inst);
2532
		*inst = TEST_rm_r;
2533
	}
2534
	return SLJIT_SUCCESS;
2535
}
2536

2537
static sljit_s32 emit_shift(struct sljit_compiler *compiler,
2538
	sljit_u8 mode,
2539
	sljit_s32 dst, sljit_sw dstw,
2540
	sljit_s32 src1, sljit_sw src1w,
2541
	sljit_s32 src2, sljit_sw src2w)
2542
{
2543
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2544
	sljit_s32 mode32;
2545
#endif
2546
	sljit_u8* inst;
2547

2548
	if (src2 == SLJIT_IMM || src2 == SLJIT_PREF_SHIFT_REG) {
2549
		if (dst == src1 && dstw == src1w) {
2550
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, dstw);
2551
			FAIL_IF(!inst);
2552
			inst[1] |= mode;
2553
			return SLJIT_SUCCESS;
2554
		}
2555
		if (dst == SLJIT_PREF_SHIFT_REG && src2 == SLJIT_PREF_SHIFT_REG) {
2556
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2557
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2558
			FAIL_IF(!inst);
2559
			inst[1] |= mode;
2560
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2561
			return SLJIT_SUCCESS;
2562
		}
2563
		if (FAST_IS_REG(dst)) {
2564
			EMIT_MOV(compiler, dst, 0, src1, src1w);
2565
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, 0);
2566
			FAIL_IF(!inst);
2567
			inst[1] |= mode;
2568
			return SLJIT_SUCCESS;
2569
		}
2570

2571
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2572
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REG1, 0);
2573
		FAIL_IF(!inst);
2574
		inst[1] |= mode;
2575
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2576
		return SLJIT_SUCCESS;
2577
	}
2578

2579
	if (dst == SLJIT_PREF_SHIFT_REG) {
2580
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2581
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2582
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2583
		FAIL_IF(!inst);
2584
		inst[1] |= mode;
2585
		return emit_mov(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2586
	}
2587

2588
	if (FAST_IS_REG(dst) && dst != src2 && dst != TMP_REG1 && !ADDRESSING_DEPENDS_ON(src2, dst)) {
2589
		if (src1 != dst)
2590
			EMIT_MOV(compiler, dst, 0, src1, src1w);
2591
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2592
		mode32 = compiler->mode32;
2593
		compiler->mode32 = 0;
2594
#endif
2595
		EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2596
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2597
		compiler->mode32 = mode32;
2598
#endif
2599
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2600
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, dst, 0);
2601
		FAIL_IF(!inst);
2602
		inst[1] |= mode;
2603
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2604
		compiler->mode32 = 0;
2605
#endif
2606
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2607
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2608
		compiler->mode32 = mode32;
2609
#endif
2610
		return SLJIT_SUCCESS;
2611
	}
2612

2613
	/* This case is complex since ecx itself may be used for
2614
	   addressing, and this case must be supported as well. */
2615
	EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2616
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2617
	EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2618
#else /* !SLJIT_CONFIG_X86_32 */
2619
	mode32 = compiler->mode32;
2620
	compiler->mode32 = 0;
2621
	EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_PREF_SHIFT_REG, 0);
2622
	compiler->mode32 = mode32;
2623
#endif /* SLJIT_CONFIG_X86_32 */
2624

2625
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2626
	inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2627
	FAIL_IF(!inst);
2628
	inst[1] |= mode;
2629

2630
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2631
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, SLJIT_MEM1(SLJIT_SP), 0);
2632
#else
2633
	compiler->mode32 = 0;
2634
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG2, 0);
2635
	compiler->mode32 = mode32;
2636
#endif /* SLJIT_CONFIG_X86_32 */
2637

2638
	if (dst != TMP_REG1)
2639
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2640

2641
	return SLJIT_SUCCESS;
2642
}
2643

2644
static sljit_s32 emit_shift_with_flags(struct sljit_compiler *compiler,
2645
	sljit_u8 mode, sljit_s32 set_flags,
2646
	sljit_s32 dst, sljit_sw dstw,
2647
	sljit_s32 src1, sljit_sw src1w,
2648
	sljit_s32 src2, sljit_sw src2w)
2649
{
2650
	/* The CPU does not set flags if the shift count is 0. */
2651
	if (src2 == SLJIT_IMM) {
2652
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2653
		src2w &= compiler->mode32 ? 0x1f : 0x3f;
2654
#else /* !SLJIT_CONFIG_X86_64 */
2655
		src2w &= 0x1f;
2656
#endif /* SLJIT_CONFIG_X86_64 */
2657
		if (src2w != 0)
2658
			return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2659

2660
		if (!set_flags)
2661
			return emit_mov(compiler, dst, dstw, src1, src1w);
2662
		/* OR dst, src, 0 */
2663
		return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2664
			dst, dstw, src1, src1w, SLJIT_IMM, 0);
2665
	}
2666

2667
	if (!set_flags)
2668
		return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2669

2670
	if (!FAST_IS_REG(dst))
2671
		FAIL_IF(emit_cmp_binary(compiler, src1, src1w, SLJIT_IMM, 0));
2672

2673
	FAIL_IF(emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w));
2674

2675
	if (FAST_IS_REG(dst))
2676
		return emit_cmp_binary(compiler, dst, dstw, SLJIT_IMM, 0);
2677
	return SLJIT_SUCCESS;
2678
}
2679

2680
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2681
	sljit_s32 dst, sljit_sw dstw,
2682
	sljit_s32 src1, sljit_sw src1w,
2683
	sljit_s32 src2, sljit_sw src2w)
2684
{
2685
	CHECK_ERROR();
2686
	CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2687
	ADJUST_LOCAL_OFFSET(dst, dstw);
2688
	ADJUST_LOCAL_OFFSET(src1, src1w);
2689
	ADJUST_LOCAL_OFFSET(src2, src2w);
2690

2691
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
2692
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2693
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2694
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2695
	compiler->mode32 = op & SLJIT_32;
2696
#endif
2697

2698
	switch (GET_OPCODE(op)) {
2699
	case SLJIT_ADD:
2700
		if (!HAS_FLAGS(op)) {
2701
			if (emit_lea_binary(compiler, dst, dstw, src1, src1w, src2, src2w) != SLJIT_ERR_UNSUPPORTED)
2702
				return compiler->error;
2703
		}
2704
		return emit_cum_binary(compiler, BINARY_OPCODE(ADD),
2705
			dst, dstw, src1, src1w, src2, src2w);
2706
	case SLJIT_ADDC:
2707
		return emit_cum_binary(compiler, BINARY_OPCODE(ADC),
2708
			dst, dstw, src1, src1w, src2, src2w);
2709
	case SLJIT_SUB:
2710
		if (src1 == SLJIT_IMM && src1w == 0)
2711
			return emit_unary(compiler, NEG_rm, dst, dstw, src2, src2w);
2712

2713
		if (!HAS_FLAGS(op)) {
2714
			if (src2 == SLJIT_IMM && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED)
2715
				return compiler->error;
2716
			if (FAST_IS_REG(dst) && src2 == dst) {
2717
				FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB), dst, 0, dst, 0, src1, src1w));
2718
				return emit_unary(compiler, NEG_rm, dst, 0, dst, 0);
2719
			}
2720
		}
2721

2722
		return emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
2723
			dst, dstw, src1, src1w, src2, src2w);
2724
	case SLJIT_SUBC:
2725
		return emit_non_cum_binary(compiler, BINARY_OPCODE(SBB),
2726
			dst, dstw, src1, src1w, src2, src2w);
2727
	case SLJIT_MUL:
2728
		return emit_mul(compiler, dst, dstw, src1, src1w, src2, src2w);
2729
	case SLJIT_AND:
2730
		return emit_cum_binary(compiler, BINARY_OPCODE(AND),
2731
			dst, dstw, src1, src1w, src2, src2w);
2732
	case SLJIT_OR:
2733
		return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2734
			dst, dstw, src1, src1w, src2, src2w);
2735
	case SLJIT_XOR:
2736
		if (!HAS_FLAGS(op)) {
2737
			if (src2 == SLJIT_IMM && src2w == -1)
2738
				return emit_unary(compiler, NOT_rm, dst, dstw, src1, src1w);
2739
			if (src1 == SLJIT_IMM && src1w == -1)
2740
				return emit_unary(compiler, NOT_rm, dst, dstw, src2, src2w);
2741
		}
2742

2743
		return emit_cum_binary(compiler, BINARY_OPCODE(XOR),
2744
			dst, dstw, src1, src1w, src2, src2w);
2745
	case SLJIT_SHL:
2746
	case SLJIT_MSHL:
2747
		return emit_shift_with_flags(compiler, SHL, HAS_FLAGS(op),
2748
			dst, dstw, src1, src1w, src2, src2w);
2749
	case SLJIT_LSHR:
2750
	case SLJIT_MLSHR:
2751
		return emit_shift_with_flags(compiler, SHR, HAS_FLAGS(op),
2752
			dst, dstw, src1, src1w, src2, src2w);
2753
	case SLJIT_ASHR:
2754
	case SLJIT_MASHR:
2755
		return emit_shift_with_flags(compiler, SAR, HAS_FLAGS(op),
2756
			dst, dstw, src1, src1w, src2, src2w);
2757
	case SLJIT_ROTL:
2758
		return emit_shift_with_flags(compiler, ROL, 0,
2759
			dst, dstw, src1, src1w, src2, src2w);
2760
	case SLJIT_ROTR:
2761
		return emit_shift_with_flags(compiler, ROR, 0,
2762
			dst, dstw, src1, src1w, src2, src2w);
2763
	}
2764

2765
	return SLJIT_SUCCESS;
2766
}
2767

2768
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2769
	sljit_s32 src1, sljit_sw src1w,
2770
	sljit_s32 src2, sljit_sw src2w)
2771
{
2772
	sljit_s32 opcode = GET_OPCODE(op);
2773

2774
	CHECK_ERROR();
2775
	CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2776

2777
	if (opcode != SLJIT_SUB && opcode != SLJIT_AND) {
2778
		SLJIT_SKIP_CHECKS(compiler);
2779
		return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
2780
	}
2781

2782
	ADJUST_LOCAL_OFFSET(src1, src1w);
2783
	ADJUST_LOCAL_OFFSET(src2, src2w);
2784

2785
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2786
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2787
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2788
	compiler->mode32 = op & SLJIT_32;
2789
#endif
2790

2791
	if (opcode == SLJIT_SUB)
2792
		return emit_cmp_binary(compiler, src1, src1w, src2, src2w);
2793

2794
	return emit_test_binary(compiler, src1, src1w, src2, src2w);
2795
}
2796

2797
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2798
	sljit_s32 dst_reg,
2799
	sljit_s32 src1, sljit_sw src1w,
2800
	sljit_s32 src2, sljit_sw src2w)
2801
{
2802
	sljit_u8* inst;
2803
	sljit_sw dstw = 0;
2804

2805
	CHECK_ERROR();
2806
	CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2807
	ADJUST_LOCAL_OFFSET(src1, src1w);
2808
	ADJUST_LOCAL_OFFSET(src2, src2w);
2809

2810
	CHECK_EXTRA_REGS(dst_reg, dstw, (void)0);
2811
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2812
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2813
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2814
	compiler->mode32 = op & SLJIT_32;
2815
#endif
2816

2817
	switch (GET_OPCODE(op)) {
2818
	case SLJIT_MULADD:
2819
		FAIL_IF(emit_mul(compiler, TMP_REG1, 0, src1, src1w, src2, src2w));
2820
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst_reg, dstw);
2821
		FAIL_IF(!inst);
2822
		*inst = ADD_rm_r;
2823
		return SLJIT_SUCCESS;
2824
	}
2825

2826
	return SLJIT_SUCCESS;
2827
}
2828

2829
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2830
	sljit_s32 dst_reg,
2831
	sljit_s32 src1_reg,
2832
	sljit_s32 src2_reg,
2833
	sljit_s32 src3, sljit_sw src3w)
2834
{
2835
	sljit_s32 is_rotate, is_left, move_src1;
2836
	sljit_u8* inst;
2837
	sljit_sw src1w = 0;
2838
	sljit_sw dstw = 0;
2839
	/* The whole register must be saved even for 32 bit operations. */
2840
	sljit_u8 restore_ecx = 0;
2841
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2842
	sljit_sw src2w = 0;
2843
	sljit_s32 restore_sp4 = 0;
2844
#endif /* SLJIT_CONFIG_X86_32 */
2845

2846
	CHECK_ERROR();
2847
	CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2848
	ADJUST_LOCAL_OFFSET(src3, src3w);
2849

2850
	CHECK_EXTRA_REGS(dst_reg, dstw, (void)0);
2851
	CHECK_EXTRA_REGS(src3, src3w, (void)0);
2852

2853
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2854
	compiler->mode32 = op & SLJIT_32;
2855
#endif /* SLJIT_CONFIG_X86_64 */
2856

2857
	if (src3 == SLJIT_IMM) {
2858
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2859
		src3w &= 0x1f;
2860
#else /* !SLJIT_CONFIG_X86_32 */
2861
		src3w &= (op & SLJIT_32) ? 0x1f : 0x3f;
2862
#endif /* SLJIT_CONFIG_X86_32 */
2863

2864
		if (src3w == 0)
2865
			return SLJIT_SUCCESS;
2866
	}
2867

2868
	is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
2869

2870
	is_rotate = (src1_reg == src2_reg);
2871
	CHECK_EXTRA_REGS(src1_reg, src1w, (void)0);
2872
	CHECK_EXTRA_REGS(src2_reg, src2w, (void)0);
2873

2874
	if (is_rotate)
2875
		return emit_shift(compiler, is_left ? ROL : ROR, dst_reg, dstw, src1_reg, src1w, src3, src3w);
2876

2877
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2878
	if (src2_reg & SLJIT_MEM) {
2879
		EMIT_MOV(compiler, TMP_REG1, 0, src2_reg, src2w);
2880
		src2_reg = TMP_REG1;
2881
	}
2882
#endif /* SLJIT_CONFIG_X86_32 */
2883

2884
	if (dst_reg == SLJIT_PREF_SHIFT_REG && src3 != SLJIT_IMM && (src3 != SLJIT_PREF_SHIFT_REG || src1_reg != SLJIT_PREF_SHIFT_REG)) {
2885
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2886
		EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2887
		src1_reg = TMP_REG1;
2888
		src1w = 0;
2889
#else /* !SLJIT_CONFIG_X86_64 */
2890
		if (src2_reg != TMP_REG1) {
2891
			EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2892
			src1_reg = TMP_REG1;
2893
			src1w = 0;
2894
		} else if ((src1_reg & SLJIT_MEM) || src1_reg == SLJIT_PREF_SHIFT_REG) {
2895
			restore_sp4 = (src3 == SLJIT_R0) ? SLJIT_R1 : SLJIT_R0;
2896
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), restore_sp4, 0);
2897
			EMIT_MOV(compiler, restore_sp4, 0, src1_reg, src1w);
2898
			src1_reg = restore_sp4;
2899
			src1w = 0;
2900
		} else {
2901
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), src1_reg, 0);
2902
			restore_sp4 = src1_reg;
2903
		}
2904
#endif /* SLJIT_CONFIG_X86_64 */
2905

2906
		if (src3 != SLJIT_PREF_SHIFT_REG)
2907
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src3, src3w);
2908
	} else {
2909
		if (src2_reg == SLJIT_PREF_SHIFT_REG && src3 != SLJIT_IMM && src3 != SLJIT_PREF_SHIFT_REG) {
2910
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2911
			compiler->mode32 = 0;
2912
#endif /* SLJIT_CONFIG_X86_64 */
2913
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2914
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2915
			compiler->mode32 = op & SLJIT_32;
2916
#endif /* SLJIT_CONFIG_X86_64 */
2917
			src2_reg = TMP_REG1;
2918
			restore_ecx = 1;
2919
		}
2920

2921
		move_src1 = 0;
2922
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2923
		if (dst_reg != src1_reg) {
2924
			if (dst_reg != src3) {
2925
				EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2926
				src1_reg = dst_reg;
2927
				src1w = 0;
2928
			} else
2929
				move_src1 = 1;
2930
		}
2931
#else /* !SLJIT_CONFIG_X86_64 */
2932
		if (dst_reg & SLJIT_MEM) {
2933
			if (src2_reg != TMP_REG1) {
2934
				EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2935
				src1_reg = TMP_REG1;
2936
				src1w = 0;
2937
			} else if ((src1_reg & SLJIT_MEM) || src1_reg == SLJIT_PREF_SHIFT_REG) {
2938
				restore_sp4 = (src3 == SLJIT_R0) ? SLJIT_R1 : SLJIT_R0;
2939
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), restore_sp4, 0);
2940
				EMIT_MOV(compiler, restore_sp4, 0, src1_reg, src1w);
2941
				src1_reg = restore_sp4;
2942
				src1w = 0;
2943
			} else {
2944
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), src1_reg, 0);
2945
				restore_sp4 = src1_reg;
2946
			}
2947
		} else if (dst_reg != src1_reg) {
2948
			if (dst_reg != src3) {
2949
				EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2950
				src1_reg = dst_reg;
2951
				src1w = 0;
2952
			} else
2953
				move_src1 = 1;
2954
		}
2955
#endif /* SLJIT_CONFIG_X86_64 */
2956

2957
		if (src3 != SLJIT_IMM && src3 != SLJIT_PREF_SHIFT_REG) {
2958
			if (!restore_ecx) {
2959
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2960
				compiler->mode32 = 0;
2961
				EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2962
				compiler->mode32 = op & SLJIT_32;
2963
				restore_ecx = 1;
2964
#else /* !SLJIT_CONFIG_X86_64 */
2965
				if (src1_reg != TMP_REG1 && src2_reg != TMP_REG1) {
2966
					EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2967
					restore_ecx = 1;
2968
				} else {
2969
					EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2970
					restore_ecx = 2;
2971
				}
2972
#endif /* SLJIT_CONFIG_X86_64 */
2973
			}
2974
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src3, src3w);
2975
		}
2976

2977
		if (move_src1) {
2978
			EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2979
			src1_reg = dst_reg;
2980
			src1w = 0;
2981
		}
2982
	}
2983

2984
	inst = emit_x86_instruction(compiler, 2, src2_reg, 0, src1_reg, src1w);
2985
	FAIL_IF(!inst);
2986
	inst[0] = GROUP_0F;
2987

2988
	if (src3 == SLJIT_IMM) {
2989
		inst[1] = U8((is_left ? SHLD : SHRD) - 1);
2990

2991
		/* Immediate argument is added separately. */
2992
		FAIL_IF(emit_byte(compiler, U8(src3w)));
2993
	} else
2994
		inst[1] = U8(is_left ? SHLD : SHRD);
2995

2996
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2997
	if (restore_ecx) {
2998
		compiler->mode32 = 0;
2999
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
3000
	}
3001

3002
	if (src1_reg != dst_reg) {
3003
		compiler->mode32 = op & SLJIT_32;
3004
		return emit_mov(compiler, dst_reg, dstw, src1_reg, 0);
3005
	}
3006
#else /* !SLJIT_CONFIG_X86_64 */
3007
	if (restore_ecx)
3008
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, restore_ecx == 1 ? TMP_REG1 : SLJIT_MEM1(SLJIT_SP), 0);
3009

3010
	if (src1_reg != dst_reg)
3011
		EMIT_MOV(compiler, dst_reg, dstw, src1_reg, 0);
3012

3013
	if (restore_sp4)
3014
		return emit_mov(compiler, restore_sp4, 0, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32));
3015
#endif /* SLJIT_CONFIG_X86_32 */
3016

3017
	return SLJIT_SUCCESS;
3018
}
3019

3020
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
3021
	sljit_s32 dst, sljit_sw dstw,
3022
	sljit_s32 src1, sljit_sw src1w,
3023
	sljit_s32 src2, sljit_sw src2w,
3024
	sljit_sw shift_arg)
3025
{
3026
	sljit_s32 dst_r;
3027
	int use_lea = 0;
3028
	sljit_u8* inst;
3029

3030
	CHECK_ERROR();
3031
	CHECK(check_sljit_emit_op2_shift(compiler, op, dst, dstw, src1, src1w, src2, src2w, shift_arg));
3032
	ADJUST_LOCAL_OFFSET(dst, dstw);
3033
	ADJUST_LOCAL_OFFSET(src1, src1w);
3034
	ADJUST_LOCAL_OFFSET(src2, src2w);
3035

3036
	shift_arg &= (sljit_sw)((sizeof(sljit_sw) * 8) - 1);
3037

3038
	if (src2 == SLJIT_IMM) {
3039
		src2w = src2w << shift_arg;
3040
		shift_arg = 0;
3041
	}
3042

3043
	if (shift_arg == 0) {
3044
		SLJIT_SKIP_CHECKS(compiler);
3045
		return sljit_emit_op2(compiler, GET_OPCODE(op), dst, dstw, src1, src1w, src2, src2w);
3046
	}
3047

3048
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3049
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
3050
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
3051

3052
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3053
	compiler->mode32 = 0;
3054
#endif
3055

3056
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3057
	if (shift_arg <= 3) {
3058
		use_lea = 1;
3059
		if (!FAST_IS_REG(src2)) {
3060
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
3061
			src2 = TMP_REG1;
3062
		}
3063

3064
		if (!FAST_IS_REG(src1)) {
3065
			EMIT_MOV(compiler, src2 == TMP_REG1 ? TMP_REG2 : TMP_REG1, 0, src1, src1w);
3066
			src1 = src2 == TMP_REG1 ? TMP_REG2 : TMP_REG1;
3067
		}
3068
	}
3069
#else /* !SLJIT_CONFIG_X86_64 */
3070
	if (shift_arg <= 3 && (FAST_IS_REG(src1) || (FAST_IS_REG(src2) && src2 != TMP_REG1))) {
3071
		use_lea = 1;
3072
		if (!FAST_IS_REG(src2)) {
3073
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
3074
			src2 = TMP_REG1;
3075
		}
3076

3077
		if (!FAST_IS_REG(src1)) {
3078
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
3079
			src1 = TMP_REG1;
3080
		}
3081
	}
3082
#endif /* SLJIT_CONFIG_X86_64 */
3083

3084
	if (use_lea) {
3085
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3086

3087
		inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), shift_arg);
3088
		FAIL_IF(!inst);
3089
		*inst = LEA_r_m;
3090

3091
		if (!FAST_IS_REG(dst))
3092
			return emit_mov(compiler, dst, dstw, dst_r, 0);
3093

3094
		return SLJIT_SUCCESS;
3095
	}
3096

3097
	if ((op & SLJIT_SRC2_UNDEFINED) != 0 && FAST_IS_REG(src2) && src1 != src2)
3098
		dst_r = src2;
3099
	else {
3100
		dst_r = FAST_IS_REG(dst) && (dst != src1) ? dst : TMP_REG1;
3101

3102
		if (src2 != dst_r) {
3103
			EMIT_MOV(compiler, dst_r, 0, src2, src2w);
3104
		}
3105
	}
3106

3107
	inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, shift_arg, dst_r, 0);
3108
	FAIL_IF(!inst);
3109
	inst[1] |= SHL;
3110

3111
	if (dst == src1 && dstw == src1w) {
3112
		inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
3113
		FAIL_IF(!inst);
3114
		*inst = ADD_rm_r;
3115
		return SLJIT_SUCCESS;
3116
	}
3117

3118
	if (FAST_IS_REG(dst) && FAST_IS_REG(src1)) {
3119
		inst = emit_x86_instruction(compiler, 1, dst, 0, SLJIT_MEM2(src1, dst_r), 0);
3120
		FAIL_IF(!inst);
3121
		*inst = LEA_r_m;
3122
		return SLJIT_SUCCESS;
3123
	}
3124

3125
	if (src1 == SLJIT_IMM) {
3126
		BINARY_IMM(ADD, ADD_rm_r, src1w, dst_r, 0);
3127
	} else {
3128
		inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
3129
		FAIL_IF(!inst);
3130
		*inst = ADD_r_rm;
3131
	}
3132

3133
	if (dst != dst_r)
3134
		return emit_mov(compiler, dst, dstw, dst_r, 0);
3135

3136
	return SLJIT_SUCCESS;
3137
}
3138

3139
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
3140
	sljit_s32 src, sljit_sw srcw)
3141
{
3142
	CHECK_ERROR();
3143
	CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
3144
	ADJUST_LOCAL_OFFSET(src, srcw);
3145

3146
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3147

3148
	switch (op) {
3149
	case SLJIT_FAST_RETURN:
3150
		return emit_fast_return(compiler, src, srcw);
3151
	case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
3152
		/* Don't adjust shadow stack if it isn't enabled.  */
3153
		if (!cpu_has_shadow_stack ())
3154
			return SLJIT_SUCCESS;
3155
		return adjust_shadow_stack(compiler, src, srcw);
3156
	case SLJIT_PREFETCH_L1:
3157
	case SLJIT_PREFETCH_L2:
3158
	case SLJIT_PREFETCH_L3:
3159
	case SLJIT_PREFETCH_ONCE:
3160
		return emit_prefetch(compiler, op, src, srcw);
3161
	}
3162

3163
	return SLJIT_SUCCESS;
3164
}
3165

3166
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
3167
	sljit_s32 dst, sljit_sw dstw)
3168
{
3169
	CHECK_ERROR();
3170
	CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
3171
	ADJUST_LOCAL_OFFSET(dst, dstw);
3172

3173
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3174

3175
	switch (op) {
3176
	case SLJIT_FAST_ENTER:
3177
		return emit_fast_enter(compiler, dst, dstw);
3178
	case SLJIT_GET_RETURN_ADDRESS:
3179
		return sljit_emit_get_return_address(compiler, dst, dstw);
3180
	}
3181

3182
	return SLJIT_SUCCESS;
3183
}
3184

3185
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
3186
{
3187
	CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
3188

3189
	if (type == SLJIT_GP_REGISTER) {
3190
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3191
		if (reg >= SLJIT_R3 && reg <= SLJIT_R8)
3192
			return -1;
3193
#endif /* SLJIT_CONFIG_X86_32 */
3194
		return reg_map[reg];
3195
	}
3196

3197
	if (type != SLJIT_FLOAT_REGISTER && type != SLJIT_SIMD_REG_128 && type != SLJIT_SIMD_REG_256 && type != SLJIT_SIMD_REG_512)
3198
		return -1;
3199

3200
	return freg_map[reg];
3201
}
3202

3203
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
3204
	void *instruction, sljit_u32 size)
3205
{
3206
	sljit_u8 *inst;
3207

3208
	CHECK_ERROR();
3209
	CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
3210

3211
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3212
	FAIL_IF(!inst);
3213
	INC_SIZE(size);
3214
	SLJIT_MEMCPY(inst, instruction, size);
3215
	return SLJIT_SUCCESS;
3216
}
3217

3218
/* --------------------------------------------------------------------- */
3219
/*  Floating point operators                                             */
3220
/* --------------------------------------------------------------------- */
3221

3222
/* Alignment(3) + 4 * 16 bytes. */
3223
static sljit_u32 sse2_data[3 + (4 * 4)];
3224
static sljit_u32 *sse2_buffer;
3225

3226
static void init_compiler(void)
3227
{
3228
	get_cpu_features();
3229

3230
	/* Align to 16 bytes. */
3231
	sse2_buffer = (sljit_u32*)(((sljit_uw)sse2_data + 15) & ~(sljit_uw)0xf);
3232

3233
	/* Single precision constants (each constant is 16 byte long). */
3234
	sse2_buffer[0] = 0x80000000;
3235
	sse2_buffer[4] = 0x7fffffff;
3236
	/* Double precision constants (each constant is 16 byte long). */
3237
	sse2_buffer[8] = 0;
3238
	sse2_buffer[9] = 0x80000000;
3239
	sse2_buffer[12] = 0xffffffff;
3240
	sse2_buffer[13] = 0x7fffffff;
3241
}
3242

3243
static sljit_s32 emit_groupf(struct sljit_compiler *compiler,
3244
	sljit_uw op,
3245
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3246
{
3247
	sljit_u8 *inst = emit_x86_instruction(compiler, 2 | (op & ~(sljit_uw)0xff), dst, 0, src, srcw);
3248
	FAIL_IF(!inst);
3249
	inst[0] = GROUP_0F;
3250
	inst[1] = op & 0xff;
3251
	return SLJIT_SUCCESS;
3252
}
3253

3254
static sljit_s32 emit_groupf_ext(struct sljit_compiler *compiler,
3255
	sljit_uw op,
3256
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3257
{
3258
	sljit_u8 *inst;
3259

3260
	SLJIT_ASSERT((op & EX86_SSE2) && ((op & VEX_OP_0F38) || (op & VEX_OP_0F3A)));
3261

3262
	inst = emit_x86_instruction(compiler, 3 | (op & ~((sljit_uw)0xff | VEX_OP_0F38 | VEX_OP_0F3A)), dst, 0, src, srcw);
3263
	FAIL_IF(!inst);
3264
	inst[0] = GROUP_0F;
3265
	inst[1] = U8((op & VEX_OP_0F38) ? 0x38 : 0x3A);
3266
	inst[2] = op & 0xff;
3267
	return SLJIT_SUCCESS;
3268
}
3269

3270
static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
3271
	sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3272
{
3273
	return emit_groupf(compiler, MOVSD_x_xm | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, dst, src, srcw);
3274
}
3275

3276
static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
3277
	sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src)
3278
{
3279
	return emit_groupf(compiler, MOVSD_xm_x | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, src, dst, dstw);
3280
}
3281

3282
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
3283
	sljit_s32 dst, sljit_sw dstw,
3284
	sljit_s32 src, sljit_sw srcw)
3285
{
3286
	sljit_s32 dst_r;
3287

3288
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3289
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3290

3291
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3292
	if (GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)
3293
		compiler->mode32 = 0;
3294
#endif
3295

3296
	FAIL_IF(emit_groupf(compiler, CVTTSD2SI_r_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP2, dst_r, src, srcw));
3297

3298
	if (dst & SLJIT_MEM)
3299
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3300
	return SLJIT_SUCCESS;
3301
}
3302

3303
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
3304
	sljit_s32 dst, sljit_sw dstw,
3305
	sljit_s32 src, sljit_sw srcw)
3306
{
3307
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
3308

3309
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3310

3311
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3312
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
3313
		compiler->mode32 = 0;
3314
#endif
3315

3316
	if (src == SLJIT_IMM) {
3317
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3318
		if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32)
3319
			srcw = (sljit_s32)srcw;
3320
#endif
3321
		EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
3322
		src = TMP_REG1;
3323
		srcw = 0;
3324
	}
3325

3326
	FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, src, srcw));
3327

3328
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3329
	compiler->mode32 = 1;
3330
#endif
3331
	if (dst_r == TMP_FREG)
3332
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3333
	return SLJIT_SUCCESS;
3334
}
3335

3336
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
3337
	sljit_s32 src1, sljit_sw src1w,
3338
	sljit_s32 src2, sljit_sw src2w)
3339
{
3340
	switch (GET_FLAG_TYPE(op)) {
3341
	case SLJIT_ORDERED_EQUAL:
3342
		/* Also: SLJIT_UNORDERED_OR_NOT_EQUAL */
3343
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3344
		FAIL_IF(emit_groupf(compiler, CMPS_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, TMP_FREG, src2, src2w));
3345

3346
		/* EQ */
3347
		FAIL_IF(emit_byte(compiler, 0));
3348

3349
		src1 = TMP_FREG;
3350
		src2 = TMP_FREG;
3351
		src2w = 0;
3352
		break;
3353

3354
	case SLJIT_ORDERED_LESS:
3355
	case SLJIT_UNORDERED_OR_GREATER:
3356
		/* Also: SLJIT_UNORDERED_OR_GREATER_EQUAL, SLJIT_ORDERED_LESS_EQUAL  */
3357
		if (!FAST_IS_REG(src2)) {
3358
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src2, src2w));
3359
			src2 = TMP_FREG;
3360
		}
3361

3362
		return emit_groupf(compiler, UCOMISD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, src2, src1, src1w);
3363
	}
3364

3365
	if (!FAST_IS_REG(src1)) {
3366
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3367
		src1 = TMP_FREG;
3368
	}
3369

3370
	return emit_groupf(compiler, UCOMISD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, src1, src2, src2w);
3371
}
3372

3373
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
3374
	sljit_s32 dst, sljit_sw dstw,
3375
	sljit_s32 src, sljit_sw srcw)
3376
{
3377
	sljit_s32 dst_r;
3378
	sljit_u8 *inst;
3379

3380
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3381
	compiler->mode32 = 1;
3382
#endif
3383

3384
	CHECK_ERROR();
3385
	SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
3386

3387
	if (GET_OPCODE(op) == SLJIT_MOV_F64) {
3388
		if (FAST_IS_REG(dst))
3389
			return emit_sse2_load(compiler, op & SLJIT_32, dst, src, srcw);
3390
		if (FAST_IS_REG(src))
3391
			return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, src);
3392
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3393
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3394
	}
3395

3396
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) {
3397
		dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
3398
		if (FAST_IS_REG(src)) {
3399
			/* We overwrite the high bits of source. From SLJIT point of view,
3400
			   this is not an issue.
3401
			   Note: In SSE3, we could also use MOVDDUP and MOVSLDUP. */
3402
			FAIL_IF(emit_groupf(compiler, UNPCKLPD_x_xm | ((op & SLJIT_32) ? EX86_PREF_66 : 0) | EX86_SSE2, src, src, 0));
3403
		} else {
3404
			FAIL_IF(emit_sse2_load(compiler, !(op & SLJIT_32), TMP_FREG, src, srcw));
3405
			src = TMP_FREG;
3406
		}
3407

3408
		FAIL_IF(emit_groupf(compiler, CVTPD2PS_x_xm | ((op & SLJIT_32) ? EX86_PREF_66 : 0) | EX86_SSE2, dst_r, src, 0));
3409
		if (dst_r == TMP_FREG)
3410
			return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3411
		return SLJIT_SUCCESS;
3412
	}
3413

3414
	if (FAST_IS_REG(dst)) {
3415
		dst_r = (dst == src) ? TMP_FREG : dst;
3416

3417
		if (src & SLJIT_MEM)
3418
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3419

3420
		FAIL_IF(emit_groupf(compiler, PCMPEQD_x_xm | EX86_PREF_66 | EX86_SSE2, dst_r, dst_r, 0));
3421

3422
		inst = emit_x86_instruction(compiler, 2 | EX86_PREF_66 | EX86_SSE2_OP2, 0, 0, dst_r, 0);
3423
		inst[0] = GROUP_0F;
3424
		/* Same as PSRLD_x / PSRLQ_x */
3425
		inst[1] = (op & SLJIT_32) ? PSLLD_x_i8 : PSLLQ_x_i8;
3426

3427
		if (GET_OPCODE(op) == SLJIT_ABS_F64) {
3428
			inst[2] |= 2 << 3;
3429
			FAIL_IF(emit_byte(compiler, 1));
3430
		} else {
3431
			inst[2] |= 6 << 3;
3432
			FAIL_IF(emit_byte(compiler, ((op & SLJIT_32) ? 31 : 63)));
3433
		}
3434

3435
		if (dst_r != TMP_FREG)
3436
			dst_r = (src & SLJIT_MEM) ? TMP_FREG : src;
3437
		return emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_NEG_F64 ? XORPD_x_xm : ANDPD_x_xm) | EX86_SSE2, dst, dst_r, 0);
3438
	}
3439

3440
	FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3441

3442
	switch (GET_OPCODE(op)) {
3443
	case SLJIT_NEG_F64:
3444
		FAIL_IF(emit_groupf(compiler, XORPD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3445
		break;
3446

3447
	case SLJIT_ABS_F64:
3448
		FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer + 4 : sse2_buffer + 12)));
3449
		break;
3450
	}
3451

3452
	return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3453
}
3454

3455
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
3456
	sljit_s32 dst, sljit_sw dstw,
3457
	sljit_s32 src1, sljit_sw src1w,
3458
	sljit_s32 src2, sljit_sw src2w)
3459
{
3460
	sljit_s32 dst_r;
3461

3462
	CHECK_ERROR();
3463
	CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3464
	ADJUST_LOCAL_OFFSET(dst, dstw);
3465
	ADJUST_LOCAL_OFFSET(src1, src1w);
3466
	ADJUST_LOCAL_OFFSET(src2, src2w);
3467

3468
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3469
	compiler->mode32 = 1;
3470
#endif
3471

3472
	if (FAST_IS_REG(dst)) {
3473
		dst_r = dst;
3474
		if (dst == src1)
3475
			; /* Do nothing here. */
3476
		else if (dst == src2 && (GET_OPCODE(op) == SLJIT_ADD_F64 || GET_OPCODE(op) == SLJIT_MUL_F64)) {
3477
			/* Swap arguments. */
3478
			src2 = src1;
3479
			src2w = src1w;
3480
		} else if (dst != src2)
3481
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_r, src1, src1w));
3482
		else {
3483
			dst_r = TMP_FREG;
3484
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3485
		}
3486
	} else {
3487
		dst_r = TMP_FREG;
3488
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3489
	}
3490

3491
	switch (GET_OPCODE(op)) {
3492
	case SLJIT_ADD_F64:
3493
		FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3494
		break;
3495

3496
	case SLJIT_SUB_F64:
3497
		FAIL_IF(emit_groupf(compiler, SUBSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3498
		break;
3499

3500
	case SLJIT_MUL_F64:
3501
		FAIL_IF(emit_groupf(compiler, MULSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3502
		break;
3503

3504
	case SLJIT_DIV_F64:
3505
		FAIL_IF(emit_groupf(compiler, DIVSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3506
		break;
3507
	}
3508

3509
	if (dst_r != dst)
3510
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3511
	return SLJIT_SUCCESS;
3512
}
3513

3514
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3515
	sljit_s32 dst_freg,
3516
	sljit_s32 src1, sljit_sw src1w,
3517
	sljit_s32 src2, sljit_sw src2w)
3518
{
3519
	sljit_uw pref;
3520

3521
	CHECK_ERROR();
3522
	CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3523
	ADJUST_LOCAL_OFFSET(src1, src1w);
3524
	ADJUST_LOCAL_OFFSET(src2, src2w);
3525

3526
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3527
	compiler->mode32 = 1;
3528
#endif
3529

3530
	if (dst_freg == src1) {
3531
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src2, src2w));
3532
		pref = EX86_SELECT_66(op) | EX86_SSE2;
3533
		FAIL_IF(emit_groupf(compiler, XORPD_x_xm | pref, TMP_FREG, src1, src1w));
3534
		FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | pref, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3535
		return emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, TMP_FREG, 0);
3536
	}
3537

3538
	if (src1 & SLJIT_MEM) {
3539
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3540
		src1 = TMP_FREG;
3541
		src1w = 0;
3542
	}
3543

3544
	if (dst_freg != src2)
3545
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_freg, src2, src2w));
3546

3547
	pref = EX86_SELECT_66(op) | EX86_SSE2;
3548
	FAIL_IF(emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, src1, src1w));
3549
	FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | pref, dst_freg, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3550
	return emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, src1, src1w);
3551
}
3552

3553
/* --------------------------------------------------------------------- */
3554
/*  Conditional instructions                                             */
3555
/* --------------------------------------------------------------------- */
3556

3557
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
3558
{
3559
	sljit_u8 *inst;
3560
	struct sljit_label *label;
3561

3562
	CHECK_ERROR_PTR();
3563
	CHECK_PTR(check_sljit_emit_label(compiler));
3564

3565
	if (compiler->last_label && compiler->last_label->size == compiler->size)
3566
		return compiler->last_label;
3567

3568
	label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
3569
	PTR_FAIL_IF(!label);
3570
	set_label(label, compiler);
3571

3572
	inst = (sljit_u8*)ensure_buf(compiler, 1);
3573
	PTR_FAIL_IF(!inst);
3574
	inst[0] = SLJIT_INST_LABEL;
3575

3576
	return label;
3577
}
3578

3579
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler,
3580
	sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
3581
{
3582
	sljit_uw mask, size;
3583
	sljit_u8 *inst;
3584
	struct sljit_label *label;
3585
	struct sljit_label *next_label;
3586
	struct sljit_extended_label *ext_label;
3587

3588
	CHECK_ERROR_PTR();
3589
	CHECK_PTR(check_sljit_emit_aligned_label(compiler, alignment, buffers));
3590

3591
	sljit_reset_read_only_buffers(buffers);
3592

3593
	if (alignment <= SLJIT_LABEL_ALIGN_1) {
3594
		SLJIT_SKIP_CHECKS(compiler);
3595
		label = sljit_emit_label(compiler);
3596
		PTR_FAIL_IF(!label);
3597
	} else {
3598
		/* The used space is filled with NOPs. */
3599
		mask = ((sljit_uw)1 << alignment) - 1;
3600
		compiler->size += mask;
3601

3602
		inst = (sljit_u8*)ensure_buf(compiler, 1);
3603
		PTR_FAIL_IF(!inst);
3604
		inst[0] = SLJIT_INST_LABEL;
3605

3606
		ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
3607
		PTR_FAIL_IF(!ext_label);
3608
		set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
3609
		label = &ext_label->label;
3610
	}
3611

3612
	if (buffers == NULL)
3613
		return label;
3614

3615
	next_label = label;
3616

3617
	while (1) {
3618
		buffers->u.label = next_label;
3619
		size = buffers->size;
3620

3621
		while (size >= 4) {
3622
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
3623
			PTR_FAIL_IF(!inst);
3624
			INC_SIZE(4);
3625
			inst[0] = NOP;
3626
			inst[1] = NOP;
3627
			inst[2] = NOP;
3628
			inst[3] = NOP;
3629
			size -= 4;
3630
		}
3631

3632
		if (size > 0) {
3633
			inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3634
			PTR_FAIL_IF(!inst);
3635
			INC_SIZE(size);
3636

3637
			do {
3638
				*inst++ = NOP;
3639
			} while (--size != 0);
3640
		}
3641

3642
		buffers = buffers->next;
3643

3644
		if (buffers == NULL)
3645
			break;
3646

3647
		SLJIT_SKIP_CHECKS(compiler);
3648
		next_label = sljit_emit_label(compiler);
3649
		PTR_FAIL_IF(!next_label);
3650
	}
3651

3652
	return label;
3653
}
3654

3655
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
3656
{
3657
	sljit_u8 *inst;
3658
	struct sljit_jump *jump;
3659

3660
	CHECK_ERROR_PTR();
3661
	CHECK_PTR(check_sljit_emit_jump(compiler, type));
3662

3663
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3664
	PTR_FAIL_IF_NULL(jump);
3665
	set_jump(jump, compiler, (sljit_u32)((type & SLJIT_REWRITABLE_JUMP) | ((type & 0xff) << TYPE_SHIFT)));
3666
	type &= 0xff;
3667

3668
	jump->addr = compiler->size;
3669
	/* Worst case size. */
3670
	compiler->size += (type >= SLJIT_JUMP) ? JUMP_MAX_SIZE : CJUMP_MAX_SIZE;
3671
	inst = (sljit_u8*)ensure_buf(compiler, 1);
3672
	PTR_FAIL_IF_NULL(inst);
3673

3674
	inst[0] = SLJIT_INST_JUMP;
3675
	return jump;
3676
}
3677

3678
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
3679
{
3680
	sljit_u8 *inst;
3681
	struct sljit_jump *jump;
3682

3683
	CHECK_ERROR();
3684
	CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
3685
	ADJUST_LOCAL_OFFSET(src, srcw);
3686

3687
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3688

3689
	if (src == SLJIT_IMM) {
3690
		jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3691
		FAIL_IF_NULL(jump);
3692
		set_jump(jump, compiler, (sljit_u32)(JUMP_ADDR | (type << TYPE_SHIFT)));
3693
		jump->u.target = (sljit_uw)srcw;
3694

3695
		jump->addr = compiler->size;
3696
		/* Worst case size. */
3697
		compiler->size += JUMP_MAX_SIZE;
3698
		inst = (sljit_u8*)ensure_buf(compiler, 1);
3699
		FAIL_IF_NULL(inst);
3700

3701
		inst[0] = SLJIT_INST_JUMP;
3702
	} else {
3703
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3704
		/* REX_W is not necessary (src is not immediate). */
3705
		compiler->mode32 = 1;
3706
#endif
3707
		inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
3708
		FAIL_IF(!inst);
3709
		inst[0] = GROUP_FF;
3710
		inst[1] = U8(inst[1] | ((type >= SLJIT_FAST_CALL) ? CALL_rm : JMP_rm));
3711
	}
3712
	return SLJIT_SUCCESS;
3713
}
3714

3715
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
3716
	sljit_s32 dst, sljit_sw dstw,
3717
	sljit_s32 type)
3718
{
3719
	sljit_u8 *inst;
3720
	sljit_u8 cond_set;
3721
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3722
	sljit_s32 reg;
3723
	sljit_uw size;
3724
#endif /* !SLJIT_CONFIG_X86_64 */
3725
	/* ADJUST_LOCAL_OFFSET and CHECK_EXTRA_REGS might overwrite these values. */
3726
	sljit_s32 dst_save = dst;
3727
	sljit_sw dstw_save = dstw;
3728

3729
	CHECK_ERROR();
3730
	CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
3731

3732
	ADJUST_LOCAL_OFFSET(dst, dstw);
3733
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3734

3735
	/* setcc = jcc + 0x10. */
3736
	cond_set = U8(get_jump_code((sljit_uw)type) + 0x10);
3737

3738
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3739
	if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst)) {
3740
		size = 3 + 2;
3741
		if (reg_map[TMP_REG1] >= 4)
3742
			size += 1 + 1;
3743
		else if (reg_map[dst] >= 4)
3744
			size++;
3745

3746
		inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3747
		FAIL_IF(!inst);
3748
		INC_SIZE(size);
3749
		/* Set low register to conditional flag. */
3750
		if (reg_map[TMP_REG1] >= 4)
3751
			*inst++ = (reg_map[TMP_REG1] <= 7) ? REX : REX_B;
3752

3753
		inst[0] = GROUP_0F;
3754
		inst[1] = cond_set;
3755
		inst[2] = MOD_REG | reg_lmap[TMP_REG1];
3756
		inst += 3;
3757

3758
		if (reg_map[TMP_REG1] >= 4 || reg_map[dst] >= 4)
3759
			*inst++ = U8(REX | (reg_map[TMP_REG1] <= 7 ? 0 : REX_R) | (reg_map[dst] <= 7 ? 0 : REX_B));
3760

3761
		inst[0] = OR_rm8_r8;
3762
		inst[1] = U8(MOD_REG | (reg_lmap[TMP_REG1] << 3) | reg_lmap[dst]);
3763
		return SLJIT_SUCCESS;
3764
	}
3765

3766
	reg = (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG1;
3767

3768
	size = 3 + (reg_map[reg] >= 4) + 4;
3769
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3770
	FAIL_IF(!inst);
3771
	INC_SIZE(size);
3772
	/* Set low register to conditional flag. */
3773

3774
	if (reg_map[reg] >= 4)
3775
		*inst++ = (reg_map[reg] <= 7) ? REX : REX_B;
3776

3777
	inst[0] = GROUP_0F;
3778
	inst[1] = cond_set;
3779
	inst[2] = MOD_REG | reg_lmap[reg];
3780

3781
	inst[3] = REX_W | (reg_map[reg] <= 7 ? 0 : (REX_B | REX_R));
3782
	/* The movzx instruction does not affect flags. */
3783
	inst[4] = GROUP_0F;
3784
	inst[5] = MOVZX_r_rm8;
3785
	inst[6] = U8(MOD_REG | (reg_lmap[reg] << 3) | reg_lmap[reg]);
3786

3787
	if (reg != TMP_REG1)
3788
		return SLJIT_SUCCESS;
3789

3790
	if (GET_OPCODE(op) < SLJIT_ADD) {
3791
		compiler->mode32 = GET_OPCODE(op) != SLJIT_MOV;
3792
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3793
	}
3794

3795
	SLJIT_SKIP_CHECKS(compiler);
3796
	return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
3797

3798
#else /* !SLJIT_CONFIG_X86_64 */
3799
	SLJIT_ASSERT(reg_map[TMP_REG1] < 4);
3800

3801
	/* The SLJIT_CONFIG_X86_32 code path starts here. */
3802
	if (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
3803
		/* Low byte is accessible. */
3804
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
3805
		FAIL_IF(!inst);
3806
		INC_SIZE(3 + 3);
3807
		/* Set low byte to conditional flag. */
3808
		inst[0] = GROUP_0F;
3809
		inst[1] = cond_set;
3810
		inst[2] = U8(MOD_REG | reg_map[dst]);
3811

3812
		inst[3] = GROUP_0F;
3813
		inst[4] = MOVZX_r_rm8;
3814
		inst[5] = U8(MOD_REG | (reg_map[dst] << 3) | reg_map[dst]);
3815
		return SLJIT_SUCCESS;
3816
	}
3817

3818
	if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
3819
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 2);
3820
		FAIL_IF(!inst);
3821
		INC_SIZE(3 + 2);
3822

3823
		/* Set low byte to conditional flag. */
3824
		inst[0] = GROUP_0F;
3825
		inst[1] = cond_set;
3826
		inst[2] = U8(MOD_REG | reg_map[TMP_REG1]);
3827

3828
		inst[3] = OR_rm8_r8;
3829
		inst[4] = U8(MOD_REG | (reg_map[TMP_REG1] << 3) | reg_map[dst]);
3830
		return SLJIT_SUCCESS;
3831
	}
3832

3833
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
3834
	FAIL_IF(!inst);
3835
	INC_SIZE(3 + 3);
3836
	/* Set low byte to conditional flag. */
3837
	inst[0] = GROUP_0F;
3838
	inst[1] = cond_set;
3839
	inst[2] = U8(MOD_REG | reg_map[TMP_REG1]);
3840

3841
	inst[3] = GROUP_0F;
3842
	inst[4] = MOVZX_r_rm8;
3843
	inst[5] = U8(MOD_REG | (reg_map[TMP_REG1] << 3) | reg_map[TMP_REG1]);
3844

3845
	if (GET_OPCODE(op) < SLJIT_ADD)
3846
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3847

3848
	SLJIT_SKIP_CHECKS(compiler);
3849
	return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
3850
#endif /* SLJIT_CONFIG_X86_64 */
3851
}
3852

3853
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
3854
	sljit_s32 dst_freg,
3855
	sljit_s32 src1, sljit_sw src1w,
3856
	sljit_s32 src2_freg)
3857
{
3858
	sljit_u8* inst;
3859
	sljit_uw size;
3860

3861
	CHECK_ERROR();
3862
	CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
3863

3864
	ADJUST_LOCAL_OFFSET(src1, src1w);
3865

3866
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3867
	compiler->mode32 = 1;
3868
#endif /* SLJIT_CONFIG_X86_64 */
3869

3870
	if (dst_freg != src2_freg) {
3871
		if (dst_freg == src1) {
3872
			src1 = src2_freg;
3873
			src1w = 0;
3874
			type ^= 0x1;
3875
		} else
3876
			FAIL_IF(emit_sse2_load(compiler, type & SLJIT_32, dst_freg, src2_freg, 0));
3877
	}
3878

3879
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
3880
	FAIL_IF(!inst);
3881
	INC_SIZE(2);
3882
	inst[0] = U8(get_jump_code((sljit_uw)(type & ~SLJIT_32) ^ 0x1) - 0x10);
3883

3884
	size = compiler->size;
3885
	FAIL_IF(emit_sse2_load(compiler, type & SLJIT_32, dst_freg, src1, src1w));
3886

3887
	inst[1] = U8(compiler->size - size);
3888
	return SLJIT_SUCCESS;
3889
}
3890

3891
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3892
	sljit_s32 vreg,
3893
	sljit_s32 srcdst, sljit_sw srcdstw)
3894
{
3895
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3896
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3897
	sljit_s32 alignment = SLJIT_SIMD_GET_ELEM2_SIZE(type);
3898
	sljit_uw op;
3899

3900
	CHECK_ERROR();
3901
	CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3902

3903
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3904

3905
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3906
	compiler->mode32 = 1;
3907
#endif /* SLJIT_CONFIG_X86_64 */
3908

3909
	switch (reg_size) {
3910
	case 4:
3911
		op = EX86_SSE2;
3912
		break;
3913
	case 5:
3914
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
3915
			return SLJIT_ERR_UNSUPPORTED;
3916
		op = EX86_SSE2 | VEX_256;
3917
		break;
3918
	default:
3919
		return SLJIT_ERR_UNSUPPORTED;
3920
	}
3921

3922
	if (!(srcdst & SLJIT_MEM))
3923
		alignment = reg_size;
3924

3925
	if (type & SLJIT_SIMD_FLOAT) {
3926
		if (elem_size == 2 || elem_size == 3) {
3927
			op |= alignment >= reg_size ? MOVAPS_x_xm : MOVUPS_x_xm;
3928

3929
			if (elem_size == 3)
3930
				op |= EX86_PREF_66;
3931

3932
			if (type & SLJIT_SIMD_STORE)
3933
				op += 1;
3934
		} else
3935
			return SLJIT_ERR_UNSUPPORTED;
3936
	} else {
3937
		op |= ((type & SLJIT_SIMD_STORE) ? MOVDQA_xm_x : MOVDQA_x_xm)
3938
			| (alignment >= reg_size ? EX86_PREF_66 : EX86_PREF_F3);
3939
	}
3940

3941
	if (type & SLJIT_SIMD_TEST)
3942
		return SLJIT_SUCCESS;
3943

3944
	if ((op & VEX_256) || ((cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX)))
3945
		return emit_vex_instruction(compiler, op, vreg, 0, srcdst, srcdstw);
3946

3947
	return emit_groupf(compiler, op, vreg, srcdst, srcdstw);
3948
}
3949

3950
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3951
	sljit_s32 vreg,
3952
	sljit_s32 src, sljit_sw srcw)
3953
{
3954
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3955
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3956
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
3957
	sljit_u8 *inst;
3958
	sljit_u8 opcode = 0;
3959
	sljit_uw op;
3960

3961
	CHECK_ERROR();
3962
	CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3963

3964
	ADJUST_LOCAL_OFFSET(src, srcw);
3965

3966
	if (!(type & SLJIT_SIMD_FLOAT)) {
3967
		CHECK_EXTRA_REGS(src, srcw, (void)0);
3968
	}
3969

3970
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3971
	if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : (elem_size > 2))
3972
		return SLJIT_ERR_UNSUPPORTED;
3973
#else /* !SLJIT_CONFIG_X86_32 */
3974
	compiler->mode32 = 1;
3975

3976
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
3977
		return SLJIT_ERR_UNSUPPORTED;
3978
#endif /* SLJIT_CONFIG_X86_32 */
3979

3980
	if (reg_size != 4 && (reg_size != 5 || !(cpu_feature_list & CPU_FEATURE_AVX2)))
3981
		return SLJIT_ERR_UNSUPPORTED;
3982

3983
	if (type & SLJIT_SIMD_TEST)
3984
		return SLJIT_SUCCESS;
3985

3986
	if (reg_size == 5)
3987
		use_vex = 1;
3988

3989
	if (use_vex && src != SLJIT_IMM) {
3990
		op = 0;
3991

3992
		switch (elem_size) {
3993
		case 0:
3994
			if (cpu_feature_list & CPU_FEATURE_AVX2)
3995
				op = VPBROADCASTB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3996
			break;
3997
		case 1:
3998
			if (cpu_feature_list & CPU_FEATURE_AVX2)
3999
				op = VPBROADCASTW_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4000
			break;
4001
		case 2:
4002
			if (type & SLJIT_SIMD_FLOAT) {
4003
				if ((cpu_feature_list & CPU_FEATURE_AVX2) || ((cpu_feature_list & CPU_FEATURE_AVX) && (src & SLJIT_MEM)))
4004
					op = VBROADCASTSS_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4005
			} else if (cpu_feature_list & CPU_FEATURE_AVX2)
4006
				op = VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4007
			break;
4008
		default:
4009
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4010
			if (!(type & SLJIT_SIMD_FLOAT)) {
4011
				if (cpu_feature_list & CPU_FEATURE_AVX2)
4012
					op = VPBROADCASTQ_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4013
				break;
4014
			}
4015
#endif /* SLJIT_CONFIG_X86_64 */
4016

4017
			if (reg_size == 5)
4018
				op = VBROADCASTSD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4019
			break;
4020
		}
4021

4022
		if (op != 0) {
4023
			if (!(src & SLJIT_MEM) && !(type & SLJIT_SIMD_FLOAT)) {
4024
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4025
				if (elem_size >= 3)
4026
					compiler->mode32 = 0;
4027
#endif /* SLJIT_CONFIG_X86_64 */
4028
				FAIL_IF(emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw));
4029
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4030
				compiler->mode32 = 1;
4031
#endif /* SLJIT_CONFIG_X86_64 */
4032
				src = vreg;
4033
				srcw = 0;
4034
			}
4035

4036
			if (reg_size == 5)
4037
				op |= VEX_256;
4038

4039
			return emit_vex_instruction(compiler, op, vreg, 0, src, srcw);
4040
		}
4041
	}
4042

4043
	if (type & SLJIT_SIMD_FLOAT) {
4044
		if (src == SLJIT_IMM) {
4045
			if (use_vex)
4046
				return emit_vex_instruction(compiler, XORPD_x_xm | (reg_size == 5 ? VEX_256 : 0) | (elem_size == 3 ? EX86_PREF_66 : 0) | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0);
4047

4048
			return emit_groupf(compiler, XORPD_x_xm | (elem_size == 3 ? EX86_PREF_66 : 0) | EX86_SSE2, vreg, vreg, 0);
4049
		}
4050

4051
		SLJIT_ASSERT(reg_size == 4);
4052

4053
		if (use_vex) {
4054
			if (elem_size == 3)
4055
				return emit_vex_instruction(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, src, srcw);
4056

4057
			SLJIT_ASSERT(!(src & SLJIT_MEM));
4058
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | EX86_SSE2 | VEX_SSE2_OPV, vreg, src, src, 0));
4059
			return emit_byte(compiler, 0);
4060
		}
4061

4062
		if (elem_size == 2 && vreg != src) {
4063
			FAIL_IF(emit_sse2_load(compiler, 1, vreg, src, srcw));
4064
			src = vreg;
4065
			srcw = 0;
4066
		}
4067

4068
		op = (elem_size == 2 ? SHUFPS_x_xm : MOVDDUP_x_xm) | (elem_size == 2 ? 0 : EX86_PREF_F2) | EX86_SSE2;
4069
		FAIL_IF(emit_groupf(compiler, op, vreg, src, srcw));
4070

4071
		if (elem_size == 2)
4072
			return emit_byte(compiler, 0);
4073
		return SLJIT_SUCCESS;
4074
	}
4075

4076
	if (src == SLJIT_IMM) {
4077
		if (elem_size == 0) {
4078
			srcw = (sljit_u8)srcw;
4079
			srcw |= srcw << 8;
4080
			srcw |= srcw << 16;
4081
			elem_size = 2;
4082
		} else if (elem_size == 1) {
4083
			srcw = (sljit_u16)srcw;
4084
			srcw |= srcw << 16;
4085
			elem_size = 2;
4086
		}
4087

4088
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4089
		if (elem_size == 2 && (sljit_s32)srcw == -1)
4090
			srcw = -1;
4091
#endif /* SLJIT_CONFIG_X86_64 */
4092

4093
		if (srcw == 0 || srcw == -1) {
4094
			if (use_vex)
4095
				return emit_vex_instruction(compiler, (srcw == 0 ? PXOR_x_xm : PCMPEQD_x_xm) | (reg_size == 5 ? VEX_256 : 0) | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0);
4096

4097
			return emit_groupf(compiler, (srcw == 0 ? PXOR_x_xm : PCMPEQD_x_xm) | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0);
4098
		}
4099

4100
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4101
		if (elem_size == 3)
4102
			FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
4103
		else
4104
#endif /* SLJIT_CONFIG_X86_64 */
4105
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcw);
4106

4107
		src = TMP_REG1;
4108
		srcw = 0;
4109

4110
	}
4111

4112
	op = 2;
4113
	opcode = MOVD_x_rm;
4114

4115
	switch (elem_size) {
4116
	case 0:
4117
		if (!FAST_IS_REG(src)) {
4118
			opcode = 0x3a /* Prefix of PINSRB_x_rm_i8. */;
4119
			op = 3;
4120
		}
4121
		break;
4122
	case 1:
4123
		if (!FAST_IS_REG(src))
4124
			opcode = PINSRW_x_rm_i8;
4125
		break;
4126
	case 2:
4127
		break;
4128
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4129
	case 3:
4130
		/* MOVQ */
4131
		compiler->mode32 = 0;
4132
		break;
4133
#endif /* SLJIT_CONFIG_X86_64 */
4134
	}
4135

4136
	if (use_vex) {
4137
		if (opcode != MOVD_x_rm) {
4138
			op = (opcode == 0x3a) ? (PINSRB_x_rm_i8 | VEX_OP_0F3A) : opcode;
4139
			FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1 | VEX_SSE2_OPV, vreg, vreg, src, srcw));
4140
		} else
4141
			FAIL_IF(emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw));
4142
	} else {
4143
		inst = emit_x86_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw);
4144
		FAIL_IF(!inst);
4145
		inst[0] = GROUP_0F;
4146
		inst[1] = opcode;
4147

4148
		if (op == 3) {
4149
			SLJIT_ASSERT(opcode == 0x3a);
4150
			inst[2] = PINSRB_x_rm_i8;
4151
		}
4152
	}
4153

4154
	if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && elem_size >= 2) {
4155
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4156
		op = VPBROADCASTD_x_xm;
4157
#else /* !SLJIT_CONFIG_X86_32 */
4158
		op = (elem_size == 3) ? VPBROADCASTQ_x_xm : VPBROADCASTD_x_xm;
4159
#endif /* SLJIT_CONFIG_X86_32 */
4160
		return emit_vex_instruction(compiler, op | ((reg_size == 5) ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
4161
	}
4162

4163
	SLJIT_ASSERT(reg_size == 4);
4164

4165
	if (opcode != MOVD_x_rm)
4166
		FAIL_IF(emit_byte(compiler, 0));
4167

4168
	switch (elem_size) {
4169
	case 0:
4170
		if (use_vex) {
4171
			FAIL_IF(emit_vex_instruction(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, TMP_FREG, 0));
4172
			return emit_vex_instruction(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, TMP_FREG, 0);
4173
		}
4174
		FAIL_IF(emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, TMP_FREG, 0));
4175
		return emit_groupf_ext(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, TMP_FREG, 0);
4176
	case 1:
4177
		if (use_vex)
4178
			FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, vreg, 0));
4179
		else
4180
			FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, vreg, 0));
4181
		FAIL_IF(emit_byte(compiler, 0));
4182
		SLJIT_FALLTHROUGH
4183
	default:
4184
		if (use_vex)
4185
			FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, vreg, 0));
4186
		else
4187
			FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0));
4188
		return emit_byte(compiler, 0);
4189
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4190
	case 3:
4191
		compiler->mode32 = 1;
4192
		if (use_vex)
4193
			FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, vreg, 0));
4194
		else
4195
			FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0));
4196
		return emit_byte(compiler, 0x44);
4197
#endif /* SLJIT_CONFIG_X86_64 */
4198
	}
4199
}
4200

4201
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
4202
	sljit_s32 vreg, sljit_s32 lane_index,
4203
	sljit_s32 srcdst, sljit_sw srcdstw)
4204
{
4205
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4206
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4207
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4208
	sljit_u8 *inst;
4209
	sljit_u8 opcode = 0;
4210
	sljit_uw op;
4211
	sljit_s32 vreg_orig = vreg;
4212
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4213
	sljit_s32 srcdst_is_ereg = 0;
4214
	sljit_s32 srcdst_orig = 0;
4215
	sljit_sw srcdstw_orig = 0;
4216
#endif /* SLJIT_CONFIG_X86_32 */
4217

4218
	CHECK_ERROR();
4219
	CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
4220

4221
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
4222

4223
	if (reg_size == 5) {
4224
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4225
			return SLJIT_ERR_UNSUPPORTED;
4226
		use_vex = 1;
4227
	} else if (reg_size != 4)
4228
		return SLJIT_ERR_UNSUPPORTED;
4229

4230
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4231
	if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : elem_size > 2)
4232
		return SLJIT_ERR_UNSUPPORTED;
4233
#else /* SLJIT_CONFIG_X86_32 */
4234
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
4235
		return SLJIT_ERR_UNSUPPORTED;
4236
#endif /* SLJIT_CONFIG_X86_32 */
4237

4238
	if (type & SLJIT_SIMD_TEST)
4239
		return SLJIT_SUCCESS;
4240

4241
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4242
	compiler->mode32 = 1;
4243
#else /* !SLJIT_CONFIG_X86_64 */
4244
	if (!(type & SLJIT_SIMD_FLOAT)) {
4245
		CHECK_EXTRA_REGS(srcdst, srcdstw, srcdst_is_ereg = 1);
4246

4247
		if ((type & SLJIT_SIMD_STORE) && ((srcdst_is_ereg && elem_size < 2) || (elem_size == 0 && (type & SLJIT_SIMD_LANE_SIGNED) && FAST_IS_REG(srcdst) && reg_map[srcdst] >= 4))) {
4248
			srcdst_orig = srcdst;
4249
			srcdstw_orig = srcdstw;
4250
			srcdst = TMP_REG1;
4251
			srcdstw = 0;
4252
		}
4253
	}
4254
#endif /* SLJIT_CONFIG_X86_64 */
4255

4256
	if (type & SLJIT_SIMD_LANE_ZERO) {
4257
		if (lane_index == 0) {
4258
			if (!(type & SLJIT_SIMD_FLOAT)) {
4259
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4260
				if (elem_size == 3) {
4261
					compiler->mode32 = 0;
4262
					elem_size = 2;
4263
				}
4264
#endif /* SLJIT_CONFIG_X86_64 */
4265
				if (srcdst == SLJIT_IMM) {
4266
					if (elem_size == 0)
4267
						srcdstw = (sljit_u8)srcdstw;
4268
					else if (elem_size == 1)
4269
						srcdstw = (sljit_u16)srcdstw;
4270

4271
					EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcdstw);
4272
					srcdst = TMP_REG1;
4273
					srcdstw = 0;
4274
					elem_size = 2;
4275
				}
4276

4277
				if (elem_size == 2) {
4278
					if (use_vex)
4279
						return emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, srcdst, srcdstw);
4280
					return emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, vreg, srcdst, srcdstw);
4281
				}
4282
			} else if (srcdst & SLJIT_MEM) {
4283
				SLJIT_ASSERT(elem_size == 2 || elem_size == 3);
4284

4285
				if (use_vex)
4286
					return emit_vex_instruction(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, vreg, 0, srcdst, srcdstw);
4287
				return emit_groupf(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, vreg, srcdst, srcdstw);
4288
			} else if (elem_size == 3) {
4289
				if (use_vex)
4290
					return emit_vex_instruction(compiler, MOVQ_x_xm | EX86_PREF_F3 | EX86_SSE2, vreg, 0, srcdst, 0);
4291
				return emit_groupf(compiler, MOVQ_x_xm | EX86_PREF_F3 | EX86_SSE2, vreg, srcdst, 0);
4292
			} else if (use_vex) {
4293
				FAIL_IF(emit_vex_instruction(compiler, XORPD_x_xm | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, TMP_FREG, 0));
4294
				return emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F3 | EX86_SSE2 | VEX_SSE2_OPV, vreg, TMP_FREG, srcdst, 0);
4295
			}
4296
		}
4297

4298
		if (reg_size == 5 && lane_index >= (1 << (4 - elem_size))) {
4299
			vreg = TMP_FREG;
4300
			lane_index -= (1 << (4 - elem_size));
4301
		} else if ((type & SLJIT_SIMD_FLOAT) && vreg == srcdst) {
4302
			if (use_vex)
4303
				FAIL_IF(emit_vex_instruction(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, srcdst, srcdstw));
4304
			else
4305
				FAIL_IF(emit_sse2_load(compiler, elem_size == 2, TMP_FREG, srcdst, srcdstw));
4306
			srcdst = TMP_FREG;
4307
			srcdstw = 0;
4308
		}
4309

4310
		op = ((!(type & SLJIT_SIMD_FLOAT) || elem_size != 2) ? EX86_PREF_66 : 0)
4311
			| ((type & SLJIT_SIMD_FLOAT) ? XORPD_x_xm : PXOR_x_xm) | EX86_SSE2;
4312

4313
		if (use_vex)
4314
			FAIL_IF(emit_vex_instruction(compiler, op | (reg_size == 5 ? VEX_256 : 0) | VEX_SSE2_OPV, vreg, vreg, vreg, 0));
4315
		else
4316
			FAIL_IF(emit_groupf(compiler, op, vreg, vreg, 0));
4317
	} else if (reg_size == 5 && lane_index >= (1 << (4 - elem_size))) {
4318
		FAIL_IF(emit_vex_instruction(compiler, ((type & SLJIT_SIMD_FLOAT) ? VEXTRACTF128_x_ym : VEXTRACTI128_x_ym) | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, TMP_FREG, 0));
4319
		FAIL_IF(emit_byte(compiler, 1));
4320

4321
		vreg = TMP_FREG;
4322
		lane_index -= (1 << (4 - elem_size));
4323
	}
4324

4325
	if (type & SLJIT_SIMD_FLOAT) {
4326
		if (elem_size == 3) {
4327
			if (srcdst & SLJIT_MEM) {
4328
				if (type & SLJIT_SIMD_STORE)
4329
					op = lane_index == 0 ? MOVLPD_m_x : MOVHPD_m_x;
4330
				else
4331
					op = lane_index == 0 ? MOVLPD_x_m : MOVHPD_x_m;
4332

4333
				/* VEX prefix clears upper bits of the target register. */
4334
				if (use_vex && ((type & SLJIT_SIMD_STORE) || reg_size == 4 || vreg == TMP_FREG))
4335
					FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2
4336
						| ((type & SLJIT_SIMD_STORE) ? 0 : VEX_SSE2_OPV), vreg, (type & SLJIT_SIMD_STORE) ? 0 : vreg, srcdst, srcdstw));
4337
				else
4338
					FAIL_IF(emit_groupf(compiler, op | EX86_PREF_66 | EX86_SSE2, vreg, srcdst, srcdstw));
4339

4340
				/* In case of store, vreg is not TMP_FREG. */
4341
			} else if (type & SLJIT_SIMD_STORE) {
4342
				if (lane_index == 1) {
4343
					if (use_vex)
4344
						return emit_vex_instruction(compiler, MOVHLPS_x_x | EX86_SSE2 | VEX_SSE2_OPV, srcdst, srcdst, vreg, 0);
4345
					return emit_groupf(compiler, MOVHLPS_x_x | EX86_SSE2, srcdst, vreg, 0);
4346
				}
4347
				if (use_vex)
4348
					return emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F2 | EX86_SSE2 | VEX_SSE2_OPV, srcdst, srcdst, vreg, 0);
4349
				return emit_sse2_load(compiler, 0, srcdst, vreg, 0);
4350
			} else if (use_vex && (reg_size == 4 || vreg == TMP_FREG)) {
4351
				if (lane_index == 1)
4352
					FAIL_IF(emit_vex_instruction(compiler, MOVLHPS_x_x | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, srcdst, 0));
4353
				else
4354
					FAIL_IF(emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F2 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, srcdst, 0));
4355
			} else {
4356
				if (lane_index == 1)
4357
					FAIL_IF(emit_groupf(compiler, MOVLHPS_x_x | EX86_SSE2, vreg, srcdst, 0));
4358
				else
4359
					FAIL_IF(emit_sse2_load(compiler, 0, vreg, srcdst, 0));
4360
			}
4361
		} else if (type & SLJIT_SIMD_STORE) {
4362
			if (lane_index == 0) {
4363
				if (use_vex)
4364
					return emit_vex_instruction(compiler, MOVSD_xm_x | EX86_PREF_F3 | EX86_SSE2 | ((srcdst & SLJIT_MEM) ? 0 : VEX_SSE2_OPV),
4365
						vreg, ((srcdst & SLJIT_MEM) ? 0 : srcdst), srcdst, srcdstw);
4366
				return emit_sse2_store(compiler, 1, srcdst, srcdstw, vreg);
4367
			}
4368

4369
			if (srcdst & SLJIT_MEM) {
4370
				if (use_vex)
4371
					FAIL_IF(emit_vex_instruction(compiler, EXTRACTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, srcdst, srcdstw));
4372
				else
4373
					FAIL_IF(emit_groupf_ext(compiler, EXTRACTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, srcdst, srcdstw));
4374
				return emit_byte(compiler, U8(lane_index));
4375
			}
4376

4377
			if (use_vex) {
4378
				FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | EX86_SSE2 | VEX_SSE2_OPV, srcdst, vreg, vreg, 0));
4379
				return emit_byte(compiler, U8(lane_index));
4380
			}
4381

4382
			if (srcdst == vreg)
4383
				op = SHUFPS_x_xm | EX86_SSE2;
4384
			else {
4385
				switch (lane_index) {
4386
				case 1:
4387
					op = MOVSHDUP_x_xm | EX86_PREF_F3 | EX86_SSE2;
4388
					break;
4389
				case 2:
4390
					op = MOVHLPS_x_x | EX86_SSE2;
4391
					break;
4392
				default:
4393
					SLJIT_ASSERT(lane_index == 3);
4394
					op = PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2;
4395
					break;
4396
				}
4397
			}
4398

4399
			FAIL_IF(emit_groupf(compiler, op, srcdst, vreg, 0));
4400

4401
			op &= 0xff;
4402
			if (op == SHUFPS_x_xm || op == PSHUFD_x_xm)
4403
				return emit_byte(compiler, U8(lane_index));
4404

4405
			return SLJIT_SUCCESS;
4406
		} else {
4407
			if (lane_index != 0 || (srcdst & SLJIT_MEM)) {
4408
				FAIL_IF(emit_groupf_ext(compiler, INSERTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, srcdst, srcdstw));
4409
				FAIL_IF(emit_byte(compiler, U8(lane_index << 4)));
4410
			} else
4411
				FAIL_IF(emit_sse2_store(compiler, 1, vreg, 0, srcdst));
4412
		}
4413

4414
		if (vreg != TMP_FREG || (type & SLJIT_SIMD_STORE))
4415
			return SLJIT_SUCCESS;
4416

4417
		SLJIT_ASSERT(reg_size == 5);
4418

4419
		if (type & SLJIT_SIMD_LANE_ZERO) {
4420
			FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg_orig, 0, TMP_FREG, 0));
4421
			return emit_byte(compiler, 0x4e);
4422
		}
4423

4424
		FAIL_IF(emit_vex_instruction(compiler, VINSERTF128_y_y_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2 | VEX_SSE2_OPV, vreg_orig, vreg_orig, TMP_FREG, 0));
4425
		return emit_byte(compiler, 1);
4426
	}
4427

4428
	if (srcdst == SLJIT_IMM) {
4429
		EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcdstw);
4430
		srcdst = TMP_REG1;
4431
		srcdstw = 0;
4432
	}
4433

4434
	op = 3;
4435

4436
	switch (elem_size) {
4437
	case 0:
4438
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRB_rm_x_i8 : PINSRB_x_rm_i8;
4439
		break;
4440
	case 1:
4441
		if (!(type & SLJIT_SIMD_STORE)) {
4442
			op = 2;
4443
			opcode = PINSRW_x_rm_i8;
4444
		} else
4445
			opcode = PEXTRW_rm_x_i8;
4446
		break;
4447
	case 2:
4448
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRD_rm_x_i8 : PINSRD_x_rm_i8;
4449
		break;
4450
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4451
	case 3:
4452
		/* PINSRQ / PEXTRQ */
4453
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRD_rm_x_i8 : PINSRD_x_rm_i8;
4454
		compiler->mode32 = 0;
4455
		break;
4456
#endif /* SLJIT_CONFIG_X86_64 */
4457
	}
4458

4459
	if (use_vex && (type & SLJIT_SIMD_STORE)) {
4460
		op = opcode | ((op == 3) ? VEX_OP_0F3A : 0);
4461
		FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | VEX_AUTO_W | EX86_SSE2_OP1 | VEX_SSE2_OPV, vreg, 0, srcdst, srcdstw));
4462
	} else {
4463
		inst = emit_x86_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, srcdst, srcdstw);
4464
		FAIL_IF(!inst);
4465
		inst[0] = GROUP_0F;
4466

4467
		if (op == 3) {
4468
			inst[1] = 0x3a;
4469
			inst[2] = opcode;
4470
		} else
4471
			inst[1] = opcode;
4472
	}
4473

4474
	FAIL_IF(emit_byte(compiler, U8(lane_index)));
4475

4476
	if (!(type & SLJIT_SIMD_LANE_SIGNED) || (srcdst & SLJIT_MEM)) {
4477
		if (vreg == TMP_FREG && !(type & SLJIT_SIMD_STORE)) {
4478
			SLJIT_ASSERT(reg_size == 5);
4479

4480
			if (type & SLJIT_SIMD_LANE_ZERO) {
4481
				FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg_orig, 0, TMP_FREG, 0));
4482
				return emit_byte(compiler, 0x4e);
4483
			}
4484

4485
			FAIL_IF(emit_vex_instruction(compiler, VINSERTI128_y_y_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2 | VEX_SSE2_OPV, vreg_orig, vreg_orig, TMP_FREG, 0));
4486
			return emit_byte(compiler, 1);
4487
		}
4488

4489
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4490
		if (srcdst_orig & SLJIT_MEM)
4491
			return emit_mov(compiler, srcdst_orig, srcdstw_orig, TMP_REG1, 0);
4492
#endif /* SLJIT_CONFIG_X86_32 */
4493
		return SLJIT_SUCCESS;
4494
	}
4495

4496
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4497
	if (elem_size >= 3)
4498
		return SLJIT_SUCCESS;
4499

4500
	compiler->mode32 = (type & SLJIT_32);
4501

4502
	op = 2;
4503

4504
	if (elem_size == 0)
4505
		op |= EX86_REX;
4506

4507
	if (elem_size == 2) {
4508
		if (type & SLJIT_32)
4509
			return SLJIT_SUCCESS;
4510

4511
		SLJIT_ASSERT(!(compiler->mode32));
4512
		op = 1;
4513
	}
4514

4515
	inst = emit_x86_instruction(compiler, op, srcdst, 0, srcdst, 0);
4516
	FAIL_IF(!inst);
4517

4518
	if (op != 1) {
4519
		inst[0] = GROUP_0F;
4520
		inst[1] = U8((elem_size == 0) ? MOVSX_r_rm8 : MOVSX_r_rm16);
4521
	} else
4522
		inst[0] = MOVSXD_r_rm;
4523
#else /* !SLJIT_CONFIG_X86_64 */
4524
	if (elem_size >= 2)
4525
		return SLJIT_SUCCESS;
4526

4527
	FAIL_IF(emit_groupf(compiler, (elem_size == 0) ? MOVSX_r_rm8 : MOVSX_r_rm16,
4528
		(srcdst_orig != 0 && FAST_IS_REG(srcdst_orig)) ? srcdst_orig : srcdst, srcdst, 0));
4529

4530
	if (srcdst_orig & SLJIT_MEM)
4531
		return emit_mov(compiler, srcdst_orig, srcdstw_orig, TMP_REG1, 0);
4532
#endif /* SLJIT_CONFIG_X86_64 */
4533
	return SLJIT_SUCCESS;
4534
}
4535

4536
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
4537
	sljit_s32 vreg,
4538
	sljit_s32 src, sljit_s32 src_lane_index)
4539
{
4540
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4541
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4542
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4543
	sljit_uw pref;
4544
	sljit_u8 byte;
4545
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4546
	sljit_s32 opcode3 = TMP_REG1;
4547
#else /* !SLJIT_CONFIG_X86_32 */
4548
	sljit_s32 opcode3 = SLJIT_S0;
4549
#endif /* SLJIT_CONFIG_X86_32 */
4550

4551
	CHECK_ERROR();
4552
	CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
4553

4554
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4555
	compiler->mode32 = 1;
4556
#endif /* SLJIT_CONFIG_X86_64 */
4557
	SLJIT_ASSERT(reg_map[opcode3] == 3);
4558

4559
	if (reg_size == 5) {
4560
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4561
			return SLJIT_ERR_UNSUPPORTED;
4562
		use_vex = 1;
4563
	} else if (reg_size != 4)
4564
		return SLJIT_ERR_UNSUPPORTED;
4565

4566
	if (type & SLJIT_SIMD_FLOAT) {
4567
		pref = 0;
4568
		byte = U8(src_lane_index);
4569

4570
		if (elem_size == 3) {
4571
			if (type & SLJIT_SIMD_TEST)
4572
				return SLJIT_SUCCESS;
4573

4574
			if (reg_size == 5) {
4575
				if (src_lane_index == 0)
4576
					return emit_vex_instruction(compiler, VBROADCASTSD_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4577

4578
				FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4579

4580
				byte = U8(byte | (byte << 2));
4581
				return emit_byte(compiler, U8(byte | (byte << 4)));
4582
			}
4583

4584
			if (src_lane_index == 0) {
4585
				if (use_vex)
4586
					return emit_vex_instruction(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, src, 0);
4587
				return emit_groupf(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, src, 0);
4588
			}
4589

4590
			/* Changes it to SHUFPD_x_xm. */
4591
			pref = EX86_PREF_66;
4592
		} else if (elem_size != 2)
4593
			return SLJIT_ERR_UNSUPPORTED;
4594
		else if (type & SLJIT_SIMD_TEST)
4595
			return SLJIT_SUCCESS;
4596

4597
		if (reg_size == 5) {
4598
			SLJIT_ASSERT(elem_size == 2);
4599

4600
			if (src_lane_index == 0)
4601
				return emit_vex_instruction(compiler, VBROADCASTSS_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4602

4603
			FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4604

4605
			byte = 0x44;
4606
			if (src_lane_index >= 4) {
4607
				byte = 0xee;
4608
				src_lane_index -= 4;
4609
			}
4610

4611
			FAIL_IF(emit_byte(compiler, byte));
4612
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | VEX_256 | pref | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0));
4613
			byte = U8(src_lane_index);
4614
		} else if (use_vex) {
4615
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | pref | EX86_SSE2 | VEX_SSE2_OPV, vreg, src, src, 0));
4616
		} else {
4617
			if (vreg != src)
4618
				FAIL_IF(emit_groupf(compiler, MOVAPS_x_xm | pref | EX86_SSE2, vreg, src, 0));
4619

4620
			FAIL_IF(emit_groupf(compiler, SHUFPS_x_xm | pref | EX86_SSE2, vreg, vreg, 0));
4621
		}
4622

4623
		if (elem_size == 2) {
4624
			byte = U8(byte | (byte << 2));
4625
			byte = U8(byte | (byte << 4));
4626
		} else
4627
			byte = U8(byte | (byte << 1));
4628

4629
		return emit_byte(compiler, U8(byte));
4630
	}
4631

4632
	if (type & SLJIT_SIMD_TEST)
4633
		return SLJIT_SUCCESS;
4634

4635
	if (elem_size == 0) {
4636
		if (reg_size == 5 && src_lane_index >= 16) {
4637
			FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4638
			FAIL_IF(emit_byte(compiler, src_lane_index >= 24 ? 0xff : 0xaa));
4639
			src_lane_index &= 0x7;
4640
			src = vreg;
4641
		}
4642

4643
		if (src_lane_index != 0 || (vreg != src && (!(cpu_feature_list & CPU_FEATURE_AVX2) || !use_vex))) {
4644
			pref = 0;
4645

4646
			if ((src_lane_index & 0x3) == 0) {
4647
				pref = EX86_PREF_66;
4648
				byte = U8(src_lane_index >> 2);
4649
			} else if (src_lane_index < 8 && (src_lane_index & 0x1) == 0) {
4650
				pref = EX86_PREF_F2;
4651
				byte = U8(src_lane_index >> 1);
4652
			} else {
4653
				if (!use_vex) {
4654
					if (vreg != src)
4655
						FAIL_IF(emit_groupf(compiler, MOVDQA_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, src, 0));
4656

4657
					FAIL_IF(emit_groupf(compiler, PSRLDQ_x | EX86_PREF_66 | EX86_SSE2_OP2, opcode3, vreg, 0));
4658
				} else
4659
					FAIL_IF(emit_vex_instruction(compiler, PSRLDQ_x | EX86_PREF_66 | EX86_SSE2_OP2 | VEX_SSE2_OPV, opcode3, vreg, src, 0));
4660

4661
				FAIL_IF(emit_byte(compiler, U8(src_lane_index)));
4662
			}
4663

4664
			if (pref != 0) {
4665
				if (use_vex)
4666
					FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, 0, src, 0));
4667
				else
4668
					FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, src, 0));
4669
				FAIL_IF(emit_byte(compiler, byte));
4670
			}
4671

4672
			src = vreg;
4673
		}
4674

4675
		if (use_vex && (cpu_feature_list & CPU_FEATURE_AVX2))
4676
			return emit_vex_instruction(compiler, VPBROADCASTB_x_xm | (reg_size == 5 ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4677

4678
		SLJIT_ASSERT(reg_size == 4);
4679
		FAIL_IF(emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, TMP_FREG, 0));
4680
		return emit_groupf_ext(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, TMP_FREG, 0);
4681
	}
4682

4683
	if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && src_lane_index == 0 && elem_size <= 3) {
4684
		switch (elem_size) {
4685
		case 1:
4686
			pref = VPBROADCASTW_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4687
			break;
4688
		case 2:
4689
			pref = VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4690
			break;
4691
		default:
4692
			pref = VPBROADCASTQ_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4693
			break;
4694
		}
4695

4696
		if (reg_size == 5)
4697
			pref |= VEX_256;
4698

4699
		return emit_vex_instruction(compiler, pref, vreg, 0, src, 0);
4700
	}
4701

4702
	if (reg_size == 5) {
4703
		switch (elem_size) {
4704
		case 1:
4705
			byte = U8(src_lane_index & 0x3);
4706
			src_lane_index >>= 2;
4707
			pref = PSHUFLW_x_xm | VEX_256 | ((src_lane_index & 1) == 0 ? EX86_PREF_F2 : EX86_PREF_F3) | EX86_SSE2;
4708
			break;
4709
		case 2:
4710
			byte = U8(src_lane_index & 0x3);
4711
			src_lane_index >>= 1;
4712
			pref = PSHUFD_x_xm | VEX_256 | EX86_PREF_66 | EX86_SSE2;
4713
			break;
4714
		case 3:
4715
			pref = 0;
4716
			break;
4717
		default:
4718
			FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4719
			return emit_byte(compiler, U8(src_lane_index == 0 ? 0x44 : 0xee));
4720
		}
4721

4722
		if (pref != 0) {
4723
			FAIL_IF(emit_vex_instruction(compiler, pref, vreg, 0, src, 0));
4724
			byte = U8(byte | (byte << 2));
4725
			FAIL_IF(emit_byte(compiler, U8(byte | (byte << 4))));
4726

4727
			if (src_lane_index == 0)
4728
				return emit_vex_instruction(compiler, VPBROADCASTQ_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
4729

4730
			src = vreg;
4731
		}
4732

4733
		FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4734
		byte = U8(src_lane_index);
4735
		byte = U8(byte | (byte << 2));
4736
		return emit_byte(compiler, U8(byte | (byte << 4)));
4737
	}
4738

4739
	switch (elem_size) {
4740
	case 1:
4741
		byte = U8(src_lane_index & 0x3);
4742
		src_lane_index >>= 1;
4743
		pref = (src_lane_index & 2) == 0 ? EX86_PREF_F2 : EX86_PREF_F3;
4744

4745
		if (use_vex)
4746
			FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, 0, src, 0));
4747
		else
4748
			FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, src, 0));
4749
		byte = U8(byte | (byte << 2));
4750
		FAIL_IF(emit_byte(compiler, U8(byte | (byte << 4))));
4751

4752
		if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && pref == EX86_PREF_F2)
4753
			return emit_vex_instruction(compiler, VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
4754

4755
		src = vreg;
4756
		SLJIT_FALLTHROUGH
4757
	case 2:
4758
		byte = U8(src_lane_index);
4759
		byte = U8(byte | (byte << 2));
4760
		break;
4761
	default:
4762
		byte = U8(src_lane_index << 1);
4763
		byte = U8(byte | (byte << 2) | 0x4);
4764
		break;
4765
	}
4766

4767
	if (use_vex)
4768
		FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, src, 0));
4769
	else
4770
		FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, src, 0));
4771
	return emit_byte(compiler, U8(byte | (byte << 4)));
4772
}
4773

4774
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
4775
	sljit_s32 vreg,
4776
	sljit_s32 src, sljit_sw srcw)
4777
{
4778
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4779
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4780
	sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
4781
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4782
	sljit_u8 opcode;
4783

4784
	CHECK_ERROR();
4785
	CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
4786

4787
	ADJUST_LOCAL_OFFSET(src, srcw);
4788

4789
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4790
	compiler->mode32 = 1;
4791
#endif /* SLJIT_CONFIG_X86_64 */
4792

4793
	if (reg_size == 5) {
4794
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4795
			return SLJIT_ERR_UNSUPPORTED;
4796
		use_vex = 1;
4797
	} else if (reg_size != 4)
4798
		return SLJIT_ERR_UNSUPPORTED;
4799

4800
	if (type & SLJIT_SIMD_FLOAT) {
4801
		if (elem_size != 2 || elem2_size != 3)
4802
			return SLJIT_ERR_UNSUPPORTED;
4803

4804
		if (type & SLJIT_SIMD_TEST)
4805
			return SLJIT_SUCCESS;
4806

4807
		if (use_vex)
4808
			return emit_vex_instruction(compiler, CVTPS2PD_x_xm | ((reg_size == 5) ? VEX_256 : 0) | EX86_SSE2, vreg, 0, src, srcw);
4809
		return emit_groupf(compiler, CVTPS2PD_x_xm | EX86_SSE2, vreg, src, srcw);
4810
	}
4811

4812
	switch (elem_size) {
4813
	case 0:
4814
		if (elem2_size == 1)
4815
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBW_x_xm : PMOVZXBW_x_xm;
4816
		else if (elem2_size == 2)
4817
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBD_x_xm : PMOVZXBD_x_xm;
4818
		else if (elem2_size == 3)
4819
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBQ_x_xm : PMOVZXBQ_x_xm;
4820
		else
4821
			return SLJIT_ERR_UNSUPPORTED;
4822
		break;
4823
	case 1:
4824
		if (elem2_size == 2)
4825
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXWD_x_xm : PMOVZXWD_x_xm;
4826
		else if (elem2_size == 3)
4827
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXWQ_x_xm : PMOVZXWQ_x_xm;
4828
		else
4829
			return SLJIT_ERR_UNSUPPORTED;
4830
		break;
4831
	case 2:
4832
		if (elem2_size == 3)
4833
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXDQ_x_xm : PMOVZXDQ_x_xm;
4834
		else
4835
			return SLJIT_ERR_UNSUPPORTED;
4836
		break;
4837
	default:
4838
		return SLJIT_ERR_UNSUPPORTED;
4839
	}
4840

4841
	if (type & SLJIT_SIMD_TEST)
4842
		return SLJIT_SUCCESS;
4843

4844
	if (use_vex)
4845
		return emit_vex_instruction(compiler, opcode | ((reg_size == 5) ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, srcw);
4846
	return emit_groupf_ext(compiler, opcode | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, src, srcw);
4847
}
4848

4849
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
4850
	sljit_s32 vreg,
4851
	sljit_s32 dst, sljit_sw dstw)
4852
{
4853
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4854
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4855
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4856
	sljit_s32 dst_r;
4857
	sljit_uw op;
4858
	sljit_u8 *inst;
4859

4860
	CHECK_ERROR();
4861
	CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
4862

4863
	ADJUST_LOCAL_OFFSET(dst, dstw);
4864

4865
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
4866
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4867
	compiler->mode32 = 1;
4868
#endif /* SLJIT_CONFIG_X86_64 */
4869

4870
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
4871
		return SLJIT_ERR_UNSUPPORTED;
4872

4873
	if (reg_size == 4) {
4874
		if (type & SLJIT_SIMD_TEST)
4875
			return SLJIT_SUCCESS;
4876

4877
		op = EX86_PREF_66 | EX86_SSE2_OP2;
4878

4879
		switch (elem_size) {
4880
		case 1:
4881
			if (use_vex)
4882
				FAIL_IF(emit_vex_instruction(compiler, PACKSSWB_x_xm | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, vreg, vreg, 0));
4883
			else
4884
				FAIL_IF(emit_groupf(compiler, PACKSSWB_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, vreg, 0));
4885
			vreg = TMP_FREG;
4886
			break;
4887
		case 2:
4888
			op = EX86_SSE2_OP2;
4889
			break;
4890
		}
4891

4892
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4893
		op |= (elem_size < 2) ? PMOVMSKB_r_x : MOVMSKPS_r_x;
4894

4895
		if (use_vex)
4896
			FAIL_IF(emit_vex_instruction(compiler, op, dst_r, 0, vreg, 0));
4897
		else
4898
			FAIL_IF(emit_groupf(compiler, op, dst_r, vreg, 0));
4899

4900
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4901
		compiler->mode32 = type & SLJIT_32;
4902
#endif /* SLJIT_CONFIG_X86_64 */
4903

4904
		if (elem_size == 1) {
4905
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 8, dst_r, 0);
4906
			FAIL_IF(!inst);
4907
			inst[1] |= SHR;
4908
		}
4909

4910
		if (dst_r == TMP_REG1)
4911
			return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
4912

4913
		return SLJIT_SUCCESS;
4914
	}
4915

4916
	if (reg_size != 5 || !(cpu_feature_list & CPU_FEATURE_AVX2))
4917
		return SLJIT_ERR_UNSUPPORTED;
4918

4919
	if (type & SLJIT_SIMD_TEST)
4920
		return SLJIT_SUCCESS;
4921

4922
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4923

4924
	if (elem_size == 1) {
4925
		FAIL_IF(emit_vex_instruction(compiler, VEXTRACTI128_x_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, TMP_FREG, 0));
4926
		FAIL_IF(emit_byte(compiler, 1));
4927
		FAIL_IF(emit_vex_instruction(compiler, PACKSSWB_x_xm | VEX_256 | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, vreg, TMP_FREG, 0));
4928
		FAIL_IF(emit_groupf(compiler, PMOVMSKB_r_x | EX86_PREF_66 | EX86_SSE2_OP2, dst_r, TMP_FREG, 0));
4929
	} else {
4930
		op = MOVMSKPS_r_x | VEX_256 | EX86_SSE2_OP2;
4931

4932
		if (elem_size == 0)
4933
			op = PMOVMSKB_r_x | VEX_256 | EX86_PREF_66 | EX86_SSE2_OP2;
4934
		else if (elem_size == 3)
4935
			op |= EX86_PREF_66;
4936

4937
		FAIL_IF(emit_vex_instruction(compiler, op, dst_r, 0, vreg, 0));
4938
	}
4939

4940
	if (dst_r == TMP_REG1) {
4941
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4942
		compiler->mode32 = type & SLJIT_32;
4943
#endif /* SLJIT_CONFIG_X86_64 */
4944
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
4945
	}
4946

4947
	return SLJIT_SUCCESS;
4948
}
4949

4950
static sljit_s32 emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
4951
	sljit_s32 dst_vreg, sljit_s32 src_vreg)
4952
{
4953
	sljit_uw op = ((type & SLJIT_SIMD_FLOAT) ? MOVAPS_x_xm : MOVDQA_x_xm) | EX86_SSE2;
4954

4955
	SLJIT_ASSERT(SLJIT_SIMD_GET_REG_SIZE(type) == 4);
4956

4957
	if (!(type & SLJIT_SIMD_FLOAT) || SLJIT_SIMD_GET_ELEM_SIZE(type) == 3)
4958
		op |= EX86_PREF_66;
4959

4960
	return emit_groupf(compiler, op, dst_vreg, src_vreg, 0);
4961
}
4962

4963
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
4964
	sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
4965
{
4966
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4967
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4968
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4969
	sljit_uw op = 0;
4970
	sljit_uw mov_op = 0;
4971

4972
	CHECK_ERROR();
4973
	CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
4974
	ADJUST_LOCAL_OFFSET(src2, src2w);
4975

4976
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4977
	compiler->mode32 = 1;
4978
#endif /* SLJIT_CONFIG_X86_64 */
4979

4980
	if (reg_size == 5) {
4981
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4982
			return SLJIT_ERR_UNSUPPORTED;
4983
	} else if (reg_size != 4)
4984
		return SLJIT_ERR_UNSUPPORTED;
4985

4986
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4987
		return SLJIT_ERR_UNSUPPORTED;
4988

4989
	switch (SLJIT_SIMD_GET_OPCODE(type)) {
4990
	case SLJIT_SIMD_OP2_AND:
4991
		op = (type & SLJIT_SIMD_FLOAT) ? ANDPD_x_xm : PAND_x_xm;
4992

4993
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
4994
			op |= EX86_PREF_66;
4995
		break;
4996
	case SLJIT_SIMD_OP2_OR:
4997
		op = (type & SLJIT_SIMD_FLOAT) ? ORPD_x_xm : POR_x_xm;
4998

4999
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
5000
			op |= EX86_PREF_66;
5001
		break;
5002
	case SLJIT_SIMD_OP2_XOR:
5003
		op = (type & SLJIT_SIMD_FLOAT) ? XORPD_x_xm : PXOR_x_xm;
5004

5005
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
5006
			op |= EX86_PREF_66;
5007
		break;
5008

5009
	case SLJIT_SIMD_OP2_SHUFFLE:
5010
		if (reg_size != 4)
5011
			return SLJIT_ERR_UNSUPPORTED;
5012

5013
		op = PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38;
5014
		break;
5015
	}
5016

5017
	if (type & SLJIT_SIMD_TEST)
5018
		return SLJIT_SUCCESS;
5019

5020
	if ((src2 & SLJIT_MEM) && SLJIT_SIMD_GET_ELEM2_SIZE(type) < reg_size) {
5021
		mov_op = ((type & SLJIT_SIMD_FLOAT) ? (MOVUPS_x_xm | (elem_size == 3 ? EX86_PREF_66 : 0)) : (MOVDQU_x_xm | EX86_PREF_F3)) | EX86_SSE2;
5022
		if (use_vex)
5023
			FAIL_IF(emit_vex_instruction(compiler, mov_op, TMP_FREG, 0, src2, src2w));
5024
		else
5025
			FAIL_IF(emit_groupf(compiler, mov_op, TMP_FREG, src2, src2w));
5026

5027
		src2 = TMP_FREG;
5028
		src2w = 0;
5029
	}
5030

5031
	if (reg_size == 5 || use_vex) {
5032
		if (reg_size == 5)
5033
			op |= VEX_256;
5034

5035
		return emit_vex_instruction(compiler, op | EX86_SSE2 | VEX_SSE2_OPV, dst_vreg, src1_vreg, src2, src2w);
5036
	}
5037

5038
	if (dst_vreg != src1_vreg) {
5039
		if (dst_vreg == src2) {
5040
			if (SLJIT_SIMD_GET_OPCODE(type) == SLJIT_SIMD_OP2_SHUFFLE) {
5041
				FAIL_IF(emit_simd_mov(compiler, type, TMP_FREG, src2));
5042
				FAIL_IF(emit_simd_mov(compiler, type, dst_vreg, src1_vreg));
5043
				src2 = TMP_FREG;
5044
				src2w = 0;
5045
			} else
5046
				src2 = src1_vreg;
5047
		} else
5048
			FAIL_IF(emit_simd_mov(compiler, type, dst_vreg, src1_vreg));
5049
	}
5050

5051
	if (op & (VEX_OP_0F38 | VEX_OP_0F3A))
5052
		return emit_groupf_ext(compiler, op | EX86_SSE2, dst_vreg, src2, src2w);
5053
	return emit_groupf(compiler, op | EX86_SSE2, dst_vreg, src2, src2w);
5054
}
5055

5056
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
5057
	sljit_s32 dst_reg,
5058
	sljit_s32 mem_reg)
5059
{
5060
	CHECK_ERROR();
5061
	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
5062

5063
	if ((op & SLJIT_ATOMIC_USE_LS) || GET_OPCODE(op) == SLJIT_MOV_S8 || GET_OPCODE(op) == SLJIT_MOV_S16 || GET_OPCODE(op) == SLJIT_MOV_S32)
5064
		return SLJIT_ERR_UNSUPPORTED;
5065

5066
	if (op & SLJIT_ATOMIC_TEST)
5067
		return SLJIT_SUCCESS;
5068

5069
	SLJIT_SKIP_CHECKS(compiler);
5070
	return sljit_emit_op1(compiler, op & ~SLJIT_ATOMIC_USE_CAS, dst_reg, 0, SLJIT_MEM1(mem_reg), 0);
5071
}
5072

5073
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
5074
	sljit_s32 src_reg,
5075
	sljit_s32 mem_reg,
5076
	sljit_s32 temp_reg)
5077
{
5078
	sljit_uw pref;
5079
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5080
	sljit_s32 saved_reg = TMP_REG1;
5081
	sljit_s32 swap_tmp = 0;
5082
	sljit_sw srcw = 0;
5083
	sljit_sw tempw = 0;
5084
#endif /* SLJIT_CONFIG_X86_32 */
5085

5086
	CHECK_ERROR();
5087
	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
5088
	CHECK_EXTRA_REGS(src_reg, srcw, (void)0);
5089
	CHECK_EXTRA_REGS(temp_reg, tempw, (void)0);
5090

5091
	SLJIT_ASSERT(FAST_IS_REG(src_reg) || src_reg == SLJIT_MEM1(SLJIT_SP));
5092
	SLJIT_ASSERT(FAST_IS_REG(temp_reg) || temp_reg == SLJIT_MEM1(SLJIT_SP));
5093

5094
	if ((op & SLJIT_ATOMIC_USE_LS) || GET_OPCODE(op) == SLJIT_MOV_S8 || GET_OPCODE(op) == SLJIT_MOV_S16 || GET_OPCODE(op) == SLJIT_MOV_S32)
5095
		return SLJIT_ERR_UNSUPPORTED;
5096

5097
	if (op & SLJIT_ATOMIC_TEST)
5098
		return SLJIT_SUCCESS;
5099

5100
	op = GET_OPCODE(op);
5101

5102
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5103
	if (temp_reg == SLJIT_TMP_DEST_REG) {
5104
		FAIL_IF(emit_byte(compiler, XCHG_EAX_r | reg_map[TMP_REG1]));
5105

5106
		if (src_reg == SLJIT_R0)
5107
			src_reg = TMP_REG1;
5108
		if (mem_reg == SLJIT_R0)
5109
			mem_reg = TMP_REG1;
5110

5111
		temp_reg = SLJIT_R0;
5112
		swap_tmp = 1;
5113
	}
5114

5115
	/* Src is virtual register or its low byte is not accessible. */
5116
	if ((src_reg & SLJIT_MEM) || (op == SLJIT_MOV_U8 && reg_map[src_reg] >= 4)) {
5117
		SLJIT_ASSERT(src_reg != SLJIT_R1 && temp_reg != SLJIT_TMP_DEST_REG);
5118

5119
		if (swap_tmp) {
5120
			saved_reg = (mem_reg != SLJIT_R1) ? SLJIT_R1 : SLJIT_R2;
5121

5122
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, saved_reg, 0);
5123
			EMIT_MOV(compiler, saved_reg, 0, src_reg, srcw);
5124
		} else
5125
			EMIT_MOV(compiler, TMP_REG1, 0, src_reg, srcw);
5126

5127
		src_reg = saved_reg;
5128

5129
		if (mem_reg == src_reg)
5130
			mem_reg = saved_reg;
5131
	}
5132
#endif /* SLJIT_CONFIG_X86_32 */
5133

5134
	if (temp_reg != SLJIT_R0) {
5135
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5136
		compiler->mode32 = 0;
5137

5138
		EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_R0, 0);
5139
		EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, 0);
5140

5141
		if (src_reg == SLJIT_R0)
5142
			src_reg = TMP_REG2;
5143
		if (mem_reg == SLJIT_R0)
5144
			mem_reg = TMP_REG2;
5145
#else /* !SLJIT_CONFIG_X86_64 */
5146
		SLJIT_ASSERT(!swap_tmp);
5147

5148
		if (src_reg == TMP_REG1) {
5149
			if (mem_reg == SLJIT_R0) {
5150
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R1, 0);
5151
				EMIT_MOV(compiler, SLJIT_R1, 0, SLJIT_R0, 0);
5152
				EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
5153

5154
				mem_reg = SLJIT_R1;
5155
				saved_reg = SLJIT_R1;
5156
			} else {
5157
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
5158
				EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
5159
				saved_reg = SLJIT_R0;
5160
			}
5161
		} else {
5162
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R0, 0);
5163
			EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
5164

5165
			if (src_reg == SLJIT_R0)
5166
				src_reg = TMP_REG1;
5167
			if (mem_reg == SLJIT_R0)
5168
				mem_reg = TMP_REG1;
5169
		}
5170
#endif /* SLJIT_CONFIG_X86_64 */
5171
	}
5172

5173
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5174
	compiler->mode32 = op != SLJIT_MOV && op != SLJIT_MOV_P;
5175
#endif /* SLJIT_CONFIG_X86_64 */
5176

5177
	/* Lock prefix. */
5178
	FAIL_IF(emit_byte(compiler, GROUP_LOCK));
5179

5180
	pref = 0;
5181
	if (op == SLJIT_MOV_U16)
5182
		pref = EX86_HALF_ARG | EX86_PREF_66;
5183
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5184
	if (op == SLJIT_MOV_U8)
5185
		pref = EX86_REX;
5186
#endif /* SLJIT_CONFIG_X86_64 */
5187

5188
	FAIL_IF(emit_groupf(compiler, (op == SLJIT_MOV_U8 ? CMPXCHG_rm8_r : CMPXCHG_rm_r) | pref, src_reg, SLJIT_MEM1(mem_reg), 0));
5189

5190
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5191
	if (swap_tmp) {
5192
		SLJIT_ASSERT(temp_reg == SLJIT_R0);
5193
		FAIL_IF(emit_byte(compiler, XCHG_EAX_r | reg_map[TMP_REG1]));
5194

5195
		if (saved_reg != TMP_REG1)
5196
			return emit_mov(compiler, saved_reg, 0, SLJIT_MEM1(SLJIT_SP), 0);
5197
		return SLJIT_SUCCESS;
5198
	}
5199
#endif /* SLJIT_CONFIG_X86_32 */
5200

5201
	if (temp_reg != SLJIT_R0) {
5202
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5203
		compiler->mode32 = 0;
5204
		return emit_mov(compiler, SLJIT_R0, 0, TMP_REG2, 0);
5205
#else /* !SLJIT_CONFIG_X86_64 */
5206
		EMIT_MOV(compiler, SLJIT_R0, 0, (saved_reg == SLJIT_R0) ? SLJIT_MEM1(SLJIT_SP) : saved_reg, 0);
5207
		if (saved_reg == SLJIT_R1)
5208
			return emit_mov(compiler, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_SP), 0);
5209
#endif /* SLJIT_CONFIG_X86_64 */
5210
	}
5211
	return SLJIT_SUCCESS;
5212
}
5213

5214
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
5215
{
5216
	CHECK_ERROR();
5217
	CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
5218
	ADJUST_LOCAL_OFFSET(dst, dstw);
5219
	ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
5220

5221
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
5222

5223
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5224
	compiler->mode32 = 0;
5225
#endif
5226

5227
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5228
	if (NOT_HALFWORD(offset)) {
5229
		FAIL_IF(emit_load_imm64(compiler, TMP_REG1, offset));
5230
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
5231
		SLJIT_ASSERT(emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0) != SLJIT_ERR_UNSUPPORTED);
5232
		return compiler->error;
5233
#else
5234
		return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0);
5235
#endif
5236
	}
5237
#endif
5238

5239
	if (offset != 0)
5240
		return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
5241
	return emit_mov(compiler, dst, dstw, SLJIT_SP, 0);
5242
}
5243

5244
SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
5245
	sljit_s32 dst, sljit_sw dstw,
5246
	sljit_sw init_value)
5247
{
5248
	sljit_u8 *inst;
5249
	struct sljit_const *const_;
5250
	sljit_s32 reg;
5251
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5252
	sljit_s32 dst_is_ereg = 0;
5253
#endif /* !SLJIT_CONFIG_X86_32 */
5254

5255
	CHECK_ERROR_PTR();
5256
	CHECK_PTR(check_sljit_emit_const(compiler, op, dst, dstw, init_value));
5257
	ADJUST_LOCAL_OFFSET(dst, dstw);
5258

5259
	CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
5260

5261
	const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
5262
	PTR_FAIL_IF(!const_);
5263
	set_const(const_, compiler);
5264

5265
	switch (GET_OPCODE(op)) {
5266
	case SLJIT_MOV_U8:
5267
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5268
		compiler->mode32 = (op & SLJIT_32);
5269
#endif /* SLJIT_CONFIG_X86_64 */
5270

5271
		if ((init_value & 0x100) != 0)
5272
			init_value = init_value | -(sljit_sw)0x100;
5273
		else
5274
			init_value = (sljit_u8)init_value;
5275

5276
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5277
		if (dst_is_ereg) {
5278
			if (emit_mov(compiler, dst, dstw, SLJIT_IMM, (sljit_s32)init_value))
5279
				return NULL;
5280
			dst = 0;
5281
			break;
5282
		}
5283
#endif /* !SLJIT_CONFIG_X86_32 */
5284

5285
		reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
5286

5287
		if (emit_mov(compiler, reg, 0, SLJIT_IMM, init_value))
5288
			return NULL;
5289
		break;
5290
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5291
	case SLJIT_MOV:
5292
		compiler->mode32 = 0;
5293
		reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
5294

5295
		if (emit_load_imm64(compiler, reg, init_value))
5296
			return NULL;
5297
		break;
5298
#endif /* SLJIT_CONFIG_X86_64 */
5299
	default:
5300
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5301
		compiler->mode32 = (op == SLJIT_MOV32);
5302
#endif /* SLJIT_CONFIG_X86_64 */
5303

5304
		if (emit_mov(compiler, dst, dstw, SLJIT_IMM, (sljit_s32)init_value))
5305
			return NULL;
5306
		dst = 0;
5307
		break;
5308
	}
5309

5310
	inst = (sljit_u8*)ensure_buf(compiler, 1);
5311
	PTR_FAIL_IF(!inst);
5312

5313
	inst[0] = SLJIT_INST_CONST;
5314

5315
	if (dst & SLJIT_MEM) {
5316
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5317
		if (op == SLJIT_MOV) {
5318
			if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
5319
				return NULL;
5320
			return const_;
5321
		}
5322
#endif
5323

5324
		if (emit_mov_byte(compiler, 0, dst, dstw, TMP_REG1, 0))
5325
			return NULL;
5326
	}
5327

5328
	return const_;
5329
}
5330

5331
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
5332
	sljit_s32 dst, sljit_sw dstw)
5333
{
5334
	struct sljit_jump *jump;
5335
	sljit_u8 *inst;
5336
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5337
	sljit_s32 reg;
5338
#endif /* SLJIT_CONFIG_X86_64 */
5339
	SLJIT_UNUSED_ARG(op);
5340

5341
	CHECK_ERROR_PTR();
5342
	CHECK_PTR(check_sljit_emit_op_addr(compiler, op, dst, dstw));
5343
	ADJUST_LOCAL_OFFSET(dst, dstw);
5344

5345
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
5346

5347
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
5348
	PTR_FAIL_IF(!jump);
5349
	set_mov_addr(jump, compiler, 0);
5350

5351
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5352
	compiler->mode32 = 0;
5353
	if (dst & SLJIT_MEM)
5354
		reg = TMP_REG1;
5355
	else
5356
		reg = (op != SLJIT_ADD_ABS_ADDR) ? dst : TMP_REG2;
5357

5358
	PTR_FAIL_IF(emit_load_imm64(compiler, reg, 0));
5359
	jump->addr = compiler->size;
5360

5361
	if (reg_map[reg] >= 8)
5362
		jump->flags |= MOV_ADDR_HI;
5363
#else /* !SLJIT_CONFIG_X86_64 */
5364
	if (op == SLJIT_ADD_ABS_ADDR) {
5365
		if (dst != SLJIT_R0) {
5366
			/* Must not be a signed byte argument. */
5367
			inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0x100, dst, dstw);
5368
			PTR_FAIL_IF(!inst);
5369
			*(inst + 1) |= ADD;
5370
		} else
5371
			PTR_FAIL_IF(emit_do_imm(compiler, ADD_EAX_i32, 0));
5372
	} else {
5373
		PTR_FAIL_IF(emit_mov(compiler, dst, dstw, SLJIT_IMM, 0));
5374
	}
5375
#endif /* SLJIT_CONFIG_X86_64 */
5376

5377
	inst = (sljit_u8*)ensure_buf(compiler, 1);
5378
	PTR_FAIL_IF(!inst);
5379

5380
	inst[0] = SLJIT_INST_MOV_ADDR;
5381

5382
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5383
	if (op == SLJIT_ADD_ABS_ADDR) {
5384
		inst = emit_x86_instruction(compiler, 1, reg, 0, dst, dstw);
5385
		PTR_FAIL_IF(!inst);
5386
		*inst = ADD_rm_r;
5387
	} else if (dst & SLJIT_MEM)
5388
		PTR_FAIL_IF(emit_mov(compiler, dst, dstw, TMP_REG1, 0));
5389
#endif /* SLJIT_CONFIG_X86_64 */
5390

5391
	return jump;
5392
}
5393

5394
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
5395
{
5396
	SLJIT_UNUSED_ARG(executable_offset);
5397

5398
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0);
5399
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5400
	sljit_unaligned_store_sw((void*)addr, (sljit_sw)(new_target - (addr + 4) - (sljit_uw)executable_offset));
5401
#else
5402
	sljit_unaligned_store_sw((void*)addr, (sljit_sw)new_target);
5403
#endif
5404
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1);
5405
}
5406

5407
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_s32 op, sljit_sw new_constant, sljit_sw executable_offset)
5408
{
5409
	void *start_addr;
5410
	SLJIT_UNUSED_ARG(executable_offset);
5411

5412
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5413
	if (op == SLJIT_MOV) {
5414
		start_addr = (void*)(addr - sizeof(sljit_sw));
5415
		SLJIT_UPDATE_WX_FLAGS(start_addr, (void*)addr, 0);
5416
		sljit_unaligned_store_sw(start_addr, new_constant);
5417
		SLJIT_UPDATE_WX_FLAGS(start_addr, (void*)addr, 1);
5418
		return;
5419
	}
5420
#endif
5421

5422
	start_addr = (void*)(addr - sizeof(sljit_s32));
5423

5424
	if ((op | SLJIT_32) == SLJIT_MOV32_U8) {
5425
		if ((new_constant & 0x100) != 0)
5426
			new_constant = new_constant | -(sljit_sw)0x100;
5427
		else
5428
			new_constant = (sljit_u8)new_constant;
5429
	}
5430

5431
	SLJIT_UPDATE_WX_FLAGS(start_addr, (void*)addr, 0);
5432
	sljit_unaligned_store_s32(start_addr, (sljit_s32)new_constant);
5433
	SLJIT_UPDATE_WX_FLAGS(start_addr, (void*)addr, 1);
5434
}
5435

5436
Product

Resources

Company
