CoCalc -- sljitNativeX86

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitNativeX86_common.c
⁹⁹¹³ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28
{
29
	return "x86" SLJIT_CPUINFO;
30
}
31

32
/*
33
   32b register indexes:
34
     0 - EAX
35
     1 - ECX
36
     2 - EDX
37
     3 - EBX
38
     4 - ESP
39
     5 - EBP
40
     6 - ESI
41
     7 - EDI
42
*/
43

44
/*
45
   64b register indexes:
46
     0 - RAX
47
     1 - RCX
48
     2 - RDX
49
     3 - RBX
50
     4 - RSP
51
     5 - RBP
52
     6 - RSI
53
     7 - RDI
54
     8 - R8   - From now on REX prefix is required
55
     9 - R9
56
    10 - R10
57
    11 - R11
58
    12 - R12
59
    13 - R13
60
    14 - R14
61
    15 - R15
62
*/
63

64
#define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
65
#define TMP_FREG	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
66

67
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
68

69
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 3] = {
70
	0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 5, 7, 6, 4, 3
71
};
72

73
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
74
	0, 1, 2, 3, 4, 5, 6, 7, 0
75
};
76

77
#define CHECK_EXTRA_REGS(p, w, do) \
78
	if (p >= SLJIT_R3 && p <= SLJIT_S3) { \
79
		w = (2 * SSIZE_OF(sw)) + ((p) - SLJIT_R3) * SSIZE_OF(sw); \
80
		p = SLJIT_MEM1(SLJIT_SP); \
81
		do; \
82
	}
83

84
#else /* SLJIT_CONFIG_X86_32 */
85

86
#define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
87

88
/* Note: r12 & 0x7 == 0b100, which decoded as SIB byte present
89
   Note: avoid to use r12 and r13 for memory addressing
90
   therefore r12 is better to be a higher saved register. */
91
#ifndef _WIN64
92
/* Args: rdi(=7), rsi(=6), rdx(=2), rcx(=1), r8, r9. Scratches: rax(=0), r10, r11 */
93
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
94
	0, 0, 6, 7, 1, 8, 11, 10, 12, 5, 13, 14, 15, 3, 4, 2, 9
95
};
96
/* low-map. reg_map & 0x7. */
97
static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
98
	0, 0, 6, 7, 1, 0,  3,  2,  4, 5,  5,  6,  7, 3, 4, 2, 1
99
};
100
#else
101
/* Args: rcx(=1), rdx(=2), r8, r9. Scratches: rax(=0), r10, r11 */
102
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = {
103
	0, 0, 2, 8, 1, 11, 12, 5, 13, 14, 15, 7, 6, 3, 4, 9, 10
104
};
105
/* low-map. reg_map & 0x7. */
106
static const sljit_u8 reg_lmap[SLJIT_NUMBER_OF_REGISTERS + 4] = {
107
	0, 0, 2, 0, 1,  3,  4, 5,  5,  6,  7, 7, 6, 3, 4, 1,  2
108
};
109
#endif
110

111
/* Args: xmm0-xmm3 */
112
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
113
	0, 0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4
114
};
115
/* low-map. freg_map & 0x7. */
116
static const sljit_u8 freg_lmap[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2] = {
117
	0, 0, 1, 2, 3, 5, 6, 7, 0, 1,  2,  3,  4,  5,  6,  7, 4
118
};
119

120
#define REX_W		0x48
121
#define REX_R		0x44
122
#define REX_X		0x42
123
#define REX_B		0x41
124
#define REX		0x40
125

126
#ifndef _WIN64
127
#define HALFWORD_MAX 0x7fffffffl
128
#define HALFWORD_MIN -0x80000000l
129
#else
130
#define HALFWORD_MAX 0x7fffffffll
131
#define HALFWORD_MIN -0x80000000ll
132
#endif
133

134
#define IS_HALFWORD(x)		((x) <= HALFWORD_MAX && (x) >= HALFWORD_MIN)
135
#define NOT_HALFWORD(x)		((x) > HALFWORD_MAX || (x) < HALFWORD_MIN)
136

137
#define CHECK_EXTRA_REGS(p, w, do)
138

139
#endif /* SLJIT_CONFIG_X86_32 */
140

141
#define U8(v)			((sljit_u8)(v))
142

143
/* Size flags for emit_x86_instruction: */
144
#define EX86_BIN_INS		((sljit_uw)0x000010)
145
#define EX86_SHIFT_INS		((sljit_uw)0x000020)
146
#define EX86_BYTE_ARG		((sljit_uw)0x000040)
147
#define EX86_HALF_ARG		((sljit_uw)0x000080)
148
/* Size flags for both emit_x86_instruction and emit_vex_instruction: */
149
#define EX86_REX		((sljit_uw)0x000100)
150
#define EX86_NO_REXW		((sljit_uw)0x000200)
151
#define EX86_PREF_66		((sljit_uw)0x000400)
152
#define EX86_PREF_F2		((sljit_uw)0x000800)
153
#define EX86_PREF_F3		((sljit_uw)0x001000)
154
#define EX86_SSE2_OP1		((sljit_uw)0x002000)
155
#define EX86_SSE2_OP2		((sljit_uw)0x004000)
156
#define EX86_SSE2		(EX86_SSE2_OP1 | EX86_SSE2_OP2)
157
#define EX86_VEX_EXT		((sljit_uw)0x008000)
158
/* Op flags for emit_vex_instruction: */
159
#define VEX_OP_0F38		((sljit_uw)0x010000)
160
#define VEX_OP_0F3A		((sljit_uw)0x020000)
161
#define VEX_SSE2_OPV		((sljit_uw)0x040000)
162
#define VEX_AUTO_W		((sljit_uw)0x080000)
163
#define VEX_W			((sljit_uw)0x100000)
164
#define VEX_256			((sljit_uw)0x200000)
165

166
#define EX86_SELECT_66(op)	(((op) & SLJIT_32) ? 0 : EX86_PREF_66)
167
#define EX86_SELECT_F2_F3(op)	(((op) & SLJIT_32) ? EX86_PREF_F3 : EX86_PREF_F2)
168

169
/* --------------------------------------------------------------------- */
170
/*  Instruction forms                                                    */
171
/* --------------------------------------------------------------------- */
172

173
#define ADD			(/* BINARY */ 0 << 3)
174
#define ADD_EAX_i32		0x05
175
#define ADD_r_rm		0x03
176
#define ADD_rm_r		0x01
177
#define ADDSD_x_xm		0x58
178
#define ADC			(/* BINARY */ 2 << 3)
179
#define ADC_EAX_i32		0x15
180
#define ADC_r_rm		0x13
181
#define ADC_rm_r		0x11
182
#define AND			(/* BINARY */ 4 << 3)
183
#define AND_EAX_i32		0x25
184
#define AND_r_rm		0x23
185
#define AND_rm_r		0x21
186
#define ANDPD_x_xm		0x54
187
#define BSR_r_rm		(/* GROUP_0F */ 0xbd)
188
#define BSF_r_rm		(/* GROUP_0F */ 0xbc)
189
#define BSWAP_r			(/* GROUP_0F */ 0xc8)
190
#define CALL_i32		0xe8
191
#define CALL_rm			(/* GROUP_FF */ 2 << 3)
192
#define CDQ			0x99
193
#define CMOVE_r_rm		(/* GROUP_0F */ 0x44)
194
#define CMP			(/* BINARY */ 7 << 3)
195
#define CMP_EAX_i32		0x3d
196
#define CMP_r_rm		0x3b
197
#define CMP_rm_r		0x39
198
#define CMPS_x_xm		0xc2
199
#define CMPXCHG_rm_r		0xb1
200
#define CMPXCHG_rm8_r		0xb0
201
#define CVTPD2PS_x_xm		0x5a
202
#define CVTPS2PD_x_xm		0x5a
203
#define CVTSI2SD_x_rm		0x2a
204
#define CVTTSD2SI_r_xm		0x2c
205
#define DIV			(/* GROUP_F7 */ 6 << 3)
206
#define DIVSD_x_xm		0x5e
207
#define EXTRACTPS_x_xm		0x17
208
#define FLDS			0xd9
209
#define FLDL			0xdd
210
#define FSTPS			0xd9
211
#define FSTPD			0xdd
212
#define INSERTPS_x_xm		0x21
213
#define INT3			0xcc
214
#define IDIV			(/* GROUP_F7 */ 7 << 3)
215
#define IMUL			(/* GROUP_F7 */ 5 << 3)
216
#define IMUL_r_rm		(/* GROUP_0F */ 0xaf)
217
#define IMUL_r_rm_i8		0x6b
218
#define IMUL_r_rm_i32		0x69
219
#define JL_i8			0x7c
220
#define JE_i8			0x74
221
#define JNC_i8			0x73
222
#define JNE_i8			0x75
223
#define JMP_i8			0xeb
224
#define JMP_i32			0xe9
225
#define JMP_rm			(/* GROUP_FF */ 4 << 3)
226
#define LEA_r_m			0x8d
227
#define LOOP_i8			0xe2
228
#define LZCNT_r_rm		(/* GROUP_F3 */ /* GROUP_0F */ 0xbd)
229
#define MOV_r_rm		0x8b
230
#define MOV_r_i32		0xb8
231
#define MOV_rm_r		0x89
232
#define MOV_rm_i32		0xc7
233
#define MOV_rm8_i8		0xc6
234
#define MOV_rm8_r8		0x88
235
#define MOVAPS_x_xm		0x28
236
#define MOVAPS_xm_x		0x29
237
#define MOVD_x_rm		0x6e
238
#define MOVD_rm_x		0x7e
239
#define MOVDDUP_x_xm		0x12
240
#define MOVDQA_x_xm		0x6f
241
#define MOVDQA_xm_x		0x7f
242
#define MOVDQU_x_xm		0x6f
243
#define MOVHLPS_x_x		0x12
244
#define MOVHPD_m_x		0x17
245
#define MOVHPD_x_m		0x16
246
#define MOVLHPS_x_x		0x16
247
#define MOVLPD_m_x		0x13
248
#define MOVLPD_x_m		0x12
249
#define MOVMSKPS_r_x		(/* GROUP_0F */ 0x50)
250
#define MOVQ_x_xm		(/* GROUP_0F */ 0x7e)
251
#define MOVSD_x_xm		0x10
252
#define MOVSD_xm_x		0x11
253
#define MOVSHDUP_x_xm		0x16
254
#define MOVSXD_r_rm		0x63
255
#define MOVSX_r_rm8		(/* GROUP_0F */ 0xbe)
256
#define MOVSX_r_rm16		(/* GROUP_0F */ 0xbf)
257
#define MOVUPS_x_xm		0x10
258
#define MOVZX_r_rm8		(/* GROUP_0F */ 0xb6)
259
#define MOVZX_r_rm16		(/* GROUP_0F */ 0xb7)
260
#define MUL			(/* GROUP_F7 */ 4 << 3)
261
#define MULSD_x_xm		0x59
262
#define NEG_rm			(/* GROUP_F7 */ 3 << 3)
263
#define NOP			0x90
264
#define NOT_rm			(/* GROUP_F7 */ 2 << 3)
265
#define OR			(/* BINARY */ 1 << 3)
266
#define OR_r_rm			0x0b
267
#define OR_EAX_i32		0x0d
268
#define OR_rm_r			0x09
269
#define OR_rm8_r8		0x08
270
#define ORPD_x_xm		0x56
271
#define PACKSSWB_x_xm		(/* GROUP_0F */ 0x63)
272
#define PAND_x_xm		0xdb
273
#define PCMPEQD_x_xm		0x76
274
#define PINSRB_x_rm_i8		0x20
275
#define PINSRW_x_rm_i8		0xc4
276
#define PINSRD_x_rm_i8		0x22
277
#define PEXTRB_rm_x_i8		0x14
278
#define PEXTRW_rm_x_i8		0x15
279
#define PEXTRD_rm_x_i8		0x16
280
#define PMOVMSKB_r_x		(/* GROUP_0F */ 0xd7)
281
#define PMOVSXBD_x_xm		0x21
282
#define PMOVSXBQ_x_xm		0x22
283
#define PMOVSXBW_x_xm		0x20
284
#define PMOVSXDQ_x_xm		0x25
285
#define PMOVSXWD_x_xm		0x23
286
#define PMOVSXWQ_x_xm		0x24
287
#define PMOVZXBD_x_xm		0x31
288
#define PMOVZXBQ_x_xm		0x32
289
#define PMOVZXBW_x_xm		0x30
290
#define PMOVZXDQ_x_xm		0x35
291
#define PMOVZXWD_x_xm		0x33
292
#define PMOVZXWQ_x_xm		0x34
293
#define POP_r			0x58
294
#define POP_rm			0x8f
295
#define POPF			0x9d
296
#define POR_x_xm		0xeb
297
#define PREFETCH		0x18
298
#define PSHUFB_x_xm		0x00
299
#define PSHUFD_x_xm		0x70
300
#define PSHUFLW_x_xm		0x70
301
#define PSRLDQ_x		0x73
302
#define PSLLD_x_i8		0x72
303
#define PSLLQ_x_i8		0x73
304
#define PUSH_i32		0x68
305
#define PUSH_r			0x50
306
#define PUSH_rm			(/* GROUP_FF */ 6 << 3)
307
#define PUSHF			0x9c
308
#define PXOR_x_xm		0xef
309
#define ROL			(/* SHIFT */ 0 << 3)
310
#define ROR			(/* SHIFT */ 1 << 3)
311
#define RET_near		0xc3
312
#define RET_i16			0xc2
313
#define SBB			(/* BINARY */ 3 << 3)
314
#define SBB_EAX_i32		0x1d
315
#define SBB_r_rm		0x1b
316
#define SBB_rm_r		0x19
317
#define SAR			(/* SHIFT */ 7 << 3)
318
#define SHL			(/* SHIFT */ 4 << 3)
319
#define SHLD			(/* GROUP_0F */ 0xa5)
320
#define SHRD			(/* GROUP_0F */ 0xad)
321
#define SHR			(/* SHIFT */ 5 << 3)
322
#define SHUFPS_x_xm		0xc6
323
#define SUB			(/* BINARY */ 5 << 3)
324
#define SUB_EAX_i32		0x2d
325
#define SUB_r_rm		0x2b
326
#define SUB_rm_r		0x29
327
#define SUBSD_x_xm		0x5c
328
#define TEST_EAX_i32		0xa9
329
#define TEST_rm_r		0x85
330
#define TZCNT_r_rm		(/* GROUP_F3 */ /* GROUP_0F */ 0xbc)
331
#define UCOMISD_x_xm		0x2e
332
#define UNPCKLPD_x_xm		0x14
333
#define UNPCKLPS_x_xm		0x14
334
#define VBROADCASTSD_x_xm	0x19
335
#define VBROADCASTSS_x_xm	0x18
336
#define VEXTRACTF128_x_ym	0x19
337
#define VEXTRACTI128_x_ym	0x39
338
#define VINSERTF128_y_y_xm	0x18
339
#define VINSERTI128_y_y_xm	0x38
340
#define VPBROADCASTB_x_xm	0x78
341
#define VPBROADCASTD_x_xm	0x58
342
#define VPBROADCASTQ_x_xm	0x59
343
#define VPBROADCASTW_x_xm	0x79
344
#define VPERMPD_y_ym		0x01
345
#define VPERMQ_y_ym		0x00
346
#define XCHG_EAX_r		0x90
347
#define XCHG_r_rm		0x87
348
#define XOR			(/* BINARY */ 6 << 3)
349
#define XOR_EAX_i32		0x35
350
#define XOR_r_rm		0x33
351
#define XOR_rm_r		0x31
352
#define XORPD_x_xm		0x57
353

354
#define GROUP_0F		0x0f
355
#define GROUP_66		0x66
356
#define GROUP_F3		0xf3
357
#define GROUP_F7		0xf7
358
#define GROUP_FF		0xff
359
#define GROUP_BINARY_81		0x81
360
#define GROUP_BINARY_83		0x83
361
#define GROUP_SHIFT_1		0xd1
362
#define GROUP_SHIFT_N		0xc1
363
#define GROUP_SHIFT_CL		0xd3
364
#define GROUP_LOCK		0xf0
365

366
#define MOD_REG			0xc0
367
#define MOD_DISP8		0x40
368

369
#define INC_SIZE(s)		(*inst++ = U8(s), compiler->size += (s))
370

371
#define PUSH_REG(r)		(*inst++ = U8(PUSH_r + (r)))
372
#define POP_REG(r)		(*inst++ = U8(POP_r + (r)))
373
#define RET()			(*inst++ = RET_near)
374
#define RET_I16(n)		(*inst++ = RET_i16, *inst++ = U8(n), *inst++ = 0)
375

376
#define SLJIT_INST_LABEL	255
377
#define SLJIT_INST_JUMP		254
378
#define SLJIT_INST_MOV_ADDR	253
379
#define SLJIT_INST_CONST	252
380

381
/* Multithreading does not affect these static variables, since they store
382
   built-in CPU features. Therefore they can be overwritten by different threads
383
   if they detect the CPU features in the same time. */
384
#define CPU_FEATURE_DETECTED		0x001
385
#if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
386
#define CPU_FEATURE_SSE2		0x002
387
#endif
388
#define CPU_FEATURE_SSE41		0x004
389
#define CPU_FEATURE_LZCNT		0x008
390
#define CPU_FEATURE_TZCNT		0x010
391
#define CPU_FEATURE_CMOV		0x020
392
#define CPU_FEATURE_AVX			0x040
393
#define CPU_FEATURE_AVX2		0x080
394
#define CPU_FEATURE_OSXSAVE		0x100
395

396
static sljit_u32 cpu_feature_list = 0;
397

398
#ifdef _WIN32_WCE
399
#include <cmnintrin.h>
400
#elif defined(_MSC_VER) && _MSC_VER >= 1400
401
#include <intrin.h>
402
#elif defined(__INTEL_COMPILER)
403
#include <cpuid.h>
404
#endif
405

406
#if (defined(_MSC_VER) && _MSC_VER >= 1400) || defined(__INTEL_COMPILER) \
407
	|| (defined(__INTEL_LLVM_COMPILER) && defined(__XSAVE__))
408
#include <immintrin.h>
409
#endif
410

411
/******************************************************/
412
/*    Unaligned-store functions                       */
413
/******************************************************/
414

415
static SLJIT_INLINE void sljit_unaligned_store_s16(void *addr, sljit_s16 value)
416
{
417
	SLJIT_MEMCPY(addr, &value, sizeof(value));
418
}
419

420
static SLJIT_INLINE void sljit_unaligned_store_s32(void *addr, sljit_s32 value)
421
{
422
	SLJIT_MEMCPY(addr, &value, sizeof(value));
423
}
424

425
static SLJIT_INLINE void sljit_unaligned_store_sw(void *addr, sljit_sw value)
426
{
427
	SLJIT_MEMCPY(addr, &value, sizeof(value));
428
}
429

430
/******************************************************/
431
/*    Utility functions                               */
432
/******************************************************/
433

434
static void execute_cpu_id(sljit_u32 info[4])
435
{
436
#if (defined(_MSC_VER) && _MSC_VER >= 1400) \
437
	|| (defined(__INTEL_COMPILER) && __INTEL_COMPILER == 2021 && __INTEL_COMPILER_UPDATE >= 7)
438

439
	__cpuidex((int*)info, (int)info[0], (int)info[2]);
440

441
#elif (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900)
442

443
	__get_cpuid_count(info[0], info[2], info, info + 1, info + 2, info + 3);
444

445
#elif (defined(_MSC_VER) || defined(__INTEL_COMPILER)) \
446
	&& (defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32)
447

448
	/* Intel syntax. */
449
	__asm {
450
		mov esi, info
451
		mov eax, [esi]
452
		mov ecx, [esi + 8]
453
		cpuid
454
		mov [esi], eax
455
		mov [esi + 4], ebx
456
		mov [esi + 8], ecx
457
		mov [esi + 12], edx
458
	}
459

460
#else
461

462
	__asm__ __volatile__ (
463
		"cpuid\n"
464
		: "=a" (info[0]), "=b" (info[1]), "=c" (info[2]), "=d" (info[3])
465
		: "0" (info[0]), "2" (info[2])
466
	);
467

468
#endif
469
}
470

471
static sljit_u32 execute_get_xcr0_low(void)
472
{
473
	sljit_u32 xcr0;
474

475
#if (defined(_MSC_VER) && _MSC_VER >= 1400) || defined(__INTEL_COMPILER) \
476
	|| (defined(__INTEL_LLVM_COMPILER) && defined(__XSAVE__))
477

478
	xcr0 = (sljit_u32)_xgetbv(0);
479

480
#elif defined(__TINYC__)
481

482
	__asm__ (
483
		"xorl %%ecx, %%ecx\n"
484
		".byte 0x0f\n"
485
		".byte 0x01\n"
486
		".byte 0xd0\n"
487
		: "=a" (xcr0)
488
		:
489
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
490
		: "ecx", "edx"
491
#else /* !SLJIT_CONFIG_X86_32 */
492
		: "rcx", "rdx"
493
#endif /* SLJIT_CONFIG_X86_32 */
494
	);
495

496
#elif (defined(__INTEL_LLVM_COMPILER) && __INTEL_LLVM_COMPILER < 20220100) \
497
	|| (defined(__clang__) && __clang_major__ < 14) \
498
	|| (defined(__GNUC__) && __GNUC__ < 3) \
499
	|| defined(__SUNPRO_C) || defined(__SUNPRO_CC)
500

501
	/* AT&T syntax. */
502
	__asm__ (
503
		"xorl %%ecx, %%ecx\n"
504
		"xgetbv\n"
505
		: "=a" (xcr0)
506
		:
507
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
508
		: "ecx", "edx"
509
#else /* !SLJIT_CONFIG_X86_32 */
510
		: "rcx", "rdx"
511
#endif /* SLJIT_CONFIG_X86_32 */
512
	);
513

514
#elif defined(_MSC_VER)
515

516
	/* Intel syntax. */
517
	__asm {
518
		xor ecx, ecx
519
		xgetbv
520
		mov xcr0, eax
521
	}
522

523
#else
524

525
	__asm__ (
526
		"xor{l %%ecx, %%ecx | ecx, ecx}\n"
527
		"xgetbv\n"
528
		: "=a" (xcr0)
529
		:
530
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
531
		: "ecx", "edx"
532
#else /* !SLJIT_CONFIG_X86_32 */
533
		: "rcx", "rdx"
534
#endif /* SLJIT_CONFIG_X86_32 */
535
	);
536

537
#endif
538
	return xcr0;
539
}
540

541
static void get_cpu_features(void)
542
{
543
	sljit_u32 feature_list = CPU_FEATURE_DETECTED;
544
	sljit_u32 info[4] = {0};
545
	sljit_u32 max_id;
546

547
	execute_cpu_id(info);
548
	max_id = info[0];
549

550
	if (max_id >= 7) {
551
		info[0] = 7;
552
		info[2] = 0;
553
		execute_cpu_id(info);
554

555
		if (info[1] & 0x8)
556
			feature_list |= CPU_FEATURE_TZCNT;
557
		if (info[1] & 0x20)
558
			feature_list |= CPU_FEATURE_AVX2;
559
	}
560

561
	if (max_id >= 1) {
562
		info[0] = 1;
563
#if defined(SLJIT_CONFIG_X86_32) && SLJIT_CONFIG_X86_32
564
		/* Winchip 2 and Cyrix MII bugs */
565
		info[1] = info[2] = 0;
566
#endif
567
		execute_cpu_id(info);
568

569
		if (info[2] & 0x80000)
570
			feature_list |= CPU_FEATURE_SSE41;
571
		if (info[2] & 0x8000000)
572
			feature_list |= CPU_FEATURE_OSXSAVE;
573
		if (info[2] & 0x10000000)
574
			feature_list |= CPU_FEATURE_AVX;
575
#if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
576
		if (info[3] & 0x4000000)
577
			feature_list |= CPU_FEATURE_SSE2;
578
#endif
579
		if (info[3] & 0x8000)
580
			feature_list |= CPU_FEATURE_CMOV;
581
	}
582

583
	info[0] = 0x80000000;
584
	execute_cpu_id(info);
585
	max_id = info[0];
586

587
	if (max_id >= 0x80000001) {
588
		info[0] = 0x80000001;
589
		execute_cpu_id(info);
590

591
		if (info[2] & 0x20)
592
			feature_list |= CPU_FEATURE_LZCNT;
593
	}
594

595
	if ((feature_list & CPU_FEATURE_OSXSAVE) && (execute_get_xcr0_low() & 0x4) == 0)
596
		feature_list &= ~(sljit_u32)(CPU_FEATURE_AVX | CPU_FEATURE_AVX2);
597

598
	cpu_feature_list = feature_list;
599
}
600

601
static sljit_u8 get_jump_code(sljit_uw type)
602
{
603
	switch (type) {
604
	case SLJIT_EQUAL:
605
	case SLJIT_ATOMIC_STORED:
606
	case SLJIT_F_EQUAL:
607
	case SLJIT_UNORDERED_OR_EQUAL:
608
		return 0x84 /* je */;
609

610
	case SLJIT_NOT_EQUAL:
611
	case SLJIT_ATOMIC_NOT_STORED:
612
	case SLJIT_F_NOT_EQUAL:
613
	case SLJIT_ORDERED_NOT_EQUAL:
614
		return 0x85 /* jne */;
615

616
	case SLJIT_LESS:
617
	case SLJIT_CARRY:
618
	case SLJIT_F_LESS:
619
	case SLJIT_UNORDERED_OR_LESS:
620
	case SLJIT_UNORDERED_OR_GREATER:
621
		return 0x82 /* jc */;
622

623
	case SLJIT_GREATER_EQUAL:
624
	case SLJIT_NOT_CARRY:
625
	case SLJIT_F_GREATER_EQUAL:
626
	case SLJIT_ORDERED_GREATER_EQUAL:
627
	case SLJIT_ORDERED_LESS_EQUAL:
628
		return 0x83 /* jae */;
629

630
	case SLJIT_GREATER:
631
	case SLJIT_F_GREATER:
632
	case SLJIT_ORDERED_LESS:
633
	case SLJIT_ORDERED_GREATER:
634
		return 0x87 /* jnbe */;
635

636
	case SLJIT_LESS_EQUAL:
637
	case SLJIT_F_LESS_EQUAL:
638
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
639
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
640
		return 0x86 /* jbe */;
641

642
	case SLJIT_SIG_LESS:
643
		return 0x8c /* jl */;
644

645
	case SLJIT_SIG_GREATER_EQUAL:
646
		return 0x8d /* jnl */;
647

648
	case SLJIT_SIG_GREATER:
649
		return 0x8f /* jnle */;
650

651
	case SLJIT_SIG_LESS_EQUAL:
652
		return 0x8e /* jle */;
653

654
	case SLJIT_OVERFLOW:
655
		return 0x80 /* jo */;
656

657
	case SLJIT_NOT_OVERFLOW:
658
		return 0x81 /* jno */;
659

660
	case SLJIT_UNORDERED:
661
	case SLJIT_ORDERED_EQUAL: /* NaN. */
662
		return 0x8a /* jp */;
663

664
	case SLJIT_ORDERED:
665
	case SLJIT_UNORDERED_OR_NOT_EQUAL: /* Not NaN. */
666
		return 0x8b /* jpo */;
667
	}
668
	return 0;
669
}
670

671
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
672
static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset);
673
#else /* !SLJIT_CONFIG_X86_32 */
674
static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr);
675
static sljit_u8* generate_mov_addr_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset);
676
#endif /* SLJIT_CONFIG_X86_32 */
677

678
static sljit_u8* detect_near_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset)
679
{
680
	sljit_uw type = jump->flags >> TYPE_SHIFT;
681
	sljit_s32 short_jump;
682
	sljit_uw label_addr;
683
	sljit_uw jump_addr;
684

685
	jump_addr = (sljit_uw)code_ptr;
686
	if (!(jump->flags & JUMP_ADDR)) {
687
		label_addr = (sljit_uw)(code + jump->u.label->size);
688

689
		if (jump->u.label->size > jump->addr)
690
			jump_addr = (sljit_uw)(code + jump->addr);
691
	} else
692
		label_addr = jump->u.target - (sljit_uw)executable_offset;
693

694
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
695
	if ((sljit_sw)(label_addr - (jump_addr + 6)) > HALFWORD_MAX || (sljit_sw)(label_addr - (jump_addr + 5)) < HALFWORD_MIN)
696
		return detect_far_jump_type(jump, code_ptr);
697
#endif /* SLJIT_CONFIG_X86_64 */
698

699
	short_jump = (sljit_sw)(label_addr - (jump_addr + 2)) >= -0x80 && (sljit_sw)(label_addr - (jump_addr + 2)) <= 0x7f;
700

701
	if (type == SLJIT_JUMP) {
702
		if (short_jump)
703
			*code_ptr++ = JMP_i8;
704
		else
705
			*code_ptr++ = JMP_i32;
706
	} else if (type > SLJIT_JUMP) {
707
		short_jump = 0;
708
		*code_ptr++ = CALL_i32;
709
	} else if (short_jump) {
710
		*code_ptr++ = U8(get_jump_code(type) - 0x10);
711
	} else {
712
		*code_ptr++ = GROUP_0F;
713
		*code_ptr++ = get_jump_code(type);
714
	}
715

716
	jump->addr = (sljit_uw)code_ptr;
717

718
	if (short_jump) {
719
		jump->flags |= PATCH_MB;
720
		code_ptr += sizeof(sljit_s8);
721
	} else {
722
		jump->flags |= PATCH_MW;
723
		code_ptr += sizeof(sljit_s32);
724
	}
725

726
	return code_ptr;
727
}
728

729
static void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
730
{
731
	sljit_uw flags = jump->flags;
732
	sljit_uw addr = (flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
733
	sljit_uw jump_addr = jump->addr;
734
	SLJIT_UNUSED_ARG(executable_offset);
735

736
	if (SLJIT_UNLIKELY(flags & JUMP_MOV_ADDR)) {
737
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
738
		sljit_unaligned_store_sw((void*)(jump_addr - sizeof(sljit_sw)), (sljit_sw)addr);
739
#else /* SLJIT_CONFIG_X86_32 */
740
		if (flags & PATCH_MD) {
741
			SLJIT_ASSERT(addr > HALFWORD_MAX);
742
			sljit_unaligned_store_sw((void*)(jump_addr - sizeof(sljit_sw)), (sljit_sw)addr);
743
			return;
744
		}
745

746
		if (flags & PATCH_MW) {
747
			addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET((sljit_u8*)jump_addr, executable_offset);
748
			SLJIT_ASSERT((sljit_sw)addr <= HALFWORD_MAX && (sljit_sw)addr >= HALFWORD_MIN);
749
		} else {
750
			SLJIT_ASSERT(addr <= HALFWORD_MAX);
751
		}
752
		sljit_unaligned_store_s32((void*)(jump_addr - sizeof(sljit_s32)), (sljit_s32)addr);
753
#endif /* !SLJIT_CONFIG_X86_32 */
754
		return;
755
	}
756

757
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
758
	if (SLJIT_UNLIKELY(flags & PATCH_MD)) {
759
		SLJIT_ASSERT(!(flags & JUMP_ADDR));
760
		sljit_unaligned_store_sw((void*)jump_addr, (sljit_sw)addr);
761
		return;
762
	}
763
#endif /* SLJIT_CONFIG_X86_64 */
764

765
	addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET((sljit_u8*)jump_addr, executable_offset);
766

767
	if (flags & PATCH_MB) {
768
		addr -= sizeof(sljit_s8);
769
		SLJIT_ASSERT((sljit_sw)addr <= 0x7f && (sljit_sw)addr >= -0x80);
770
		*(sljit_u8*)jump_addr = U8(addr);
771
		return;
772
	} else if (flags & PATCH_MW) {
773
		addr -= sizeof(sljit_s32);
774
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
775
		sljit_unaligned_store_sw((void*)jump_addr, (sljit_sw)addr);
776
#else /* !SLJIT_CONFIG_X86_32 */
777
		SLJIT_ASSERT((sljit_sw)addr <= HALFWORD_MAX && (sljit_sw)addr >= HALFWORD_MIN);
778
		sljit_unaligned_store_s32((void*)jump_addr, (sljit_s32)addr);
779
#endif /* SLJIT_CONFIG_X86_32 */
780
	}
781
}
782

783
static void reduce_code_size(struct sljit_compiler *compiler)
784
{
785
	struct sljit_label *label;
786
	struct sljit_jump *jump;
787
	sljit_uw next_label_size;
788
	sljit_uw next_jump_addr;
789
	sljit_uw next_min_addr;
790
	sljit_uw size_reduce = 0;
791
	sljit_sw diff;
792
	sljit_uw type;
793
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
794
	sljit_uw size_reduce_max;
795
#endif /* SLJIT_DEBUG */
796

797
	label = compiler->labels;
798
	jump = compiler->jumps;
799

800
	next_label_size = SLJIT_GET_NEXT_SIZE(label);
801
	next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
802

803
	while (1) {
804
		next_min_addr = next_label_size;
805
		if (next_jump_addr < next_min_addr)
806
			next_min_addr = next_jump_addr;
807

808
		if (next_min_addr == SLJIT_MAX_ADDRESS)
809
			break;
810

811
		if (next_min_addr == next_label_size) {
812
			label->size -= size_reduce;
813

814
			label = label->next;
815
			next_label_size = SLJIT_GET_NEXT_SIZE(label);
816
		}
817

818
		if (next_min_addr != next_jump_addr)
819
			continue;
820

821
		jump->addr -= size_reduce;
822
		if (!(jump->flags & JUMP_MOV_ADDR)) {
823
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
824
			size_reduce_max = size_reduce + (((jump->flags >> TYPE_SHIFT) < SLJIT_JUMP) ? CJUMP_MAX_SIZE : JUMP_MAX_SIZE);
825
#endif /* SLJIT_DEBUG */
826

827
			if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
828
				if (jump->flags & JUMP_ADDR) {
829
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
830
					if (jump->u.target <= 0xffffffffl)
831
						size_reduce += sizeof(sljit_s32);
832
#endif /* SLJIT_CONFIG_X86_64 */
833
				} else {
834
					/* Unit size: instruction. */
835
					diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
836
					if (jump->u.label->size > jump->addr) {
837
						SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
838
						diff -= (sljit_sw)size_reduce;
839
					}
840
					type = jump->flags >> TYPE_SHIFT;
841

842
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
843
					if (type == SLJIT_JUMP) {
844
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
845
							size_reduce += JUMP_MAX_SIZE - 2;
846
						else if (diff <= HALFWORD_MAX + 5 && diff >= HALFWORD_MIN + 5)
847
							size_reduce += JUMP_MAX_SIZE - 5;
848
					} else if (type < SLJIT_JUMP) {
849
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
850
							size_reduce += CJUMP_MAX_SIZE - 2;
851
						else if (diff <= HALFWORD_MAX + 6 && diff >= HALFWORD_MIN + 6)
852
							size_reduce += CJUMP_MAX_SIZE - 6;
853
					} else  {
854
						if (diff <= HALFWORD_MAX + 5 && diff >= HALFWORD_MIN + 5)
855
							size_reduce += JUMP_MAX_SIZE - 5;
856
					}
857
#else /* !SLJIT_CONFIG_X86_64 */
858
					if (type == SLJIT_JUMP) {
859
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
860
							size_reduce += JUMP_MAX_SIZE - 2;
861
					} else if (type < SLJIT_JUMP) {
862
						if (diff <= 0x7f + 2 && diff >= -0x80 + 2)
863
							size_reduce += CJUMP_MAX_SIZE - 2;
864
					}
865
#endif /* SLJIT_CONFIG_X86_64 */
866
				}
867
			}
868

869
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
870
			jump->flags |= (size_reduce_max - size_reduce) << JUMP_SIZE_SHIFT;
871
#endif /* SLJIT_DEBUG */
872
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
873
		} else {
874
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
875
			size_reduce_max = size_reduce + 10;
876
#endif /* SLJIT_DEBUG */
877

878
			if (!(jump->flags & JUMP_ADDR)) {
879
				diff = (sljit_sw)jump->u.label->size - (sljit_sw)(jump->addr - 3);
880

881
				if (diff <= HALFWORD_MAX && diff >= HALFWORD_MIN)
882
					size_reduce += 3;
883
			} else if (jump->u.target <= 0xffffffffl)
884
				size_reduce += (jump->flags & MOV_ADDR_HI) ? 4 : 5;
885

886
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
887
			jump->flags |= (size_reduce_max - size_reduce) << JUMP_SIZE_SHIFT;
888
#endif /* SLJIT_DEBUG */
889
#endif /* SLJIT_CONFIG_X86_64 */
890
		}
891

892
		jump = jump->next;
893
		next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
894
	}
895

896
	compiler->size -= size_reduce;
897
}
898

899
SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
900
{
901
	struct sljit_memory_fragment *buf;
902
	sljit_u8 *code;
903
	sljit_u8 *code_ptr;
904
	sljit_u8 *buf_ptr;
905
	sljit_u8 *buf_end;
906
	sljit_u8 len;
907
	sljit_sw executable_offset;
908
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
909
	sljit_uw addr;
910
#endif /* SLJIT_DEBUG */
911

912
	struct sljit_label *label;
913
	struct sljit_jump *jump;
914
	struct sljit_const *const_;
915

916
	CHECK_ERROR_PTR();
917
	CHECK_PTR(check_sljit_generate_code(compiler));
918

919
	reduce_code_size(compiler);
920

921
	/* Second code generation pass. */
922
	code = (sljit_u8*)allocate_executable_memory(compiler->size, options, exec_allocator_data, &executable_offset);
923
	PTR_FAIL_WITH_EXEC_IF(code);
924

925
	reverse_buf(compiler);
926
	buf = compiler->buf;
927

928
	code_ptr = code;
929
	label = compiler->labels;
930
	jump = compiler->jumps;
931
	const_ = compiler->consts;
932

933
	do {
934
		buf_ptr = buf->memory;
935
		buf_end = buf_ptr + buf->used_size;
936
		do {
937
			len = *buf_ptr++;
938
			SLJIT_ASSERT(len > 0);
939
			if (len < SLJIT_INST_CONST) {
940
				/* The code is already generated. */
941
				SLJIT_MEMCPY(code_ptr, buf_ptr, len);
942
				code_ptr += len;
943
				buf_ptr += len;
944
			} else {
945
				switch (len) {
946
				case SLJIT_INST_LABEL:
947
					label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
948
					label->size = (sljit_uw)(code_ptr - code);
949
					label = label->next;
950
					break;
951
				case SLJIT_INST_JUMP:
952
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
953
					addr = (sljit_uw)code_ptr;
954
#endif /* SLJIT_DEBUG */
955
					if (!(jump->flags & SLJIT_REWRITABLE_JUMP))
956
						code_ptr = detect_near_jump_type(jump, code_ptr, code, executable_offset);
957
					else {
958
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
959
						code_ptr = detect_far_jump_type(jump, code_ptr, executable_offset);
960
#else /* !SLJIT_CONFIG_X86_32 */
961
						code_ptr = detect_far_jump_type(jump, code_ptr);
962
#endif /* SLJIT_CONFIG_X86_32 */
963
					}
964

965
					SLJIT_ASSERT((sljit_uw)code_ptr - addr <= ((jump->flags >> JUMP_SIZE_SHIFT) & 0x1f));
966
					jump = jump->next;
967
					break;
968
				case SLJIT_INST_MOV_ADDR:
969
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
970
					code_ptr = generate_mov_addr_code(jump, code_ptr, code, executable_offset);
971
#endif /* SLJIT_CONFIG_X86_64 */
972
					jump->addr = (sljit_uw)code_ptr;
973
					jump = jump->next;
974
					break;
975
				default:
976
					SLJIT_ASSERT(len == SLJIT_INST_CONST);
977
					const_->addr = ((sljit_uw)code_ptr) - sizeof(sljit_sw);
978
					const_ = const_->next;
979
					break;
980
				}
981
			}
982
		} while (buf_ptr < buf_end);
983

984
		SLJIT_ASSERT(buf_ptr == buf_end);
985
		buf = buf->next;
986
	} while (buf);
987

988
	SLJIT_ASSERT(!label);
989
	SLJIT_ASSERT(!jump);
990
	SLJIT_ASSERT(!const_);
991
	SLJIT_ASSERT(code_ptr <= code + compiler->size);
992

993
	jump = compiler->jumps;
994
	while (jump) {
995
		generate_jump_or_mov_addr(jump, executable_offset);
996
		jump = jump->next;
997
	}
998

999
	compiler->error = SLJIT_ERR_COMPILED;
1000
	compiler->executable_offset = executable_offset;
1001
	compiler->executable_size = (sljit_uw)(code_ptr - code);
1002

1003
	code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
1004

1005
	SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1);
1006
	return (void*)code;
1007
}
1008

1009
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
1010
{
1011
	switch (feature_type) {
1012
	case SLJIT_HAS_FPU:
1013
#ifdef SLJIT_IS_FPU_AVAILABLE
1014
		return (SLJIT_IS_FPU_AVAILABLE) != 0;
1015
#elif (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2)
1016
		if (cpu_feature_list == 0)
1017
			get_cpu_features();
1018
		return (cpu_feature_list & CPU_FEATURE_SSE2) != 0;
1019
#else /* SLJIT_DETECT_SSE2 */
1020
		return 1;
1021
#endif /* SLJIT_DETECT_SSE2 */
1022

1023
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1024
	case SLJIT_HAS_VIRTUAL_REGISTERS:
1025
		return 1;
1026
#endif /* SLJIT_CONFIG_X86_32 */
1027

1028
	case SLJIT_HAS_CLZ:
1029
		if (cpu_feature_list == 0)
1030
			get_cpu_features();
1031

1032
		return (cpu_feature_list & CPU_FEATURE_LZCNT) ? 1 : 2;
1033

1034
	case SLJIT_HAS_CTZ:
1035
		if (cpu_feature_list == 0)
1036
			get_cpu_features();
1037

1038
		return (cpu_feature_list & CPU_FEATURE_TZCNT) ? 1 : 2;
1039

1040
	case SLJIT_HAS_CMOV:
1041
		if (cpu_feature_list == 0)
1042
			get_cpu_features();
1043
		return (cpu_feature_list & CPU_FEATURE_CMOV) != 0;
1044

1045
	case SLJIT_HAS_REV:
1046
	case SLJIT_HAS_ROT:
1047
	case SLJIT_HAS_PREFETCH:
1048
	case SLJIT_HAS_COPY_F32:
1049
	case SLJIT_HAS_COPY_F64:
1050
	case SLJIT_HAS_ATOMIC:
1051
	case SLJIT_HAS_MEMORY_BARRIER:
1052
		return 1;
1053

1054
#if !(defined SLJIT_IS_FPU_AVAILABLE) || SLJIT_IS_FPU_AVAILABLE
1055
	case SLJIT_HAS_AVX:
1056
		if (cpu_feature_list == 0)
1057
			get_cpu_features();
1058
		return (cpu_feature_list & CPU_FEATURE_AVX) != 0;
1059
	case SLJIT_HAS_AVX2:
1060
		if (cpu_feature_list == 0)
1061
			get_cpu_features();
1062
		return (cpu_feature_list & CPU_FEATURE_AVX2) != 0;
1063
	case SLJIT_HAS_SIMD:
1064
		if (cpu_feature_list == 0)
1065
			get_cpu_features();
1066
		return (cpu_feature_list & CPU_FEATURE_SSE41) != 0;
1067
#endif /* SLJIT_IS_FPU_AVAILABLE */
1068
	default:
1069
		return 0;
1070
	}
1071
}
1072

1073
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
1074
{
1075
	switch (type) {
1076
	case SLJIT_ORDERED_EQUAL:
1077
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
1078
		return 2;
1079
	}
1080

1081
	return 0;
1082
}
1083

1084
/* --------------------------------------------------------------------- */
1085
/*  Operators                                                            */
1086
/* --------------------------------------------------------------------- */
1087

1088
#define BINARY_OPCODE(opcode) (((opcode ## _EAX_i32) << 24) | ((opcode ## _r_rm) << 16) | ((opcode ## _rm_r) << 8) | (opcode))
1089

1090
#define BINARY_IMM32(op_imm, immw, arg, argw) \
1091
	do { \
1092
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, immw, arg, argw); \
1093
		FAIL_IF(!inst); \
1094
		*(inst + 1) |= (op_imm); \
1095
	} while (0)
1096

1097
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1098

1099
#define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1100
	do { \
1101
		if (IS_HALFWORD(immw) || compiler->mode32) { \
1102
			BINARY_IMM32(op_imm, immw, arg, argw); \
1103
		} \
1104
		else { \
1105
			FAIL_IF(emit_load_imm64(compiler, FAST_IS_REG(arg) ? TMP_REG2 : TMP_REG1, immw)); \
1106
			inst = emit_x86_instruction(compiler, 1, FAST_IS_REG(arg) ? TMP_REG2 : TMP_REG1, 0, arg, argw); \
1107
			FAIL_IF(!inst); \
1108
			*inst = (op_mr); \
1109
		} \
1110
	} while (0)
1111

1112
#define BINARY_EAX_IMM(op_eax_imm, immw) \
1113
	FAIL_IF(emit_do_imm32(compiler, (!compiler->mode32) ? REX_W : 0, (op_eax_imm), immw))
1114

1115
#else /* !SLJIT_CONFIG_X86_64 */
1116

1117
#define BINARY_IMM(op_imm, op_mr, immw, arg, argw) \
1118
	BINARY_IMM32(op_imm, immw, arg, argw)
1119

1120
#define BINARY_EAX_IMM(op_eax_imm, immw) \
1121
	FAIL_IF(emit_do_imm(compiler, (op_eax_imm), immw))
1122

1123
#endif /* SLJIT_CONFIG_X86_64 */
1124

1125
static sljit_s32 emit_byte(struct sljit_compiler *compiler, sljit_u8 byte)
1126
{
1127
	sljit_u8 *inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1128
	FAIL_IF(!inst);
1129
	INC_SIZE(1);
1130
	*inst = byte;
1131
	return SLJIT_SUCCESS;
1132
}
1133

1134
static sljit_s32 emit_mov(struct sljit_compiler *compiler,
1135
	sljit_s32 dst, sljit_sw dstw,
1136
	sljit_s32 src, sljit_sw srcw);
1137

1138
#define EMIT_MOV(compiler, dst, dstw, src, srcw) \
1139
	FAIL_IF(emit_mov(compiler, dst, dstw, src, srcw));
1140

1141
static sljit_s32 emit_groupf(struct sljit_compiler *compiler,
1142
	sljit_uw op,
1143
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1144

1145
static sljit_s32 emit_groupf_ext(struct sljit_compiler *compiler,
1146
	sljit_uw op,
1147
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1148

1149
static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
1150
	sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src);
1151

1152
static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
1153
	sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw);
1154

1155
static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
1156
	sljit_s32 src1, sljit_sw src1w,
1157
	sljit_s32 src2, sljit_sw src2w);
1158

1159
static sljit_s32 emit_cmov_generic(struct sljit_compiler *compiler, sljit_s32 type,
1160
	sljit_s32 dst_reg,
1161
	sljit_s32 src, sljit_sw srcw);
1162

1163
static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler)
1164
{
1165
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET)
1166
	/* Emit endbr32/endbr64 when CET is enabled.  */
1167
	sljit_u8 *inst;
1168
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1169
	FAIL_IF(!inst);
1170
	INC_SIZE(4);
1171
	inst[0] = GROUP_F3;
1172
	inst[1] = GROUP_0F;
1173
	inst[2] = 0x1e;
1174
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1175
	inst[3] = 0xfb;
1176
#else /* !SLJIT_CONFIG_X86_32 */
1177
	inst[3] = 0xfa;
1178
#endif /* SLJIT_CONFIG_X86_32 */
1179
#else /* !SLJIT_CONFIG_X86_CET */
1180
	SLJIT_UNUSED_ARG(compiler);
1181
#endif /* SLJIT_CONFIG_X86_CET */
1182
	return SLJIT_SUCCESS;
1183
}
1184

1185
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1186

1187
static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg)
1188
{
1189
	sljit_u8 *inst;
1190
	sljit_s32 size;
1191

1192
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1193
	size = 5;
1194
#else
1195
	size = 4;
1196
#endif
1197

1198
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1199
	FAIL_IF(!inst);
1200
	INC_SIZE(size);
1201
	*inst++ = GROUP_F3;
1202
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1203
	*inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
1204
#endif
1205
	inst[0] = GROUP_0F;
1206
	inst[1] = 0x1e;
1207
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1208
	inst[2] = U8(MOD_REG | (0x1 << 3) | reg_lmap[reg]);
1209
#else
1210
	inst[2] = U8(MOD_REG | (0x1 << 3) | reg_map[reg]);
1211
#endif
1212
	return SLJIT_SUCCESS;
1213
}
1214

1215
static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg)
1216
{
1217
	sljit_u8 *inst;
1218
	sljit_s32 size;
1219

1220
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1221
	size = 5;
1222
#else
1223
	size = 4;
1224
#endif
1225

1226
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1227
	FAIL_IF(!inst);
1228
	INC_SIZE(size);
1229
	*inst++ = GROUP_F3;
1230
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1231
	*inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B);
1232
#endif
1233
	inst[0] = GROUP_0F;
1234
	inst[1] = 0xae;
1235
	inst[2] = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7);
1236
	return SLJIT_SUCCESS;
1237
}
1238

1239
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1240

1241
static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void)
1242
{
1243
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1244
	return _get_ssp() != 0;
1245
#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
1246
	return 0;
1247
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1248
}
1249

1250
static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler,
1251
	sljit_s32 src, sljit_sw srcw)
1252
{
1253
#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__)
1254
	sljit_u8 *inst, *jz_after_cmp_inst;
1255
	sljit_uw size_jz_after_cmp_inst;
1256

1257
	sljit_uw size_before_rdssp_inst = compiler->size;
1258

1259
	/* Generate "RDSSP TMP_REG1". */
1260
	FAIL_IF(emit_rdssp(compiler, TMP_REG1));
1261

1262
	/* Load return address on shadow stack into TMP_REG1. */
1263
	EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), 0);
1264

1265
	/* Compare return address against TMP_REG1. */
1266
	FAIL_IF(emit_cmp_binary (compiler, TMP_REG1, 0, src, srcw));
1267

1268
	/* Generate JZ to skip shadow stack ajdustment when shadow
1269
	   stack matches normal stack. */
1270
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1271
	FAIL_IF(!inst);
1272
	INC_SIZE(2);
1273
	*inst++ = get_jump_code(SLJIT_EQUAL) - 0x10;
1274
	size_jz_after_cmp_inst = compiler->size;
1275
	jz_after_cmp_inst = inst;
1276

1277
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1278
	/* REX_W is not necessary. */
1279
	compiler->mode32 = 1;
1280
#endif
1281
	/* Load 1 into TMP_REG1. */
1282
	EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1);
1283

1284
	/* Generate "INCSSP TMP_REG1". */
1285
	FAIL_IF(emit_incssp(compiler, TMP_REG1));
1286

1287
	/* Jump back to "RDSSP TMP_REG1" to check shadow stack again. */
1288
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1289
	FAIL_IF(!inst);
1290
	INC_SIZE(2);
1291
	inst[0] = JMP_i8;
1292
	inst[1] = size_before_rdssp_inst - compiler->size;
1293

1294
	*jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst;
1295
#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */
1296
	SLJIT_UNUSED_ARG(compiler);
1297
	SLJIT_UNUSED_ARG(src);
1298
	SLJIT_UNUSED_ARG(srcw);
1299
#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */
1300
	return SLJIT_SUCCESS;
1301
}
1302

1303
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1304
#include "sljitNativeX86_32.c"
1305
#else
1306
#include "sljitNativeX86_64.c"
1307
#endif
1308

1309
static sljit_s32 emit_mov(struct sljit_compiler *compiler,
1310
	sljit_s32 dst, sljit_sw dstw,
1311
	sljit_s32 src, sljit_sw srcw)
1312
{
1313
	sljit_u8* inst;
1314

1315
	if (FAST_IS_REG(src)) {
1316
		inst = emit_x86_instruction(compiler, 1, src, 0, dst, dstw);
1317
		FAIL_IF(!inst);
1318
		*inst = MOV_rm_r;
1319
		return SLJIT_SUCCESS;
1320
	}
1321

1322
	if (src == SLJIT_IMM) {
1323
		if (FAST_IS_REG(dst)) {
1324
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1325
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1326
#else
1327
			if (!compiler->mode32) {
1328
				if (NOT_HALFWORD(srcw))
1329
					return emit_load_imm64(compiler, dst, srcw);
1330
			}
1331
			else
1332
				return emit_do_imm32(compiler, (reg_map[dst] >= 8) ? REX_B : 0, U8(MOV_r_i32 | reg_lmap[dst]), srcw);
1333
#endif
1334
		}
1335
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1336
		if (!compiler->mode32 && NOT_HALFWORD(srcw)) {
1337
			/* Immediate to memory move. Only SLJIT_MOV operation copies
1338
			   an immediate directly into memory so TMP_REG1 can be used. */
1339
			FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
1340
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1341
			FAIL_IF(!inst);
1342
			*inst = MOV_rm_r;
1343
			return SLJIT_SUCCESS;
1344
		}
1345
#endif
1346
		inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, dstw);
1347
		FAIL_IF(!inst);
1348
		*inst = MOV_rm_i32;
1349
		return SLJIT_SUCCESS;
1350
	}
1351
	if (FAST_IS_REG(dst)) {
1352
		inst = emit_x86_instruction(compiler, 1, dst, 0, src, srcw);
1353
		FAIL_IF(!inst);
1354
		*inst = MOV_r_rm;
1355
		return SLJIT_SUCCESS;
1356
	}
1357

1358
	/* Memory to memory move. Only SLJIT_MOV operation copies
1359
	   data from memory to memory so TMP_REG1 can be used. */
1360
	inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src, srcw);
1361
	FAIL_IF(!inst);
1362
	*inst = MOV_r_rm;
1363
	inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
1364
	FAIL_IF(!inst);
1365
	*inst = MOV_rm_r;
1366
	return SLJIT_SUCCESS;
1367
}
1368

1369
static sljit_s32 emit_cmov_generic(struct sljit_compiler *compiler, sljit_s32 type,
1370
	sljit_s32 dst_reg,
1371
	sljit_s32 src, sljit_sw srcw)
1372
{
1373
	sljit_u8* inst;
1374
	sljit_uw size;
1375

1376
	SLJIT_ASSERT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
1377

1378
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1379
	FAIL_IF(!inst);
1380
	INC_SIZE(2);
1381
	inst[0] = U8(get_jump_code((sljit_uw)type ^ 0x1) - 0x10);
1382

1383
	size = compiler->size;
1384
	EMIT_MOV(compiler, dst_reg, 0, src, srcw);
1385

1386
	inst[1] = U8(compiler->size - size);
1387
	return SLJIT_SUCCESS;
1388
}
1389

1390
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1391
{
1392
	sljit_u8 *inst;
1393
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1394
	sljit_uw size;
1395
#endif
1396

1397
	CHECK_ERROR();
1398
	CHECK(check_sljit_emit_op0(compiler, op));
1399

1400
	switch (GET_OPCODE(op)) {
1401
	case SLJIT_BREAKPOINT:
1402
		return emit_byte(compiler, INT3);
1403
	case SLJIT_NOP:
1404
		return emit_byte(compiler, NOP);
1405
	case SLJIT_LMUL_UW:
1406
	case SLJIT_LMUL_SW:
1407
	case SLJIT_DIVMOD_UW:
1408
	case SLJIT_DIVMOD_SW:
1409
	case SLJIT_DIV_UW:
1410
	case SLJIT_DIV_SW:
1411
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1412
#ifdef _WIN64
1413
		SLJIT_ASSERT(
1414
			reg_map[SLJIT_R0] == 0
1415
			&& reg_map[SLJIT_R1] == 2
1416
			&& reg_map[TMP_REG1] > 7);
1417
#else
1418
		SLJIT_ASSERT(
1419
			reg_map[SLJIT_R0] == 0
1420
			&& reg_map[SLJIT_R1] < 7
1421
			&& reg_map[TMP_REG1] == 2);
1422
#endif
1423
		compiler->mode32 = op & SLJIT_32;
1424
#endif
1425
		SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1426

1427
		op = GET_OPCODE(op);
1428
		if ((op | 0x2) == SLJIT_DIV_UW) {
1429
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1430
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1431
			inst = emit_x86_instruction(compiler, 1, SLJIT_R1, 0, SLJIT_R1, 0);
1432
#else
1433
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, TMP_REG1, 0);
1434
#endif
1435
			FAIL_IF(!inst);
1436
			*inst = XOR_r_rm;
1437
		}
1438

1439
		if ((op | 0x2) == SLJIT_DIV_SW) {
1440
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) || defined(_WIN64)
1441
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R1, 0);
1442
#endif
1443

1444
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1445
			FAIL_IF(emit_byte(compiler, CDQ));
1446
#else
1447
			if (!compiler->mode32) {
1448
				inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1449
				FAIL_IF(!inst);
1450
				INC_SIZE(2);
1451
				inst[0] = REX_W;
1452
				inst[1] = CDQ;
1453
			} else
1454
				FAIL_IF(emit_byte(compiler, CDQ));
1455
#endif
1456
		}
1457

1458
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1459
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1460
		FAIL_IF(!inst);
1461
		INC_SIZE(2);
1462
		inst[0] = GROUP_F7;
1463
		inst[1] = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_map[TMP_REG1] : reg_map[SLJIT_R1]);
1464
#else /* !SLJIT_CONFIG_X86_32 */
1465
#ifdef _WIN64
1466
		size = (!compiler->mode32 || op >= SLJIT_DIVMOD_UW) ? 3 : 2;
1467
#else /* !_WIN64 */
1468
		size = (!compiler->mode32) ? 3 : 2;
1469
#endif /* _WIN64 */
1470
		inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1471
		FAIL_IF(!inst);
1472
		INC_SIZE(size);
1473
#ifdef _WIN64
1474
		if (!compiler->mode32)
1475
			*inst++ = REX_W | ((op >= SLJIT_DIVMOD_UW) ? REX_B : 0);
1476
		else if (op >= SLJIT_DIVMOD_UW)
1477
			*inst++ = REX_B;
1478
		inst[0] = GROUP_F7;
1479
		inst[1] = MOD_REG | ((op >= SLJIT_DIVMOD_UW) ? reg_lmap[TMP_REG1] : reg_lmap[SLJIT_R1]);
1480
#else /* !_WIN64 */
1481
		if (!compiler->mode32)
1482
			*inst++ = REX_W;
1483
		inst[0] = GROUP_F7;
1484
		inst[1] = MOD_REG | reg_map[SLJIT_R1];
1485
#endif /* _WIN64 */
1486
#endif /* SLJIT_CONFIG_X86_32 */
1487
		switch (op) {
1488
		case SLJIT_LMUL_UW:
1489
			inst[1] |= MUL;
1490
			break;
1491
		case SLJIT_LMUL_SW:
1492
			inst[1] |= IMUL;
1493
			break;
1494
		case SLJIT_DIVMOD_UW:
1495
		case SLJIT_DIV_UW:
1496
			inst[1] |= DIV;
1497
			break;
1498
		case SLJIT_DIVMOD_SW:
1499
		case SLJIT_DIV_SW:
1500
			inst[1] |= IDIV;
1501
			break;
1502
		}
1503
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) && !defined(_WIN64)
1504
		if (op <= SLJIT_DIVMOD_SW)
1505
			EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1506
#else
1507
		if (op >= SLJIT_DIV_UW)
1508
			EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0);
1509
#endif
1510
		break;
1511
	case SLJIT_MEMORY_BARRIER:
1512
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1513
		FAIL_IF(!inst);
1514
		INC_SIZE(3);
1515
		inst[0] = GROUP_0F;
1516
		inst[1] = 0xae;
1517
		inst[2] = 0xf0;
1518
		return SLJIT_SUCCESS;
1519
	case SLJIT_ENDBR:
1520
		return emit_endbranch(compiler);
1521
	case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1522
		return skip_frames_before_return(compiler);
1523
	}
1524

1525
	return SLJIT_SUCCESS;
1526
}
1527

1528
static sljit_s32 emit_mov_byte(struct sljit_compiler *compiler, sljit_s32 sign,
1529
	sljit_s32 dst, sljit_sw dstw,
1530
	sljit_s32 src, sljit_sw srcw)
1531
{
1532
	sljit_u8* inst;
1533
	sljit_s32 dst_r;
1534

1535
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1536
	compiler->mode32 = 0;
1537
#endif
1538

1539
	if (src == SLJIT_IMM) {
1540
		if (FAST_IS_REG(dst)) {
1541
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1542
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1543
#else
1544
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1545
			FAIL_IF(!inst);
1546
			*inst = MOV_rm_i32;
1547
			return SLJIT_SUCCESS;
1548
#endif
1549
		}
1550
		inst = emit_x86_instruction(compiler, 1 | EX86_BYTE_ARG | EX86_NO_REXW, SLJIT_IMM, srcw, dst, dstw);
1551
		FAIL_IF(!inst);
1552
		*inst = MOV_rm8_i8;
1553
		return SLJIT_SUCCESS;
1554
	}
1555

1556
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1557

1558
	if ((dst & SLJIT_MEM) && FAST_IS_REG(src)) {
1559
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1560
		if (reg_map[src] >= 4) {
1561
			SLJIT_ASSERT(dst_r == TMP_REG1);
1562
			EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1563
		} else
1564
			dst_r = src;
1565
#else
1566
		dst_r = src;
1567
#endif
1568
	} else {
1569
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1570
		if (FAST_IS_REG(src) && reg_map[src] >= 4) {
1571
			/* Both src and dst are registers. */
1572
			SLJIT_ASSERT(FAST_IS_REG(dst));
1573

1574
			if (src == dst && !sign) {
1575
				inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 0xff, dst, 0);
1576
				FAIL_IF(!inst);
1577
				*(inst + 1) |= AND;
1578
				return SLJIT_SUCCESS;
1579
			}
1580

1581
			EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1582
			src = TMP_REG1;
1583
			srcw = 0;
1584
		}
1585
#endif /* !SLJIT_CONFIG_X86_32 */
1586

1587
		/* src can be memory addr or reg_map[src] < 4 on x86_32 architectures. */
1588
		FAIL_IF(emit_groupf(compiler, sign ? MOVSX_r_rm8 : MOVZX_r_rm8, dst_r, src, srcw));
1589
	}
1590

1591
	if (dst & SLJIT_MEM) {
1592
		inst = emit_x86_instruction(compiler, 1 | EX86_REX | EX86_NO_REXW, dst_r, 0, dst, dstw);
1593
		FAIL_IF(!inst);
1594
		*inst = MOV_rm8_r8;
1595
	}
1596

1597
	return SLJIT_SUCCESS;
1598
}
1599

1600
static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 op,
1601
	sljit_s32 src, sljit_sw srcw)
1602
{
1603
	sljit_u8* inst;
1604

1605
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1606
	compiler->mode32 = 1;
1607
#endif
1608

1609
	inst = emit_x86_instruction(compiler, 2, 0, 0, src, srcw);
1610
	FAIL_IF(!inst);
1611
	inst[0] = GROUP_0F;
1612
	inst[1] = PREFETCH;
1613

1614
	if (op == SLJIT_PREFETCH_L1)
1615
		inst[2] |= (1 << 3);
1616
	else if (op == SLJIT_PREFETCH_L2)
1617
		inst[2] |= (2 << 3);
1618
	else if (op == SLJIT_PREFETCH_L3)
1619
		inst[2] |= (3 << 3);
1620

1621
	return SLJIT_SUCCESS;
1622
}
1623

1624
static sljit_s32 emit_mov_half(struct sljit_compiler *compiler, sljit_s32 sign,
1625
	sljit_s32 dst, sljit_sw dstw,
1626
	sljit_s32 src, sljit_sw srcw)
1627
{
1628
	sljit_u8* inst;
1629
	sljit_s32 dst_r;
1630

1631
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1632
	compiler->mode32 = 0;
1633
#endif
1634

1635
	if (src == SLJIT_IMM) {
1636
		if (FAST_IS_REG(dst)) {
1637
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1638
			return emit_do_imm(compiler, MOV_r_i32 | reg_map[dst], srcw);
1639
#else
1640
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, srcw, dst, 0);
1641
			FAIL_IF(!inst);
1642
			*inst = MOV_rm_i32;
1643
			return SLJIT_SUCCESS;
1644
#endif
1645
		}
1646
		inst = emit_x86_instruction(compiler, 1 | EX86_HALF_ARG | EX86_NO_REXW | EX86_PREF_66, SLJIT_IMM, srcw, dst, dstw);
1647
		FAIL_IF(!inst);
1648
		*inst = MOV_rm_i32;
1649
		return SLJIT_SUCCESS;
1650
	}
1651

1652
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1653

1654
	if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
1655
		dst_r = src;
1656
	else
1657
		FAIL_IF(emit_groupf(compiler, sign ? MOVSX_r_rm16 : MOVZX_r_rm16, dst_r, src, srcw));
1658

1659
	if (dst & SLJIT_MEM) {
1660
		inst = emit_x86_instruction(compiler, 1 | EX86_NO_REXW | EX86_PREF_66, dst_r, 0, dst, dstw);
1661
		FAIL_IF(!inst);
1662
		*inst = MOV_rm_r;
1663
	}
1664

1665
	return SLJIT_SUCCESS;
1666
}
1667

1668
static sljit_s32 emit_unary(struct sljit_compiler *compiler, sljit_u8 opcode,
1669
	sljit_s32 dst, sljit_sw dstw,
1670
	sljit_s32 src, sljit_sw srcw)
1671
{
1672
	sljit_u8* inst;
1673

1674
	if (dst == src && dstw == srcw) {
1675
		/* Same input and output */
1676
		inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1677
		FAIL_IF(!inst);
1678
		inst[0] = GROUP_F7;
1679
		inst[1] |= opcode;
1680
		return SLJIT_SUCCESS;
1681
	}
1682

1683
	if (FAST_IS_REG(dst)) {
1684
		EMIT_MOV(compiler, dst, 0, src, srcw);
1685
		inst = emit_x86_instruction(compiler, 1, 0, 0, dst, 0);
1686
		FAIL_IF(!inst);
1687
		inst[0] = GROUP_F7;
1688
		inst[1] |= opcode;
1689
		return SLJIT_SUCCESS;
1690
	}
1691

1692
	EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1693
	inst = emit_x86_instruction(compiler, 1, 0, 0, TMP_REG1, 0);
1694
	FAIL_IF(!inst);
1695
	inst[0] = GROUP_F7;
1696
	inst[1] |= opcode;
1697
	EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1698
	return SLJIT_SUCCESS;
1699
}
1700

1701
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1702
static const sljit_sw emit_clz_arg = 32 + 31;
1703
static const sljit_sw emit_ctz_arg = 32;
1704
#endif
1705

1706
static sljit_s32 emit_clz_ctz(struct sljit_compiler *compiler, sljit_s32 is_clz,
1707
	sljit_s32 dst, sljit_sw dstw,
1708
	sljit_s32 src, sljit_sw srcw)
1709
{
1710
	sljit_u8* inst;
1711
	sljit_s32 dst_r;
1712
	sljit_sw max;
1713

1714
	SLJIT_ASSERT(cpu_feature_list != 0);
1715

1716
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1717

1718
	if (is_clz ? (cpu_feature_list & CPU_FEATURE_LZCNT) : (cpu_feature_list & CPU_FEATURE_TZCNT)) {
1719
		FAIL_IF(emit_groupf(compiler, (is_clz ? LZCNT_r_rm : TZCNT_r_rm) | EX86_PREF_F3, dst_r, src, srcw));
1720

1721
		if (dst & SLJIT_MEM)
1722
			EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1723
		return SLJIT_SUCCESS;
1724
	}
1725

1726
	FAIL_IF(emit_groupf(compiler, is_clz ? BSR_r_rm : BSF_r_rm, dst_r, src, srcw));
1727

1728
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1729
	max = is_clz ? (32 + 31) : 32;
1730

1731
	if (cpu_feature_list & CPU_FEATURE_CMOV) {
1732
		if (dst_r != TMP_REG1) {
1733
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, max);
1734
			inst = emit_x86_instruction(compiler, 2, dst_r, 0, TMP_REG1, 0);
1735
		}
1736
		else
1737
			inst = emit_x86_instruction(compiler, 2, dst_r, 0, SLJIT_MEM0(), is_clz ? (sljit_sw)&emit_clz_arg : (sljit_sw)&emit_ctz_arg);
1738

1739
		FAIL_IF(!inst);
1740
		inst[0] = GROUP_0F;
1741
		inst[1] = CMOVE_r_rm;
1742
	}
1743
	else
1744
		FAIL_IF(emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, max));
1745

1746
	if (is_clz) {
1747
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, 31, dst_r, 0);
1748
		FAIL_IF(!inst);
1749
		*(inst + 1) |= XOR;
1750
	}
1751
#else
1752
	if (is_clz)
1753
		max = compiler->mode32 ? (32 + 31) : (64 + 63);
1754
	else
1755
		max = compiler->mode32 ? 32 : 64;
1756

1757
	if (cpu_feature_list & CPU_FEATURE_CMOV) {
1758
		EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_IMM, max);
1759
		FAIL_IF(emit_groupf(compiler, CMOVE_r_rm, dst_r, TMP_REG2, 0));
1760
	} else
1761
		FAIL_IF(emit_cmov_generic(compiler, SLJIT_EQUAL, dst_r, SLJIT_IMM, max));
1762

1763
	if (is_clz) {
1764
		inst = emit_x86_instruction(compiler, 1 | EX86_BIN_INS, SLJIT_IMM, max >> 1, dst_r, 0);
1765
		FAIL_IF(!inst);
1766
		*(inst + 1) |= XOR;
1767
	}
1768
#endif
1769

1770
	if (dst & SLJIT_MEM)
1771
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
1772
	return SLJIT_SUCCESS;
1773
}
1774

1775
static sljit_s32 emit_bswap(struct sljit_compiler *compiler,
1776
	sljit_s32 op,
1777
	sljit_s32 dst, sljit_sw dstw,
1778
	sljit_s32 src, sljit_sw srcw)
1779
{
1780
	sljit_u8 *inst;
1781
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1782
	sljit_uw size;
1783
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1784
	sljit_u8 rex = 0;
1785
#else /* !SLJIT_CONFIG_X86_64 */
1786
	sljit_s32 dst_is_ereg = op & SLJIT_32;
1787
#endif /* SLJIT_CONFIG_X86_64 */
1788

1789
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1790
	if (op == SLJIT_REV_U32 || op == SLJIT_REV_S32)
1791
		compiler->mode32 = 1;
1792
#else /* !SLJIT_CONFIG_X86_64 */
1793
	op &= ~SLJIT_32;
1794
#endif /* SLJIT_CONFIG_X86_64 */
1795

1796
	if (src != dst_r) {
1797
		/* Only the lower 16 bit is read for eregs. */
1798
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16)
1799
			FAIL_IF(emit_mov_half(compiler, 0, dst_r, 0, src, srcw));
1800
		else
1801
			EMIT_MOV(compiler, dst_r, 0, src, srcw);
1802
	}
1803

1804
	size = 2;
1805
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1806
	if (!compiler->mode32)
1807
		rex = REX_W;
1808

1809
	if (reg_map[dst_r] >= 8)
1810
		rex |= REX_B;
1811

1812
	if (rex != 0)
1813
		size++;
1814
#endif /* SLJIT_CONFIG_X86_64 */
1815

1816
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
1817
	FAIL_IF(!inst);
1818
	INC_SIZE(size);
1819

1820
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1821
	if (rex != 0)
1822
		*inst++ = rex;
1823

1824
	inst[0] = GROUP_0F;
1825
	inst[1] = BSWAP_r | reg_lmap[dst_r];
1826
#else /* !SLJIT_CONFIG_X86_64 */
1827
	inst[0] = GROUP_0F;
1828
	inst[1] = BSWAP_r | reg_map[dst_r];
1829
#endif /* SLJIT_CONFIG_X86_64 */
1830

1831
	if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1832
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1833
		size = compiler->mode32 ? 16 : 48;
1834
#else /* !SLJIT_CONFIG_X86_64 */
1835
		size = 16;
1836
#endif /* SLJIT_CONFIG_X86_64 */
1837

1838
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, (sljit_sw)size, dst_r, 0);
1839
		FAIL_IF(!inst);
1840
		if (op == SLJIT_REV_U16)
1841
			inst[1] |= SHR;
1842
		else
1843
			inst[1] |= SAR;
1844
	}
1845

1846
	if (dst & SLJIT_MEM) {
1847
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1848
		if (dst_is_ereg)
1849
			op = SLJIT_REV;
1850
#endif /* SLJIT_CONFIG_X86_32 */
1851
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16)
1852
			return emit_mov_half(compiler, 0, dst, dstw, TMP_REG1, 0);
1853

1854
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1855
	}
1856

1857
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1858
	if (op == SLJIT_REV_S32) {
1859
		compiler->mode32 = 0;
1860
		inst = emit_x86_instruction(compiler, 1, dst, 0, dst, 0);
1861
		FAIL_IF(!inst);
1862
		*inst = MOVSXD_r_rm;
1863
	}
1864
#endif /* SLJIT_CONFIG_X86_64 */
1865

1866
	return SLJIT_SUCCESS;
1867
}
1868

1869
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1870
	sljit_s32 dst, sljit_sw dstw,
1871
	sljit_s32 src, sljit_sw srcw)
1872
{
1873
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1874
	sljit_s32 dst_is_ereg = 0;
1875
#else /* !SLJIT_CONFIG_X86_32 */
1876
	sljit_s32 op_flags = GET_ALL_FLAGS(op);
1877
#endif /* SLJIT_CONFIG_X86_32 */
1878

1879
	CHECK_ERROR();
1880
	CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1881
	ADJUST_LOCAL_OFFSET(dst, dstw);
1882
	ADJUST_LOCAL_OFFSET(src, srcw);
1883

1884
	CHECK_EXTRA_REGS(dst, dstw, dst_is_ereg = 1);
1885
	CHECK_EXTRA_REGS(src, srcw, (void)0);
1886
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1887
	compiler->mode32 = op_flags & SLJIT_32;
1888
#endif /* SLJIT_CONFIG_X86_64 */
1889

1890
	op = GET_OPCODE(op);
1891

1892
	if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1893
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1894
		compiler->mode32 = 0;
1895
#endif /* SLJIT_CONFIG_X86_64 */
1896

1897
		if (FAST_IS_REG(src) && src == dst) {
1898
			if (!TYPE_CAST_NEEDED(op))
1899
				return SLJIT_SUCCESS;
1900
		}
1901

1902
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1903
		if (op_flags & SLJIT_32) {
1904
			if (src & SLJIT_MEM) {
1905
				if (op == SLJIT_MOV_S32)
1906
					op = SLJIT_MOV_U32;
1907
			}
1908
			else if (src == SLJIT_IMM) {
1909
				if (op == SLJIT_MOV_U32)
1910
					op = SLJIT_MOV_S32;
1911
			}
1912
		}
1913
#endif /* SLJIT_CONFIG_X86_64 */
1914

1915
		if (src == SLJIT_IMM) {
1916
			switch (op) {
1917
			case SLJIT_MOV_U8:
1918
				srcw = (sljit_u8)srcw;
1919
				break;
1920
			case SLJIT_MOV_S8:
1921
				srcw = (sljit_s8)srcw;
1922
				break;
1923
			case SLJIT_MOV_U16:
1924
				srcw = (sljit_u16)srcw;
1925
				break;
1926
			case SLJIT_MOV_S16:
1927
				srcw = (sljit_s16)srcw;
1928
				break;
1929
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1930
			case SLJIT_MOV_U32:
1931
				srcw = (sljit_u32)srcw;
1932
				break;
1933
			case SLJIT_MOV_S32:
1934
				srcw = (sljit_s32)srcw;
1935
				break;
1936
#endif /* SLJIT_CONFIG_X86_64 */
1937
			}
1938
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1939
			if (SLJIT_UNLIKELY(dst_is_ereg))
1940
				return emit_mov(compiler, dst, dstw, src, srcw);
1941
#endif /* SLJIT_CONFIG_X86_32 */
1942
		}
1943

1944
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1945
		if (SLJIT_UNLIKELY(dst_is_ereg) && (!(op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P) || (src & SLJIT_MEM))) {
1946
			SLJIT_ASSERT(dst == SLJIT_MEM1(SLJIT_SP));
1947
			dst = TMP_REG1;
1948
		}
1949
#endif /* SLJIT_CONFIG_X86_32 */
1950

1951
		switch (op) {
1952
		case SLJIT_MOV:
1953
		case SLJIT_MOV_P:
1954
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1955
		case SLJIT_MOV_U32:
1956
		case SLJIT_MOV_S32:
1957
		case SLJIT_MOV32:
1958
#endif /* SLJIT_CONFIG_X86_32 */
1959
			EMIT_MOV(compiler, dst, dstw, src, srcw);
1960
			break;
1961
		case SLJIT_MOV_U8:
1962
			FAIL_IF(emit_mov_byte(compiler, 0, dst, dstw, src, srcw));
1963
			break;
1964
		case SLJIT_MOV_S8:
1965
			FAIL_IF(emit_mov_byte(compiler, 1, dst, dstw, src, srcw));
1966
			break;
1967
		case SLJIT_MOV_U16:
1968
			FAIL_IF(emit_mov_half(compiler, 0, dst, dstw, src, srcw));
1969
			break;
1970
		case SLJIT_MOV_S16:
1971
			FAIL_IF(emit_mov_half(compiler, 1, dst, dstw, src, srcw));
1972
			break;
1973
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
1974
		case SLJIT_MOV_U32:
1975
			FAIL_IF(emit_mov_int(compiler, 0, dst, dstw, src, srcw));
1976
			break;
1977
		case SLJIT_MOV_S32:
1978
			FAIL_IF(emit_mov_int(compiler, 1, dst, dstw, src, srcw));
1979
			break;
1980
		case SLJIT_MOV32:
1981
			compiler->mode32 = 1;
1982
			EMIT_MOV(compiler, dst, dstw, src, srcw);
1983
			compiler->mode32 = 0;
1984
			break;
1985
#endif /* SLJIT_CONFIG_X86_64 */
1986
		}
1987

1988
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
1989
		if (SLJIT_UNLIKELY(dst_is_ereg) && dst == TMP_REG1)
1990
			return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), dstw, TMP_REG1, 0);
1991
#endif /* SLJIT_CONFIG_X86_32 */
1992
		return SLJIT_SUCCESS;
1993
	}
1994

1995
	switch (op) {
1996
	case SLJIT_CLZ:
1997
	case SLJIT_CTZ:
1998
		return emit_clz_ctz(compiler, (op == SLJIT_CLZ), dst, dstw, src, srcw);
1999
	case SLJIT_REV:
2000
	case SLJIT_REV_U16:
2001
	case SLJIT_REV_S16:
2002
	case SLJIT_REV_U32:
2003
	case SLJIT_REV_S32:
2004
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2005
		if (dst_is_ereg)
2006
			op |= SLJIT_32;
2007
#endif /* SLJIT_CONFIG_X86_32 */
2008
		return emit_bswap(compiler, op, dst, dstw, src, srcw);
2009
	}
2010

2011
	return SLJIT_SUCCESS;
2012
}
2013

2014
static sljit_s32 emit_cum_binary(struct sljit_compiler *compiler,
2015
	sljit_u32 op_types,
2016
	sljit_s32 dst, sljit_sw dstw,
2017
	sljit_s32 src1, sljit_sw src1w,
2018
	sljit_s32 src2, sljit_sw src2w)
2019
{
2020
	sljit_u8* inst;
2021
	sljit_u8 op_eax_imm = U8(op_types >> 24);
2022
	sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
2023
	sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
2024
	sljit_u8 op_imm = U8(op_types & 0xff);
2025

2026
	if (dst == src1 && dstw == src1w) {
2027
		if (src2 == SLJIT_IMM) {
2028
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2029
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2030
#else
2031
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
2032
#endif
2033
				BINARY_EAX_IMM(op_eax_imm, src2w);
2034
			}
2035
			else {
2036
				BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
2037
			}
2038
		}
2039
		else if (FAST_IS_REG(dst)) {
2040
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
2041
			FAIL_IF(!inst);
2042
			*inst = op_rm;
2043
		}
2044
		else if (FAST_IS_REG(src2)) {
2045
			/* Special exception for sljit_emit_op_flags. */
2046
			inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
2047
			FAIL_IF(!inst);
2048
			*inst = op_mr;
2049
		}
2050
		else {
2051
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
2052
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2053
			FAIL_IF(!inst);
2054
			*inst = op_mr;
2055
		}
2056
		return SLJIT_SUCCESS;
2057
	}
2058

2059
	/* Only for cumulative operations. */
2060
	if (dst == src2 && dstw == src2w) {
2061
		if (src1 == SLJIT_IMM) {
2062
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2063
			if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2064
#else
2065
			if ((dst == SLJIT_R0) && (src1w > 127 || src1w < -128)) {
2066
#endif
2067
				BINARY_EAX_IMM(op_eax_imm, src1w);
2068
			}
2069
			else {
2070
				BINARY_IMM(op_imm, op_mr, src1w, dst, dstw);
2071
			}
2072
		}
2073
		else if (FAST_IS_REG(dst)) {
2074
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src1, src1w);
2075
			FAIL_IF(!inst);
2076
			*inst = op_rm;
2077
		}
2078
		else if (FAST_IS_REG(src1)) {
2079
			inst = emit_x86_instruction(compiler, 1, src1, src1w, dst, dstw);
2080
			FAIL_IF(!inst);
2081
			*inst = op_mr;
2082
		}
2083
		else {
2084
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2085
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2086
			FAIL_IF(!inst);
2087
			*inst = op_mr;
2088
		}
2089
		return SLJIT_SUCCESS;
2090
	}
2091

2092
	/* General version. */
2093
	if (FAST_IS_REG(dst)) {
2094
		EMIT_MOV(compiler, dst, 0, src1, src1w);
2095
		if (src2 == SLJIT_IMM) {
2096
			BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
2097
		}
2098
		else {
2099
			inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
2100
			FAIL_IF(!inst);
2101
			*inst = op_rm;
2102
		}
2103
	}
2104
	else {
2105
		/* This version requires less memory writing. */
2106
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2107
		if (src2 == SLJIT_IMM) {
2108
			BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
2109
		}
2110
		else {
2111
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2112
			FAIL_IF(!inst);
2113
			*inst = op_rm;
2114
		}
2115
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2116
	}
2117

2118
	return SLJIT_SUCCESS;
2119
}
2120

2121
static sljit_s32 emit_non_cum_binary(struct sljit_compiler *compiler,
2122
	sljit_u32 op_types,
2123
	sljit_s32 dst, sljit_sw dstw,
2124
	sljit_s32 src1, sljit_sw src1w,
2125
	sljit_s32 src2, sljit_sw src2w)
2126
{
2127
	sljit_u8* inst;
2128
	sljit_u8 op_eax_imm = U8(op_types >> 24);
2129
	sljit_u8 op_rm = U8((op_types >> 16) & 0xff);
2130
	sljit_u8 op_mr = U8((op_types >> 8) & 0xff);
2131
	sljit_u8 op_imm = U8(op_types & 0xff);
2132

2133
	if (dst == src1 && dstw == src1w) {
2134
		if (src2 == SLJIT_IMM) {
2135
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2136
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2137
#else
2138
			if ((dst == SLJIT_R0) && (src2w > 127 || src2w < -128)) {
2139
#endif
2140
				BINARY_EAX_IMM(op_eax_imm, src2w);
2141
			}
2142
			else {
2143
				BINARY_IMM(op_imm, op_mr, src2w, dst, dstw);
2144
			}
2145
		}
2146
		else if (FAST_IS_REG(dst)) {
2147
			inst = emit_x86_instruction(compiler, 1, dst, dstw, src2, src2w);
2148
			FAIL_IF(!inst);
2149
			*inst = op_rm;
2150
		}
2151
		else if (FAST_IS_REG(src2)) {
2152
			inst = emit_x86_instruction(compiler, 1, src2, src2w, dst, dstw);
2153
			FAIL_IF(!inst);
2154
			*inst = op_mr;
2155
		}
2156
		else {
2157
			EMIT_MOV(compiler, TMP_REG1, 0, src2, src2w);
2158
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst, dstw);
2159
			FAIL_IF(!inst);
2160
			*inst = op_mr;
2161
		}
2162
		return SLJIT_SUCCESS;
2163
	}
2164

2165
	/* General version. */
2166
	if (FAST_IS_REG(dst) && dst != src2) {
2167
		EMIT_MOV(compiler, dst, 0, src1, src1w);
2168
		if (src2 == SLJIT_IMM) {
2169
			BINARY_IMM(op_imm, op_mr, src2w, dst, 0);
2170
		}
2171
		else {
2172
			inst = emit_x86_instruction(compiler, 1, dst, 0, src2, src2w);
2173
			FAIL_IF(!inst);
2174
			*inst = op_rm;
2175
		}
2176
	}
2177
	else {
2178
		/* This version requires less memory writing. */
2179
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2180
		if (src2 == SLJIT_IMM) {
2181
			BINARY_IMM(op_imm, op_mr, src2w, TMP_REG1, 0);
2182
		}
2183
		else {
2184
			inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2185
			FAIL_IF(!inst);
2186
			*inst = op_rm;
2187
		}
2188
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2189
	}
2190

2191
	return SLJIT_SUCCESS;
2192
}
2193

2194
static sljit_s32 emit_mul(struct sljit_compiler *compiler,
2195
	sljit_s32 dst, sljit_sw dstw,
2196
	sljit_s32 src1, sljit_sw src1w,
2197
	sljit_s32 src2, sljit_sw src2w)
2198
{
2199
	sljit_u8* inst;
2200
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2201

2202
	/* Register destination. */
2203
	if (dst_r == src1 && src2 != SLJIT_IMM) {
2204
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src2, src2w));
2205
	} else if (dst_r == src2 && src1 != SLJIT_IMM) {
2206
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src1, src1w));
2207
	} else if (src1 == SLJIT_IMM) {
2208
		if (src2 == SLJIT_IMM) {
2209
			EMIT_MOV(compiler, dst_r, 0, SLJIT_IMM, src2w);
2210
			src2 = dst_r;
2211
			src2w = 0;
2212
		}
2213

2214
		if (src1w <= 127 && src1w >= -128) {
2215
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2216
			FAIL_IF(!inst);
2217
			*inst = IMUL_r_rm_i8;
2218

2219
			FAIL_IF(emit_byte(compiler, U8(src1w)));
2220
		}
2221
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2222
		else {
2223
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2224
			FAIL_IF(!inst);
2225
			*inst = IMUL_r_rm_i32;
2226
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2227
			FAIL_IF(!inst);
2228
			INC_SIZE(4);
2229
			sljit_unaligned_store_sw(inst, src1w);
2230
		}
2231
#else
2232
		else if (IS_HALFWORD(src1w)) {
2233
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src2, src2w);
2234
			FAIL_IF(!inst);
2235
			*inst = IMUL_r_rm_i32;
2236
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2237
			FAIL_IF(!inst);
2238
			INC_SIZE(4);
2239
			sljit_unaligned_store_s32(inst, (sljit_s32)src1w);
2240
		}
2241
		else {
2242
			if (dst_r != src2)
2243
				EMIT_MOV(compiler, dst_r, 0, src2, src2w);
2244
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src1w));
2245
			FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, TMP_REG2, 0));
2246
		}
2247
#endif
2248
	}
2249
	else if (src2 == SLJIT_IMM) {
2250
		/* Note: src1 is NOT immediate. */
2251

2252
		if (src2w <= 127 && src2w >= -128) {
2253
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2254
			FAIL_IF(!inst);
2255
			*inst = IMUL_r_rm_i8;
2256

2257
			FAIL_IF(emit_byte(compiler, U8(src2w)));
2258
		}
2259
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2260
		else {
2261
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2262
			FAIL_IF(!inst);
2263
			*inst = IMUL_r_rm_i32;
2264

2265
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2266
			FAIL_IF(!inst);
2267
			INC_SIZE(4);
2268
			sljit_unaligned_store_sw(inst, src2w);
2269
		}
2270
#else
2271
		else if (IS_HALFWORD(src2w)) {
2272
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, src1, src1w);
2273
			FAIL_IF(!inst);
2274
			*inst = IMUL_r_rm_i32;
2275

2276
			inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
2277
			FAIL_IF(!inst);
2278
			INC_SIZE(4);
2279
			sljit_unaligned_store_s32(inst, (sljit_s32)src2w);
2280
		} else {
2281
			if (dst_r != src1)
2282
				EMIT_MOV(compiler, dst_r, 0, src1, src1w);
2283
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2284
			FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, TMP_REG2, 0));
2285
		}
2286
#endif
2287
	} else {
2288
		/* Neither argument is immediate. */
2289
		if (ADDRESSING_DEPENDS_ON(src2, dst_r))
2290
			dst_r = TMP_REG1;
2291
		EMIT_MOV(compiler, dst_r, 0, src1, src1w);
2292
		FAIL_IF(emit_groupf(compiler, IMUL_r_rm, dst_r, src2, src2w));
2293
	}
2294

2295
	if (dst & SLJIT_MEM)
2296
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2297

2298
	return SLJIT_SUCCESS;
2299
}
2300

2301
static sljit_s32 emit_lea_binary(struct sljit_compiler *compiler,
2302
	sljit_s32 dst, sljit_sw dstw,
2303
	sljit_s32 src1, sljit_sw src1w,
2304
	sljit_s32 src2, sljit_sw src2w)
2305
{
2306
	sljit_u8* inst;
2307
	sljit_s32 dst_r, done = 0;
2308

2309
	/* These cases better be left to handled by normal way. */
2310
	if (dst == src1 && dstw == src1w)
2311
		return SLJIT_ERR_UNSUPPORTED;
2312
	if (dst == src2 && dstw == src2w)
2313
		return SLJIT_ERR_UNSUPPORTED;
2314

2315
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2316

2317
	if (FAST_IS_REG(src1)) {
2318
		if (FAST_IS_REG(src2)) {
2319
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM2(src1, src2), 0);
2320
			FAIL_IF(!inst);
2321
			*inst = LEA_r_m;
2322
			done = 1;
2323
		}
2324
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2325
		if (src2 == SLJIT_IMM && (compiler->mode32 || IS_HALFWORD(src2w))) {
2326
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), (sljit_s32)src2w);
2327
#else
2328
		if (src2 == SLJIT_IMM) {
2329
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src1), src2w);
2330
#endif
2331
			FAIL_IF(!inst);
2332
			*inst = LEA_r_m;
2333
			done = 1;
2334
		}
2335
	}
2336
	else if (FAST_IS_REG(src2)) {
2337
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2338
		if (src1 == SLJIT_IMM && (compiler->mode32 || IS_HALFWORD(src1w))) {
2339
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), (sljit_s32)src1w);
2340
#else
2341
		if (src1 == SLJIT_IMM) {
2342
			inst = emit_x86_instruction(compiler, 1, dst_r, 0, SLJIT_MEM1(src2), src1w);
2343
#endif
2344
			FAIL_IF(!inst);
2345
			*inst = LEA_r_m;
2346
			done = 1;
2347
		}
2348
	}
2349

2350
	if (done) {
2351
		if (dst_r == TMP_REG1)
2352
			return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2353
		return SLJIT_SUCCESS;
2354
	}
2355
	return SLJIT_ERR_UNSUPPORTED;
2356
}
2357

2358
static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler,
2359
	sljit_s32 src1, sljit_sw src1w,
2360
	sljit_s32 src2, sljit_sw src2w)
2361
{
2362
	sljit_u8* inst;
2363

2364
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2365
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2366
#else
2367
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128)) {
2368
#endif
2369
		BINARY_EAX_IMM(CMP_EAX_i32, src2w);
2370
		return SLJIT_SUCCESS;
2371
	}
2372

2373
	if (FAST_IS_REG(src1)) {
2374
		if (src2 == SLJIT_IMM) {
2375
			BINARY_IMM(CMP, CMP_rm_r, src2w, src1, 0);
2376
		}
2377
		else {
2378
			inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2379
			FAIL_IF(!inst);
2380
			*inst = CMP_r_rm;
2381
		}
2382
		return SLJIT_SUCCESS;
2383
	}
2384

2385
	if (FAST_IS_REG(src2) && src1 != SLJIT_IMM) {
2386
		inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2387
		FAIL_IF(!inst);
2388
		*inst = CMP_rm_r;
2389
		return SLJIT_SUCCESS;
2390
	}
2391

2392
	if (src2 == SLJIT_IMM) {
2393
		if (src1 == SLJIT_IMM) {
2394
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2395
			src1 = TMP_REG1;
2396
			src1w = 0;
2397
		}
2398
		BINARY_IMM(CMP, CMP_rm_r, src2w, src1, src1w);
2399
	}
2400
	else {
2401
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2402
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2403
		FAIL_IF(!inst);
2404
		*inst = CMP_r_rm;
2405
	}
2406
	return SLJIT_SUCCESS;
2407
}
2408

2409
static sljit_s32 emit_test_binary(struct sljit_compiler *compiler,
2410
	sljit_s32 src1, sljit_sw src1w,
2411
	sljit_s32 src2, sljit_sw src2w)
2412
{
2413
	sljit_u8* inst;
2414

2415
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2416
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128) && (compiler->mode32 || IS_HALFWORD(src2w))) {
2417
#else
2418
	if (src1 == SLJIT_R0 && src2 == SLJIT_IMM && (src2w > 127 || src2w < -128)) {
2419
#endif
2420
		BINARY_EAX_IMM(TEST_EAX_i32, src2w);
2421
		return SLJIT_SUCCESS;
2422
	}
2423

2424
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2425
	if (src2 == SLJIT_R0 && src1 == SLJIT_IMM && (src1w > 127 || src1w < -128) && (compiler->mode32 || IS_HALFWORD(src1w))) {
2426
#else
2427
	if (src2 == SLJIT_R0 && src1 == SLJIT_IMM && (src1w > 127 || src1w < -128)) {
2428
#endif
2429
		BINARY_EAX_IMM(TEST_EAX_i32, src1w);
2430
		return SLJIT_SUCCESS;
2431
	}
2432

2433
	if (src1 != SLJIT_IMM) {
2434
		if (src2 == SLJIT_IMM) {
2435
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2436
			if (IS_HALFWORD(src2w) || compiler->mode32) {
2437
				inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2438
				FAIL_IF(!inst);
2439
				*inst = GROUP_F7;
2440
			} else {
2441
				FAIL_IF(emit_load_imm64(compiler, FAST_IS_REG(src1) ? TMP_REG2 : TMP_REG1, src2w));
2442
				inst = emit_x86_instruction(compiler, 1, FAST_IS_REG(src1) ? TMP_REG2 : TMP_REG1, 0, src1, src1w);
2443
				FAIL_IF(!inst);
2444
				*inst = TEST_rm_r;
2445
			}
2446
#else
2447
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, src1, src1w);
2448
			FAIL_IF(!inst);
2449
			*inst = GROUP_F7;
2450
#endif
2451
			return SLJIT_SUCCESS;
2452
		}
2453
		else if (FAST_IS_REG(src1)) {
2454
			inst = emit_x86_instruction(compiler, 1, src1, 0, src2, src2w);
2455
			FAIL_IF(!inst);
2456
			*inst = TEST_rm_r;
2457
			return SLJIT_SUCCESS;
2458
		}
2459
	}
2460

2461
	if (src2 != SLJIT_IMM) {
2462
		if (src1 == SLJIT_IMM) {
2463
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2464
			if (IS_HALFWORD(src1w) || compiler->mode32) {
2465
				inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src1w, src2, src2w);
2466
				FAIL_IF(!inst);
2467
				*inst = GROUP_F7;
2468
			}
2469
			else {
2470
				FAIL_IF(emit_load_imm64(compiler, TMP_REG1, src1w));
2471
				inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2472
				FAIL_IF(!inst);
2473
				*inst = TEST_rm_r;
2474
			}
2475
#else
2476
			inst = emit_x86_instruction(compiler, 1, src1, src1w, src2, src2w);
2477
			FAIL_IF(!inst);
2478
			*inst = GROUP_F7;
2479
#endif
2480
			return SLJIT_SUCCESS;
2481
		}
2482
		else if (FAST_IS_REG(src2)) {
2483
			inst = emit_x86_instruction(compiler, 1, src2, 0, src1, src1w);
2484
			FAIL_IF(!inst);
2485
			*inst = TEST_rm_r;
2486
			return SLJIT_SUCCESS;
2487
		}
2488
	}
2489

2490
	EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2491
	if (src2 == SLJIT_IMM) {
2492
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2493
		if (IS_HALFWORD(src2w) || compiler->mode32) {
2494
			inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2495
			FAIL_IF(!inst);
2496
			*inst = GROUP_F7;
2497
		}
2498
		else {
2499
			FAIL_IF(emit_load_imm64(compiler, TMP_REG2, src2w));
2500
			inst = emit_x86_instruction(compiler, 1, TMP_REG2, 0, TMP_REG1, 0);
2501
			FAIL_IF(!inst);
2502
			*inst = TEST_rm_r;
2503
		}
2504
#else
2505
		inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, src2w, TMP_REG1, 0);
2506
		FAIL_IF(!inst);
2507
		*inst = GROUP_F7;
2508
#endif
2509
	}
2510
	else {
2511
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, src2, src2w);
2512
		FAIL_IF(!inst);
2513
		*inst = TEST_rm_r;
2514
	}
2515
	return SLJIT_SUCCESS;
2516
}
2517

2518
static sljit_s32 emit_shift(struct sljit_compiler *compiler,
2519
	sljit_u8 mode,
2520
	sljit_s32 dst, sljit_sw dstw,
2521
	sljit_s32 src1, sljit_sw src1w,
2522
	sljit_s32 src2, sljit_sw src2w)
2523
{
2524
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2525
	sljit_s32 mode32;
2526
#endif
2527
	sljit_u8* inst;
2528

2529
	if (src2 == SLJIT_IMM || src2 == SLJIT_PREF_SHIFT_REG) {
2530
		if (dst == src1 && dstw == src1w) {
2531
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, dstw);
2532
			FAIL_IF(!inst);
2533
			inst[1] |= mode;
2534
			return SLJIT_SUCCESS;
2535
		}
2536
		if (dst == SLJIT_PREF_SHIFT_REG && src2 == SLJIT_PREF_SHIFT_REG) {
2537
			EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2538
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2539
			FAIL_IF(!inst);
2540
			inst[1] |= mode;
2541
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2542
			return SLJIT_SUCCESS;
2543
		}
2544
		if (FAST_IS_REG(dst)) {
2545
			EMIT_MOV(compiler, dst, 0, src1, src1w);
2546
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, dst, 0);
2547
			FAIL_IF(!inst);
2548
			inst[1] |= mode;
2549
			return SLJIT_SUCCESS;
2550
		}
2551

2552
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2553
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, src2, src2w, TMP_REG1, 0);
2554
		FAIL_IF(!inst);
2555
		inst[1] |= mode;
2556
		EMIT_MOV(compiler, dst, dstw, TMP_REG1, 0);
2557
		return SLJIT_SUCCESS;
2558
	}
2559

2560
	if (dst == SLJIT_PREF_SHIFT_REG) {
2561
		EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2562
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2563
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2564
		FAIL_IF(!inst);
2565
		inst[1] |= mode;
2566
		return emit_mov(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2567
	}
2568

2569
	if (FAST_IS_REG(dst) && dst != src2 && dst != TMP_REG1 && !ADDRESSING_DEPENDS_ON(src2, dst)) {
2570
		if (src1 != dst)
2571
			EMIT_MOV(compiler, dst, 0, src1, src1w);
2572
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2573
		mode32 = compiler->mode32;
2574
		compiler->mode32 = 0;
2575
#endif
2576
		EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2577
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2578
		compiler->mode32 = mode32;
2579
#endif
2580
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2581
		inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, dst, 0);
2582
		FAIL_IF(!inst);
2583
		inst[1] |= mode;
2584
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2585
		compiler->mode32 = 0;
2586
#endif
2587
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2588
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2589
		compiler->mode32 = mode32;
2590
#endif
2591
		return SLJIT_SUCCESS;
2592
	}
2593

2594
	/* This case is complex since ecx itself may be used for
2595
	   addressing, and this case must be supported as well. */
2596
	EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
2597
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2598
	EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2599
#else /* !SLJIT_CONFIG_X86_32 */
2600
	mode32 = compiler->mode32;
2601
	compiler->mode32 = 0;
2602
	EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_PREF_SHIFT_REG, 0);
2603
	compiler->mode32 = mode32;
2604
#endif /* SLJIT_CONFIG_X86_32 */
2605

2606
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src2, src2w);
2607
	inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2608
	FAIL_IF(!inst);
2609
	inst[1] |= mode;
2610

2611
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2612
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, SLJIT_MEM1(SLJIT_SP), 0);
2613
#else
2614
	compiler->mode32 = 0;
2615
	EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG2, 0);
2616
	compiler->mode32 = mode32;
2617
#endif /* SLJIT_CONFIG_X86_32 */
2618

2619
	if (dst != TMP_REG1)
2620
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
2621

2622
	return SLJIT_SUCCESS;
2623
}
2624

2625
static sljit_s32 emit_shift_with_flags(struct sljit_compiler *compiler,
2626
	sljit_u8 mode, sljit_s32 set_flags,
2627
	sljit_s32 dst, sljit_sw dstw,
2628
	sljit_s32 src1, sljit_sw src1w,
2629
	sljit_s32 src2, sljit_sw src2w)
2630
{
2631
	/* The CPU does not set flags if the shift count is 0. */
2632
	if (src2 == SLJIT_IMM) {
2633
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2634
		src2w &= compiler->mode32 ? 0x1f : 0x3f;
2635
#else /* !SLJIT_CONFIG_X86_64 */
2636
		src2w &= 0x1f;
2637
#endif /* SLJIT_CONFIG_X86_64 */
2638
		if (src2w != 0)
2639
			return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2640

2641
		if (!set_flags)
2642
			return emit_mov(compiler, dst, dstw, src1, src1w);
2643
		/* OR dst, src, 0 */
2644
		return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2645
			dst, dstw, src1, src1w, SLJIT_IMM, 0);
2646
	}
2647

2648
	if (!set_flags)
2649
		return emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w);
2650

2651
	if (!FAST_IS_REG(dst))
2652
		FAIL_IF(emit_cmp_binary(compiler, src1, src1w, SLJIT_IMM, 0));
2653

2654
	FAIL_IF(emit_shift(compiler, mode, dst, dstw, src1, src1w, src2, src2w));
2655

2656
	if (FAST_IS_REG(dst))
2657
		return emit_cmp_binary(compiler, dst, dstw, SLJIT_IMM, 0);
2658
	return SLJIT_SUCCESS;
2659
}
2660

2661
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2662
	sljit_s32 dst, sljit_sw dstw,
2663
	sljit_s32 src1, sljit_sw src1w,
2664
	sljit_s32 src2, sljit_sw src2w)
2665
{
2666
	CHECK_ERROR();
2667
	CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2668
	ADJUST_LOCAL_OFFSET(dst, dstw);
2669
	ADJUST_LOCAL_OFFSET(src1, src1w);
2670
	ADJUST_LOCAL_OFFSET(src2, src2w);
2671

2672
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
2673
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2674
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2675
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2676
	compiler->mode32 = op & SLJIT_32;
2677
#endif
2678

2679
	switch (GET_OPCODE(op)) {
2680
	case SLJIT_ADD:
2681
		if (!HAS_FLAGS(op)) {
2682
			if (emit_lea_binary(compiler, dst, dstw, src1, src1w, src2, src2w) != SLJIT_ERR_UNSUPPORTED)
2683
				return compiler->error;
2684
		}
2685
		return emit_cum_binary(compiler, BINARY_OPCODE(ADD),
2686
			dst, dstw, src1, src1w, src2, src2w);
2687
	case SLJIT_ADDC:
2688
		return emit_cum_binary(compiler, BINARY_OPCODE(ADC),
2689
			dst, dstw, src1, src1w, src2, src2w);
2690
	case SLJIT_SUB:
2691
		if (src1 == SLJIT_IMM && src1w == 0)
2692
			return emit_unary(compiler, NEG_rm, dst, dstw, src2, src2w);
2693

2694
		if (!HAS_FLAGS(op)) {
2695
			if (src2 == SLJIT_IMM && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED)
2696
				return compiler->error;
2697
			if (FAST_IS_REG(dst) && src2 == dst) {
2698
				FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB), dst, 0, dst, 0, src1, src1w));
2699
				return emit_unary(compiler, NEG_rm, dst, 0, dst, 0);
2700
			}
2701
		}
2702

2703
		return emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
2704
			dst, dstw, src1, src1w, src2, src2w);
2705
	case SLJIT_SUBC:
2706
		return emit_non_cum_binary(compiler, BINARY_OPCODE(SBB),
2707
			dst, dstw, src1, src1w, src2, src2w);
2708
	case SLJIT_MUL:
2709
		return emit_mul(compiler, dst, dstw, src1, src1w, src2, src2w);
2710
	case SLJIT_AND:
2711
		return emit_cum_binary(compiler, BINARY_OPCODE(AND),
2712
			dst, dstw, src1, src1w, src2, src2w);
2713
	case SLJIT_OR:
2714
		return emit_cum_binary(compiler, BINARY_OPCODE(OR),
2715
			dst, dstw, src1, src1w, src2, src2w);
2716
	case SLJIT_XOR:
2717
		if (!HAS_FLAGS(op)) {
2718
			if (src2 == SLJIT_IMM && src2w == -1)
2719
				return emit_unary(compiler, NOT_rm, dst, dstw, src1, src1w);
2720
			if (src1 == SLJIT_IMM && src1w == -1)
2721
				return emit_unary(compiler, NOT_rm, dst, dstw, src2, src2w);
2722
		}
2723

2724
		return emit_cum_binary(compiler, BINARY_OPCODE(XOR),
2725
			dst, dstw, src1, src1w, src2, src2w);
2726
	case SLJIT_SHL:
2727
	case SLJIT_MSHL:
2728
		return emit_shift_with_flags(compiler, SHL, HAS_FLAGS(op),
2729
			dst, dstw, src1, src1w, src2, src2w);
2730
	case SLJIT_LSHR:
2731
	case SLJIT_MLSHR:
2732
		return emit_shift_with_flags(compiler, SHR, HAS_FLAGS(op),
2733
			dst, dstw, src1, src1w, src2, src2w);
2734
	case SLJIT_ASHR:
2735
	case SLJIT_MASHR:
2736
		return emit_shift_with_flags(compiler, SAR, HAS_FLAGS(op),
2737
			dst, dstw, src1, src1w, src2, src2w);
2738
	case SLJIT_ROTL:
2739
		return emit_shift_with_flags(compiler, ROL, 0,
2740
			dst, dstw, src1, src1w, src2, src2w);
2741
	case SLJIT_ROTR:
2742
		return emit_shift_with_flags(compiler, ROR, 0,
2743
			dst, dstw, src1, src1w, src2, src2w);
2744
	}
2745

2746
	return SLJIT_SUCCESS;
2747
}
2748

2749
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2750
	sljit_s32 src1, sljit_sw src1w,
2751
	sljit_s32 src2, sljit_sw src2w)
2752
{
2753
	sljit_s32 opcode = GET_OPCODE(op);
2754

2755
	CHECK_ERROR();
2756
	CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2757

2758
	if (opcode != SLJIT_SUB && opcode != SLJIT_AND) {
2759
		SLJIT_SKIP_CHECKS(compiler);
2760
		return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
2761
	}
2762

2763
	ADJUST_LOCAL_OFFSET(src1, src1w);
2764
	ADJUST_LOCAL_OFFSET(src2, src2w);
2765

2766
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2767
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2768
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2769
	compiler->mode32 = op & SLJIT_32;
2770
#endif
2771

2772
	if (opcode == SLJIT_SUB)
2773
		return emit_cmp_binary(compiler, src1, src1w, src2, src2w);
2774

2775
	return emit_test_binary(compiler, src1, src1w, src2, src2w);
2776
}
2777

2778
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2779
	sljit_s32 dst_reg,
2780
	sljit_s32 src1, sljit_sw src1w,
2781
	sljit_s32 src2, sljit_sw src2w)
2782
{
2783
	sljit_u8* inst;
2784
	sljit_sw dstw = 0;
2785

2786
	CHECK_ERROR();
2787
	CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2788
	ADJUST_LOCAL_OFFSET(src1, src1w);
2789
	ADJUST_LOCAL_OFFSET(src2, src2w);
2790

2791
	CHECK_EXTRA_REGS(dst_reg, dstw, (void)0);
2792
	CHECK_EXTRA_REGS(src1, src1w, (void)0);
2793
	CHECK_EXTRA_REGS(src2, src2w, (void)0);
2794
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2795
	compiler->mode32 = op & SLJIT_32;
2796
#endif
2797

2798
	switch (GET_OPCODE(op)) {
2799
	case SLJIT_MULADD:
2800
		FAIL_IF(emit_mul(compiler, TMP_REG1, 0, src1, src1w, src2, src2w));
2801
		inst = emit_x86_instruction(compiler, 1, TMP_REG1, 0, dst_reg, dstw);
2802
		FAIL_IF(!inst);
2803
		*inst = ADD_rm_r;
2804
		return SLJIT_SUCCESS;
2805
	}
2806

2807
	return SLJIT_SUCCESS;
2808
}
2809

2810
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2811
	sljit_s32 dst_reg,
2812
	sljit_s32 src1_reg,
2813
	sljit_s32 src2_reg,
2814
	sljit_s32 src3, sljit_sw src3w)
2815
{
2816
	sljit_s32 is_rotate, is_left, move_src1;
2817
	sljit_u8* inst;
2818
	sljit_sw src1w = 0;
2819
	sljit_sw dstw = 0;
2820
	/* The whole register must be saved even for 32 bit operations. */
2821
	sljit_u8 restore_ecx = 0;
2822
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2823
	sljit_sw src2w = 0;
2824
	sljit_s32 restore_sp4 = 0;
2825
#endif /* SLJIT_CONFIG_X86_32 */
2826

2827
	CHECK_ERROR();
2828
	CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2829
	ADJUST_LOCAL_OFFSET(src3, src3w);
2830

2831
	CHECK_EXTRA_REGS(dst_reg, dstw, (void)0);
2832
	CHECK_EXTRA_REGS(src3, src3w, (void)0);
2833

2834
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2835
	compiler->mode32 = op & SLJIT_32;
2836
#endif /* SLJIT_CONFIG_X86_64 */
2837

2838
	if (src3 == SLJIT_IMM) {
2839
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2840
		src3w &= 0x1f;
2841
#else /* !SLJIT_CONFIG_X86_32 */
2842
		src3w &= (op & SLJIT_32) ? 0x1f : 0x3f;
2843
#endif /* SLJIT_CONFIG_X86_32 */
2844

2845
		if (src3w == 0)
2846
			return SLJIT_SUCCESS;
2847
	}
2848

2849
	is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
2850

2851
	is_rotate = (src1_reg == src2_reg);
2852
	CHECK_EXTRA_REGS(src1_reg, src1w, (void)0);
2853
	CHECK_EXTRA_REGS(src2_reg, src2w, (void)0);
2854

2855
	if (is_rotate)
2856
		return emit_shift(compiler, is_left ? ROL : ROR, dst_reg, dstw, src1_reg, src1w, src3, src3w);
2857

2858
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
2859
	if (src2_reg & SLJIT_MEM) {
2860
		EMIT_MOV(compiler, TMP_REG1, 0, src2_reg, src2w);
2861
		src2_reg = TMP_REG1;
2862
	}
2863
#endif /* SLJIT_CONFIG_X86_32 */
2864

2865
	if (dst_reg == SLJIT_PREF_SHIFT_REG && src3 != SLJIT_IMM && (src3 != SLJIT_PREF_SHIFT_REG || src1_reg != SLJIT_PREF_SHIFT_REG)) {
2866
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2867
		EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2868
		src1_reg = TMP_REG1;
2869
		src1w = 0;
2870
#else /* !SLJIT_CONFIG_X86_64 */
2871
		if (src2_reg != TMP_REG1) {
2872
			EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2873
			src1_reg = TMP_REG1;
2874
			src1w = 0;
2875
		} else if ((src1_reg & SLJIT_MEM) || src1_reg == SLJIT_PREF_SHIFT_REG) {
2876
			restore_sp4 = (src3 == SLJIT_R0) ? SLJIT_R1 : SLJIT_R0;
2877
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), restore_sp4, 0);
2878
			EMIT_MOV(compiler, restore_sp4, 0, src1_reg, src1w);
2879
			src1_reg = restore_sp4;
2880
			src1w = 0;
2881
		} else {
2882
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), src1_reg, 0);
2883
			restore_sp4 = src1_reg;
2884
		}
2885
#endif /* SLJIT_CONFIG_X86_64 */
2886

2887
		if (src3 != SLJIT_PREF_SHIFT_REG)
2888
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src3, src3w);
2889
	} else {
2890
		if (src2_reg == SLJIT_PREF_SHIFT_REG && src3 != SLJIT_IMM && src3 != SLJIT_PREF_SHIFT_REG) {
2891
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2892
			compiler->mode32 = 0;
2893
#endif /* SLJIT_CONFIG_X86_64 */
2894
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2895
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2896
			compiler->mode32 = op & SLJIT_32;
2897
#endif /* SLJIT_CONFIG_X86_64 */
2898
			src2_reg = TMP_REG1;
2899
			restore_ecx = 1;
2900
		}
2901

2902
		move_src1 = 0;
2903
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2904
		if (dst_reg != src1_reg) {
2905
			if (dst_reg != src3) {
2906
				EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2907
				src1_reg = dst_reg;
2908
				src1w = 0;
2909
			} else
2910
				move_src1 = 1;
2911
		}
2912
#else /* !SLJIT_CONFIG_X86_64 */
2913
		if (dst_reg & SLJIT_MEM) {
2914
			if (src2_reg != TMP_REG1) {
2915
				EMIT_MOV(compiler, TMP_REG1, 0, src1_reg, src1w);
2916
				src1_reg = TMP_REG1;
2917
				src1w = 0;
2918
			} else if ((src1_reg & SLJIT_MEM) || src1_reg == SLJIT_PREF_SHIFT_REG) {
2919
				restore_sp4 = (src3 == SLJIT_R0) ? SLJIT_R1 : SLJIT_R0;
2920
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), restore_sp4, 0);
2921
				EMIT_MOV(compiler, restore_sp4, 0, src1_reg, src1w);
2922
				src1_reg = restore_sp4;
2923
				src1w = 0;
2924
			} else {
2925
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32), src1_reg, 0);
2926
				restore_sp4 = src1_reg;
2927
			}
2928
		} else if (dst_reg != src1_reg) {
2929
			if (dst_reg != src3) {
2930
				EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2931
				src1_reg = dst_reg;
2932
				src1w = 0;
2933
			} else
2934
				move_src1 = 1;
2935
		}
2936
#endif /* SLJIT_CONFIG_X86_64 */
2937

2938
		if (src3 != SLJIT_IMM && src3 != SLJIT_PREF_SHIFT_REG) {
2939
			if (!restore_ecx) {
2940
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2941
				compiler->mode32 = 0;
2942
				EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2943
				compiler->mode32 = op & SLJIT_32;
2944
				restore_ecx = 1;
2945
#else /* !SLJIT_CONFIG_X86_64 */
2946
				if (src1_reg != TMP_REG1 && src2_reg != TMP_REG1) {
2947
					EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_PREF_SHIFT_REG, 0);
2948
					restore_ecx = 1;
2949
				} else {
2950
					EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_PREF_SHIFT_REG, 0);
2951
					restore_ecx = 2;
2952
				}
2953
#endif /* SLJIT_CONFIG_X86_64 */
2954
			}
2955
			EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, src3, src3w);
2956
		}
2957

2958
		if (move_src1) {
2959
			EMIT_MOV(compiler, dst_reg, 0, src1_reg, src1w);
2960
			src1_reg = dst_reg;
2961
			src1w = 0;
2962
		}
2963
	}
2964

2965
	inst = emit_x86_instruction(compiler, 2, src2_reg, 0, src1_reg, src1w);
2966
	FAIL_IF(!inst);
2967
	inst[0] = GROUP_0F;
2968

2969
	if (src3 == SLJIT_IMM) {
2970
		inst[1] = U8((is_left ? SHLD : SHRD) - 1);
2971

2972
		/* Immediate argument is added separately. */
2973
		FAIL_IF(emit_byte(compiler, U8(src3w)));
2974
	} else
2975
		inst[1] = U8(is_left ? SHLD : SHRD);
2976

2977
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
2978
	if (restore_ecx) {
2979
		compiler->mode32 = 0;
2980
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, TMP_REG1, 0);
2981
	}
2982

2983
	if (src1_reg != dst_reg) {
2984
		compiler->mode32 = op & SLJIT_32;
2985
		return emit_mov(compiler, dst_reg, dstw, src1_reg, 0);
2986
	}
2987
#else /* !SLJIT_CONFIG_X86_64 */
2988
	if (restore_ecx)
2989
		EMIT_MOV(compiler, SLJIT_PREF_SHIFT_REG, 0, restore_ecx == 1 ? TMP_REG1 : SLJIT_MEM1(SLJIT_SP), 0);
2990

2991
	if (src1_reg != dst_reg)
2992
		EMIT_MOV(compiler, dst_reg, dstw, src1_reg, 0);
2993

2994
	if (restore_sp4)
2995
		return emit_mov(compiler, restore_sp4, 0, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_s32));
2996
#endif /* SLJIT_CONFIG_X86_32 */
2997

2998
	return SLJIT_SUCCESS;
2999
}
3000

3001
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
3002
	sljit_s32 src, sljit_sw srcw)
3003
{
3004
	CHECK_ERROR();
3005
	CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
3006
	ADJUST_LOCAL_OFFSET(src, srcw);
3007

3008
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3009

3010
	switch (op) {
3011
	case SLJIT_FAST_RETURN:
3012
		return emit_fast_return(compiler, src, srcw);
3013
	case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
3014
		/* Don't adjust shadow stack if it isn't enabled.  */
3015
		if (!cpu_has_shadow_stack ())
3016
			return SLJIT_SUCCESS;
3017
		return adjust_shadow_stack(compiler, src, srcw);
3018
	case SLJIT_PREFETCH_L1:
3019
	case SLJIT_PREFETCH_L2:
3020
	case SLJIT_PREFETCH_L3:
3021
	case SLJIT_PREFETCH_ONCE:
3022
		return emit_prefetch(compiler, op, src, srcw);
3023
	}
3024

3025
	return SLJIT_SUCCESS;
3026
}
3027

3028
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
3029
	sljit_s32 dst, sljit_sw dstw)
3030
{
3031
	CHECK_ERROR();
3032
	CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
3033
	ADJUST_LOCAL_OFFSET(dst, dstw);
3034

3035
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3036

3037
	switch (op) {
3038
	case SLJIT_FAST_ENTER:
3039
		return emit_fast_enter(compiler, dst, dstw);
3040
	case SLJIT_GET_RETURN_ADDRESS:
3041
		return sljit_emit_get_return_address(compiler, dst, dstw);
3042
	}
3043

3044
	return SLJIT_SUCCESS;
3045
}
3046

3047
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
3048
{
3049
	CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
3050

3051
	if (type == SLJIT_GP_REGISTER) {
3052
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3053
		if (reg >= SLJIT_R3 && reg <= SLJIT_R8)
3054
			return -1;
3055
#endif /* SLJIT_CONFIG_X86_32 */
3056
		return reg_map[reg];
3057
	}
3058

3059
	if (type != SLJIT_FLOAT_REGISTER && type != SLJIT_SIMD_REG_128 && type != SLJIT_SIMD_REG_256 && type != SLJIT_SIMD_REG_512)
3060
		return -1;
3061

3062
	return freg_map[reg];
3063
}
3064

3065
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
3066
	void *instruction, sljit_u32 size)
3067
{
3068
	sljit_u8 *inst;
3069

3070
	CHECK_ERROR();
3071
	CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
3072

3073
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3074
	FAIL_IF(!inst);
3075
	INC_SIZE(size);
3076
	SLJIT_MEMCPY(inst, instruction, size);
3077
	return SLJIT_SUCCESS;
3078
}
3079

3080
/* --------------------------------------------------------------------- */
3081
/*  Floating point operators                                             */
3082
/* --------------------------------------------------------------------- */
3083

3084
/* Alignment(3) + 4 * 16 bytes. */
3085
static sljit_u32 sse2_data[3 + (4 * 4)];
3086
static sljit_u32 *sse2_buffer;
3087

3088
static void init_compiler(void)
3089
{
3090
	get_cpu_features();
3091

3092
	/* Align to 16 bytes. */
3093
	sse2_buffer = (sljit_u32*)(((sljit_uw)sse2_data + 15) & ~(sljit_uw)0xf);
3094

3095
	/* Single precision constants (each constant is 16 byte long). */
3096
	sse2_buffer[0] = 0x80000000;
3097
	sse2_buffer[4] = 0x7fffffff;
3098
	/* Double precision constants (each constant is 16 byte long). */
3099
	sse2_buffer[8] = 0;
3100
	sse2_buffer[9] = 0x80000000;
3101
	sse2_buffer[12] = 0xffffffff;
3102
	sse2_buffer[13] = 0x7fffffff;
3103
}
3104

3105
static sljit_s32 emit_groupf(struct sljit_compiler *compiler,
3106
	sljit_uw op,
3107
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3108
{
3109
	sljit_u8 *inst = emit_x86_instruction(compiler, 2 | (op & ~(sljit_uw)0xff), dst, 0, src, srcw);
3110
	FAIL_IF(!inst);
3111
	inst[0] = GROUP_0F;
3112
	inst[1] = op & 0xff;
3113
	return SLJIT_SUCCESS;
3114
}
3115

3116
static sljit_s32 emit_groupf_ext(struct sljit_compiler *compiler,
3117
	sljit_uw op,
3118
	sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3119
{
3120
	sljit_u8 *inst;
3121

3122
	SLJIT_ASSERT((op & EX86_SSE2) && ((op & VEX_OP_0F38) || (op & VEX_OP_0F3A)));
3123

3124
	inst = emit_x86_instruction(compiler, 3 | (op & ~((sljit_uw)0xff | VEX_OP_0F38 | VEX_OP_0F3A)), dst, 0, src, srcw);
3125
	FAIL_IF(!inst);
3126
	inst[0] = GROUP_0F;
3127
	inst[1] = U8((op & VEX_OP_0F38) ? 0x38 : 0x3A);
3128
	inst[2] = op & 0xff;
3129
	return SLJIT_SUCCESS;
3130
}
3131

3132
static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler,
3133
	sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw)
3134
{
3135
	return emit_groupf(compiler, MOVSD_x_xm | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, dst, src, srcw);
3136
}
3137

3138
static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler,
3139
	sljit_s32 single, sljit_s32 dst, sljit_sw dstw, sljit_s32 src)
3140
{
3141
	return emit_groupf(compiler, MOVSD_xm_x | (single ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, src, dst, dstw);
3142
}
3143

3144
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
3145
	sljit_s32 dst, sljit_sw dstw,
3146
	sljit_s32 src, sljit_sw srcw)
3147
{
3148
	sljit_s32 dst_r;
3149

3150
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3151
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3152

3153
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3154
	if (GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)
3155
		compiler->mode32 = 0;
3156
#endif
3157

3158
	FAIL_IF(emit_groupf(compiler, CVTTSD2SI_r_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP2, dst_r, src, srcw));
3159

3160
	if (dst & SLJIT_MEM)
3161
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3162
	return SLJIT_SUCCESS;
3163
}
3164

3165
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
3166
	sljit_s32 dst, sljit_sw dstw,
3167
	sljit_s32 src, sljit_sw srcw)
3168
{
3169
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
3170

3171
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3172

3173
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3174
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
3175
		compiler->mode32 = 0;
3176
#endif
3177

3178
	if (src == SLJIT_IMM) {
3179
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3180
		if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32)
3181
			srcw = (sljit_s32)srcw;
3182
#endif
3183
		EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
3184
		src = TMP_REG1;
3185
		srcw = 0;
3186
	}
3187

3188
	FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, src, srcw));
3189

3190
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3191
	compiler->mode32 = 1;
3192
#endif
3193
	if (dst_r == TMP_FREG)
3194
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3195
	return SLJIT_SUCCESS;
3196
}
3197

3198
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
3199
	sljit_s32 src1, sljit_sw src1w,
3200
	sljit_s32 src2, sljit_sw src2w)
3201
{
3202
	switch (GET_FLAG_TYPE(op)) {
3203
	case SLJIT_ORDERED_EQUAL:
3204
		/* Also: SLJIT_UNORDERED_OR_NOT_EQUAL */
3205
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3206
		FAIL_IF(emit_groupf(compiler, CMPS_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, TMP_FREG, src2, src2w));
3207

3208
		/* EQ */
3209
		FAIL_IF(emit_byte(compiler, 0));
3210

3211
		src1 = TMP_FREG;
3212
		src2 = TMP_FREG;
3213
		src2w = 0;
3214
		break;
3215

3216
	case SLJIT_ORDERED_LESS:
3217
	case SLJIT_UNORDERED_OR_GREATER:
3218
		/* Also: SLJIT_UNORDERED_OR_GREATER_EQUAL, SLJIT_ORDERED_LESS_EQUAL  */
3219
		if (!FAST_IS_REG(src2)) {
3220
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src2, src2w));
3221
			src2 = TMP_FREG;
3222
		}
3223

3224
		return emit_groupf(compiler, UCOMISD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, src2, src1, src1w);
3225
	}
3226

3227
	if (!FAST_IS_REG(src1)) {
3228
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3229
		src1 = TMP_FREG;
3230
	}
3231

3232
	return emit_groupf(compiler, UCOMISD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, src1, src2, src2w);
3233
}
3234

3235
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
3236
	sljit_s32 dst, sljit_sw dstw,
3237
	sljit_s32 src, sljit_sw srcw)
3238
{
3239
	sljit_s32 dst_r;
3240
	sljit_u8 *inst;
3241

3242
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3243
	compiler->mode32 = 1;
3244
#endif
3245

3246
	CHECK_ERROR();
3247
	SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
3248

3249
	if (GET_OPCODE(op) == SLJIT_MOV_F64) {
3250
		if (FAST_IS_REG(dst))
3251
			return emit_sse2_load(compiler, op & SLJIT_32, dst, src, srcw);
3252
		if (FAST_IS_REG(src))
3253
			return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, src);
3254
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3255
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3256
	}
3257

3258
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) {
3259
		dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
3260
		if (FAST_IS_REG(src)) {
3261
			/* We overwrite the high bits of source. From SLJIT point of view,
3262
			   this is not an issue.
3263
			   Note: In SSE3, we could also use MOVDDUP and MOVSLDUP. */
3264
			FAIL_IF(emit_groupf(compiler, UNPCKLPD_x_xm | ((op & SLJIT_32) ? EX86_PREF_66 : 0) | EX86_SSE2, src, src, 0));
3265
		} else {
3266
			FAIL_IF(emit_sse2_load(compiler, !(op & SLJIT_32), TMP_FREG, src, srcw));
3267
			src = TMP_FREG;
3268
		}
3269

3270
		FAIL_IF(emit_groupf(compiler, CVTPD2PS_x_xm | ((op & SLJIT_32) ? EX86_PREF_66 : 0) | EX86_SSE2, dst_r, src, 0));
3271
		if (dst_r == TMP_FREG)
3272
			return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3273
		return SLJIT_SUCCESS;
3274
	}
3275

3276
	if (FAST_IS_REG(dst)) {
3277
		dst_r = (dst == src) ? TMP_FREG : dst;
3278

3279
		if (src & SLJIT_MEM)
3280
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3281

3282
		FAIL_IF(emit_groupf(compiler, PCMPEQD_x_xm | EX86_PREF_66 | EX86_SSE2, dst_r, dst_r, 0));
3283

3284
		inst = emit_x86_instruction(compiler, 2 | EX86_PREF_66 | EX86_SSE2_OP2, 0, 0, dst_r, 0);
3285
		inst[0] = GROUP_0F;
3286
		/* Same as PSRLD_x / PSRLQ_x */
3287
		inst[1] = (op & SLJIT_32) ? PSLLD_x_i8 : PSLLQ_x_i8;
3288

3289
		if (GET_OPCODE(op) == SLJIT_ABS_F64) {
3290
			inst[2] |= 2 << 3;
3291
			FAIL_IF(emit_byte(compiler, 1));
3292
		} else {
3293
			inst[2] |= 6 << 3;
3294
			FAIL_IF(emit_byte(compiler, ((op & SLJIT_32) ? 31 : 63)));
3295
		}
3296

3297
		if (dst_r != TMP_FREG)
3298
			dst_r = (src & SLJIT_MEM) ? TMP_FREG : src;
3299
		return emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_NEG_F64 ? XORPD_x_xm : ANDPD_x_xm) | EX86_SSE2, dst, dst_r, 0);
3300
	}
3301

3302
	FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src, srcw));
3303

3304
	switch (GET_OPCODE(op)) {
3305
	case SLJIT_NEG_F64:
3306
		FAIL_IF(emit_groupf(compiler, XORPD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3307
		break;
3308

3309
	case SLJIT_ABS_F64:
3310
		FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | EX86_SELECT_66(op) | EX86_SSE2, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer + 4 : sse2_buffer + 12)));
3311
		break;
3312
	}
3313

3314
	return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3315
}
3316

3317
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
3318
	sljit_s32 dst, sljit_sw dstw,
3319
	sljit_s32 src1, sljit_sw src1w,
3320
	sljit_s32 src2, sljit_sw src2w)
3321
{
3322
	sljit_s32 dst_r;
3323

3324
	CHECK_ERROR();
3325
	CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3326
	ADJUST_LOCAL_OFFSET(dst, dstw);
3327
	ADJUST_LOCAL_OFFSET(src1, src1w);
3328
	ADJUST_LOCAL_OFFSET(src2, src2w);
3329

3330
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3331
	compiler->mode32 = 1;
3332
#endif
3333

3334
	if (FAST_IS_REG(dst)) {
3335
		dst_r = dst;
3336
		if (dst == src1)
3337
			; /* Do nothing here. */
3338
		else if (dst == src2 && (GET_OPCODE(op) == SLJIT_ADD_F64 || GET_OPCODE(op) == SLJIT_MUL_F64)) {
3339
			/* Swap arguments. */
3340
			src2 = src1;
3341
			src2w = src1w;
3342
		} else if (dst != src2)
3343
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_r, src1, src1w));
3344
		else {
3345
			dst_r = TMP_FREG;
3346
			FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3347
		}
3348
	} else {
3349
		dst_r = TMP_FREG;
3350
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3351
	}
3352

3353
	switch (GET_OPCODE(op)) {
3354
	case SLJIT_ADD_F64:
3355
		FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3356
		break;
3357

3358
	case SLJIT_SUB_F64:
3359
		FAIL_IF(emit_groupf(compiler, SUBSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3360
		break;
3361

3362
	case SLJIT_MUL_F64:
3363
		FAIL_IF(emit_groupf(compiler, MULSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3364
		break;
3365

3366
	case SLJIT_DIV_F64:
3367
		FAIL_IF(emit_groupf(compiler, DIVSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, src2, src2w));
3368
		break;
3369
	}
3370

3371
	if (dst_r != dst)
3372
		return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
3373
	return SLJIT_SUCCESS;
3374
}
3375

3376
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3377
	sljit_s32 dst_freg,
3378
	sljit_s32 src1, sljit_sw src1w,
3379
	sljit_s32 src2, sljit_sw src2w)
3380
{
3381
	sljit_uw pref;
3382

3383
	CHECK_ERROR();
3384
	CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3385
	ADJUST_LOCAL_OFFSET(src1, src1w);
3386
	ADJUST_LOCAL_OFFSET(src2, src2w);
3387

3388
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3389
	compiler->mode32 = 1;
3390
#endif
3391

3392
	if (dst_freg == src1) {
3393
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src2, src2w));
3394
		pref = EX86_SELECT_66(op) | EX86_SSE2;
3395
		FAIL_IF(emit_groupf(compiler, XORPD_x_xm | pref, TMP_FREG, src1, src1w));
3396
		FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | pref, TMP_FREG, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3397
		return emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, TMP_FREG, 0);
3398
	}
3399

3400
	if (src1 & SLJIT_MEM) {
3401
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, TMP_FREG, src1, src1w));
3402
		src1 = TMP_FREG;
3403
		src1w = 0;
3404
	}
3405

3406
	if (dst_freg != src2)
3407
		FAIL_IF(emit_sse2_load(compiler, op & SLJIT_32, dst_freg, src2, src2w));
3408

3409
	pref = EX86_SELECT_66(op) | EX86_SSE2;
3410
	FAIL_IF(emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, src1, src1w));
3411
	FAIL_IF(emit_groupf(compiler, ANDPD_x_xm | pref, dst_freg, SLJIT_MEM0(), (sljit_sw)((op & SLJIT_32) ? sse2_buffer : sse2_buffer + 8)));
3412
	return emit_groupf(compiler, XORPD_x_xm | pref, dst_freg, src1, src1w);
3413
}
3414

3415
/* --------------------------------------------------------------------- */
3416
/*  Conditional instructions                                             */
3417
/* --------------------------------------------------------------------- */
3418

3419
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
3420
{
3421
	sljit_u8 *inst;
3422
	struct sljit_label *label;
3423

3424
	CHECK_ERROR_PTR();
3425
	CHECK_PTR(check_sljit_emit_label(compiler));
3426

3427
	if (compiler->last_label && compiler->last_label->size == compiler->size)
3428
		return compiler->last_label;
3429

3430
	label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
3431
	PTR_FAIL_IF(!label);
3432
	set_label(label, compiler);
3433

3434
	inst = (sljit_u8*)ensure_buf(compiler, 1);
3435
	PTR_FAIL_IF(!inst);
3436
	inst[0] = SLJIT_INST_LABEL;
3437

3438
	return label;
3439
}
3440

3441
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
3442
{
3443
	sljit_u8 *inst;
3444
	struct sljit_jump *jump;
3445

3446
	CHECK_ERROR_PTR();
3447
	CHECK_PTR(check_sljit_emit_jump(compiler, type));
3448

3449
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3450
	PTR_FAIL_IF_NULL(jump);
3451
	set_jump(jump, compiler, (sljit_u32)((type & SLJIT_REWRITABLE_JUMP) | ((type & 0xff) << TYPE_SHIFT)));
3452
	type &= 0xff;
3453

3454
	jump->addr = compiler->size;
3455
	/* Worst case size. */
3456
	compiler->size += (type >= SLJIT_JUMP) ? JUMP_MAX_SIZE : CJUMP_MAX_SIZE;
3457
	inst = (sljit_u8*)ensure_buf(compiler, 1);
3458
	PTR_FAIL_IF_NULL(inst);
3459

3460
	inst[0] = SLJIT_INST_JUMP;
3461
	return jump;
3462
}
3463

3464
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
3465
{
3466
	sljit_u8 *inst;
3467
	struct sljit_jump *jump;
3468

3469
	CHECK_ERROR();
3470
	CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
3471
	ADJUST_LOCAL_OFFSET(src, srcw);
3472

3473
	CHECK_EXTRA_REGS(src, srcw, (void)0);
3474

3475
	if (src == SLJIT_IMM) {
3476
		jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3477
		FAIL_IF_NULL(jump);
3478
		set_jump(jump, compiler, (sljit_u32)(JUMP_ADDR | (type << TYPE_SHIFT)));
3479
		jump->u.target = (sljit_uw)srcw;
3480

3481
		jump->addr = compiler->size;
3482
		/* Worst case size. */
3483
		compiler->size += JUMP_MAX_SIZE;
3484
		inst = (sljit_u8*)ensure_buf(compiler, 1);
3485
		FAIL_IF_NULL(inst);
3486

3487
		inst[0] = SLJIT_INST_JUMP;
3488
	} else {
3489
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3490
		/* REX_W is not necessary (src is not immediate). */
3491
		compiler->mode32 = 1;
3492
#endif
3493
		inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
3494
		FAIL_IF(!inst);
3495
		inst[0] = GROUP_FF;
3496
		inst[1] = U8(inst[1] | ((type >= SLJIT_FAST_CALL) ? CALL_rm : JMP_rm));
3497
	}
3498
	return SLJIT_SUCCESS;
3499
}
3500

3501
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
3502
	sljit_s32 dst, sljit_sw dstw,
3503
	sljit_s32 type)
3504
{
3505
	sljit_u8 *inst;
3506
	sljit_u8 cond_set;
3507
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3508
	sljit_s32 reg;
3509
	sljit_uw size;
3510
#endif /* !SLJIT_CONFIG_X86_64 */
3511
	/* ADJUST_LOCAL_OFFSET and CHECK_EXTRA_REGS might overwrite these values. */
3512
	sljit_s32 dst_save = dst;
3513
	sljit_sw dstw_save = dstw;
3514

3515
	CHECK_ERROR();
3516
	CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
3517

3518
	ADJUST_LOCAL_OFFSET(dst, dstw);
3519
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
3520

3521
	/* setcc = jcc + 0x10. */
3522
	cond_set = U8(get_jump_code((sljit_uw)type) + 0x10);
3523

3524
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3525
	if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst)) {
3526
		size = 3 + 2;
3527
		if (reg_map[TMP_REG1] >= 4)
3528
			size += 1 + 1;
3529
		else if (reg_map[dst] >= 4)
3530
			size++;
3531

3532
		inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3533
		FAIL_IF(!inst);
3534
		INC_SIZE(size);
3535
		/* Set low register to conditional flag. */
3536
		if (reg_map[TMP_REG1] >= 4)
3537
			*inst++ = (reg_map[TMP_REG1] <= 7) ? REX : REX_B;
3538

3539
		inst[0] = GROUP_0F;
3540
		inst[1] = cond_set;
3541
		inst[2] = MOD_REG | reg_lmap[TMP_REG1];
3542
		inst += 3;
3543

3544
		if (reg_map[TMP_REG1] >= 4 || reg_map[dst] >= 4)
3545
			*inst++ = U8(REX | (reg_map[TMP_REG1] <= 7 ? 0 : REX_R) | (reg_map[dst] <= 7 ? 0 : REX_B));
3546

3547
		inst[0] = OR_rm8_r8;
3548
		inst[1] = U8(MOD_REG | (reg_lmap[TMP_REG1] << 3) | reg_lmap[dst]);
3549
		return SLJIT_SUCCESS;
3550
	}
3551

3552
	reg = (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG1;
3553

3554
	size = 3 + (reg_map[reg] >= 4) + 4;
3555
	inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
3556
	FAIL_IF(!inst);
3557
	INC_SIZE(size);
3558
	/* Set low register to conditional flag. */
3559

3560
	if (reg_map[reg] >= 4)
3561
		*inst++ = (reg_map[reg] <= 7) ? REX : REX_B;
3562

3563
	inst[0] = GROUP_0F;
3564
	inst[1] = cond_set;
3565
	inst[2] = MOD_REG | reg_lmap[reg];
3566

3567
	inst[3] = REX_W | (reg_map[reg] <= 7 ? 0 : (REX_B | REX_R));
3568
	/* The movzx instruction does not affect flags. */
3569
	inst[4] = GROUP_0F;
3570
	inst[5] = MOVZX_r_rm8;
3571
	inst[6] = U8(MOD_REG | (reg_lmap[reg] << 3) | reg_lmap[reg]);
3572

3573
	if (reg != TMP_REG1)
3574
		return SLJIT_SUCCESS;
3575

3576
	if (GET_OPCODE(op) < SLJIT_ADD) {
3577
		compiler->mode32 = GET_OPCODE(op) != SLJIT_MOV;
3578
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3579
	}
3580

3581
	SLJIT_SKIP_CHECKS(compiler);
3582
	return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
3583

3584
#else /* !SLJIT_CONFIG_X86_64 */
3585
	SLJIT_ASSERT(reg_map[TMP_REG1] < 4);
3586

3587
	/* The SLJIT_CONFIG_X86_32 code path starts here. */
3588
	if (GET_OPCODE(op) < SLJIT_ADD && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
3589
		/* Low byte is accessible. */
3590
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
3591
		FAIL_IF(!inst);
3592
		INC_SIZE(3 + 3);
3593
		/* Set low byte to conditional flag. */
3594
		inst[0] = GROUP_0F;
3595
		inst[1] = cond_set;
3596
		inst[2] = U8(MOD_REG | reg_map[dst]);
3597

3598
		inst[3] = GROUP_0F;
3599
		inst[4] = MOVZX_r_rm8;
3600
		inst[5] = U8(MOD_REG | (reg_map[dst] << 3) | reg_map[dst]);
3601
		return SLJIT_SUCCESS;
3602
	}
3603

3604
	if (GET_OPCODE(op) == SLJIT_OR && !GET_ALL_FLAGS(op) && FAST_IS_REG(dst) && reg_map[dst] <= 4) {
3605
		inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 2);
3606
		FAIL_IF(!inst);
3607
		INC_SIZE(3 + 2);
3608

3609
		/* Set low byte to conditional flag. */
3610
		inst[0] = GROUP_0F;
3611
		inst[1] = cond_set;
3612
		inst[2] = U8(MOD_REG | reg_map[TMP_REG1]);
3613

3614
		inst[3] = OR_rm8_r8;
3615
		inst[4] = U8(MOD_REG | (reg_map[TMP_REG1] << 3) | reg_map[dst]);
3616
		return SLJIT_SUCCESS;
3617
	}
3618

3619
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 3 + 3);
3620
	FAIL_IF(!inst);
3621
	INC_SIZE(3 + 3);
3622
	/* Set low byte to conditional flag. */
3623
	inst[0] = GROUP_0F;
3624
	inst[1] = cond_set;
3625
	inst[2] = U8(MOD_REG | reg_map[TMP_REG1]);
3626

3627
	inst[3] = GROUP_0F;
3628
	inst[4] = MOVZX_r_rm8;
3629
	inst[5] = U8(MOD_REG | (reg_map[TMP_REG1] << 3) | reg_map[TMP_REG1]);
3630

3631
	if (GET_OPCODE(op) < SLJIT_ADD)
3632
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
3633

3634
	SLJIT_SKIP_CHECKS(compiler);
3635
	return sljit_emit_op2(compiler, op, dst_save, dstw_save, dst_save, dstw_save, TMP_REG1, 0);
3636
#endif /* SLJIT_CONFIG_X86_64 */
3637
}
3638

3639
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
3640
	sljit_s32 dst_freg,
3641
	sljit_s32 src1, sljit_sw src1w,
3642
	sljit_s32 src2_freg)
3643
{
3644
	sljit_u8* inst;
3645
	sljit_uw size;
3646

3647
	CHECK_ERROR();
3648
	CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
3649

3650
	ADJUST_LOCAL_OFFSET(src1, src1w);
3651

3652
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3653
	compiler->mode32 = 1;
3654
#endif /* SLJIT_CONFIG_X86_64 */
3655

3656
	if (dst_freg != src2_freg) {
3657
		if (dst_freg == src1) {
3658
			src1 = src2_freg;
3659
			src1w = 0;
3660
			type ^= 0x1;
3661
		} else
3662
			FAIL_IF(emit_sse2_load(compiler, type & SLJIT_32, dst_freg, src2_freg, 0));
3663
	}
3664

3665
	inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
3666
	FAIL_IF(!inst);
3667
	INC_SIZE(2);
3668
	inst[0] = U8(get_jump_code((sljit_uw)(type & ~SLJIT_32) ^ 0x1) - 0x10);
3669

3670
	size = compiler->size;
3671
	FAIL_IF(emit_sse2_load(compiler, type & SLJIT_32, dst_freg, src1, src1w));
3672

3673
	inst[1] = U8(compiler->size - size);
3674
	return SLJIT_SUCCESS;
3675
}
3676

3677
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3678
	sljit_s32 vreg,
3679
	sljit_s32 srcdst, sljit_sw srcdstw)
3680
{
3681
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3682
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3683
	sljit_s32 alignment = SLJIT_SIMD_GET_ELEM2_SIZE(type);
3684
	sljit_uw op;
3685

3686
	CHECK_ERROR();
3687
	CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3688

3689
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3690

3691
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3692
	compiler->mode32 = 1;
3693
#endif /* SLJIT_CONFIG_X86_64 */
3694

3695
	switch (reg_size) {
3696
	case 4:
3697
		op = EX86_SSE2;
3698
		break;
3699
	case 5:
3700
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
3701
			return SLJIT_ERR_UNSUPPORTED;
3702
		op = EX86_SSE2 | VEX_256;
3703
		break;
3704
	default:
3705
		return SLJIT_ERR_UNSUPPORTED;
3706
	}
3707

3708
	if (!(srcdst & SLJIT_MEM))
3709
		alignment = reg_size;
3710

3711
	if (type & SLJIT_SIMD_FLOAT) {
3712
		if (elem_size == 2 || elem_size == 3) {
3713
			op |= alignment >= reg_size ? MOVAPS_x_xm : MOVUPS_x_xm;
3714

3715
			if (elem_size == 3)
3716
				op |= EX86_PREF_66;
3717

3718
			if (type & SLJIT_SIMD_STORE)
3719
				op += 1;
3720
		} else
3721
			return SLJIT_ERR_UNSUPPORTED;
3722
	} else {
3723
		op |= ((type & SLJIT_SIMD_STORE) ? MOVDQA_xm_x : MOVDQA_x_xm)
3724
			| (alignment >= reg_size ? EX86_PREF_66 : EX86_PREF_F3);
3725
	}
3726

3727
	if (type & SLJIT_SIMD_TEST)
3728
		return SLJIT_SUCCESS;
3729

3730
	if ((op & VEX_256) || ((cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX)))
3731
		return emit_vex_instruction(compiler, op, vreg, 0, srcdst, srcdstw);
3732

3733
	return emit_groupf(compiler, op, vreg, srcdst, srcdstw);
3734
}
3735

3736
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3737
	sljit_s32 vreg,
3738
	sljit_s32 src, sljit_sw srcw)
3739
{
3740
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3741
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3742
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
3743
	sljit_u8 *inst;
3744
	sljit_u8 opcode = 0;
3745
	sljit_uw op;
3746

3747
	CHECK_ERROR();
3748
	CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3749

3750
	ADJUST_LOCAL_OFFSET(src, srcw);
3751

3752
	if (!(type & SLJIT_SIMD_FLOAT)) {
3753
		CHECK_EXTRA_REGS(src, srcw, (void)0);
3754
	}
3755

3756
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3757
	if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : (elem_size > 2))
3758
		return SLJIT_ERR_UNSUPPORTED;
3759
#else /* !SLJIT_CONFIG_X86_32 */
3760
	compiler->mode32 = 1;
3761

3762
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
3763
		return SLJIT_ERR_UNSUPPORTED;
3764
#endif /* SLJIT_CONFIG_X86_32 */
3765

3766
	if (reg_size != 4 && (reg_size != 5 || !(cpu_feature_list & CPU_FEATURE_AVX2)))
3767
		return SLJIT_ERR_UNSUPPORTED;
3768

3769
	if (type & SLJIT_SIMD_TEST)
3770
		return SLJIT_SUCCESS;
3771

3772
	if (reg_size == 5)
3773
		use_vex = 1;
3774

3775
	if (use_vex && src != SLJIT_IMM) {
3776
		op = 0;
3777

3778
		switch (elem_size) {
3779
		case 0:
3780
			if (cpu_feature_list & CPU_FEATURE_AVX2)
3781
				op = VPBROADCASTB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3782
			break;
3783
		case 1:
3784
			if (cpu_feature_list & CPU_FEATURE_AVX2)
3785
				op = VPBROADCASTW_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3786
			break;
3787
		case 2:
3788
			if (type & SLJIT_SIMD_FLOAT) {
3789
				if ((cpu_feature_list & CPU_FEATURE_AVX2) || ((cpu_feature_list & CPU_FEATURE_AVX) && (src & SLJIT_MEM)))
3790
					op = VBROADCASTSS_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3791
			} else if (cpu_feature_list & CPU_FEATURE_AVX2)
3792
				op = VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3793
			break;
3794
		default:
3795
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3796
			if (!(type & SLJIT_SIMD_FLOAT)) {
3797
				if (cpu_feature_list & CPU_FEATURE_AVX2)
3798
					op = VPBROADCASTQ_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3799
				break;
3800
			}
3801
#endif /* SLJIT_CONFIG_X86_64 */
3802

3803
			if (reg_size == 5)
3804
				op = VBROADCASTSD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
3805
			break;
3806
		}
3807

3808
		if (op != 0) {
3809
			if (!(src & SLJIT_MEM) && !(type & SLJIT_SIMD_FLOAT)) {
3810
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3811
				if (elem_size >= 3)
3812
					compiler->mode32 = 0;
3813
#endif /* SLJIT_CONFIG_X86_64 */
3814
				FAIL_IF(emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw));
3815
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3816
				compiler->mode32 = 1;
3817
#endif /* SLJIT_CONFIG_X86_64 */
3818
				src = vreg;
3819
				srcw = 0;
3820
			}
3821

3822
			if (reg_size == 5)
3823
				op |= VEX_256;
3824

3825
			return emit_vex_instruction(compiler, op, vreg, 0, src, srcw);
3826
		}
3827
	}
3828

3829
	if (type & SLJIT_SIMD_FLOAT) {
3830
		if (src == SLJIT_IMM) {
3831
			if (use_vex)
3832
				return emit_vex_instruction(compiler, XORPD_x_xm | (reg_size == 5 ? VEX_256 : 0) | (elem_size == 3 ? EX86_PREF_66 : 0) | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0);
3833

3834
			return emit_groupf(compiler, XORPD_x_xm | (elem_size == 3 ? EX86_PREF_66 : 0) | EX86_SSE2, vreg, vreg, 0);
3835
		}
3836

3837
		SLJIT_ASSERT(reg_size == 4);
3838

3839
		if (use_vex) {
3840
			if (elem_size == 3)
3841
				return emit_vex_instruction(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, src, srcw);
3842

3843
			SLJIT_ASSERT(!(src & SLJIT_MEM));
3844
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | EX86_SSE2 | VEX_SSE2_OPV, vreg, src, src, 0));
3845
			return emit_byte(compiler, 0);
3846
		}
3847

3848
		if (elem_size == 2 && vreg != src) {
3849
			FAIL_IF(emit_sse2_load(compiler, 1, vreg, src, srcw));
3850
			src = vreg;
3851
			srcw = 0;
3852
		}
3853

3854
		op = (elem_size == 2 ? SHUFPS_x_xm : MOVDDUP_x_xm) | (elem_size == 2 ? 0 : EX86_PREF_F2) | EX86_SSE2;
3855
		FAIL_IF(emit_groupf(compiler, op, vreg, src, srcw));
3856

3857
		if (elem_size == 2)
3858
			return emit_byte(compiler, 0);
3859
		return SLJIT_SUCCESS;
3860
	}
3861

3862
	if (src == SLJIT_IMM) {
3863
		if (elem_size == 0) {
3864
			srcw = (sljit_u8)srcw;
3865
			srcw |= srcw << 8;
3866
			srcw |= srcw << 16;
3867
			elem_size = 2;
3868
		} else if (elem_size == 1) {
3869
			srcw = (sljit_u16)srcw;
3870
			srcw |= srcw << 16;
3871
			elem_size = 2;
3872
		}
3873

3874
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3875
		if (elem_size == 2 && (sljit_s32)srcw == -1)
3876
			srcw = -1;
3877
#endif /* SLJIT_CONFIG_X86_64 */
3878

3879
		if (srcw == 0 || srcw == -1) {
3880
			if (use_vex)
3881
				return emit_vex_instruction(compiler, (srcw == 0 ? PXOR_x_xm : PCMPEQD_x_xm) | (reg_size == 5 ? VEX_256 : 0) | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0);
3882

3883
			return emit_groupf(compiler, (srcw == 0 ? PXOR_x_xm : PCMPEQD_x_xm) | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0);
3884
		}
3885

3886
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3887
		if (elem_size == 3)
3888
			FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
3889
		else
3890
#endif /* SLJIT_CONFIG_X86_64 */
3891
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcw);
3892

3893
		src = TMP_REG1;
3894
		srcw = 0;
3895

3896
	}
3897

3898
	op = 2;
3899
	opcode = MOVD_x_rm;
3900

3901
	switch (elem_size) {
3902
	case 0:
3903
		if (!FAST_IS_REG(src)) {
3904
			opcode = 0x3a /* Prefix of PINSRB_x_rm_i8. */;
3905
			op = 3;
3906
		}
3907
		break;
3908
	case 1:
3909
		if (!FAST_IS_REG(src))
3910
			opcode = PINSRW_x_rm_i8;
3911
		break;
3912
	case 2:
3913
		break;
3914
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3915
	case 3:
3916
		/* MOVQ */
3917
		compiler->mode32 = 0;
3918
		break;
3919
#endif /* SLJIT_CONFIG_X86_64 */
3920
	}
3921

3922
	if (use_vex) {
3923
		if (opcode != MOVD_x_rm) {
3924
			op = (opcode == 0x3a) ? (PINSRB_x_rm_i8 | VEX_OP_0F3A) : opcode;
3925
			FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1 | VEX_SSE2_OPV, vreg, vreg, src, srcw));
3926
		} else
3927
			FAIL_IF(emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw));
3928
	} else {
3929
		inst = emit_x86_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, src, srcw);
3930
		FAIL_IF(!inst);
3931
		inst[0] = GROUP_0F;
3932
		inst[1] = opcode;
3933

3934
		if (op == 3) {
3935
			SLJIT_ASSERT(opcode == 0x3a);
3936
			inst[2] = PINSRB_x_rm_i8;
3937
		}
3938
	}
3939

3940
	if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && elem_size >= 2) {
3941
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3942
		op = VPBROADCASTD_x_xm;
3943
#else /* !SLJIT_CONFIG_X86_32 */
3944
		op = (elem_size == 3) ? VPBROADCASTQ_x_xm : VPBROADCASTD_x_xm;
3945
#endif /* SLJIT_CONFIG_X86_32 */
3946
		return emit_vex_instruction(compiler, op | ((reg_size == 5) ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
3947
	}
3948

3949
	SLJIT_ASSERT(reg_size == 4);
3950

3951
	if (opcode != MOVD_x_rm)
3952
		FAIL_IF(emit_byte(compiler, 0));
3953

3954
	switch (elem_size) {
3955
	case 0:
3956
		if (use_vex) {
3957
			FAIL_IF(emit_vex_instruction(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, TMP_FREG, 0));
3958
			return emit_vex_instruction(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, TMP_FREG, 0);
3959
		}
3960
		FAIL_IF(emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, TMP_FREG, 0));
3961
		return emit_groupf_ext(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, TMP_FREG, 0);
3962
	case 1:
3963
		if (use_vex)
3964
			FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, vreg, 0));
3965
		else
3966
			FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, vreg, 0));
3967
		FAIL_IF(emit_byte(compiler, 0));
3968
		/* fallthrough */
3969
	default:
3970
		if (use_vex)
3971
			FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, vreg, 0));
3972
		else
3973
			FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0));
3974
		return emit_byte(compiler, 0);
3975
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
3976
	case 3:
3977
		compiler->mode32 = 1;
3978
		if (use_vex)
3979
			FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, vreg, 0));
3980
		else
3981
			FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, vreg, 0));
3982
		return emit_byte(compiler, 0x44);
3983
#endif /* SLJIT_CONFIG_X86_64 */
3984
	}
3985
}
3986

3987
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3988
	sljit_s32 vreg, sljit_s32 lane_index,
3989
	sljit_s32 srcdst, sljit_sw srcdstw)
3990
{
3991
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3992
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3993
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
3994
	sljit_u8 *inst;
3995
	sljit_u8 opcode = 0;
3996
	sljit_uw op;
3997
	sljit_s32 vreg_orig = vreg;
3998
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
3999
	sljit_s32 srcdst_is_ereg = 0;
4000
	sljit_s32 srcdst_orig = 0;
4001
	sljit_sw srcdstw_orig = 0;
4002
#endif /* SLJIT_CONFIG_X86_32 */
4003

4004
	CHECK_ERROR();
4005
	CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
4006

4007
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
4008

4009
	if (reg_size == 5) {
4010
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4011
			return SLJIT_ERR_UNSUPPORTED;
4012
		use_vex = 1;
4013
	} else if (reg_size != 4)
4014
		return SLJIT_ERR_UNSUPPORTED;
4015

4016
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4017
	if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : elem_size > 2)
4018
		return SLJIT_ERR_UNSUPPORTED;
4019
#else /* SLJIT_CONFIG_X86_32 */
4020
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
4021
		return SLJIT_ERR_UNSUPPORTED;
4022
#endif /* SLJIT_CONFIG_X86_32 */
4023

4024
	if (type & SLJIT_SIMD_TEST)
4025
		return SLJIT_SUCCESS;
4026

4027
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4028
	compiler->mode32 = 1;
4029
#else /* !SLJIT_CONFIG_X86_64 */
4030
	if (!(type & SLJIT_SIMD_FLOAT)) {
4031
		CHECK_EXTRA_REGS(srcdst, srcdstw, srcdst_is_ereg = 1);
4032

4033
		if ((type & SLJIT_SIMD_STORE) && ((srcdst_is_ereg && elem_size < 2) || (elem_size == 0 && (type & SLJIT_SIMD_LANE_SIGNED) && FAST_IS_REG(srcdst) && reg_map[srcdst] >= 4))) {
4034
			srcdst_orig = srcdst;
4035
			srcdstw_orig = srcdstw;
4036
			srcdst = TMP_REG1;
4037
			srcdstw = 0;
4038
		}
4039
	}
4040
#endif /* SLJIT_CONFIG_X86_64 */
4041

4042
	if (type & SLJIT_SIMD_LANE_ZERO) {
4043
		if (lane_index == 0) {
4044
			if (!(type & SLJIT_SIMD_FLOAT)) {
4045
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4046
				if (elem_size == 3) {
4047
					compiler->mode32 = 0;
4048
					elem_size = 2;
4049
				}
4050
#endif /* SLJIT_CONFIG_X86_64 */
4051
				if (srcdst == SLJIT_IMM) {
4052
					if (elem_size == 0)
4053
						srcdstw = (sljit_u8)srcdstw;
4054
					else if (elem_size == 1)
4055
						srcdstw = (sljit_u16)srcdstw;
4056

4057
					EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcdstw);
4058
					srcdst = TMP_REG1;
4059
					srcdstw = 0;
4060
					elem_size = 2;
4061
				}
4062

4063
				if (elem_size == 2) {
4064
					if (use_vex)
4065
						return emit_vex_instruction(compiler, MOVD_x_rm | VEX_AUTO_W | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, srcdst, srcdstw);
4066
					return emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, vreg, srcdst, srcdstw);
4067
				}
4068
			} else if (srcdst & SLJIT_MEM) {
4069
				SLJIT_ASSERT(elem_size == 2 || elem_size == 3);
4070

4071
				if (use_vex)
4072
					return emit_vex_instruction(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, vreg, 0, srcdst, srcdstw);
4073
				return emit_groupf(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2, vreg, srcdst, srcdstw);
4074
			} else if (elem_size == 3) {
4075
				if (use_vex)
4076
					return emit_vex_instruction(compiler, MOVQ_x_xm | EX86_PREF_F3 | EX86_SSE2, vreg, 0, srcdst, 0);
4077
				return emit_groupf(compiler, MOVQ_x_xm | EX86_PREF_F3 | EX86_SSE2, vreg, srcdst, 0);
4078
			} else if (use_vex) {
4079
				FAIL_IF(emit_vex_instruction(compiler, XORPD_x_xm | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, TMP_FREG, 0));
4080
				return emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F3 | EX86_SSE2 | VEX_SSE2_OPV, vreg, TMP_FREG, srcdst, 0);
4081
			}
4082
		}
4083

4084
		if (reg_size == 5 && lane_index >= (1 << (4 - elem_size))) {
4085
			vreg = TMP_FREG;
4086
			lane_index -= (1 << (4 - elem_size));
4087
		} else if ((type & SLJIT_SIMD_FLOAT) && vreg == srcdst) {
4088
			if (use_vex)
4089
				FAIL_IF(emit_vex_instruction(compiler, MOVSD_x_xm | (elem_size == 2 ? EX86_PREF_F3 : EX86_PREF_F2) | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, TMP_FREG, srcdst, srcdstw));
4090
			else
4091
				FAIL_IF(emit_sse2_load(compiler, elem_size == 2, TMP_FREG, srcdst, srcdstw));
4092
			srcdst = TMP_FREG;
4093
			srcdstw = 0;
4094
		}
4095

4096
		op = ((!(type & SLJIT_SIMD_FLOAT) || elem_size != 2) ? EX86_PREF_66 : 0)
4097
			| ((type & SLJIT_SIMD_FLOAT) ? XORPD_x_xm : PXOR_x_xm) | EX86_SSE2;
4098

4099
		if (use_vex)
4100
			FAIL_IF(emit_vex_instruction(compiler, op | (reg_size == 5 ? VEX_256 : 0) | VEX_SSE2_OPV, vreg, vreg, vreg, 0));
4101
		else
4102
			FAIL_IF(emit_groupf(compiler, op, vreg, vreg, 0));
4103
	} else if (reg_size == 5 && lane_index >= (1 << (4 - elem_size))) {
4104
		FAIL_IF(emit_vex_instruction(compiler, ((type & SLJIT_SIMD_FLOAT) ? VEXTRACTF128_x_ym : VEXTRACTI128_x_ym) | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, TMP_FREG, 0));
4105
		FAIL_IF(emit_byte(compiler, 1));
4106

4107
		vreg = TMP_FREG;
4108
		lane_index -= (1 << (4 - elem_size));
4109
	}
4110

4111
	if (type & SLJIT_SIMD_FLOAT) {
4112
		if (elem_size == 3) {
4113
			if (srcdst & SLJIT_MEM) {
4114
				if (type & SLJIT_SIMD_STORE)
4115
					op = lane_index == 0 ? MOVLPD_m_x : MOVHPD_m_x;
4116
				else
4117
					op = lane_index == 0 ? MOVLPD_x_m : MOVHPD_x_m;
4118

4119
				/* VEX prefix clears upper bits of the target register. */
4120
				if (use_vex && ((type & SLJIT_SIMD_STORE) || reg_size == 4 || vreg == TMP_FREG))
4121
					FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2
4122
						| ((type & SLJIT_SIMD_STORE) ? 0 : VEX_SSE2_OPV), vreg, (type & SLJIT_SIMD_STORE) ? 0 : vreg, srcdst, srcdstw));
4123
				else
4124
					FAIL_IF(emit_groupf(compiler, op | EX86_PREF_66 | EX86_SSE2, vreg, srcdst, srcdstw));
4125

4126
				/* In case of store, vreg is not TMP_FREG. */
4127
			} else if (type & SLJIT_SIMD_STORE) {
4128
				if (lane_index == 1) {
4129
					if (use_vex)
4130
						return emit_vex_instruction(compiler, MOVHLPS_x_x | EX86_SSE2 | VEX_SSE2_OPV, srcdst, srcdst, vreg, 0);
4131
					return emit_groupf(compiler, MOVHLPS_x_x | EX86_SSE2, srcdst, vreg, 0);
4132
				}
4133
				if (use_vex)
4134
					return emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F2 | EX86_SSE2 | VEX_SSE2_OPV, srcdst, srcdst, vreg, 0);
4135
				return emit_sse2_load(compiler, 0, srcdst, vreg, 0);
4136
			} else if (use_vex && (reg_size == 4 || vreg == TMP_FREG)) {
4137
				if (lane_index == 1)
4138
					FAIL_IF(emit_vex_instruction(compiler, MOVLHPS_x_x | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, srcdst, 0));
4139
				else
4140
					FAIL_IF(emit_vex_instruction(compiler, MOVSD_x_xm | EX86_PREF_F2 | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, srcdst, 0));
4141
			} else {
4142
				if (lane_index == 1)
4143
					FAIL_IF(emit_groupf(compiler, MOVLHPS_x_x | EX86_SSE2, vreg, srcdst, 0));
4144
				else
4145
					FAIL_IF(emit_sse2_load(compiler, 0, vreg, srcdst, 0));
4146
			}
4147
		} else if (type & SLJIT_SIMD_STORE) {
4148
			if (lane_index == 0) {
4149
				if (use_vex)
4150
					return emit_vex_instruction(compiler, MOVSD_xm_x | EX86_PREF_F3 | EX86_SSE2 | ((srcdst & SLJIT_MEM) ? 0 : VEX_SSE2_OPV),
4151
						vreg, ((srcdst & SLJIT_MEM) ? 0 : srcdst), srcdst, srcdstw);
4152
				return emit_sse2_store(compiler, 1, srcdst, srcdstw, vreg);
4153
			}
4154

4155
			if (srcdst & SLJIT_MEM) {
4156
				if (use_vex)
4157
					FAIL_IF(emit_vex_instruction(compiler, EXTRACTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, srcdst, srcdstw));
4158
				else
4159
					FAIL_IF(emit_groupf_ext(compiler, EXTRACTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, srcdst, srcdstw));
4160
				return emit_byte(compiler, U8(lane_index));
4161
			}
4162

4163
			if (use_vex) {
4164
				FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | EX86_SSE2 | VEX_SSE2_OPV, srcdst, vreg, vreg, 0));
4165
				return emit_byte(compiler, U8(lane_index));
4166
			}
4167

4168
			if (srcdst == vreg)
4169
				op = SHUFPS_x_xm | EX86_SSE2;
4170
			else {
4171
				switch (lane_index) {
4172
				case 1:
4173
					op = MOVSHDUP_x_xm | EX86_PREF_F3 | EX86_SSE2;
4174
					break;
4175
				case 2:
4176
					op = MOVHLPS_x_x | EX86_SSE2;
4177
					break;
4178
				default:
4179
					SLJIT_ASSERT(lane_index == 3);
4180
					op = PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2;
4181
					break;
4182
				}
4183
			}
4184

4185
			FAIL_IF(emit_groupf(compiler, op, srcdst, vreg, 0));
4186

4187
			op &= 0xff;
4188
			if (op == SHUFPS_x_xm || op == PSHUFD_x_xm)
4189
				return emit_byte(compiler, U8(lane_index));
4190

4191
			return SLJIT_SUCCESS;
4192
		} else {
4193
			if (lane_index != 0 || (srcdst & SLJIT_MEM)) {
4194
				FAIL_IF(emit_groupf_ext(compiler, INSERTPS_x_xm | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, srcdst, srcdstw));
4195
				FAIL_IF(emit_byte(compiler, U8(lane_index << 4)));
4196
			} else
4197
				FAIL_IF(emit_sse2_store(compiler, 1, vreg, 0, srcdst));
4198
		}
4199

4200
		if (vreg != TMP_FREG || (type & SLJIT_SIMD_STORE))
4201
			return SLJIT_SUCCESS;
4202

4203
		SLJIT_ASSERT(reg_size == 5);
4204

4205
		if (type & SLJIT_SIMD_LANE_ZERO) {
4206
			FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg_orig, 0, TMP_FREG, 0));
4207
			return emit_byte(compiler, 0x4e);
4208
		}
4209

4210
		FAIL_IF(emit_vex_instruction(compiler, VINSERTF128_y_y_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2 | VEX_SSE2_OPV, vreg_orig, vreg_orig, TMP_FREG, 0));
4211
		return emit_byte(compiler, 1);
4212
	}
4213

4214
	if (srcdst == SLJIT_IMM) {
4215
		EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, srcdstw);
4216
		srcdst = TMP_REG1;
4217
		srcdstw = 0;
4218
	}
4219

4220
	op = 3;
4221

4222
	switch (elem_size) {
4223
	case 0:
4224
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRB_rm_x_i8 : PINSRB_x_rm_i8;
4225
		break;
4226
	case 1:
4227
		if (!(type & SLJIT_SIMD_STORE)) {
4228
			op = 2;
4229
			opcode = PINSRW_x_rm_i8;
4230
		} else
4231
			opcode = PEXTRW_rm_x_i8;
4232
		break;
4233
	case 2:
4234
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRD_rm_x_i8 : PINSRD_x_rm_i8;
4235
		break;
4236
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4237
	case 3:
4238
		/* PINSRQ / PEXTRQ */
4239
		opcode = (type & SLJIT_SIMD_STORE) ? PEXTRD_rm_x_i8 : PINSRD_x_rm_i8;
4240
		compiler->mode32 = 0;
4241
		break;
4242
#endif /* SLJIT_CONFIG_X86_64 */
4243
	}
4244

4245
	if (use_vex && (type & SLJIT_SIMD_STORE)) {
4246
		op = opcode | ((op == 3) ? VEX_OP_0F3A : 0);
4247
		FAIL_IF(emit_vex_instruction(compiler, op | EX86_PREF_66 | VEX_AUTO_W | EX86_SSE2_OP1 | VEX_SSE2_OPV, vreg, 0, srcdst, srcdstw));
4248
	} else {
4249
		inst = emit_x86_instruction(compiler, op | EX86_PREF_66 | EX86_SSE2_OP1, vreg, 0, srcdst, srcdstw);
4250
		FAIL_IF(!inst);
4251
		inst[0] = GROUP_0F;
4252

4253
		if (op == 3) {
4254
			inst[1] = 0x3a;
4255
			inst[2] = opcode;
4256
		} else
4257
			inst[1] = opcode;
4258
	}
4259

4260
	FAIL_IF(emit_byte(compiler, U8(lane_index)));
4261

4262
	if (!(type & SLJIT_SIMD_LANE_SIGNED) || (srcdst & SLJIT_MEM)) {
4263
		if (vreg == TMP_FREG && !(type & SLJIT_SIMD_STORE)) {
4264
			SLJIT_ASSERT(reg_size == 5);
4265

4266
			if (type & SLJIT_SIMD_LANE_ZERO) {
4267
				FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg_orig, 0, TMP_FREG, 0));
4268
				return emit_byte(compiler, 0x4e);
4269
			}
4270

4271
			FAIL_IF(emit_vex_instruction(compiler, VINSERTI128_y_y_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2 | VEX_SSE2_OPV, vreg_orig, vreg_orig, TMP_FREG, 0));
4272
			return emit_byte(compiler, 1);
4273
		}
4274

4275
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4276
		if (srcdst_orig & SLJIT_MEM)
4277
			return emit_mov(compiler, srcdst_orig, srcdstw_orig, TMP_REG1, 0);
4278
#endif /* SLJIT_CONFIG_X86_32 */
4279
		return SLJIT_SUCCESS;
4280
	}
4281

4282
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4283
	if (elem_size >= 3)
4284
		return SLJIT_SUCCESS;
4285

4286
	compiler->mode32 = (type & SLJIT_32);
4287

4288
	op = 2;
4289

4290
	if (elem_size == 0)
4291
		op |= EX86_REX;
4292

4293
	if (elem_size == 2) {
4294
		if (type & SLJIT_32)
4295
			return SLJIT_SUCCESS;
4296

4297
		SLJIT_ASSERT(!(compiler->mode32));
4298
		op = 1;
4299
	}
4300

4301
	inst = emit_x86_instruction(compiler, op, srcdst, 0, srcdst, 0);
4302
	FAIL_IF(!inst);
4303

4304
	if (op != 1) {
4305
		inst[0] = GROUP_0F;
4306
		inst[1] = U8((elem_size == 0) ? MOVSX_r_rm8 : MOVSX_r_rm16);
4307
	} else
4308
		inst[0] = MOVSXD_r_rm;
4309
#else /* !SLJIT_CONFIG_X86_64 */
4310
	if (elem_size >= 2)
4311
		return SLJIT_SUCCESS;
4312

4313
	FAIL_IF(emit_groupf(compiler, (elem_size == 0) ? MOVSX_r_rm8 : MOVSX_r_rm16,
4314
		(srcdst_orig != 0 && FAST_IS_REG(srcdst_orig)) ? srcdst_orig : srcdst, srcdst, 0));
4315

4316
	if (srcdst_orig & SLJIT_MEM)
4317
		return emit_mov(compiler, srcdst_orig, srcdstw_orig, TMP_REG1, 0);
4318
#endif /* SLJIT_CONFIG_X86_64 */
4319
	return SLJIT_SUCCESS;
4320
}
4321

4322
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
4323
	sljit_s32 vreg,
4324
	sljit_s32 src, sljit_s32 src_lane_index)
4325
{
4326
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4327
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4328
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4329
	sljit_uw pref;
4330
	sljit_u8 byte;
4331
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4332
	sljit_s32 opcode3 = TMP_REG1;
4333
#else /* !SLJIT_CONFIG_X86_32 */
4334
	sljit_s32 opcode3 = SLJIT_S0;
4335
#endif /* SLJIT_CONFIG_X86_32 */
4336

4337
	CHECK_ERROR();
4338
	CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
4339

4340
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4341
	compiler->mode32 = 1;
4342
#endif /* SLJIT_CONFIG_X86_64 */
4343
	SLJIT_ASSERT(reg_map[opcode3] == 3);
4344

4345
	if (reg_size == 5) {
4346
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4347
			return SLJIT_ERR_UNSUPPORTED;
4348
		use_vex = 1;
4349
	} else if (reg_size != 4)
4350
		return SLJIT_ERR_UNSUPPORTED;
4351

4352
	if (type & SLJIT_SIMD_FLOAT) {
4353
		pref = 0;
4354
		byte = U8(src_lane_index);
4355

4356
		if (elem_size == 3) {
4357
			if (type & SLJIT_SIMD_TEST)
4358
				return SLJIT_SUCCESS;
4359

4360
			if (reg_size == 5) {
4361
				if (src_lane_index == 0)
4362
					return emit_vex_instruction(compiler, VBROADCASTSD_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4363

4364
				FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4365

4366
				byte = U8(byte | (byte << 2));
4367
				return emit_byte(compiler, U8(byte | (byte << 4)));
4368
			}
4369

4370
			if (src_lane_index == 0) {
4371
				if (use_vex)
4372
					return emit_vex_instruction(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, 0, src, 0);
4373
				return emit_groupf(compiler, MOVDDUP_x_xm | EX86_PREF_F2 | EX86_SSE2, vreg, src, 0);
4374
			}
4375

4376
			/* Changes it to SHUFPD_x_xm. */
4377
			pref = EX86_PREF_66;
4378
		} else if (elem_size != 2)
4379
			return SLJIT_ERR_UNSUPPORTED;
4380
		else if (type & SLJIT_SIMD_TEST)
4381
			return SLJIT_SUCCESS;
4382

4383
		if (reg_size == 5) {
4384
			SLJIT_ASSERT(elem_size == 2);
4385

4386
			if (src_lane_index == 0)
4387
				return emit_vex_instruction(compiler, VBROADCASTSS_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4388

4389
			FAIL_IF(emit_vex_instruction(compiler, VPERMPD_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4390

4391
			byte = 0x44;
4392
			if (src_lane_index >= 4) {
4393
				byte = 0xee;
4394
				src_lane_index -= 4;
4395
			}
4396

4397
			FAIL_IF(emit_byte(compiler, byte));
4398
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | VEX_256 | pref | EX86_SSE2 | VEX_SSE2_OPV, vreg, vreg, vreg, 0));
4399
			byte = U8(src_lane_index);
4400
		} else if (use_vex) {
4401
			FAIL_IF(emit_vex_instruction(compiler, SHUFPS_x_xm | pref | EX86_SSE2 | VEX_SSE2_OPV, vreg, src, src, 0));
4402
		} else {
4403
			if (vreg != src)
4404
				FAIL_IF(emit_groupf(compiler, MOVAPS_x_xm | pref | EX86_SSE2, vreg, src, 0));
4405

4406
			FAIL_IF(emit_groupf(compiler, SHUFPS_x_xm | pref | EX86_SSE2, vreg, vreg, 0));
4407
		}
4408

4409
		if (elem_size == 2) {
4410
			byte = U8(byte | (byte << 2));
4411
			byte = U8(byte | (byte << 4));
4412
		} else
4413
			byte = U8(byte | (byte << 1));
4414

4415
		return emit_byte(compiler, U8(byte));
4416
	}
4417

4418
	if (type & SLJIT_SIMD_TEST)
4419
		return SLJIT_SUCCESS;
4420

4421
	if (elem_size == 0) {
4422
		if (reg_size == 5 && src_lane_index >= 16) {
4423
			FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4424
			FAIL_IF(emit_byte(compiler, src_lane_index >= 24 ? 0xff : 0xaa));
4425
			src_lane_index &= 0x7;
4426
			src = vreg;
4427
		}
4428

4429
		if (src_lane_index != 0 || (vreg != src && (!(cpu_feature_list & CPU_FEATURE_AVX2) || !use_vex))) {
4430
			pref = 0;
4431

4432
			if ((src_lane_index & 0x3) == 0) {
4433
				pref = EX86_PREF_66;
4434
				byte = U8(src_lane_index >> 2);
4435
			} else if (src_lane_index < 8 && (src_lane_index & 0x1) == 0) {
4436
				pref = EX86_PREF_F2;
4437
				byte = U8(src_lane_index >> 1);
4438
			} else {
4439
				if (!use_vex) {
4440
					if (vreg != src)
4441
						FAIL_IF(emit_groupf(compiler, MOVDQA_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, src, 0));
4442

4443
					FAIL_IF(emit_groupf(compiler, PSRLDQ_x | EX86_PREF_66 | EX86_SSE2_OP2, opcode3, vreg, 0));
4444
				} else
4445
					FAIL_IF(emit_vex_instruction(compiler, PSRLDQ_x | EX86_PREF_66 | EX86_SSE2_OP2 | VEX_SSE2_OPV, opcode3, vreg, src, 0));
4446

4447
				FAIL_IF(emit_byte(compiler, U8(src_lane_index)));
4448
			}
4449

4450
			if (pref != 0) {
4451
				if (use_vex)
4452
					FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, 0, src, 0));
4453
				else
4454
					FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, src, 0));
4455
				FAIL_IF(emit_byte(compiler, byte));
4456
			}
4457

4458
			src = vreg;
4459
		}
4460

4461
		if (use_vex && (cpu_feature_list & CPU_FEATURE_AVX2))
4462
			return emit_vex_instruction(compiler, VPBROADCASTB_x_xm | (reg_size == 5 ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, 0);
4463

4464
		SLJIT_ASSERT(reg_size == 4);
4465
		FAIL_IF(emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, TMP_FREG, 0));
4466
		return emit_groupf_ext(compiler, PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, TMP_FREG, 0);
4467
	}
4468

4469
	if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && src_lane_index == 0 && elem_size <= 3) {
4470
		switch (elem_size) {
4471
		case 1:
4472
			pref = VPBROADCASTW_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4473
			break;
4474
		case 2:
4475
			pref = VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4476
			break;
4477
		default:
4478
			pref = VPBROADCASTQ_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2;
4479
			break;
4480
		}
4481

4482
		if (reg_size == 5)
4483
			pref |= VEX_256;
4484

4485
		return emit_vex_instruction(compiler, pref, vreg, 0, src, 0);
4486
	}
4487

4488
	if (reg_size == 5) {
4489
		switch (elem_size) {
4490
		case 1:
4491
			byte = U8(src_lane_index & 0x3);
4492
			src_lane_index >>= 2;
4493
			pref = PSHUFLW_x_xm | VEX_256 | ((src_lane_index & 1) == 0 ? EX86_PREF_F2 : EX86_PREF_F3) | EX86_SSE2;
4494
			break;
4495
		case 2:
4496
			byte = U8(src_lane_index & 0x3);
4497
			src_lane_index >>= 1;
4498
			pref = PSHUFD_x_xm | VEX_256 | EX86_PREF_66 | EX86_SSE2;
4499
			break;
4500
		case 3:
4501
			pref = 0;
4502
			break;
4503
		default:
4504
			FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4505
			return emit_byte(compiler, U8(src_lane_index == 0 ? 0x44 : 0xee));
4506
		}
4507

4508
		if (pref != 0) {
4509
			FAIL_IF(emit_vex_instruction(compiler, pref, vreg, 0, src, 0));
4510
			byte = U8(byte | (byte << 2));
4511
			FAIL_IF(emit_byte(compiler, U8(byte | (byte << 4))));
4512

4513
			if (src_lane_index == 0)
4514
				return emit_vex_instruction(compiler, VPBROADCASTQ_x_xm | VEX_256 | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
4515

4516
			src = vreg;
4517
		}
4518

4519
		FAIL_IF(emit_vex_instruction(compiler, VPERMQ_y_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | VEX_W | EX86_SSE2, vreg, 0, src, 0));
4520
		byte = U8(src_lane_index);
4521
		byte = U8(byte | (byte << 2));
4522
		return emit_byte(compiler, U8(byte | (byte << 4)));
4523
	}
4524

4525
	switch (elem_size) {
4526
	case 1:
4527
		byte = U8(src_lane_index & 0x3);
4528
		src_lane_index >>= 1;
4529
		pref = (src_lane_index & 2) == 0 ? EX86_PREF_F2 : EX86_PREF_F3;
4530

4531
		if (use_vex)
4532
			FAIL_IF(emit_vex_instruction(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, 0, src, 0));
4533
		else
4534
			FAIL_IF(emit_groupf(compiler, PSHUFLW_x_xm | pref | EX86_SSE2, vreg, src, 0));
4535
		byte = U8(byte | (byte << 2));
4536
		FAIL_IF(emit_byte(compiler, U8(byte | (byte << 4))));
4537

4538
		if ((cpu_feature_list & CPU_FEATURE_AVX2) && use_vex && pref == EX86_PREF_F2)
4539
			return emit_vex_instruction(compiler, VPBROADCASTD_x_xm | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, vreg, 0);
4540

4541
		src = vreg;
4542
		/* fallthrough */
4543
	case 2:
4544
		byte = U8(src_lane_index);
4545
		byte = U8(byte | (byte << 2));
4546
		break;
4547
	default:
4548
		byte = U8(src_lane_index << 1);
4549
		byte = U8(byte | (byte << 2) | 0x4);
4550
		break;
4551
	}
4552

4553
	if (use_vex)
4554
		FAIL_IF(emit_vex_instruction(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, 0, src, 0));
4555
	else
4556
		FAIL_IF(emit_groupf(compiler, PSHUFD_x_xm | EX86_PREF_66 | EX86_SSE2, vreg, src, 0));
4557
	return emit_byte(compiler, U8(byte | (byte << 4)));
4558
}
4559

4560
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
4561
	sljit_s32 vreg,
4562
	sljit_s32 src, sljit_sw srcw)
4563
{
4564
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4565
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4566
	sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
4567
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4568
	sljit_u8 opcode;
4569

4570
	CHECK_ERROR();
4571
	CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
4572

4573
	ADJUST_LOCAL_OFFSET(src, srcw);
4574

4575
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4576
	compiler->mode32 = 1;
4577
#endif /* SLJIT_CONFIG_X86_64 */
4578

4579
	if (reg_size == 5) {
4580
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4581
			return SLJIT_ERR_UNSUPPORTED;
4582
		use_vex = 1;
4583
	} else if (reg_size != 4)
4584
		return SLJIT_ERR_UNSUPPORTED;
4585

4586
	if (type & SLJIT_SIMD_FLOAT) {
4587
		if (elem_size != 2 || elem2_size != 3)
4588
			return SLJIT_ERR_UNSUPPORTED;
4589

4590
		if (type & SLJIT_SIMD_TEST)
4591
			return SLJIT_SUCCESS;
4592

4593
		if (use_vex)
4594
			return emit_vex_instruction(compiler, CVTPS2PD_x_xm | ((reg_size == 5) ? VEX_256 : 0) | EX86_SSE2, vreg, 0, src, srcw);
4595
		return emit_groupf(compiler, CVTPS2PD_x_xm | EX86_SSE2, vreg, src, srcw);
4596
	}
4597

4598
	switch (elem_size) {
4599
	case 0:
4600
		if (elem2_size == 1)
4601
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBW_x_xm : PMOVZXBW_x_xm;
4602
		else if (elem2_size == 2)
4603
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBD_x_xm : PMOVZXBD_x_xm;
4604
		else if (elem2_size == 3)
4605
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXBQ_x_xm : PMOVZXBQ_x_xm;
4606
		else
4607
			return SLJIT_ERR_UNSUPPORTED;
4608
		break;
4609
	case 1:
4610
		if (elem2_size == 2)
4611
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXWD_x_xm : PMOVZXWD_x_xm;
4612
		else if (elem2_size == 3)
4613
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXWQ_x_xm : PMOVZXWQ_x_xm;
4614
		else
4615
			return SLJIT_ERR_UNSUPPORTED;
4616
		break;
4617
	case 2:
4618
		if (elem2_size == 3)
4619
			opcode = (type & SLJIT_SIMD_EXTEND_SIGNED) ? PMOVSXDQ_x_xm : PMOVZXDQ_x_xm;
4620
		else
4621
			return SLJIT_ERR_UNSUPPORTED;
4622
		break;
4623
	default:
4624
		return SLJIT_ERR_UNSUPPORTED;
4625
	}
4626

4627
	if (type & SLJIT_SIMD_TEST)
4628
		return SLJIT_SUCCESS;
4629

4630
	if (use_vex)
4631
		return emit_vex_instruction(compiler, opcode | ((reg_size == 5) ? VEX_256 : 0) | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, 0, src, srcw);
4632
	return emit_groupf_ext(compiler, opcode | EX86_PREF_66 | VEX_OP_0F38 | EX86_SSE2, vreg, src, srcw);
4633
}
4634

4635
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
4636
	sljit_s32 vreg,
4637
	sljit_s32 dst, sljit_sw dstw)
4638
{
4639
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4640
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4641
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4642
	sljit_s32 dst_r;
4643
	sljit_uw op;
4644
	sljit_u8 *inst;
4645

4646
	CHECK_ERROR();
4647
	CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
4648

4649
	ADJUST_LOCAL_OFFSET(dst, dstw);
4650

4651
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
4652
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4653
	compiler->mode32 = 1;
4654
#endif /* SLJIT_CONFIG_X86_64 */
4655

4656
	if (elem_size > 3 || ((type & SLJIT_SIMD_FLOAT) && elem_size < 2))
4657
		return SLJIT_ERR_UNSUPPORTED;
4658

4659
	if (reg_size == 4) {
4660
		if (type & SLJIT_SIMD_TEST)
4661
			return SLJIT_SUCCESS;
4662

4663
		op = EX86_PREF_66 | EX86_SSE2_OP2;
4664

4665
		switch (elem_size) {
4666
		case 1:
4667
			if (use_vex)
4668
				FAIL_IF(emit_vex_instruction(compiler, PACKSSWB_x_xm | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, vreg, vreg, 0));
4669
			else
4670
				FAIL_IF(emit_groupf(compiler, PACKSSWB_x_xm | EX86_PREF_66 | EX86_SSE2, TMP_FREG, vreg, 0));
4671
			vreg = TMP_FREG;
4672
			break;
4673
		case 2:
4674
			op = EX86_SSE2_OP2;
4675
			break;
4676
		}
4677

4678
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4679
		op |= (elem_size < 2) ? PMOVMSKB_r_x : MOVMSKPS_r_x;
4680

4681
		if (use_vex)
4682
			FAIL_IF(emit_vex_instruction(compiler, op, dst_r, 0, vreg, 0));
4683
		else
4684
			FAIL_IF(emit_groupf(compiler, op, dst_r, vreg, 0));
4685

4686
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4687
		compiler->mode32 = type & SLJIT_32;
4688
#endif /* SLJIT_CONFIG_X86_64 */
4689

4690
		if (elem_size == 1) {
4691
			inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 8, dst_r, 0);
4692
			FAIL_IF(!inst);
4693
			inst[1] |= SHR;
4694
		}
4695

4696
		if (dst_r == TMP_REG1)
4697
			return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
4698

4699
		return SLJIT_SUCCESS;
4700
	}
4701

4702
	if (reg_size != 5 || !(cpu_feature_list & CPU_FEATURE_AVX2))
4703
		return SLJIT_ERR_UNSUPPORTED;
4704

4705
	if (type & SLJIT_SIMD_TEST)
4706
		return SLJIT_SUCCESS;
4707

4708
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4709

4710
	if (elem_size == 1) {
4711
		FAIL_IF(emit_vex_instruction(compiler, VEXTRACTI128_x_ym | VEX_256 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2, vreg, 0, TMP_FREG, 0));
4712
		FAIL_IF(emit_byte(compiler, 1));
4713
		FAIL_IF(emit_vex_instruction(compiler, PACKSSWB_x_xm | VEX_256 | EX86_PREF_66 | EX86_SSE2 | VEX_SSE2_OPV, TMP_FREG, vreg, TMP_FREG, 0));
4714
		FAIL_IF(emit_groupf(compiler, PMOVMSKB_r_x | EX86_PREF_66 | EX86_SSE2_OP2, dst_r, TMP_FREG, 0));
4715
	} else {
4716
		op = MOVMSKPS_r_x | VEX_256 | EX86_SSE2_OP2;
4717

4718
		if (elem_size == 0)
4719
			op = PMOVMSKB_r_x | VEX_256 | EX86_PREF_66 | EX86_SSE2_OP2;
4720
		else if (elem_size == 3)
4721
			op |= EX86_PREF_66;
4722

4723
		FAIL_IF(emit_vex_instruction(compiler, op, dst_r, 0, vreg, 0));
4724
	}
4725

4726
	if (dst_r == TMP_REG1) {
4727
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4728
		compiler->mode32 = type & SLJIT_32;
4729
#endif /* SLJIT_CONFIG_X86_64 */
4730
		return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
4731
	}
4732

4733
	return SLJIT_SUCCESS;
4734
}
4735

4736
static sljit_s32 emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
4737
	sljit_s32 dst_vreg, sljit_s32 src_vreg)
4738
{
4739
	sljit_uw op = ((type & SLJIT_SIMD_FLOAT) ? MOVAPS_x_xm : MOVDQA_x_xm) | EX86_SSE2;
4740

4741
	SLJIT_ASSERT(SLJIT_SIMD_GET_REG_SIZE(type) == 4);
4742

4743
	if (!(type & SLJIT_SIMD_FLOAT) || SLJIT_SIMD_GET_ELEM_SIZE(type) == 3)
4744
		op |= EX86_PREF_66;
4745

4746
	return emit_groupf(compiler, op, dst_vreg, src_vreg, 0);
4747
}
4748

4749
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
4750
	sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
4751
{
4752
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4753
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4754
	sljit_s32 use_vex = (cpu_feature_list & CPU_FEATURE_AVX) && (compiler->options & SLJIT_ENTER_USE_VEX);
4755
	sljit_uw op = 0;
4756
	sljit_uw mov_op = 0;
4757

4758
	CHECK_ERROR();
4759
	CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
4760
	ADJUST_LOCAL_OFFSET(src2, src2w);
4761

4762
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4763
	compiler->mode32 = 1;
4764
#endif /* SLJIT_CONFIG_X86_64 */
4765

4766
	if (reg_size == 5) {
4767
		if (!(cpu_feature_list & CPU_FEATURE_AVX2))
4768
			return SLJIT_ERR_UNSUPPORTED;
4769
	} else if (reg_size != 4)
4770
		return SLJIT_ERR_UNSUPPORTED;
4771

4772
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4773
		return SLJIT_ERR_UNSUPPORTED;
4774

4775
	switch (SLJIT_SIMD_GET_OPCODE(type)) {
4776
	case SLJIT_SIMD_OP2_AND:
4777
		op = (type & SLJIT_SIMD_FLOAT) ? ANDPD_x_xm : PAND_x_xm;
4778

4779
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
4780
			op |= EX86_PREF_66;
4781
		break;
4782
	case SLJIT_SIMD_OP2_OR:
4783
		op = (type & SLJIT_SIMD_FLOAT) ? ORPD_x_xm : POR_x_xm;
4784

4785
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
4786
			op |= EX86_PREF_66;
4787
		break;
4788
	case SLJIT_SIMD_OP2_XOR:
4789
		op = (type & SLJIT_SIMD_FLOAT) ? XORPD_x_xm : PXOR_x_xm;
4790

4791
		if (!(type & SLJIT_SIMD_FLOAT) || elem_size == 3)
4792
			op |= EX86_PREF_66;
4793
		break;
4794

4795
	case SLJIT_SIMD_OP2_SHUFFLE:
4796
		if (reg_size != 4)
4797
			return SLJIT_ERR_UNSUPPORTED;
4798

4799
		op = PSHUFB_x_xm | EX86_PREF_66 | VEX_OP_0F38;
4800
		break;
4801
	}
4802

4803
	if (type & SLJIT_SIMD_TEST)
4804
		return SLJIT_SUCCESS;
4805

4806
	if ((src2 & SLJIT_MEM) && SLJIT_SIMD_GET_ELEM2_SIZE(type) < reg_size) {
4807
		mov_op = ((type & SLJIT_SIMD_FLOAT) ? (MOVUPS_x_xm | (elem_size == 3 ? EX86_PREF_66 : 0)) : (MOVDQU_x_xm | EX86_PREF_F3)) | EX86_SSE2;
4808
		if (use_vex)
4809
			FAIL_IF(emit_vex_instruction(compiler, mov_op, TMP_FREG, 0, src2, src2w));
4810
		else
4811
			FAIL_IF(emit_groupf(compiler, mov_op, TMP_FREG, src2, src2w));
4812

4813
		src2 = TMP_FREG;
4814
		src2w = 0;
4815
	}
4816

4817
	if (reg_size == 5 || use_vex) {
4818
		if (reg_size == 5)
4819
			op |= VEX_256;
4820

4821
		return emit_vex_instruction(compiler, op | EX86_SSE2 | VEX_SSE2_OPV, dst_vreg, src1_vreg, src2, src2w);
4822
	}
4823

4824
	if (dst_vreg != src1_vreg) {
4825
		if (dst_vreg == src2) {
4826
			if (SLJIT_SIMD_GET_OPCODE(type) == SLJIT_SIMD_OP2_SHUFFLE) {
4827
				FAIL_IF(emit_simd_mov(compiler, type, TMP_FREG, src2));
4828
				FAIL_IF(emit_simd_mov(compiler, type, dst_vreg, src1_vreg));
4829
				src2 = TMP_FREG;
4830
				src2w = 0;
4831
			} else
4832
				src2 = src1_vreg;
4833
		} else
4834
			FAIL_IF(emit_simd_mov(compiler, type, dst_vreg, src1_vreg));
4835
	}
4836

4837
	if (op & (VEX_OP_0F38 | VEX_OP_0F3A))
4838
		return emit_groupf_ext(compiler, op | EX86_SSE2, dst_vreg, src2, src2w);
4839
	return emit_groupf(compiler, op | EX86_SSE2, dst_vreg, src2, src2w);
4840
}
4841

4842
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
4843
	sljit_s32 dst_reg,
4844
	sljit_s32 mem_reg)
4845
{
4846
	CHECK_ERROR();
4847
	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
4848

4849
	if ((op & SLJIT_ATOMIC_USE_LS) || GET_OPCODE(op) == SLJIT_MOV_S8 || GET_OPCODE(op) == SLJIT_MOV_S16 || GET_OPCODE(op) == SLJIT_MOV_S32)
4850
		return SLJIT_ERR_UNSUPPORTED;
4851

4852
	if (op & SLJIT_ATOMIC_TEST)
4853
		return SLJIT_SUCCESS;
4854

4855
	SLJIT_SKIP_CHECKS(compiler);
4856
	return sljit_emit_op1(compiler, op & ~SLJIT_ATOMIC_USE_CAS, dst_reg, 0, SLJIT_MEM1(mem_reg), 0);
4857
}
4858

4859
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
4860
	sljit_s32 src_reg,
4861
	sljit_s32 mem_reg,
4862
	sljit_s32 temp_reg)
4863
{
4864
	sljit_uw pref;
4865
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4866
	sljit_s32 saved_reg = TMP_REG1;
4867
	sljit_s32 swap_tmp = 0;
4868
	sljit_sw srcw = 0;
4869
	sljit_sw tempw = 0;
4870
#endif /* SLJIT_CONFIG_X86_32 */
4871

4872
	CHECK_ERROR();
4873
	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
4874
	CHECK_EXTRA_REGS(src_reg, srcw, (void)0);
4875
	CHECK_EXTRA_REGS(temp_reg, tempw, (void)0);
4876

4877
	SLJIT_ASSERT(FAST_IS_REG(src_reg) || src_reg == SLJIT_MEM1(SLJIT_SP));
4878
	SLJIT_ASSERT(FAST_IS_REG(temp_reg) || temp_reg == SLJIT_MEM1(SLJIT_SP));
4879

4880
	if ((op & SLJIT_ATOMIC_USE_LS) || GET_OPCODE(op) == SLJIT_MOV_S8 || GET_OPCODE(op) == SLJIT_MOV_S16 || GET_OPCODE(op) == SLJIT_MOV_S32)
4881
		return SLJIT_ERR_UNSUPPORTED;
4882

4883
	if (op & SLJIT_ATOMIC_TEST)
4884
		return SLJIT_SUCCESS;
4885

4886
	op = GET_OPCODE(op);
4887

4888
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4889
	if (temp_reg == SLJIT_TMP_DEST_REG) {
4890
		FAIL_IF(emit_byte(compiler, XCHG_EAX_r | reg_map[TMP_REG1]));
4891

4892
		if (src_reg == SLJIT_R0)
4893
			src_reg = TMP_REG1;
4894
		if (mem_reg == SLJIT_R0)
4895
			mem_reg = TMP_REG1;
4896

4897
		temp_reg = SLJIT_R0;
4898
		swap_tmp = 1;
4899
	}
4900

4901
	/* Src is virtual register or its low byte is not accessible. */
4902
	if ((src_reg & SLJIT_MEM) || (op == SLJIT_MOV_U8 && reg_map[src_reg] >= 4)) {
4903
		SLJIT_ASSERT(src_reg != SLJIT_R1 && temp_reg != SLJIT_TMP_DEST_REG);
4904

4905
		if (swap_tmp) {
4906
			saved_reg = (mem_reg != SLJIT_R1) ? SLJIT_R1 : SLJIT_R2;
4907

4908
			EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, saved_reg, 0);
4909
			EMIT_MOV(compiler, saved_reg, 0, src_reg, srcw);
4910
		} else
4911
			EMIT_MOV(compiler, TMP_REG1, 0, src_reg, srcw);
4912

4913
		src_reg = saved_reg;
4914

4915
		if (mem_reg == src_reg)
4916
			mem_reg = saved_reg;
4917
	}
4918
#endif /* SLJIT_CONFIG_X86_32 */
4919

4920
	if (temp_reg != SLJIT_R0) {
4921
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4922
		compiler->mode32 = 0;
4923

4924
		EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_R0, 0);
4925
		EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, 0);
4926

4927
		if (src_reg == SLJIT_R0)
4928
			src_reg = TMP_REG2;
4929
		if (mem_reg == SLJIT_R0)
4930
			mem_reg = TMP_REG2;
4931
#else /* !SLJIT_CONFIG_X86_64 */
4932
		SLJIT_ASSERT(!swap_tmp);
4933

4934
		if (src_reg == TMP_REG1) {
4935
			if (mem_reg == SLJIT_R0) {
4936
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R1, 0);
4937
				EMIT_MOV(compiler, SLJIT_R1, 0, SLJIT_R0, 0);
4938
				EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
4939

4940
				mem_reg = SLJIT_R1;
4941
				saved_reg = SLJIT_R1;
4942
			} else {
4943
				EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
4944
				EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
4945
				saved_reg = SLJIT_R0;
4946
			}
4947
		} else {
4948
			EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R0, 0);
4949
			EMIT_MOV(compiler, SLJIT_R0, 0, temp_reg, tempw);
4950

4951
			if (src_reg == SLJIT_R0)
4952
				src_reg = TMP_REG1;
4953
			if (mem_reg == SLJIT_R0)
4954
				mem_reg = TMP_REG1;
4955
		}
4956
#endif /* SLJIT_CONFIG_X86_64 */
4957
	}
4958

4959
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4960
	compiler->mode32 = op != SLJIT_MOV && op != SLJIT_MOV_P;
4961
#endif /* SLJIT_CONFIG_X86_64 */
4962

4963
	/* Lock prefix. */
4964
	FAIL_IF(emit_byte(compiler, GROUP_LOCK));
4965

4966
	pref = 0;
4967
	if (op == SLJIT_MOV_U16)
4968
		pref = EX86_HALF_ARG | EX86_PREF_66;
4969
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4970
	if (op == SLJIT_MOV_U8)
4971
		pref = EX86_REX;
4972
#endif /* SLJIT_CONFIG_X86_64 */
4973

4974
	FAIL_IF(emit_groupf(compiler, (op == SLJIT_MOV_U8 ? CMPXCHG_rm8_r : CMPXCHG_rm_r) | pref, src_reg, SLJIT_MEM1(mem_reg), 0));
4975

4976
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
4977
	if (swap_tmp) {
4978
		SLJIT_ASSERT(temp_reg == SLJIT_R0);
4979
		FAIL_IF(emit_byte(compiler, XCHG_EAX_r | reg_map[TMP_REG1]));
4980

4981
		if (saved_reg != TMP_REG1)
4982
			return emit_mov(compiler, saved_reg, 0, SLJIT_MEM1(SLJIT_SP), 0);
4983
		return SLJIT_SUCCESS;
4984
	}
4985
#endif /* SLJIT_CONFIG_X86_32 */
4986

4987
	if (temp_reg != SLJIT_R0) {
4988
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
4989
		compiler->mode32 = 0;
4990
		return emit_mov(compiler, SLJIT_R0, 0, TMP_REG2, 0);
4991
#else /* !SLJIT_CONFIG_X86_64 */
4992
		EMIT_MOV(compiler, SLJIT_R0, 0, (saved_reg == SLJIT_R0) ? SLJIT_MEM1(SLJIT_SP) : saved_reg, 0);
4993
		if (saved_reg == SLJIT_R1)
4994
			return emit_mov(compiler, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_SP), 0);
4995
#endif /* SLJIT_CONFIG_X86_64 */
4996
	}
4997
	return SLJIT_SUCCESS;
4998
}
4999

5000
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
5001
{
5002
	CHECK_ERROR();
5003
	CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
5004
	ADJUST_LOCAL_OFFSET(dst, dstw);
5005

5006
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
5007

5008
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5009
	compiler->mode32 = 0;
5010
#endif
5011

5012
	ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
5013

5014
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5015
	if (NOT_HALFWORD(offset)) {
5016
		FAIL_IF(emit_load_imm64(compiler, TMP_REG1, offset));
5017
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
5018
		SLJIT_ASSERT(emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0) != SLJIT_ERR_UNSUPPORTED);
5019
		return compiler->error;
5020
#else
5021
		return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, TMP_REG1, 0);
5022
#endif
5023
	}
5024
#endif
5025

5026
	if (offset != 0)
5027
		return emit_lea_binary(compiler, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
5028
	return emit_mov(compiler, dst, dstw, SLJIT_SP, 0);
5029
}
5030

5031
SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
5032
{
5033
	sljit_u8 *inst;
5034
	struct sljit_const *const_;
5035
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5036
	sljit_s32 reg;
5037
#endif
5038

5039
	CHECK_ERROR_PTR();
5040
	CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
5041
	ADJUST_LOCAL_OFFSET(dst, dstw);
5042

5043
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
5044

5045
	const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
5046
	PTR_FAIL_IF(!const_);
5047
	set_const(const_, compiler);
5048

5049
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5050
	compiler->mode32 = 0;
5051
	reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
5052

5053
	if (emit_load_imm64(compiler, reg, init_value))
5054
		return NULL;
5055
#else
5056
	if (emit_mov(compiler, dst, dstw, SLJIT_IMM, init_value))
5057
		return NULL;
5058
#endif
5059

5060
	inst = (sljit_u8*)ensure_buf(compiler, 1);
5061
	PTR_FAIL_IF(!inst);
5062

5063
	inst[0] = SLJIT_INST_CONST;
5064

5065
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5066
	if (dst & SLJIT_MEM)
5067
		if (emit_mov(compiler, dst, dstw, TMP_REG1, 0))
5068
			return NULL;
5069
#endif
5070

5071
	return const_;
5072
}
5073

5074
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
5075
{
5076
	struct sljit_jump *jump;
5077
	sljit_u8 *inst;
5078
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5079
	sljit_s32 reg;
5080
#endif /* SLJIT_CONFIG_X86_64 */
5081

5082
	CHECK_ERROR_PTR();
5083
	CHECK_PTR(check_sljit_emit_mov_addr(compiler, dst, dstw));
5084
	ADJUST_LOCAL_OFFSET(dst, dstw);
5085

5086
	CHECK_EXTRA_REGS(dst, dstw, (void)0);
5087

5088
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
5089
	PTR_FAIL_IF(!jump);
5090
	set_mov_addr(jump, compiler, 0);
5091

5092
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5093
	compiler->mode32 = 0;
5094
	reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
5095

5096
	PTR_FAIL_IF(emit_load_imm64(compiler, reg, 0));
5097
	jump->addr = compiler->size;
5098

5099
	if (reg_map[reg] >= 8)
5100
		jump->flags |= MOV_ADDR_HI;
5101
#else /* !SLJIT_CONFIG_X86_64 */
5102
	PTR_FAIL_IF(emit_mov(compiler, dst, dstw, SLJIT_IMM, 0));
5103
#endif /* SLJIT_CONFIG_X86_64 */
5104

5105
	inst = (sljit_u8*)ensure_buf(compiler, 1);
5106
	PTR_FAIL_IF(!inst);
5107

5108
	inst[0] = SLJIT_INST_MOV_ADDR;
5109

5110
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
5111
	if (dst & SLJIT_MEM)
5112
		PTR_FAIL_IF(emit_mov(compiler, dst, dstw, TMP_REG1, 0));
5113
#endif /* SLJIT_CONFIG_X86_64 */
5114

5115
	return jump;
5116
}
5117

5118
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
5119
{
5120
	SLJIT_UNUSED_ARG(executable_offset);
5121

5122
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0);
5123
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
5124
	sljit_unaligned_store_sw((void*)addr, (sljit_sw)(new_target - (addr + 4) - (sljit_uw)executable_offset));
5125
#else
5126
	sljit_unaligned_store_sw((void*)addr, (sljit_sw)new_target);
5127
#endif
5128
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1);
5129
}
5130

5131
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
5132
{
5133
	SLJIT_UNUSED_ARG(executable_offset);
5134

5135
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 0);
5136
	sljit_unaligned_store_sw((void*)addr, new_constant);
5137
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 1);
5138
}
5139

5140
Product

Resources

Company
