CoCalc -- sljitLir.c

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitLir.c
²²³³⁵ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
#include "sljitLir.h"
28

29
#ifdef _WIN32
30

31
#include <windows.h>
32

33
#endif /* _WIN32 */
34

35
#if !(defined SLJIT_STD_MACROS_DEFINED && SLJIT_STD_MACROS_DEFINED)
36

37
/* These libraries are needed for the macros below. */
38
#include <stdlib.h>
39
#include <string.h>
40

41
#endif /* SLJIT_STD_MACROS_DEFINED */
42

43
#define CHECK_ERROR() \
44
	do { \
45
		if (SLJIT_UNLIKELY(compiler->error)) \
46
			return compiler->error; \
47
	} while (0)
48

49
#define CHECK_ERROR_PTR() \
50
	do { \
51
		if (SLJIT_UNLIKELY(compiler->error)) \
52
			return NULL; \
53
	} while (0)
54

55
#define FAIL_IF(expr) \
56
	do { \
57
		if (SLJIT_UNLIKELY(expr)) \
58
			return compiler->error; \
59
	} while (0)
60

61
#define PTR_FAIL_IF(expr) \
62
	do { \
63
		if (SLJIT_UNLIKELY(expr)) \
64
			return NULL; \
65
	} while (0)
66

67
#define FAIL_IF_NULL(ptr) \
68
	do { \
69
		if (SLJIT_UNLIKELY(!(ptr))) { \
70
			compiler->error = SLJIT_ERR_ALLOC_FAILED; \
71
			return SLJIT_ERR_ALLOC_FAILED; \
72
		} \
73
	} while (0)
74

75
#define PTR_FAIL_IF_NULL(ptr) \
76
	do { \
77
		if (SLJIT_UNLIKELY(!(ptr))) { \
78
			compiler->error = SLJIT_ERR_ALLOC_FAILED; \
79
			return NULL; \
80
		} \
81
	} while (0)
82

83
#define PTR_FAIL_WITH_EXEC_IF(ptr) \
84
	do { \
85
		if (SLJIT_UNLIKELY(!(ptr))) { \
86
			compiler->error = SLJIT_ERR_EX_ALLOC_FAILED; \
87
			return NULL; \
88
		} \
89
	} while (0)
90

91
#if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
92

93
#define SSIZE_OF(type) ((sljit_s32)sizeof(sljit_ ## type))
94

95
#define VARIABLE_FLAG_SHIFT (10)
96
/* All variable flags are even. */
97
#define VARIABLE_FLAG_MASK (0x3e << VARIABLE_FLAG_SHIFT)
98
#define ALL_STATUS_FLAGS_MASK (VARIABLE_FLAG_MASK | SLJIT_SET_Z)
99
#define GET_FLAG_TYPE(op) ((op) >> VARIABLE_FLAG_SHIFT)
100
#define GET_FLAG_TYPE_MASK(op) (((op) >> VARIABLE_FLAG_SHIFT) & 0x3e)
101
#define GET_OPCODE(op) ((op) & 0xff)
102
#define HAS_FLAGS(op) ((op) & ALL_STATUS_FLAGS_MASK)
103
#define GET_ALL_FLAGS(op) ((op) & (SLJIT_32 | ALL_STATUS_FLAGS_MASK))
104

105
#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
106
#define TYPE_CAST_NEEDED(op) \
107
	((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S32)
108
#else /* !SLJIT_64BIT_ARCHITECTURE */
109
#define TYPE_CAST_NEEDED(op) \
110
	((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S16)
111
#endif /* SLJIT_64BIT_ARCHITECTURE */
112

113
#define BUF_SIZE	4096
114

115
#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
116
#define ABUF_SIZE	2048
117
#else /* !SLJIT_32BIT_ARCHITECTURE */
118
#define ABUF_SIZE	4096
119
#endif /* SLJIT_32BIT_ARCHITECTURE */
120

121
/* Parameter parsing. */
122
#define REG_MASK		0x7f
123
#define OFFS_REG(reg)		(((reg) >> 8) & REG_MASK)
124
#define OFFS_REG_MASK		(REG_MASK << 8)
125
#define TO_OFFS_REG(reg)	((reg) << 8)
126
#define FAST_IS_REG(reg)	((reg) < REG_MASK)
127

128
/* Mask for argument types. */
129
#define SLJIT_ARG_MASK		0x7
130
#define SLJIT_ARG_FULL_MASK	(SLJIT_ARG_MASK | SLJIT_ARG_TYPE_SCRATCH_REG)
131

132
/* Mask for register pairs. */
133
#define REG_PAIR_MASK		0x7f00
134
#define REG_PAIR_FIRST(reg)	((reg) & 0x7f)
135
#define REG_PAIR_SECOND(reg)	((reg) >> 8)
136

137
/* Mask for sljit_emit_enter. */
138
#define ENTER_GET_REGS(regs)			((regs) & 0xff)
139
#define ENTER_GET_FLOAT_REGS(regs)		(((regs) >> 8) & 0xff)
140
#define ENTER_GET_VECTOR_REGS(regs)		(((regs) >> 16) & 0xff)
141
#define SLJIT_KEPT_SAVEDS_COUNT(options)	((options) & 0x3)
142

143
/* Getters for simd operations, which returns with log2(size). */
144
#define SLJIT_SIMD_GET_OPCODE(type)		((type) & 0xff)
145
#define SLJIT_SIMD_GET_REG_SIZE(type)		(((type) >> 12) & 0x3f)
146
#define SLJIT_SIMD_GET_ELEM_SIZE(type)		(((type) >> 18) & 0x3f)
147
#define SLJIT_SIMD_GET_ELEM2_SIZE(type)		(((type) >> 24) & 0x3f)
148

149
#define SLJIT_SIMD_CHECK_REG(type) (((type) & 0x3f000) >= SLJIT_SIMD_REG_64 && ((type) & 0x3f000) <= SLJIT_SIMD_REG_512)
150
#define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
151
#define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
152

153
/* Label definitions. */
154

155
#define SLJIT_LABEL_ALIGNED ((~(sljit_uw)0) - 1)
156

157
struct sljit_extended_label {
158
	struct sljit_label label;
159
	sljit_uw index;
160
	sljit_uw data;
161
};
162

163
/* Jump definitions. */
164

165
/* Jump flag bits. */
166
#define JUMP_ADDR	0x1
167
#define JUMP_MOV_ADDR	0x2
168
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
169

170
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
171
#	define PATCH_MB		0x04
172
#	define PATCH_MW		0x08
173
#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
174
#	define PATCH_MD		0x10
175
#	define MOV_ADDR_HI	0x20
176
#	define JUMP_MAX_SIZE	((sljit_uw)(10 + 3))
177
#	define CJUMP_MAX_SIZE	((sljit_uw)(2 + 10 + 3))
178
#else /* !SLJIT_CONFIG_X86_64 */
179
#	define JUMP_MAX_SIZE	((sljit_uw)5)
180
#	define CJUMP_MAX_SIZE	((sljit_uw)6)
181
#endif /* SLJIT_CONFIG_X86_64 */
182
#	define TYPE_SHIFT	17
183
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
184
/* Bits 7..15 is for debug jump size, SLJIT_REWRITABLE_JUMP is 0x10000 */
185
#	define JUMP_SIZE_SHIFT	8
186
#endif /* SLJIT_DEBUG */
187
#endif /* SLJIT_CONFIG_X86 */
188

189
#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
190
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
191
#	define IS_BL		0x04
192
#	define PATCH_B		0x08
193
#endif /* SLJIT_CONFIG_ARM_V6 || SLJIT_CONFIG_ARM_V7 */
194

195
#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
196
#	define CPOOL_SIZE	512
197
#endif /* SLJIT_CONFIG_ARM_V6 */
198

199
#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
200
#	define JUMP_SIZE_SHIFT	26
201
#	define JUMP_MAX_SIZE	((sljit_uw)3)
202
#endif /* SLJIT_CONFIG_ARM_V7 */
203

204
#if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
205
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
206
#	define IS_COND		0x04
207
#	define IS_BL		0x08
208
	/* mov_addr does not use IS_BL */
209
#	define IS_ABS		IS_BL
210
	/* conditional + imm8 */
211
#	define PATCH_TYPE1	0x10
212
	/* conditional + imm20 */
213
#	define PATCH_TYPE2	0x20
214
	/* imm11 */
215
#	define PATCH_TYPE3	0x30
216
	/* imm24 */
217
#	define PATCH_TYPE4	0x40
218
	/* BL + imm24 */
219
#	define PATCH_TYPE5	0x50
220
	/* addwi/subwi */
221
#	define PATCH_TYPE6	0x60
222
	/* 0xf00 cc code for branches */
223
#	define JUMP_SIZE_SHIFT	26
224
#	define JUMP_MAX_SIZE	((sljit_uw)5)
225
#endif /* SLJIT_CONFIG_ARM_THUMB2 */
226

227
#if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
228
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
229
#	define IS_COND		0x004
230
#	define IS_CBZ		0x008
231
#	define IS_BL		0x010
232
#	define PATCH_COND	0x020
233
#	define PATCH_B		0x040
234
#	define PATCH_B32	0x080
235
#	define PATCH_ABS48	0x100
236
#	define PATCH_ABS64	0x200
237
#	define JUMP_SIZE_SHIFT	58
238
#	define JUMP_MAX_SIZE	((sljit_uw)5)
239
#endif /* SLJIT_CONFIG_ARM_64 */
240

241
#if (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
242
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
243
#	define IS_COND		0x004
244
#	define IS_CALL		0x008
245
#	define PATCH_B		0x010
246
#	define PATCH_ABS_B	0x020
247
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
248
#	define PATCH_ABS32	0x040
249
#	define PATCH_ABS48	0x080
250
#	define JUMP_SIZE_SHIFT	58
251
#	define JUMP_MAX_SIZE	((sljit_uw)7)
252
#else /* !SLJIT_CONFIG_PPC_64 */
253
#	define JUMP_SIZE_SHIFT	26
254
#	define JUMP_MAX_SIZE	((sljit_uw)4)
255
#endif /* SLJIT_CONFIG_PPC_64 */
256
#endif /* SLJIT_CONFIG_PPC */
257

258
#if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
259
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
260
#	define IS_MOVABLE	0x004
261
#	define IS_JAL		0x008
262
#	define IS_CALL		0x010
263
#	define IS_BIT26_COND	0x020
264
#	define IS_BIT16_COND	0x040
265
#	define IS_BIT23_COND	0x080
266

267
#	define IS_COND		(IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND)
268

269
#	define PATCH_B		0x100
270
#	define PATCH_J		0x200
271

272
#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64)
273
#	define PATCH_ABS32	0x400
274
#	define PATCH_ABS48	0x800
275
#endif /* SLJIT_CONFIG_MIPS_64 */
276

277
	/* instruction types */
278
#	define MOVABLE_INS	0
279
	/* 1 - 31 last destination register */
280
	/* no destination (i.e: store) */
281
#	define UNMOVABLE_INS	32
282
	/* FPU status register */
283
#	define FCSR_FCC		33
284
#endif /* SLJIT_CONFIG_MIPS */
285

286
#if (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
287
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
288
#	define IS_COND		0x0004
289
#	define IS_CALL		0x0008
290

291
#	define IS_COND16	0x0010
292
#	define PATCH_B		0x0020
293
#	define PATCH_J		0x0040
294
#	define PATCH_16		0x0080
295

296
#if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
297
#	define PATCH_REL32	0x0100
298
#	define PATCH_ABS32	0x0200
299
#	define PATCH_ABS44	0x0400
300
#	define PATCH_ABS52	0x0800
301
#	define JUMP_SIZE_SHIFT	58
302
#	define JUMP_MAX_SIZE	((sljit_uw)12)
303
#else /* !SLJIT_CONFIG_RISCV_64 */
304
#	define JUMP_SIZE_SHIFT	26
305
#	define JUMP_MAX_SIZE	((sljit_uw)4)
306
#endif /* SLJIT_CONFIG_RISCV_64 */
307
#endif /* SLJIT_CONFIG_RISCV */
308

309
#if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
310
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
311
#	define PATCH_POOL		0x004
312
#	define PATCH_IMM32		0x008
313
#endif /* SLJIT_CONFIG_S390X */
314

315
#if (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
316
/* SLJIT_REWRITABLE_JUMP is 0x10000. */
317
#	define IS_COND		0x004
318
#	define IS_CALL		0x008
319

320
#	define PATCH_B		0x010
321
#	define PATCH_J		0x020
322

323
#	define PATCH_REL32	0x040
324
#	define PATCH_ABS32	0x080
325
#	define PATCH_ABS52	0x100
326
#	define JUMP_SIZE_SHIFT	58
327
#	define JUMP_MAX_SIZE	((sljit_uw)4)
328

329
#endif /* SLJIT_CONFIG_LOONGARCH */
330

331
/* Stack management. */
332

333
#define GET_SAVED_REGISTERS_SIZE(scratches, saveds, extra) \
334
	(((scratches < SLJIT_NUMBER_OF_SCRATCH_REGISTERS ? 0 : (scratches - SLJIT_NUMBER_OF_SCRATCH_REGISTERS)) + \
335
		(saveds) + (sljit_s32)(extra)) * (sljit_s32)sizeof(sljit_sw))
336

337
#define GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, type) \
338
	(((fscratches < SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS ? 0 : (fscratches - SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)) + \
339
		(fsaveds)) * SSIZE_OF(type))
340

341
#define ADJUST_LOCAL_OFFSET(p, i) \
342
	if ((p) == (SLJIT_MEM1(SLJIT_SP))) \
343
		(i) += SLJIT_LOCALS_OFFSET;
344

345
#endif /* !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) */
346

347
/* Utils can still be used even if SLJIT_CONFIG_UNSUPPORTED is set. */
348
#include "sljitUtils.c"
349

350
#if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
351
#define SLJIT_CODE_TO_PTR(code) ((void*)((sljit_up)(code) & ~(sljit_up)0x1))
352
#elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
353
#define SLJIT_CODE_TO_PTR(code) ((void*)(*(sljit_up*)code))
354
#else /* !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_INDIRECT_CALL */
355
#define SLJIT_CODE_TO_PTR(code) ((void*)(code))
356
#endif /* SLJIT_CONFIG_ARM_THUMB2 || SLJIT_INDIRECT_CALL */
357

358
#if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
359

360
#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR)
361

362
#if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
363

364
#if defined(__NetBSD__)
365
#include "allocator_src/sljitProtExecAllocatorNetBSD.c"
366
#else /* !__NetBSD__ */
367
#include "allocator_src/sljitProtExecAllocatorPosix.c"
368
#endif /* __NetBSD__ */
369

370
#elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR)
371

372
#if defined(_WIN32)
373
#include "allocator_src/sljitWXExecAllocatorWindows.c"
374
#else /* !_WIN32 */
375
#include "allocator_src/sljitWXExecAllocatorPosix.c"
376
#endif /* _WIN32 */
377

378
#else /* !SLJIT_PROT_EXECUTABLAE_ALLOCATOR && !SLJIT_WX_EXECUTABLE_ALLOCATOR */
379

380
#if defined(_WIN32)
381
#include "allocator_src/sljitExecAllocatorWindows.c"
382
#elif defined(__APPLE__)
383
#include "allocator_src/sljitExecAllocatorApple.c"
384
#elif defined(__FreeBSD__)
385
#include "allocator_src/sljitExecAllocatorFreeBSD.c"
386
#else /* !_WIN32 && !__APPLE__ && !__FreeBSD__ */
387
#include "allocator_src/sljitExecAllocatorPosix.c"
388
#endif /* _WIN32 */
389

390
#endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
391

392
#else /* !SLJIT_EXECUTABLE_ALLOCATOR */
393

394
#ifndef SLJIT_UPDATE_WX_FLAGS
395
#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec)
396
#endif /* SLJIT_UPDATE_WX_FLAGS */
397

398
#endif /* SLJIT_EXECUTABLE_ALLOCATOR */
399

400
#if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
401
#define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr) + (exec_offset))
402
#else /* !SLJIT_PROT_EXECUTABLE_ALLOCATOR */
403
#define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr))
404
#endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
405

406
/* Argument checking features. */
407

408
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
409

410
/* Returns with error when an invalid argument is passed. */
411

412
#define CHECK_ARGUMENT(x) \
413
	do { \
414
		if (SLJIT_UNLIKELY(!(x))) \
415
			return 1; \
416
	} while (0)
417

418
#define CHECK_RETURN_TYPE sljit_s32
419
#define CHECK_RETURN_OK return 0
420

421
#define CHECK(x) \
422
	do { \
423
		if (SLJIT_UNLIKELY(x)) { \
424
			compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
425
			return SLJIT_ERR_BAD_ARGUMENT; \
426
		} \
427
	} while (0)
428

429
#define CHECK_PTR(x) \
430
	do { \
431
		if (SLJIT_UNLIKELY(x)) { \
432
			compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
433
			return NULL; \
434
		} \
435
	} while (0)
436

437
#define CHECK_REG_INDEX(x) \
438
	do { \
439
		if (SLJIT_UNLIKELY(x)) { \
440
			return -2; \
441
		} \
442
	} while (0)
443

444
#elif (defined SLJIT_DEBUG && SLJIT_DEBUG)
445

446
/* Assertion failure occures if an invalid argument is passed. */
447
#undef SLJIT_ARGUMENT_CHECKS
448
#define SLJIT_ARGUMENT_CHECKS 1
449

450
#define CHECK_ARGUMENT(x) SLJIT_ASSERT(x)
451
#define CHECK_RETURN_TYPE void
452
#define CHECK_RETURN_OK return
453
#define CHECK(x) x
454
#define CHECK_PTR(x) x
455
#define CHECK_REG_INDEX(x) x
456

457
#elif (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
458

459
/* Arguments are not checked. */
460
#define CHECK_RETURN_TYPE void
461
#define CHECK_RETURN_OK return
462
#define CHECK(x) x
463
#define CHECK_PTR(x) x
464
#define CHECK_REG_INDEX(x) x
465

466
#else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_DEBUG && !SLJIT_VERBOSE */
467

468
/* Arguments are not checked. */
469
#define CHECK(x)
470
#define CHECK_PTR(x)
471
#define CHECK_REG_INDEX(x)
472

473
#endif /* SLJIT_ARGUMENT_CHECKS */
474

475
/* --------------------------------------------------------------------- */
476
/*  Public functions                                                     */
477
/* --------------------------------------------------------------------- */
478

479
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) || (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
480
#define SLJIT_NEEDS_COMPILER_INIT 1
481
static sljit_s32 compiler_initialized = 0;
482
/* A thread safe initialization. */
483
static void init_compiler(void);
484
#endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_RISCV */
485

486
SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data)
487
{
488
	struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data);
489
	if (!compiler)
490
		return NULL;
491
	SLJIT_ZEROMEM(compiler, sizeof(struct sljit_compiler));
492

493
	SLJIT_COMPILE_ASSERT(
494
		sizeof(sljit_s8) == 1 && sizeof(sljit_u8) == 1
495
		&& sizeof(sljit_s16) == 2 && sizeof(sljit_u16) == 2
496
		&& sizeof(sljit_s32) == 4 && sizeof(sljit_u32) == 4
497
		&& (sizeof(sljit_up) == 4 || sizeof(sljit_up) == 8)
498
		&& sizeof(sljit_up) <= sizeof(sljit_sw)
499
		&& sizeof(sljit_up) == sizeof(sljit_sp)
500
		&& (sizeof(sljit_sw) == 4 || sizeof(sljit_sw) == 8)
501
		&& (sizeof(sljit_uw) == sizeof(sljit_sw)),
502
		invalid_integer_types);
503
	SLJIT_COMPILE_ASSERT((SLJIT_REWRITABLE_JUMP & SLJIT_32) == 0
504
			&& (ALL_STATUS_FLAGS_MASK & SLJIT_32) == 0
505
			&& (SLJIT_REWRITABLE_JUMP & ALL_STATUS_FLAGS_MASK) == 0,
506
		rewritable_jump_and_status_flag_bits_and_sljit_32_must_not_have_common_bits);
507
	SLJIT_COMPILE_ASSERT((VARIABLE_FLAG_MASK & SLJIT_SET_Z) == 0,
508
		rewritable_jump_and_status_flag_bits_must_not_have_common_bits);
509
	SLJIT_COMPILE_ASSERT(!(SLJIT_EQUAL & 0x1) && !(SLJIT_LESS & 0x1) && !(SLJIT_F_EQUAL & 0x1) && !(SLJIT_JUMP & 0x1),
510
		conditional_flags_must_be_even_numbers);
511

512
	/* Only the non-zero members must be set. */
513
	compiler->error = SLJIT_SUCCESS;
514

515
	compiler->allocator_data = allocator_data;
516
	compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data);
517
	compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data);
518

519
	if (!compiler->buf || !compiler->abuf) {
520
		if (compiler->buf)
521
			SLJIT_FREE(compiler->buf, allocator_data);
522
		if (compiler->abuf)
523
			SLJIT_FREE(compiler->abuf, allocator_data);
524
		SLJIT_FREE(compiler, allocator_data);
525
		return NULL;
526
	}
527

528
	compiler->buf->next = NULL;
529
	compiler->buf->used_size = 0;
530
	compiler->abuf->next = NULL;
531
	compiler->abuf->used_size = 0;
532

533
	compiler->scratches = -1;
534
	compiler->saveds = -1;
535
	compiler->fscratches = -1;
536
	compiler->fsaveds = -1;
537
#if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS) \
538
		|| (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
539
		|| (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
540
	compiler->vscratches = -1;
541
	compiler->vsaveds = -1;
542
#endif /* SLJIT_SEPARATE_VECTOR_REGISTERS || SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
543
	compiler->local_size = -1;
544

545
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
546
	compiler->args_size = -1;
547
#endif /* SLJIT_CONFIG_X86_32 */
548

549
#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
550
	compiler->cpool = (sljit_uw*)SLJIT_MALLOC(CPOOL_SIZE * sizeof(sljit_uw)
551
		+ CPOOL_SIZE * sizeof(sljit_u8), allocator_data);
552
	if (!compiler->cpool) {
553
		SLJIT_FREE(compiler->buf, allocator_data);
554
		SLJIT_FREE(compiler->abuf, allocator_data);
555
		SLJIT_FREE(compiler, allocator_data);
556
		return NULL;
557
	}
558
	compiler->cpool_unique = (sljit_u8*)(compiler->cpool + CPOOL_SIZE);
559
	compiler->cpool_diff = 0xffffffff;
560
#endif /* SLJIT_CONFIG_ARM_V6 */
561

562
#if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
563
	compiler->delay_slot = UNMOVABLE_INS;
564
#endif /* SLJIT_CONFIG_MIPS */
565

566
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
567
		|| (defined SLJIT_DEBUG && SLJIT_DEBUG)
568
	SLJIT_ASSERT(compiler->last_flags == 0 && compiler->logical_local_size == 0);
569
	compiler->last_return = -1;
570
#endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_DEBUG */
571

572
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
573
		|| (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
574
#if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
575
	compiler->real_fscratches = -1;
576
	compiler->real_fsaveds = -1;
577
#endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
578
	SLJIT_ASSERT(compiler->skip_checks == 0);
579
#endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
580

581
#if (defined SLJIT_NEEDS_COMPILER_INIT && SLJIT_NEEDS_COMPILER_INIT)
582
	if (!compiler_initialized) {
583
		init_compiler();
584
		compiler_initialized = 1;
585
	}
586
#endif /* SLJIT_NEEDS_COMPILER_INIT */
587

588
	return compiler;
589
}
590

591
SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler)
592
{
593
	struct sljit_memory_fragment *buf;
594
	struct sljit_memory_fragment *curr;
595
	void *allocator_data = compiler->allocator_data;
596
	SLJIT_UNUSED_ARG(allocator_data);
597

598
	buf = compiler->buf;
599
	while (buf) {
600
		curr = buf;
601
		buf = buf->next;
602
		SLJIT_FREE(curr, allocator_data);
603
	}
604

605
	buf = compiler->abuf;
606
	while (buf) {
607
		curr = buf;
608
		buf = buf->next;
609
		SLJIT_FREE(curr, allocator_data);
610
	}
611

612
#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
613
	SLJIT_FREE(compiler->cpool, allocator_data);
614
#endif /* SLJIT_CONFIG_ARM_V6 */
615
	SLJIT_FREE(compiler, allocator_data);
616
}
617

618
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler)
619
{
620
	if (compiler->error == SLJIT_SUCCESS)
621
		compiler->error = SLJIT_ERR_ALLOC_FAILED;
622
}
623

624
SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data)
625
{
626
	SLJIT_UNUSED_ARG(exec_allocator_data);
627

628
	SLJIT_FREE_EXEC(SLJIT_CODE_TO_PTR(code), exec_allocator_data);
629
}
630

631
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_label(struct sljit_jump *jump, struct sljit_label* label)
632
{
633
	if (SLJIT_LIKELY(!!jump) && SLJIT_LIKELY(!!label)) {
634
		jump->flags &= (sljit_uw)~JUMP_ADDR;
635
		jump->u.label = label;
636
	}
637
}
638

639
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw target)
640
{
641
	if (SLJIT_LIKELY(!!jump)) {
642
		jump->flags |= JUMP_ADDR;
643
		jump->u.target = target;
644
	}
645
}
646

647
#define SLJIT_CURRENT_FLAGS_ALL (SLJIT_CURRENT_FLAGS_32 | SLJIT_CURRENT_FLAGS_ADD \
648
	| SLJIT_CURRENT_FLAGS_SUB | SLJIT_CURRENT_FLAGS_COMPARE | SLJIT_CURRENT_FLAGS_OP2CMPZ)
649

650
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags)
651
{
652
	SLJIT_UNUSED_ARG(compiler);
653
	SLJIT_UNUSED_ARG(current_flags);
654

655
#if (defined SLJIT_HAS_STATUS_FLAGS_STATE && SLJIT_HAS_STATUS_FLAGS_STATE)
656
#if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
657
	if (current_flags & SLJIT_CURRENT_FLAGS_OP2CMPZ) {
658
		current_flags |= SLJIT_SET_Z;
659
	}
660
#endif /* SLJIT_CONFIG_S390X */
661

662
	compiler->status_flags_state = current_flags;
663
#endif /* SLJIT_HAS_STATUS_FLAGS_STATE */
664

665
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
666
	compiler->last_flags = 0;
667
	if ((current_flags & ~(ALL_STATUS_FLAGS_MASK | SLJIT_CURRENT_FLAGS_ALL)) == 0) {
668
		compiler->last_flags = GET_FLAG_TYPE(current_flags) | (current_flags & (SLJIT_32 | SLJIT_SET_Z));
669
	}
670
#endif /* SLJIT_ARGUMENT_CHECKS */
671
}
672

673
/* --------------------------------------------------------------------- */
674
/*  Private functions                                                    */
675
/* --------------------------------------------------------------------- */
676

677
static void* ensure_buf(struct sljit_compiler *compiler, sljit_uw size)
678
{
679
	sljit_u8 *ret;
680
	struct sljit_memory_fragment *new_frag;
681

682
	SLJIT_ASSERT(size <= 256);
683
	if (compiler->buf->used_size + size <= (BUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
684
		ret = compiler->buf->memory + compiler->buf->used_size;
685
		compiler->buf->used_size += size;
686
		return ret;
687
	}
688
	new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, compiler->allocator_data);
689
	PTR_FAIL_IF_NULL(new_frag);
690
	new_frag->next = compiler->buf;
691
	compiler->buf = new_frag;
692
	new_frag->used_size = size;
693
	return new_frag->memory;
694
}
695

696
static void* ensure_abuf(struct sljit_compiler *compiler, sljit_uw size)
697
{
698
	sljit_u8 *ret;
699
	struct sljit_memory_fragment *new_frag;
700

701
	SLJIT_ASSERT(size <= 256);
702
	if (compiler->abuf->used_size + size <= (ABUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
703
		ret = compiler->abuf->memory + compiler->abuf->used_size;
704
		compiler->abuf->used_size += size;
705
		return ret;
706
	}
707
	new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, compiler->allocator_data);
708
	PTR_FAIL_IF_NULL(new_frag);
709
	new_frag->next = compiler->abuf;
710
	compiler->abuf = new_frag;
711
	new_frag->used_size = size;
712
	return new_frag->memory;
713
}
714

715
SLJIT_API_FUNC_ATTRIBUTE void* sljit_alloc_memory(struct sljit_compiler *compiler, sljit_s32 size)
716
{
717
	CHECK_ERROR_PTR();
718

719
#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
720
	if (size <= 0 || size > 128)
721
		return NULL;
722
	size = (size + 7) & ~7;
723
#else /* !SLJIT_64BIT_ARCHITECTURE */
724
	if (size <= 0 || size > 64)
725
		return NULL;
726
	size = (size + 3) & ~3;
727
#endif /* SLJIT_64BIT_ARCHITECTURE */
728
	return ensure_abuf(compiler, (sljit_uw)size);
729
}
730

731
static SLJIT_INLINE void reverse_buf(struct sljit_compiler *compiler)
732
{
733
	struct sljit_memory_fragment *buf = compiler->buf;
734
	struct sljit_memory_fragment *prev = NULL;
735
	struct sljit_memory_fragment *tmp;
736

737
	do {
738
		tmp = buf->next;
739
		buf->next = prev;
740
		prev = buf;
741
		buf = tmp;
742
	} while (buf != NULL);
743

744
	compiler->buf = prev;
745
}
746

747
static SLJIT_INLINE void* allocate_executable_memory(sljit_uw size, sljit_s32 options,
748
	void *exec_allocator_data, sljit_sw *executable_offset)
749
{
750
	void *code;
751
	struct sljit_generate_code_buffer *buffer;
752

753
	if (SLJIT_LIKELY(!(options & SLJIT_GENERATE_CODE_BUFFER))) {
754
		code = SLJIT_MALLOC_EXEC(size, exec_allocator_data);
755
		*executable_offset = SLJIT_EXEC_OFFSET(code);
756
		return code;
757
	}
758

759
	buffer = (struct sljit_generate_code_buffer*)exec_allocator_data;
760

761
	if (size <= buffer->size) {
762
		*executable_offset = buffer->executable_offset;
763
		return buffer->buffer;
764
	}
765

766
	return NULL;
767
}
768

769
#define SLJIT_MAX_ADDRESS ~(sljit_uw)0
770

771
#define SLJIT_GET_NEXT_SIZE(ptr) (ptr != NULL) ? ((ptr)->size) : SLJIT_MAX_ADDRESS
772
#define SLJIT_GET_NEXT_ADDRESS(ptr) (ptr != NULL) ? ((ptr)->addr) : SLJIT_MAX_ADDRESS
773

774
#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
775

776
#define SLJIT_NEXT_DEFINE_TYPES \
777
	sljit_uw next_label_size; \
778
	sljit_uw next_jump_addr; \
779
	sljit_uw next_const_addr; \
780
	sljit_uw next_min_addr
781

782
#define SLJIT_NEXT_INIT_TYPES() \
783
	next_label_size = SLJIT_GET_NEXT_SIZE(label); \
784
	next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump); \
785
	next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
786

787
#define SLJIT_GET_NEXT_MIN() \
788
	next_min_addr = sljit_get_next_min(next_label_size, next_jump_addr, next_const_addr);
789

790
static SLJIT_INLINE sljit_uw sljit_get_next_min(sljit_uw next_label_size,
791
	sljit_uw next_jump_addr, sljit_uw next_const_addr)
792
{
793
	sljit_uw result = next_jump_addr;
794

795
	SLJIT_ASSERT(result == SLJIT_MAX_ADDRESS || result != next_const_addr);
796

797
	if (next_const_addr < result)
798
		result = next_const_addr;
799

800
	if (next_label_size < result)
801
		result = next_label_size;
802

803
	return result;
804
}
805

806
#endif /* !SLJIT_CONFIG_X86 */
807

808
#if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
809

810
static void update_float_register_count(struct sljit_compiler *compiler, sljit_s32 scratches, sljit_s32 saveds)
811
{
812
	sljit_s32 vscratches = ENTER_GET_VECTOR_REGS(scratches);
813
	sljit_s32 vsaveds = ENTER_GET_VECTOR_REGS(saveds);
814

815
	if (compiler->fscratches < vscratches)
816
		compiler->fscratches = vscratches;
817

818
	if (compiler->fsaveds < vsaveds)
819
		compiler->fsaveds = vsaveds;
820

821
	if (compiler->fsaveds + compiler->fscratches > SLJIT_NUMBER_OF_FLOAT_REGISTERS)
822
		compiler->fscratches = SLJIT_NUMBER_OF_FLOAT_REGISTERS - compiler->fsaveds;
823
}
824

825
#endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
826

827
static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler,
828
	sljit_s32 options, sljit_s32 args,
829
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
830
{
831
	SLJIT_UNUSED_ARG(args);
832
	SLJIT_UNUSED_ARG(local_size);
833

834
	compiler->options = options;
835
	compiler->scratches = ENTER_GET_REGS(scratches);
836
	compiler->saveds = ENTER_GET_REGS(saveds);
837
	/* These members may be copied to real_* members below. */
838
	compiler->fscratches = ENTER_GET_FLOAT_REGS(scratches);
839
	compiler->fsaveds = ENTER_GET_FLOAT_REGS(saveds);
840
#if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
841
	compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
842
	compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
843
#else /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
844
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
845
		|| (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
846
	compiler->real_fscratches = compiler->fscratches;
847
	compiler->real_fsaveds = compiler->fsaveds;
848
	compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
849
	compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
850
#endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
851
	update_float_register_count(compiler, scratches, saveds);
852
#endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
853
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
854
	compiler->last_return = args & SLJIT_ARG_MASK;
855
	compiler->logical_local_size = local_size;
856
#endif /* SLJIT_ARGUMENT_CHECKS */
857
}
858

859
static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compiler *compiler)
860
{
861
	label->next = NULL;
862
	label->u.index = compiler->label_count++;
863
	label->size = compiler->size;
864

865
	if (compiler->last_label != NULL)
866
		compiler->last_label->next = label;
867
	else
868
		compiler->labels = label;
869

870
	compiler->last_label = label;
871
}
872

873
static SLJIT_INLINE void set_extended_label(struct sljit_extended_label *ext_label, struct sljit_compiler *compiler, sljit_uw type, sljit_uw data)
874
{
875
	set_label(&ext_label->label, compiler);
876
	ext_label->index = ext_label->label.u.index;
877
	ext_label->label.u.index = type;
878
	ext_label->data = data;
879
}
880

881
static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)
882
{
883
	jump->next = NULL;
884
	jump->flags = flags;
885
	jump->u.label = NULL;
886

887
	if (compiler->last_jump != NULL)
888
		compiler->last_jump->next = jump;
889
	else
890
		compiler->jumps = jump;
891

892
	compiler->last_jump = jump;
893
}
894

895
static SLJIT_INLINE void set_mov_addr(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_uw offset)
896
{
897
	jump->next = NULL;
898
	jump->addr = compiler->size - offset;
899
	jump->flags = JUMP_MOV_ADDR;
900
	jump->u.label = NULL;
901
	if (compiler->last_jump != NULL)
902
		compiler->last_jump->next = jump;
903
	else
904
		compiler->jumps = jump;
905
	compiler->last_jump = jump;
906
}
907

908
static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_compiler *compiler)
909
{
910
	const_->next = NULL;
911
	const_->addr = compiler->size;
912
	if (compiler->last_const != NULL)
913
		compiler->last_const->next = const_;
914
	else
915
		compiler->consts = const_;
916
	compiler->last_const = const_;
917
}
918

919
#define ADDRESSING_DEPENDS_ON(exp, reg) \
920
	(((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg))
921

922
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
923

924
static sljit_s32 function_check_arguments(sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches)
925
{
926
	sljit_s32 word_arg_count, scratch_arg_end, saved_arg_count, float_arg_count, curr_type;
927

928
	curr_type = (arg_types & SLJIT_ARG_FULL_MASK);
929

930
	if (curr_type >= SLJIT_ARG_TYPE_F64) {
931
		if (curr_type > SLJIT_ARG_TYPE_F32 || fscratches == 0)
932
			return 0;
933
	} else if (curr_type >= SLJIT_ARG_TYPE_W) {
934
		if (scratches == 0)
935
			return 0;
936
	}
937

938
	arg_types >>= SLJIT_ARG_SHIFT;
939

940
	word_arg_count = 0;
941
	scratch_arg_end = 0;
942
	saved_arg_count = 0;
943
	float_arg_count = 0;
944
	while (arg_types != 0) {
945
		if (word_arg_count + float_arg_count >= 4)
946
			return 0;
947

948
		curr_type = (arg_types & SLJIT_ARG_MASK);
949

950
		if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
951
			if (saveds == -1 || curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_P)
952
				return 0;
953

954
			word_arg_count++;
955
			scratch_arg_end = word_arg_count;
956
		} else {
957
			if (curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_F32)
958
				return 0;
959

960
			if (curr_type < SLJIT_ARG_TYPE_F64) {
961
				word_arg_count++;
962
				saved_arg_count++;
963
			} else
964
				float_arg_count++;
965
		}
966

967
		arg_types >>= SLJIT_ARG_SHIFT;
968
	}
969

970
	if (saveds == -1)
971
		return (word_arg_count <= scratches && float_arg_count <= fscratches);
972

973
	return (saved_arg_count <= saveds && scratch_arg_end <= scratches && float_arg_count <= fscratches);
974
}
975

976
#define FUNCTION_CHECK_IS_REG(r) \
977
	(((r) >= SLJIT_R0 && (r) < (SLJIT_R0 + compiler->scratches)) \
978
	|| ((r) > (SLJIT_S0 - compiler->saveds) && (r) <= SLJIT_S0) \
979
	|| ((r) >= SLJIT_TMP_REGISTER_BASE && (r) < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)))
980

981
#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
982
#define CHECK_IF_VIRTUAL_REGISTER(p) ((p) <= SLJIT_S3 && (p) >= SLJIT_S8)
983
#else /* !SLJIT_CONFIG_X86_32 */
984
#define CHECK_IF_VIRTUAL_REGISTER(p) 0
985
#endif /* SLJIT_CONFIG_X86_32 */
986

987
static sljit_s32 function_check_src_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
988
{
989
	if (compiler->scratches == -1)
990
		return 0;
991

992
	if (!(p & SLJIT_MEM))
993
		return 0;
994

995
	if (p == SLJIT_MEM1(SLJIT_SP))
996
		return (i >= 0 && i < compiler->logical_local_size);
997

998
	if (!(!(p & REG_MASK) || FUNCTION_CHECK_IS_REG(p & REG_MASK)))
999
		return 0;
1000

1001
	if (CHECK_IF_VIRTUAL_REGISTER(p & REG_MASK))
1002
		return 0;
1003

1004
	if (p & OFFS_REG_MASK) {
1005
		if (!(p & REG_MASK))
1006
			return 0;
1007

1008
		if (!(FUNCTION_CHECK_IS_REG(OFFS_REG(p))))
1009
			return 0;
1010

1011
		if (CHECK_IF_VIRTUAL_REGISTER(OFFS_REG(p)))
1012
			return 0;
1013

1014
		if ((i & ~0x3) != 0)
1015
			return 0;
1016
	}
1017

1018
	return (p & ~(SLJIT_MEM | REG_MASK | OFFS_REG_MASK)) == 0;
1019
}
1020

1021
#define FUNCTION_CHECK_SRC_MEM(p, i) \
1022
	CHECK_ARGUMENT(function_check_src_mem(compiler, p, i));
1023

1024
static sljit_s32 function_check_src(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1025
{
1026
	if (compiler->scratches == -1)
1027
		return 0;
1028

1029
	if (FUNCTION_CHECK_IS_REG(p))
1030
		return (i == 0);
1031

1032
	if (p == SLJIT_IMM)
1033
		return 1;
1034

1035
	return function_check_src_mem(compiler, p, i);
1036
}
1037

1038
#define FUNCTION_CHECK_SRC(p, i) \
1039
	CHECK_ARGUMENT(function_check_src(compiler, p, i));
1040

1041
static sljit_s32 function_check_dst(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1042
{
1043
	if (compiler->scratches == -1)
1044
		return 0;
1045

1046
	if (FUNCTION_CHECK_IS_REG(p))
1047
		return (i == 0);
1048

1049
	return function_check_src_mem(compiler, p, i);
1050
}
1051

1052
#define FUNCTION_CHECK_DST(p, i) \
1053
	CHECK_ARGUMENT(function_check_dst(compiler, p, i));
1054

1055
#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1056
	|| (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1057

1058
#define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1059
	function_check_is_freg(compiler, (fr), (is_32))
1060

1061
static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32);
1062

1063
#define FUNCTION_FCHECK(p, i, is_32) \
1064
	CHECK_ARGUMENT(function_fcheck(compiler, (p), (i), (is_32)));
1065

1066
static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 is_32)
1067
{
1068
	if (compiler->scratches == -1)
1069
		return 0;
1070

1071
	if (FUNCTION_CHECK_IS_FREG(p, is_32))
1072
		return (i == 0);
1073

1074
	return function_check_src_mem(compiler, p, i);
1075
}
1076

1077
static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr, sljit_s32 type);
1078

1079
#define FUNCTION_CHECK_IS_VREG(vr, type) \
1080
	function_check_is_vreg(compiler, (vr), (type))
1081

1082
#define FUNCTION_VCHECK(p, i, type) \
1083
	CHECK_ARGUMENT(function_vcheck(compiler, (p), (i), (type)))
1084

1085
static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 type)
1086
{
1087
	if (compiler->scratches == -1)
1088
		return 0;
1089

1090
	if (FUNCTION_CHECK_IS_VREG(p, type))
1091
		return (i == 0);
1092

1093
	return function_check_src_mem(compiler, p, i);
1094
}
1095

1096
#else /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS_32 */
1097

1098
#define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1099
	function_check_is_freg(compiler, (fr))
1100

1101
static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr)
1102
{
1103
	sljit_s32 fscratches, fsaveds;
1104

1105
	if (compiler->scratches == -1)
1106
		return 0;
1107

1108
#if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
1109
	fscratches = compiler->fscratches;
1110
	fsaveds = compiler->fsaveds;
1111
#else /* SLJIT_SEPARATE_VECTOR_REGISTERS */
1112
	fscratches = compiler->real_fscratches;
1113
	fsaveds = compiler->real_fsaveds;
1114
#endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
1115

1116
	return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + fscratches))
1117
		|| (fr > (SLJIT_FS0 - fsaveds) && fr <= SLJIT_FS0)
1118
		|| (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));
1119
}
1120

1121
#define FUNCTION_FCHECK(p, i, is_32) \
1122
	CHECK_ARGUMENT(function_fcheck(compiler, (p), (i)));
1123

1124
static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1125
{
1126
	if (compiler->scratches == -1)
1127
		return 0;
1128

1129
	if (function_check_is_freg(compiler, p))
1130
		return (i == 0);
1131

1132
	return function_check_src_mem(compiler, p, i);
1133
}
1134

1135
#define FUNCTION_CHECK_IS_VREG(vr, type) \
1136
	function_check_is_vreg(compiler, (vr))
1137

1138
static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr)
1139
{
1140
	if (compiler->scratches == -1)
1141
		return 0;
1142

1143
	return (vr >= SLJIT_VR0 && vr < (SLJIT_VR0 + compiler->vscratches))
1144
		|| (vr > (SLJIT_VS0 - compiler->vsaveds) && vr <= SLJIT_VS0)
1145
		|| (vr >= SLJIT_TMP_VREGISTER_BASE && vr < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS));
1146
}
1147

1148
#define FUNCTION_VCHECK(p, i, type) \
1149
	CHECK_ARGUMENT(function_vcheck(compiler, (p), (i)))
1150

1151
static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1152
{
1153
	if (compiler->scratches == -1)
1154
		return 0;
1155

1156
	if (function_check_is_vreg(compiler, p))
1157
		return (i == 0);
1158

1159
	return function_check_src_mem(compiler, p, i);
1160
}
1161

1162
#endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1163

1164
#endif /* SLJIT_ARGUMENT_CHECKS */
1165

1166
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1167

1168
SLJIT_API_FUNC_ATTRIBUTE void sljit_compiler_verbose(struct sljit_compiler *compiler, FILE* verbose)
1169
{
1170
	compiler->verbose = verbose;
1171
}
1172

1173
#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
1174
#ifdef _WIN64
1175
#ifdef __GNUC__
1176
#	define SLJIT_PRINT_D	"ll"
1177
#else /* !__GNUC__ */
1178
#	define SLJIT_PRINT_D	"I64"
1179
#endif  /* __GNUC__ */
1180
#else /* !_WIN64 */
1181
#	define SLJIT_PRINT_D	"l"
1182
#endif /* _WIN64 */
1183
#else /* !SLJIT_64BIT_ARCHITECTURE */
1184
#	define SLJIT_PRINT_D	""
1185
#endif /* SLJIT_64BIT_ARCHITECTURE */
1186

1187
static void sljit_verbose_reg(struct sljit_compiler *compiler, sljit_s32 r)
1188
{
1189
	if (r < (SLJIT_R0 + compiler->scratches))
1190
		fprintf(compiler->verbose, "r%d", r - SLJIT_R0);
1191
	else if (r < SLJIT_SP)
1192
		fprintf(compiler->verbose, "s%d", SLJIT_NUMBER_OF_REGISTERS - r);
1193
	else if (r == SLJIT_SP)
1194
		fprintf(compiler->verbose, "sp");
1195
	else
1196
		fprintf(compiler->verbose, "t%d", r - SLJIT_TMP_REGISTER_BASE);
1197
}
1198

1199
static void sljit_verbose_freg(struct sljit_compiler *compiler, sljit_s32 r)
1200
{
1201
#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1202
		|| (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1203
	if (r >= SLJIT_F64_SECOND(SLJIT_FR0)) {
1204
		fprintf(compiler->verbose, "^");
1205
		r -= SLJIT_F64_SECOND(0);
1206
	}
1207
#endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1208

1209
	if (r < (SLJIT_FR0 + compiler->fscratches))
1210
		fprintf(compiler->verbose, "fr%d", r - SLJIT_FR0);
1211
	else if (r < SLJIT_TMP_FREGISTER_BASE)
1212
		fprintf(compiler->verbose, "fs%d", SLJIT_NUMBER_OF_FLOAT_REGISTERS - r);
1213
	else
1214
		fprintf(compiler->verbose, "ft%d", r - SLJIT_TMP_FREGISTER_BASE);
1215
}
1216

1217
static void sljit_verbose_vreg(struct sljit_compiler *compiler, sljit_s32 r)
1218
{
1219
#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1220
		|| (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1221
	if (r >= SLJIT_F64_SECOND(SLJIT_VR0)) {
1222
		fprintf(compiler->verbose, "^");
1223
		r -= SLJIT_F64_SECOND(0);
1224
	}
1225
#endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1226

1227
	if (r < (SLJIT_VR0 + compiler->vscratches))
1228
		fprintf(compiler->verbose, "vr%d", r - SLJIT_VR0);
1229
	else if (r < SLJIT_TMP_VREGISTER_BASE)
1230
		fprintf(compiler->verbose, "vs%d", SLJIT_NUMBER_OF_VECTOR_REGISTERS - r);
1231
	else
1232
		fprintf(compiler->verbose, "vt%d", r - SLJIT_TMP_VREGISTER_BASE);
1233
}
1234

1235
static void sljit_verbose_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1236
{
1237
	if (!(p & REG_MASK)) {
1238
		fprintf(compiler->verbose, "[%" SLJIT_PRINT_D "d]", i);
1239
		return;
1240
	}
1241

1242
	fputc('[', compiler->verbose);
1243
	sljit_verbose_reg(compiler, (p) & REG_MASK);
1244
	if (p & OFFS_REG_MASK) {
1245
		fprintf(compiler->verbose, " + ");
1246
		sljit_verbose_reg(compiler, OFFS_REG(p));
1247
		if (i)
1248
			fprintf(compiler->verbose, " * %d", 1 << (i));
1249
	} else if (i)
1250
		fprintf(compiler->verbose, " + %" SLJIT_PRINT_D "d", (i));
1251
	fputc(']', compiler->verbose);
1252
}
1253

1254
static void sljit_verbose_param(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1255
{
1256
	if (p == SLJIT_IMM)
1257
		fprintf(compiler->verbose, "#%" SLJIT_PRINT_D "d", i);
1258
	else if (p & SLJIT_MEM)
1259
		sljit_verbose_mem(compiler, p, i);
1260
	else
1261
		sljit_verbose_reg(compiler, p);
1262
}
1263

1264
static void sljit_verbose_fparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1265
{
1266
	if (p & SLJIT_MEM)
1267
		sljit_verbose_mem(compiler, p, i);
1268
	else
1269
		sljit_verbose_freg(compiler, p);
1270
}
1271

1272
static void sljit_verbose_vparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1273
{
1274
	if (p & SLJIT_MEM)
1275
		sljit_verbose_mem(compiler, p, i);
1276
	else
1277
		sljit_verbose_vreg(compiler, p);
1278
}
1279

1280
static const char* op0_names[] = {
1281
	"breakpoint", "nop", "lmul.uw", "lmul.sw",
1282
	"divmod.u", "divmod.s", "div.u", "div.s",
1283
	"memory_barrier", "endbr", "skip_frames_before_return"
1284
};
1285

1286
static const char* op1_names[] = {
1287
	"mov", "mov", "mov", "mov",
1288
	"mov", "mov", "mov", "mov",
1289
	"mov", "clz", "ctz", "rev",
1290
	"rev", "rev", "rev", "rev"
1291
};
1292

1293
static const char* op1_types[] = {
1294
	"", ".u8", ".s8", ".u16",
1295
	".s16", ".u32", ".s32", "32",
1296
	".p", "", "", "",
1297
	".u16", ".s16", ".u32", ".s32"
1298
};
1299

1300
static const char* op2_names[] = {
1301
	"add", "addc", "sub", "subc",
1302
	"mul", "and", "or", "xor",
1303
	"shl", "mshl", "lshr", "mlshr",
1304
	"ashr", "mashr", "rotl", "rotr"
1305
};
1306

1307
static const char* op2r_names[] = {
1308
	"muladd"
1309
};
1310

1311
static const char* op_src_dst_names[] = {
1312
	"fast_return", "skip_frames_before_fast_return",
1313
	"prefetch_l1", "prefetch_l2",
1314
	"prefetch_l3", "prefetch_once",
1315
	"fast_enter", "get_return_address"
1316
};
1317

1318
static const char* fop1_names[] = {
1319
	"mov", "conv", "conv", "conv",
1320
	"conv", "conv", "conv", "conv",
1321
	"cmp", "neg", "abs",
1322
};
1323

1324
static const char* fop1_conv_types[] = {
1325
	"sw", "s32", "sw", "s32",
1326
	"uw", "u32"
1327
};
1328

1329
static const char* fop2_names[] = {
1330
	"add", "sub", "mul", "div"
1331
};
1332

1333
static const char* fop2r_names[] = {
1334
	"copysign"
1335
};
1336

1337
static const char* simd_op2_names[] = {
1338
	"and", "or", "xor", "shuffle"
1339
};
1340

1341
static const char* jump_names[] = {
1342
	"equal", "not_equal",
1343
	"less", "greater_equal",
1344
	"greater", "less_equal",
1345
	"sig_less", "sig_greater_equal",
1346
	"sig_greater", "sig_less_equal",
1347
	"overflow", "not_overflow",
1348
	"carry", "not_carry",
1349
	"atomic_stored", "atomic_not_stored",
1350
	"f_equal", "f_not_equal",
1351
	"f_less", "f_greater_equal",
1352
	"f_greater", "f_less_equal",
1353
	"unordered", "ordered",
1354
	"ordered_equal", "unordered_or_not_equal",
1355
	"ordered_less", "unordered_or_greater_equal",
1356
	"ordered_greater", "unordered_or_less_equal",
1357
	"unordered_or_equal", "ordered_not_equal",
1358
	"unordered_or_less", "ordered_greater_equal",
1359
	"unordered_or_greater", "ordered_less_equal",
1360
	"jump", "fast_call",
1361
	"call", "call_reg_arg"
1362
};
1363

1364
static const char* call_arg_names[] = {
1365
	"void", "w", "32", "p", "f64", "f32"
1366
};
1367

1368
static const char* op_addr_types[] = {
1369
	"mov_addr", "mov_abs_addr", "add_abs_addr"
1370
};
1371

1372
#endif /* SLJIT_VERBOSE */
1373

1374
/* --------------------------------------------------------------------- */
1375
/*  Arch dependent                                                       */
1376
/* --------------------------------------------------------------------- */
1377

1378
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
1379
	|| (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1380

1381
#define SLJIT_SKIP_CHECKS(compiler) (compiler)->skip_checks = 1
1382
#define SLJIT_CHECK_OPCODE(op, flags) ((op) & ~(SLJIT_32 | ALL_STATUS_FLAGS_MASK | (flags)))
1383

1384
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options)
1385
{
1386
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1387
	struct sljit_jump *jump;
1388
#endif /* SLJIT_ARGUMENT_CHECKS */
1389

1390
	SLJIT_UNUSED_ARG(compiler);
1391
	SLJIT_UNUSED_ARG(options);
1392

1393
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1394
	CHECK_ARGUMENT(compiler->size > 0);
1395
	CHECK_ARGUMENT((options & ~(SLJIT_GENERATE_CODE_BUFFER | SLJIT_GENERATE_CODE_NO_CONTEXT)) == 0);
1396

1397
	jump = compiler->jumps;
1398
	while (jump) {
1399
		/* All jumps have target. */
1400
		CHECK_ARGUMENT((jump->flags & JUMP_ADDR) || jump->u.label != NULL);
1401
		jump = jump->next;
1402
	}
1403
#endif /* SLJIT_ARGUMENT_CHECKS */
1404
	CHECK_RETURN_OK;
1405
}
1406

1407
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1408
#define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (SLJIT_ENTER_USE_VEX)
1409
#else /* !SLJIT_CONFIG_X86 */
1410
#define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (0)
1411
#endif /* !SLJIT_CONFIG_X86 */
1412

1413
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_enter(struct sljit_compiler *compiler,
1414
	sljit_s32 options, sljit_s32 arg_types,
1415
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1416
{
1417
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1418
	sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1419
	sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1420
	sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1421
	sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1422
	sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1423
	sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1424
#endif /* SLJIT_ARGUMENT_CHECKS */
1425
	SLJIT_UNUSED_ARG(compiler);
1426

1427
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1428
	if (options & SLJIT_ENTER_REG_ARG) {
1429
		CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1430
	} else {
1431
		CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1432
	}
1433
	CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1434
	CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1435
	CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1436
	CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1437
	CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1438
	CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1439
	CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1440
	CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1441
	CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1442
	CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1443
	CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1444
	CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1445
	CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) <= SLJIT_ARG_TYPE_F32);
1446
	CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1447
		(options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1448

1449
	compiler->last_flags = 0;
1450
#endif /* SLJIT_ARGUMENT_CHECKS */
1451
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1452
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1453
		fprintf(compiler->verbose, "  enter ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1454

1455
		arg_types >>= SLJIT_ARG_SHIFT;
1456
		if (arg_types) {
1457
			fprintf(compiler->verbose, "], args[");
1458
			do {
1459
				fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1460
					(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1461
				arg_types >>= SLJIT_ARG_SHIFT;
1462
				if (arg_types)
1463
					fprintf(compiler->verbose, ",");
1464
			} while (arg_types);
1465
		}
1466

1467
		fprintf(compiler->verbose, "],");
1468

1469
		if (options & SLJIT_ENTER_REG_ARG) {
1470
			if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1471
				fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1472
			else
1473
				fprintf(compiler->verbose, " opt:reg_arg,");
1474
		}
1475

1476
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1477
		if (options & SLJIT_ENTER_USE_VEX) {
1478
				fprintf(compiler->verbose, " opt:use_vex,");
1479
		}
1480
#endif /* !SLJIT_CONFIG_X86 */
1481

1482
		fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1483
			ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1484
			ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1485
	}
1486
#endif /* SLJIT_VERBOSE */
1487
	CHECK_RETURN_OK;
1488
}
1489

1490
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_set_context(struct sljit_compiler *compiler,
1491
	sljit_s32 options, sljit_s32 arg_types,
1492
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1493
{
1494
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1495
	sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1496
	sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1497
	sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1498
	sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1499
	sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1500
	sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1501
#endif /* SLJIT_ARGUMENT_CHECKS */
1502
	SLJIT_UNUSED_ARG(compiler);
1503

1504
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1505
	if (options & SLJIT_ENTER_REG_ARG) {
1506
		CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1507
	} else {
1508
		CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1509
	}
1510
	CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1511
	CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1512
	CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1513
	CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1514
	CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1515
	CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1516
	CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1517
	CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1518
	CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1519
	CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1520
	CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1521
	CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1522
	CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) < SLJIT_ARG_TYPE_F64);
1523
	CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1524
		(options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1525

1526
	compiler->last_flags = 0;
1527
#endif /* SLJIT_ARGUMENT_CHECKS */
1528
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1529
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1530
		fprintf(compiler->verbose, "  set_context ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1531

1532
		arg_types >>= SLJIT_ARG_SHIFT;
1533
		if (arg_types) {
1534
			fprintf(compiler->verbose, "], args[");
1535
			do {
1536
				fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1537
					(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1538
				arg_types >>= SLJIT_ARG_SHIFT;
1539
				if (arg_types)
1540
					fprintf(compiler->verbose, ",");
1541
			} while (arg_types);
1542
		}
1543

1544
		fprintf(compiler->verbose, "],");
1545

1546
		if (options & SLJIT_ENTER_REG_ARG) {
1547
			if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1548
				fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1549
			else
1550
				fprintf(compiler->verbose, " opt:reg_arg,");
1551
		}
1552

1553
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1554
		if (options & SLJIT_ENTER_USE_VEX) {
1555
				fprintf(compiler->verbose, " opt:use_vex,");
1556
		}
1557
#endif /* !SLJIT_CONFIG_X86 */
1558

1559
		fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1560
			ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1561
			ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1562
	}
1563
#endif /* SLJIT_VERBOSE */
1564
	CHECK_RETURN_OK;
1565
}
1566

1567
#undef SLJIT_ENTER_CPU_SPECIFIC_OPTIONS
1568

1569
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_void(struct sljit_compiler *compiler)
1570
{
1571
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1572
		compiler->skip_checks = 0;
1573
		CHECK_RETURN_OK;
1574
	}
1575

1576
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1577
	CHECK_ARGUMENT(compiler->last_return == SLJIT_ARG_TYPE_RET_VOID);
1578
#endif /* SLJIT_ARGUMENT_CHECKS */
1579

1580
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1581
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1582
		fprintf(compiler->verbose, "  return_void\n");
1583
	}
1584
#endif /* SLJIT_VERBOSE */
1585
	CHECK_RETURN_OK;
1586
}
1587

1588
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1589
{
1590
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1591
	CHECK_ARGUMENT(compiler->scratches >= 0);
1592

1593
	switch (compiler->last_return) {
1594
	case SLJIT_ARG_TYPE_W:
1595
		CHECK_ARGUMENT(op >= SLJIT_MOV && op <= SLJIT_MOV_S32);
1596
		break;
1597
	case SLJIT_ARG_TYPE_32:
1598
		CHECK_ARGUMENT(op == SLJIT_MOV32 || (op >= SLJIT_MOV32_U8 && op <= SLJIT_MOV32_S16));
1599
		break;
1600
	case SLJIT_ARG_TYPE_P:
1601
		CHECK_ARGUMENT(op == SLJIT_MOV_P);
1602
		break;
1603
	case SLJIT_ARG_TYPE_F64:
1604
		CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1605
		CHECK_ARGUMENT(op == SLJIT_MOV_F64);
1606
		break;
1607
	case SLJIT_ARG_TYPE_F32:
1608
		CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1609
		CHECK_ARGUMENT(op == SLJIT_MOV_F32);
1610
		break;
1611
	default:
1612
		/* Context not initialized, void, etc. */
1613
		CHECK_ARGUMENT(0);
1614
		break;
1615
	}
1616

1617
	if (SLJIT_CHECK_OPCODE(op, 0) < SLJIT_MOV_F64) {
1618
		FUNCTION_CHECK_SRC(src, srcw);
1619
	} else {
1620
		FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1621
	}
1622
	compiler->last_flags = 0;
1623
#endif /* SLJIT_ARGUMENT_CHECKS */
1624
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1625
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1626
		if (GET_OPCODE(op) < SLJIT_MOV_F64) {
1627
			fprintf(compiler->verbose, "  return%s%s ", !(op & SLJIT_32) ? "" : "32",
1628
				op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1629
			sljit_verbose_param(compiler, src, srcw);
1630
		} else {
1631
			fprintf(compiler->verbose, "  return%s ", !(op & SLJIT_32) ? ".f64" : ".f32");
1632
			sljit_verbose_fparam(compiler, src, srcw);
1633
		}
1634
		fprintf(compiler->verbose, "\n");
1635
	}
1636
#endif /* SLJIT_VERBOSE */
1637
	CHECK_RETURN_OK;
1638
}
1639

1640
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_to(struct sljit_compiler *compiler,
1641
	sljit_s32 src, sljit_sw srcw)
1642
{
1643
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1644
	FUNCTION_CHECK_SRC(src, srcw);
1645
#endif /* SLJIT_ARGUMENT_CHECKS */
1646
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1647
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1648
		fprintf(compiler->verbose, "  return_to ");
1649
		sljit_verbose_param(compiler, src, srcw);
1650
		fprintf(compiler->verbose, "\n");
1651
	}
1652
#endif /* SLJIT_VERBOSE */
1653
	CHECK_RETURN_OK;
1654
}
1655

1656
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1657
{
1658
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1659
	CHECK_ARGUMENT((op >= SLJIT_BREAKPOINT && op <= SLJIT_LMUL_SW)
1660
		|| ((op & ~SLJIT_32) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_32) <= SLJIT_DIV_SW)
1661
		|| (op >= SLJIT_MEMORY_BARRIER && op <= SLJIT_SKIP_FRAMES_BEFORE_RETURN));
1662
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) < SLJIT_LMUL_UW || SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MEMORY_BARRIER || compiler->scratches >= 2);
1663
	if ((SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_LMUL_UW && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_SW) || op == SLJIT_SKIP_FRAMES_BEFORE_RETURN)
1664
		compiler->last_flags = 0;
1665
#endif /* SLJIT_ARGUMENT_CHECKS */
1666
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1667
	if (SLJIT_UNLIKELY(!!compiler->verbose))
1668
	{
1669
		fprintf(compiler->verbose, "  %s", op0_names[GET_OPCODE(op) - SLJIT_OP0_BASE]);
1670
		if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) {
1671
			fprintf(compiler->verbose, (op & SLJIT_32) ? "32" : "w");
1672
		}
1673
		fprintf(compiler->verbose, "\n");
1674
	}
1675
#endif /* SLJIT_VERBOSE */
1676
	CHECK_RETURN_OK;
1677
}
1678

1679
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1680
	sljit_s32 dst, sljit_sw dstw,
1681
	sljit_s32 src, sljit_sw srcw)
1682
{
1683
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1684
		compiler->skip_checks = 0;
1685
		CHECK_RETURN_OK;
1686
	}
1687

1688
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1689
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_REV_S32);
1690

1691
	switch (GET_OPCODE(op)) {
1692
	case SLJIT_MOV:
1693
	case SLJIT_MOV_U32:
1694
	case SLJIT_MOV_S32:
1695
	case SLJIT_MOV32:
1696
	case SLJIT_MOV_P:
1697
	case SLJIT_REV_U32:
1698
	case SLJIT_REV_S32:
1699
		/* Nothing allowed */
1700
		CHECK_ARGUMENT(!(op & (SLJIT_32 | ALL_STATUS_FLAGS_MASK)));
1701
		break;
1702
	default:
1703
		/* Only SLJIT_32 is allowed. */
1704
		CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
1705
		break;
1706
	}
1707

1708
	FUNCTION_CHECK_DST(dst, dstw);
1709
	FUNCTION_CHECK_SRC(src, srcw);
1710
#endif /* SLJIT_ARGUMENT_CHECKS */
1711
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1712
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1713
		fprintf(compiler->verbose, "  %s%s%s ", op1_names[GET_OPCODE(op) - SLJIT_OP1_BASE],
1714
			!(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1715

1716
		sljit_verbose_param(compiler, dst, dstw);
1717
		fprintf(compiler->verbose, ", ");
1718
		sljit_verbose_param(compiler, src, srcw);
1719
		fprintf(compiler->verbose, "\n");
1720
	}
1721
#endif /* SLJIT_VERBOSE */
1722
	CHECK_RETURN_OK;
1723
}
1724

1725
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
1726
	sljit_s32 dst_reg,
1727
	sljit_s32 mem_reg)
1728
{
1729
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1730
		compiler->skip_checks = 0;
1731
		CHECK_RETURN_OK;
1732
	}
1733

1734
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1735
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1736
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | ALL_STATUS_FLAGS_MASK) >= SLJIT_MOV
1737
		&& SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | ALL_STATUS_FLAGS_MASK) <= SLJIT_MOV_P);
1738
	CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1739

1740
	/* All arguments must be valid registers. */
1741
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1742
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1743

1744
	if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1745
		/* Nothing allowed. */
1746
		CHECK_ARGUMENT(!(op & SLJIT_32));
1747
	}
1748

1749
	compiler->last_flags = 0;
1750
#endif /* SLJIT_ARGUMENT_CHECKS */
1751
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1752
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1753
		if (op & SLJIT_ATOMIC_TEST)
1754
			CHECK_RETURN_OK;
1755
		if (sljit_emit_atomic_load(compiler, op | SLJIT_ATOMIC_TEST, dst_reg, mem_reg)) {
1756
			fprintf(compiler->verbose, "    # atomic_load: unsupported form, no instructions are emitted\n");
1757
			CHECK_RETURN_OK;
1758
		}
1759

1760
		fprintf(compiler->verbose, "  atomic_load");
1761
		if (op & SLJIT_ATOMIC_USE_CAS)
1762
			fprintf(compiler->verbose, "_cas");
1763
		if (op & SLJIT_ATOMIC_USE_LS)
1764
			fprintf(compiler->verbose, "_ls");
1765

1766
		fprintf(compiler->verbose, "%s%s ", !(op & SLJIT_32) ? "" : "32",
1767
				op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1768
		sljit_verbose_reg(compiler, dst_reg);
1769
		fprintf(compiler->verbose, ", [");
1770
		sljit_verbose_reg(compiler, mem_reg);
1771
		fprintf(compiler->verbose, "]\n");
1772
	}
1773
#endif /* SLJIT_VERBOSE */
1774
	CHECK_RETURN_OK;
1775
}
1776

1777
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
1778
	sljit_s32 src_reg,
1779
	sljit_s32 mem_reg,
1780
	sljit_s32 temp_reg)
1781
{
1782
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1783
		compiler->skip_checks = 0;
1784
		CHECK_RETURN_OK;
1785
	}
1786

1787
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1788
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1789
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) >= SLJIT_MOV
1790
		&& SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) <= SLJIT_MOV_P);
1791
	CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1792

1793
	/* All arguments must be valid registers. */
1794
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src_reg));
1795
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1796
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(temp_reg) && (src_reg != temp_reg || (op & SLJIT_ATOMIC_USE_LS)));
1797

1798
	CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE_MASK(op) == SLJIT_ATOMIC_STORED);
1799

1800
	if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1801
		/* Nothing allowed. */
1802
		CHECK_ARGUMENT(!(op & SLJIT_32));
1803
	}
1804

1805
	compiler->last_flags = GET_FLAG_TYPE_MASK(op) | (op & SLJIT_32);
1806
#endif /* SLJIT_ARGUMENT_CHECKS */
1807
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1808
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1809
		if (op & SLJIT_ATOMIC_TEST)
1810
			CHECK_RETURN_OK;
1811
		if (sljit_emit_atomic_store(compiler, op | SLJIT_ATOMIC_TEST, src_reg, mem_reg, temp_reg)) {
1812
			fprintf(compiler->verbose, "    # atomic_store: unsupported form, no instructions are emitted\n");
1813
			CHECK_RETURN_OK;
1814
		}
1815

1816
		fprintf(compiler->verbose, "  atomic_store");
1817
		if (op & SLJIT_ATOMIC_USE_CAS)
1818
			fprintf(compiler->verbose, "_cas");
1819
		if (op & SLJIT_ATOMIC_USE_LS)
1820
			fprintf(compiler->verbose, "_ls");
1821

1822
		fprintf(compiler->verbose, "%s%s%s ", !(op & SLJIT_32) ? "" : "32",
1823
				op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE], !(op & VARIABLE_FLAG_MASK) ? "" : ".stored");
1824
		sljit_verbose_reg(compiler, src_reg);
1825
		fprintf(compiler->verbose, ", [");
1826
		sljit_verbose_reg(compiler, mem_reg);
1827
		fprintf(compiler->verbose, "], ");
1828
		sljit_verbose_reg(compiler, temp_reg);
1829
		fprintf(compiler->verbose, "\n");
1830
	}
1831
#endif /* SLJIT_VERBOSE */
1832
	CHECK_RETURN_OK;
1833
}
1834

1835
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1836

1837
static sljit_s32 check_sljit_emit_op2_operation(struct sljit_compiler *compiler, sljit_s32 op)
1838
{
1839
	switch (GET_OPCODE(op)) {
1840
	case SLJIT_AND:
1841
	case SLJIT_OR:
1842
	case SLJIT_XOR:
1843
	case SLJIT_SHL:
1844
	case SLJIT_MSHL:
1845
	case SLJIT_LSHR:
1846
	case SLJIT_MLSHR:
1847
	case SLJIT_ASHR:
1848
	case SLJIT_MASHR:
1849
		return !(op & VARIABLE_FLAG_MASK);
1850
	case SLJIT_MUL:
1851
		return !(op & SLJIT_SET_Z) && (!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1852
	case SLJIT_ADD:
1853
		return !(op & VARIABLE_FLAG_MASK)
1854
			|| GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1855
			|| GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
1856
	case SLJIT_SUB:
1857
		return !(op & VARIABLE_FLAG_MASK)
1858
			|| (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_OVERFLOW)
1859
			|| GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY);
1860
	case SLJIT_ADDC:
1861
	case SLJIT_SUBC:
1862
		return (!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY))
1863
			&& (compiler->last_flags & 0xff) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1864
			&& (op & SLJIT_32) == (compiler->last_flags & SLJIT_32);
1865
		break;
1866
	case SLJIT_ROTL:
1867
	case SLJIT_ROTR:
1868
		return !(op & ALL_STATUS_FLAGS_MASK);
1869
	}
1870

1871
	/* Operation type should be checked earlier. */
1872
	SLJIT_UNREACHABLE();
1873
	return 1;
1874
}
1875

1876
#endif /* SLJIT_ARGUMENT_CHECKS */
1877

1878
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 unset,
1879
	sljit_s32 dst, sljit_sw dstw,
1880
	sljit_s32 src1, sljit_sw src1w,
1881
	sljit_s32 src2, sljit_sw src2w)
1882
{
1883
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1884
		compiler->skip_checks = 0;
1885
		CHECK_RETURN_OK;
1886
	}
1887

1888
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1889
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ROTR);
1890
	CHECK_ARGUMENT(check_sljit_emit_op2_operation(compiler, op));
1891

1892
	if (unset) {
1893
		CHECK_ARGUMENT(HAS_FLAGS(op));
1894
	} else {
1895
		FUNCTION_CHECK_DST(dst, dstw);
1896
	}
1897
	FUNCTION_CHECK_SRC(src1, src1w);
1898
	FUNCTION_CHECK_SRC(src2, src2w);
1899
	compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
1900
#endif /* SLJIT_ARGUMENT_CHECKS */
1901
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1902
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1903
		fprintf(compiler->verbose, "  %s%s%s%s%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1904
			!(op & SLJIT_SET_Z) ? "" : ".z", !(op & VARIABLE_FLAG_MASK) ? "" : ".",
1905
			!(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op)]);
1906
		if (unset)
1907
			fprintf(compiler->verbose, "unset");
1908
		else
1909
			sljit_verbose_param(compiler, dst, dstw);
1910
		fprintf(compiler->verbose, ", ");
1911
		sljit_verbose_param(compiler, src1, src1w);
1912
		fprintf(compiler->verbose, ", ");
1913
		sljit_verbose_param(compiler, src2, src2w);
1914
		fprintf(compiler->verbose, "\n");
1915
	}
1916
#endif /* SLJIT_VERBOSE */
1917
	CHECK_RETURN_OK;
1918
}
1919

1920
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1921
	sljit_s32 dst_reg,
1922
	sljit_s32 src1, sljit_sw src1w,
1923
	sljit_s32 src2, sljit_sw src2w)
1924
{
1925
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1926
	CHECK_ARGUMENT((op | SLJIT_32) == SLJIT_MULADD32);
1927
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1928
	FUNCTION_CHECK_SRC(src1, src1w);
1929
	FUNCTION_CHECK_SRC(src2, src2w);
1930
	compiler->last_flags = 0;
1931
#endif /* SLJIT_ARGUMENT_CHECKS */
1932
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1933
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1934
		fprintf(compiler->verbose, "  %s%s ", op2r_names[GET_OPCODE(op) - SLJIT_OP2R_BASE], !(op & SLJIT_32) ? "" : "32");
1935

1936
		sljit_verbose_reg(compiler, dst_reg);
1937
		fprintf(compiler->verbose, ", ");
1938
		sljit_verbose_param(compiler, src1, src1w);
1939
		fprintf(compiler->verbose, ", ");
1940
		sljit_verbose_param(compiler, src2, src2w);
1941
		fprintf(compiler->verbose, "\n");
1942
	}
1943
#endif /* SLJIT_VERBOSE */
1944
	CHECK_RETURN_OK;
1945
}
1946

1947
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1948
	sljit_s32 dst_reg,
1949
	sljit_s32 src1_reg,
1950
	sljit_s32 src2_reg,
1951
	sljit_s32 src3, sljit_sw src3w)
1952
{
1953
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1954
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_SHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_LSHR
1955
		|| SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MSHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MLSHR);
1956
	CHECK_ARGUMENT((op & ~(0xff | SLJIT_32 | SLJIT_SHIFT_INTO_NON_ZERO)) == 0);
1957
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1958
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src1_reg));
1959
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
1960
	FUNCTION_CHECK_SRC(src3, src3w);
1961
	CHECK_ARGUMENT(dst_reg != src2_reg);
1962
#endif /* SLJIT_ARGUMENT_CHECKS */
1963
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1964
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1965
		fprintf(compiler->verbose, "  %s%s.into%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1966
			(op & SLJIT_SHIFT_INTO_NON_ZERO) ? ".nz" : "");
1967

1968
		sljit_verbose_reg(compiler, dst_reg);
1969
		fprintf(compiler->verbose, ", ");
1970
		sljit_verbose_reg(compiler, src1_reg);
1971
		fprintf(compiler->verbose, ", ");
1972
		sljit_verbose_reg(compiler, src2_reg);
1973
		fprintf(compiler->verbose, ", ");
1974
		sljit_verbose_param(compiler, src3, src3w);
1975
		fprintf(compiler->verbose, "\n");
1976
	}
1977
#endif /* SLJIT_VERBOSE */
1978
	CHECK_RETURN_OK;
1979
}
1980

1981
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
1982
	sljit_s32 dst, sljit_sw dstw,
1983
	sljit_s32 src1, sljit_sw src1w,
1984
	sljit_s32 src2, sljit_sw src2w,
1985
	sljit_sw shift_arg)
1986
{
1987
	SLJIT_UNUSED_ARG(shift_arg);
1988
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1989
	CHECK_ARGUMENT((op & ~SLJIT_SRC2_UNDEFINED) == (SLJIT_ADD | SLJIT_SHL_IMM));
1990
	FUNCTION_CHECK_DST(dst, dstw);
1991
	FUNCTION_CHECK_SRC(src1, src1w);
1992
	FUNCTION_CHECK_SRC(src2, src2w);
1993
	compiler->last_flags = 0;
1994
#endif /* SLJIT_ARGUMENT_CHECKS */
1995
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1996
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1997
		fprintf(compiler->verbose, "  %s.shl_imm%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
1998
			(op & SLJIT_SRC2_UNDEFINED) ? ".src2und" : "");
1999

2000
		sljit_verbose_param(compiler, dst, dstw);
2001
		fprintf(compiler->verbose, ", ");
2002
		sljit_verbose_param(compiler, src1, src1w);
2003
		fprintf(compiler->verbose, ", ");
2004
		sljit_verbose_param(compiler, src2, src2w);
2005
		fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", shift_arg);
2006
	}
2007
#endif /* SLJIT_VERBOSE */
2008
	CHECK_RETURN_OK;
2009
}
2010

2011
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2012
	sljit_s32 src, sljit_sw srcw)
2013
{
2014
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2015
	CHECK_ARGUMENT(op >= SLJIT_FAST_RETURN && op <= SLJIT_PREFETCH_ONCE);
2016
	FUNCTION_CHECK_SRC(src, srcw);
2017

2018
	if (op == SLJIT_FAST_RETURN || op == SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN) {
2019
		CHECK_ARGUMENT(src != SLJIT_IMM);
2020
		compiler->last_flags = 0;
2021
	} else if (op >= SLJIT_PREFETCH_L1 && op <= SLJIT_PREFETCH_ONCE) {
2022
		CHECK_ARGUMENT(src & SLJIT_MEM);
2023
	}
2024
#endif /* SLJIT_ARGUMENT_CHECKS */
2025
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2026
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2027
		fprintf(compiler->verbose, "  %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
2028
		sljit_verbose_param(compiler, src, srcw);
2029
		fprintf(compiler->verbose, "\n");
2030
	}
2031
#endif /* SLJIT_VERBOSE */
2032
	CHECK_RETURN_OK;
2033
}
2034

2035
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2036
	sljit_s32 dst, sljit_sw dstw)
2037
{
2038
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2039
	CHECK_ARGUMENT(op >= SLJIT_FAST_ENTER && op <= SLJIT_GET_RETURN_ADDRESS);
2040
	FUNCTION_CHECK_DST(dst, dstw);
2041

2042
	if (op == SLJIT_FAST_ENTER)
2043
		compiler->last_flags = 0;
2044
#endif /* SLJIT_ARGUMENT_CHECKS */
2045
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2046
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2047
		fprintf(compiler->verbose, "  %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
2048
		sljit_verbose_param(compiler, dst, dstw);
2049
		fprintf(compiler->verbose, "\n");
2050
	}
2051
#endif /* SLJIT_VERBOSE */
2052
	CHECK_RETURN_OK;
2053
}
2054

2055
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2056
{
2057
	SLJIT_UNUSED_ARG(type);
2058
	SLJIT_UNUSED_ARG(reg);
2059
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2060
	if (type == SLJIT_GP_REGISTER) {
2061
		CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_REGISTERS)
2062
			|| (reg >= SLJIT_TMP_REGISTER_BASE && reg < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)));
2063
	}
2064
#if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
2065
	else if (((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6))) {
2066
		CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_VECTOR_REGISTERS)
2067
			|| (reg >= SLJIT_TMP_VREGISTER_BASE && reg < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS)));
2068
	}
2069
#endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
2070
	else {
2071
		CHECK_ARGUMENT(type == SLJIT_FLOAT_REGISTER || ((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6) || (type & (3 << 12)) || (type & (4 << 12)) || (type & (5 << 12)) || (type & (6 << 12))));
2072
		CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_FLOAT_REGISTERS)
2073
			|| (reg >= SLJIT_TMP_FREGISTER_BASE && reg < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS)));
2074
	}
2075
#endif /* SLJIT_ARGUMENT_CHECKS */
2076
	CHECK_RETURN_OK;
2077
}
2078

2079
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_custom(struct sljit_compiler *compiler,
2080
	void *instruction, sljit_u32 size)
2081
{
2082
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2083
	sljit_u32 i;
2084
#endif /* SLJIT_VERBOSE */
2085

2086
	SLJIT_UNUSED_ARG(compiler);
2087

2088
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2089
	CHECK_ARGUMENT(instruction);
2090

2091
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
2092
	CHECK_ARGUMENT(size > 0 && size < 16);
2093
#elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) || (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
2094
	CHECK_ARGUMENT((size == 2 && (((sljit_sw)instruction) & 0x1) == 0)
2095
		|| (size == 4 && (((sljit_sw)instruction) & 0x3) == 0));
2096
#elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
2097
	CHECK_ARGUMENT(size == 2 || size == 4 || size == 6);
2098
#else /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_CONFIG_RISCV && !SLJIT_CONFIG_S390X */
2099
	CHECK_ARGUMENT(size == 4 && (((sljit_sw)instruction) & 0x3) == 0);
2100
#endif /* SLJIT_CONFIG_X86 */
2101

2102
	compiler->last_flags = 0;
2103
#endif /* SLJIT_ARGUMENT_CHECKS */
2104
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2105
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2106
		fprintf(compiler->verbose, "  op_custom");
2107
		for (i = 0; i < size; i++)
2108
			fprintf(compiler->verbose, " 0x%x", ((sljit_u8*)instruction)[i]);
2109
		fprintf(compiler->verbose, "\n");
2110
	}
2111
#endif /* SLJIT_VERBOSE */
2112
	CHECK_RETURN_OK;
2113
}
2114

2115
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2116
	sljit_s32 dst, sljit_sw dstw,
2117
	sljit_s32 src, sljit_sw srcw)
2118
{
2119
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2120
		compiler->skip_checks = 0;
2121
		CHECK_RETURN_OK;
2122
	}
2123

2124
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2125
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2126
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ABS_F64);
2127
	CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2128
	FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2129
	FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2130
#endif /* SLJIT_ARGUMENT_CHECKS */
2131
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2132
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2133
		if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2134
			fprintf(compiler->verbose, "  %s%s ", fop1_names[SLJIT_CONV_F64_FROM_F32 - SLJIT_FOP1_BASE],
2135
				(op & SLJIT_32) ? ".f32.from.f64" : ".f64.from.f32");
2136
		else
2137
			fprintf(compiler->verbose, "  %s%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2138
				(op & SLJIT_32) ? ".f32" : ".f64");
2139

2140
		sljit_verbose_fparam(compiler, dst, dstw);
2141
		fprintf(compiler->verbose, ", ");
2142
		sljit_verbose_fparam(compiler, src, srcw);
2143
		fprintf(compiler->verbose, "\n");
2144
	}
2145
#endif /* SLJIT_VERBOSE */
2146
	CHECK_RETURN_OK;
2147
}
2148

2149
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2150
	sljit_s32 src1, sljit_sw src1w,
2151
	sljit_s32 src2, sljit_sw src2w)
2152
{
2153
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2154
	compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
2155
#endif /* SLJIT_ARGUMENT_CHECKS */
2156

2157
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2158
		compiler->skip_checks = 0;
2159
		CHECK_RETURN_OK;
2160
	}
2161

2162
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2163
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2164
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_CMP_F64);
2165
	CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
2166
	CHECK_ARGUMENT((op & VARIABLE_FLAG_MASK)
2167
		|| (GET_FLAG_TYPE(op) >= SLJIT_F_EQUAL && GET_FLAG_TYPE(op) <= SLJIT_ORDERED_LESS_EQUAL));
2168
	FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2169
	FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2170
#endif /* SLJIT_ARGUMENT_CHECKS */
2171
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2172
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2173
		fprintf(compiler->verbose, "  %s%s", fop1_names[SLJIT_CMP_F64 - SLJIT_FOP1_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2174
		if (op & VARIABLE_FLAG_MASK) {
2175
			fprintf(compiler->verbose, ".%s", jump_names[GET_FLAG_TYPE(op)]);
2176
		}
2177
		fprintf(compiler->verbose, " ");
2178
		sljit_verbose_fparam(compiler, src1, src1w);
2179
		fprintf(compiler->verbose, ", ");
2180
		sljit_verbose_fparam(compiler, src2, src2w);
2181
		fprintf(compiler->verbose, "\n");
2182
	}
2183
#endif /* SLJIT_VERBOSE */
2184
	CHECK_RETURN_OK;
2185
}
2186

2187
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2188
	sljit_s32 dst, sljit_sw dstw,
2189
	sljit_s32 src, sljit_sw srcw)
2190
{
2191
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2192
		compiler->skip_checks = 0;
2193
		CHECK_RETURN_OK;
2194
	}
2195

2196
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2197
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2198
	CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2199
	FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2200
	FUNCTION_CHECK_DST(dst, dstw);
2201
#endif /* SLJIT_ARGUMENT_CHECKS */
2202
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2203
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2204
		fprintf(compiler->verbose, "  %s%s.from%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2205
			fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64],
2206
			(op & SLJIT_32) ? ".f32" : ".f64");
2207
		sljit_verbose_param(compiler, dst, dstw);
2208
		fprintf(compiler->verbose, ", ");
2209
		sljit_verbose_fparam(compiler, src, srcw);
2210
		fprintf(compiler->verbose, "\n");
2211
	}
2212
#endif /* SLJIT_VERBOSE */
2213
	CHECK_RETURN_OK;
2214
}
2215

2216
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_s32 op,
2217
	sljit_s32 dst, sljit_sw dstw,
2218
	sljit_s32 src, sljit_sw srcw)
2219
{
2220
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2221
		compiler->skip_checks = 0;
2222
		CHECK_RETURN_OK;
2223
	}
2224

2225
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2226
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2227
	CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2228
	FUNCTION_CHECK_SRC(src, srcw);
2229
	FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2230
#endif /* SLJIT_ARGUMENT_CHECKS */
2231
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2232
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2233
		fprintf(compiler->verbose, "  %s%s.from.%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2234
			(op & SLJIT_32) ? ".f32" : ".f64",
2235
			fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64]);
2236
		sljit_verbose_fparam(compiler, dst, dstw);
2237
		fprintf(compiler->verbose, ", ");
2238
		sljit_verbose_param(compiler, src, srcw);
2239
		fprintf(compiler->verbose, "\n");
2240
	}
2241
#endif /* SLJIT_VERBOSE */
2242
	CHECK_RETURN_OK;
2243
}
2244

2245
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2246
	sljit_s32 dst, sljit_sw dstw,
2247
	sljit_s32 src1, sljit_sw src1w,
2248
	sljit_s32 src2, sljit_sw src2w)
2249
{
2250
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2251
		compiler->skip_checks = 0;
2252
		CHECK_RETURN_OK;
2253
	}
2254

2255
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2256
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2257
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_F64);
2258
	CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2259
	FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2260
	FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2261
	FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2262
#endif /* SLJIT_ARGUMENT_CHECKS */
2263
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2264
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2265
		fprintf(compiler->verbose, "  %s%s ", fop2_names[GET_OPCODE(op) - SLJIT_FOP2_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2266
		sljit_verbose_fparam(compiler, dst, dstw);
2267
		fprintf(compiler->verbose, ", ");
2268
		sljit_verbose_fparam(compiler, src1, src1w);
2269
		fprintf(compiler->verbose, ", ");
2270
		sljit_verbose_fparam(compiler, src2, src2w);
2271
		fprintf(compiler->verbose, "\n");
2272
	}
2273
#endif /* SLJIT_VERBOSE */
2274
	CHECK_RETURN_OK;
2275
}
2276

2277
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
2278
	sljit_s32 dst_freg,
2279
	sljit_s32 src1, sljit_sw src1w,
2280
	sljit_s32 src2, sljit_sw src2w)
2281
{
2282
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2283
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2284
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_COPYSIGN_F64);
2285
	FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2286
	FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2287
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, op & SLJIT_32));
2288
#endif /* SLJIT_ARGUMENT_CHECKS */
2289
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2290
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2291
		fprintf(compiler->verbose, "  %s%s ", fop2r_names[GET_OPCODE(op) - SLJIT_FOP2R_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2292
		sljit_verbose_freg(compiler, dst_freg);
2293
		fprintf(compiler->verbose, ", ");
2294
		sljit_verbose_fparam(compiler, src1, src1w);
2295
		fprintf(compiler->verbose, ", ");
2296
		sljit_verbose_fparam(compiler, src2, src2w);
2297
		fprintf(compiler->verbose, "\n");
2298
	}
2299
#endif /* SLJIT_VERBOSE */
2300
	CHECK_RETURN_OK;
2301
}
2302

2303
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset32(struct sljit_compiler *compiler,
2304
	sljit_s32 freg, sljit_f32 value)
2305
{
2306
	SLJIT_UNUSED_ARG(value);
2307

2308
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2309
		compiler->skip_checks = 0;
2310
		CHECK_RETURN_OK;
2311
	}
2312

2313
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2314
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2315
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 1));
2316
#endif /* SLJIT_ARGUMENT_CHECKS */
2317
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2318
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2319
		fprintf(compiler->verbose, "  fset32 ");
2320
		sljit_verbose_freg(compiler, freg);
2321
		fprintf(compiler->verbose, ", %f\n", value);
2322
	}
2323
#endif /* SLJIT_VERBOSE */
2324
	CHECK_RETURN_OK;
2325
}
2326

2327
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset64(struct sljit_compiler *compiler,
2328
	sljit_s32 freg, sljit_f64 value)
2329
{
2330
	SLJIT_UNUSED_ARG(value);
2331

2332
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2333
		compiler->skip_checks = 0;
2334
		CHECK_RETURN_OK;
2335
	}
2336

2337
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2338
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2339
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2340
#endif /* SLJIT_ARGUMENT_CHECKS */
2341
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2342
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2343
		fprintf(compiler->verbose, "  fset64 ");
2344
		sljit_verbose_freg(compiler, freg);
2345
		fprintf(compiler->verbose, ", %f\n", value);
2346
	}
2347
#endif /* SLJIT_VERBOSE */
2348
	CHECK_RETURN_OK;
2349
}
2350

2351
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2352
	sljit_s32 freg, sljit_s32 reg)
2353
{
2354
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2355
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2356
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_COPY_TO_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_COPY_FROM_F64);
2357
	CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2358
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, op & SLJIT_32));
2359

2360
#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
2361
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2362
#else /* !SLJIT_64BIT_ARCHITECTURE */
2363
	switch (op) {
2364
	case SLJIT_COPY32_TO_F32:
2365
	case SLJIT_COPY32_FROM_F32:
2366
		CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2367
		break;
2368
	case SLJIT_COPY_TO_F64:
2369
	case SLJIT_COPY_FROM_F64:
2370
		if (reg & REG_PAIR_MASK) {
2371
			CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2372
			CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2373

2374
			if (op == SLJIT_COPY_TO_F64)
2375
				break;
2376

2377
			CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2378
			break;
2379
		}
2380

2381
		CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2382
		break;
2383
	}
2384
#endif /* SLJIT_64BIT_ARCHITECTURE */
2385
#endif /* SLJIT_ARGUMENT_CHECKS */
2386
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2387
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2388
		fprintf(compiler->verbose, "  copy%s_%s_f%s ", (op & SLJIT_32) ? "32" : "",
2389
			GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? "to" : "from", (op & SLJIT_32) ? "32" : "64");
2390

2391
		sljit_verbose_freg(compiler, freg);
2392

2393
		if (reg & REG_PAIR_MASK) {
2394
			fprintf(compiler->verbose, ", {");
2395
			sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2396
			fprintf(compiler->verbose, ", ");
2397
			sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2398
			fprintf(compiler->verbose, "}\n");
2399
		} else {
2400
			fprintf(compiler->verbose, ", ");
2401
			sljit_verbose_reg(compiler, reg);
2402
			fprintf(compiler->verbose, "\n");
2403
		}
2404
	}
2405
#endif /* SLJIT_VERBOSE */
2406
	CHECK_RETURN_OK;
2407
}
2408

2409
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_label(struct sljit_compiler *compiler)
2410
{
2411
	SLJIT_UNUSED_ARG(compiler);
2412

2413
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2414
		compiler->skip_checks = 0;
2415
		CHECK_RETURN_OK;
2416
	}
2417

2418
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2419
	compiler->last_flags = 0;
2420
#endif /* SLJIT_ARGUMENT_CHECKS */
2421

2422
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2423
	if (SLJIT_UNLIKELY(!!compiler->verbose))
2424
		fprintf(compiler->verbose, "label:\n");
2425
#endif /* SLJIT_VERBOSE */
2426
	CHECK_RETURN_OK;
2427
}
2428

2429
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_aligned_label(struct sljit_compiler *compiler,
2430
	sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
2431
{
2432
	SLJIT_UNUSED_ARG(compiler);
2433
	SLJIT_UNUSED_ARG(buffers);
2434

2435
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2436
	CHECK_ARGUMENT(alignment >= SLJIT_LABEL_ALIGN_1 && alignment <= SLJIT_LABEL_ALIGN_16);
2437
	compiler->last_flags = 0;
2438
#endif /* SLJIT_ARGUMENT_CHECKS */
2439

2440
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2441
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2442
		fprintf(compiler->verbose, "label.al%d:%s", 1 << alignment, buffers == NULL ? "\n" : " [");
2443

2444
		if (buffers != NULL) {
2445
			fprintf(compiler->verbose, "%ld", (long int)buffers->size);
2446
			buffers = buffers->next;
2447

2448
			while (buffers != NULL) {
2449
				fprintf(compiler->verbose, ", %ld", (long int)buffers->size);
2450
				buffers = buffers->next;
2451
			}
2452

2453
			fprintf(compiler->verbose, "]\n");
2454
		}
2455
	}
2456
#endif /* SLJIT_VERBOSE */
2457
	CHECK_RETURN_OK;
2458
}
2459

2460
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2461
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
2462
	|| (defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM)
2463
#define CHECK_UNORDERED(type, last_flags) \
2464
	((((type) & 0xfe) == SLJIT_ORDERED) && \
2465
		((last_flags) & 0xff) >= SLJIT_UNORDERED && ((last_flags) & 0xff) <= SLJIT_ORDERED_LESS_EQUAL)
2466
#else /* !SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2467
#define CHECK_UNORDERED(type, last_flags) 0
2468
#endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2469
#endif /* SLJIT_ARGUMENT_CHECKS */
2470

2471
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2472
{
2473
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2474
		compiler->skip_checks = 0;
2475
		CHECK_RETURN_OK;
2476
	}
2477

2478
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2479
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP)));
2480
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_FAST_CALL);
2481

2482
	if ((type & 0xff) < SLJIT_JUMP) {
2483
		if ((type & 0xff) <= SLJIT_NOT_ZERO)
2484
			CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2485
		else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2486
			CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2487
			compiler->last_flags = 0;
2488
		} else
2489
			CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2490
				|| CHECK_UNORDERED(type, compiler->last_flags));
2491
	}
2492
#endif /* SLJIT_ARGUMENT_CHECKS */
2493
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2494
	if (SLJIT_UNLIKELY(!!compiler->verbose))
2495
		fprintf(compiler->verbose, "  jump%s %s\n", !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2496
			jump_names[type & 0xff]);
2497
#endif /* SLJIT_VERBOSE */
2498
	CHECK_RETURN_OK;
2499
}
2500

2501
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2502
	sljit_s32 arg_types)
2503
{
2504
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2505
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_CALL_RETURN)));
2506
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2507
	CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2508

2509
	if (type & SLJIT_CALL_RETURN) {
2510
		CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2511

2512
		if (compiler->options & SLJIT_ENTER_REG_ARG) {
2513
			CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2514
		} else {
2515
			CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2516
		}
2517
	}
2518
#endif /* SLJIT_ARGUMENT_CHECKS */
2519
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2520
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2521
		fprintf(compiler->verbose, "  %s%s%s ret[%s", jump_names[type & 0xff],
2522
			!(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2523
			!(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2524
			call_arg_names[arg_types & SLJIT_ARG_MASK]);
2525

2526
		arg_types >>= SLJIT_ARG_SHIFT;
2527
		if (arg_types) {
2528
			fprintf(compiler->verbose, "], args[");
2529
			do {
2530
				fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2531
				arg_types >>= SLJIT_ARG_SHIFT;
2532
				if (arg_types)
2533
					fprintf(compiler->verbose, ",");
2534
			} while (arg_types);
2535
		}
2536
		fprintf(compiler->verbose, "]\n");
2537
	}
2538
#endif /* SLJIT_VERBOSE */
2539
	CHECK_RETURN_OK;
2540
}
2541

2542
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
2543
	sljit_s32 src1, sljit_sw src1w,
2544
	sljit_s32 src2, sljit_sw src2w)
2545
{
2546
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2547
		compiler->skip_checks = 0;
2548
		CHECK_RETURN_OK;
2549
	}
2550

2551
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2552
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2553
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_SIG_LESS_EQUAL);
2554
	FUNCTION_CHECK_SRC(src1, src1w);
2555
	FUNCTION_CHECK_SRC(src2, src2w);
2556
	compiler->last_flags = 0;
2557
#endif /* SLJIT_ARGUMENT_CHECKS */
2558
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2559
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2560
		fprintf(compiler->verbose, "  cmp%s%s %s, ", (type & SLJIT_32) ? "32" : "",
2561
			!(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2562
		sljit_verbose_param(compiler, src1, src1w);
2563
		fprintf(compiler->verbose, ", ");
2564
		sljit_verbose_param(compiler, src2, src2w);
2565
		fprintf(compiler->verbose, "\n");
2566
	}
2567
#endif /* SLJIT_VERBOSE */
2568
	CHECK_RETURN_OK;
2569
}
2570

2571
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
2572
	sljit_s32 src1, sljit_sw src1w,
2573
	sljit_s32 src2, sljit_sw src2w)
2574
{
2575
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2576
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2577
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2578
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_F_EQUAL && (type & 0xff) <= SLJIT_ORDERED_LESS_EQUAL);
2579
	FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2580
	FUNCTION_FCHECK(src2, src2w, type & SLJIT_32);
2581
	compiler->last_flags = 0;
2582
#endif /* SLJIT_ARGUMENT_CHECKS */
2583
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2584
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2585
		fprintf(compiler->verbose, "  fcmp%s%s %s, ", (type & SLJIT_32) ? ".f32" : ".f64",
2586
			!(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2587
		sljit_verbose_fparam(compiler, src1, src1w);
2588
		fprintf(compiler->verbose, ", ");
2589
		sljit_verbose_fparam(compiler, src2, src2w);
2590
		fprintf(compiler->verbose, "\n");
2591
	}
2592
#endif /* SLJIT_VERBOSE */
2593
	CHECK_RETURN_OK;
2594
}
2595

2596
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
2597
	sljit_s32 dst, sljit_sw dstw,
2598
	sljit_s32 src1, sljit_sw src1w,
2599
	sljit_s32 src2, sljit_sw src2w)
2600
{
2601
	SLJIT_COMPILE_ASSERT((SLJIT_JUMP_IF_ZERO == SLJIT_SET_Z) && (SLJIT_JUMP_IF_NON_ZERO == 0),
2602
		incorrect_values_for_sljit_jump_zero_or_non_zero);
2603

2604
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2605
	CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_REWRITABLE_JUMP) >= SLJIT_ADD && SLJIT_CHECK_OPCODE(op, SLJIT_REWRITABLE_JUMP) <= SLJIT_ROTR);
2606
	/* Not all opcodes allow setting zero flags. */
2607
	CHECK_ARGUMENT(check_sljit_emit_op2_operation(compiler, ((op | SLJIT_SET_Z) & ~SLJIT_REWRITABLE_JUMP)));
2608
	FUNCTION_CHECK_DST(dst, dstw);
2609
	FUNCTION_CHECK_SRC(src1, src1w);
2610
	FUNCTION_CHECK_SRC(src2, src2w);
2611
	compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
2612
#endif /* SLJIT_ARGUMENT_CHECKS */
2613
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2614
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2615
		fprintf(compiler->verbose, "  cmp%sz.%s%s%s%s%s ", (op & SLJIT_JUMP_IF_ZERO) ? "" : "n",
2616
			op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
2617
			!(op & VARIABLE_FLAG_MASK) ? "" : ".", !(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op & ~SLJIT_REWRITABLE_JUMP)],
2618
			!(op & SLJIT_REWRITABLE_JUMP) ? "" : ".r");
2619
		sljit_verbose_param(compiler, dst, dstw);
2620
		fprintf(compiler->verbose, ", ");
2621
		sljit_verbose_param(compiler, src1, src1w);
2622
		fprintf(compiler->verbose, ", ");
2623
		sljit_verbose_param(compiler, src2, src2w);
2624
		fprintf(compiler->verbose, "\n");
2625
	}
2626
#endif /* SLJIT_VERBOSE */
2627
	CHECK_RETURN_OK;
2628
}
2629

2630
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type,
2631
	sljit_s32 src, sljit_sw srcw)
2632
{
2633
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2634
		compiler->skip_checks = 0;
2635
		CHECK_RETURN_OK;
2636
	}
2637

2638
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2639
	CHECK_ARGUMENT(type >= SLJIT_JUMP && type <= SLJIT_FAST_CALL);
2640
	FUNCTION_CHECK_SRC(src, srcw);
2641
#endif /* SLJIT_ARGUMENT_CHECKS */
2642
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2643
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2644
		fprintf(compiler->verbose, "  ijump.%s ", jump_names[type]);
2645
		sljit_verbose_param(compiler, src, srcw);
2646
		fprintf(compiler->verbose, "\n");
2647
	}
2648
#endif /* SLJIT_VERBOSE */
2649
	CHECK_RETURN_OK;
2650
}
2651

2652
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2653
	sljit_s32 arg_types,
2654
	sljit_s32 src, sljit_sw srcw)
2655
{
2656
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2657
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_CALL_RETURN)));
2658
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2659
	CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2660
	FUNCTION_CHECK_SRC(src, srcw);
2661

2662
	if (type & SLJIT_CALL_RETURN) {
2663
		CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2664

2665
		if (compiler->options & SLJIT_ENTER_REG_ARG) {
2666
			CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2667
		} else {
2668
			CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2669
		}
2670
	}
2671
#endif /* SLJIT_ARGUMENT_CHECKS */
2672
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2673
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2674
		fprintf(compiler->verbose, "  i%s%s ret[%s", jump_names[type & 0xff],
2675
			!(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2676
			call_arg_names[arg_types & SLJIT_ARG_MASK]);
2677

2678
		arg_types >>= SLJIT_ARG_SHIFT;
2679
		if (arg_types) {
2680
			fprintf(compiler->verbose, "], args[");
2681
			do {
2682
				fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2683
				arg_types >>= SLJIT_ARG_SHIFT;
2684
				if (arg_types)
2685
					fprintf(compiler->verbose, ",");
2686
			} while (arg_types);
2687
		}
2688
		fprintf(compiler->verbose, "], ");
2689
		sljit_verbose_param(compiler, src, srcw);
2690
		fprintf(compiler->verbose, "\n");
2691
	}
2692
#endif /* SLJIT_VERBOSE */
2693
	CHECK_RETURN_OK;
2694
}
2695

2696
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2697
	sljit_s32 dst, sljit_sw dstw,
2698
	sljit_s32 type)
2699
{
2700
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2701
	CHECK_ARGUMENT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
2702
	CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV32
2703
		|| (SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_AND && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_XOR));
2704
	CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
2705

2706
	if (type <= SLJIT_NOT_ZERO)
2707
		CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2708
	else
2709
		CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2710
			|| CHECK_UNORDERED(type, compiler->last_flags));
2711

2712
	FUNCTION_CHECK_DST(dst, dstw);
2713

2714
	if (GET_OPCODE(op) >= SLJIT_ADD)
2715
		compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
2716
#endif /* SLJIT_ARGUMENT_CHECKS */
2717
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2718
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2719
		fprintf(compiler->verbose, "  flags.%s%s%s ",
2720
			GET_OPCODE(op) < SLJIT_OP2_BASE ? "mov" : op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
2721
			GET_OPCODE(op) < SLJIT_OP2_BASE ? op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE] : ((op & SLJIT_32) ? "32" : ""),
2722
			!(op & SLJIT_SET_Z) ? "" : ".z");
2723
		sljit_verbose_param(compiler, dst, dstw);
2724
		fprintf(compiler->verbose, ", %s\n", jump_names[type]);
2725
	}
2726
#endif /* SLJIT_VERBOSE */
2727
	CHECK_RETURN_OK;
2728
}
2729

2730
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2731
	sljit_s32 dst_reg,
2732
	sljit_s32 src1, sljit_sw src1w,
2733
	sljit_s32 src2_reg)
2734
{
2735
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2736
	sljit_s32 cond = type & ~(SLJIT_32 | SLJIT_COMPARE_SELECT);
2737
	CHECK_ARGUMENT((type & SLJIT_COMPARE_SELECT) ? (cond >= SLJIT_LESS && cond <= SLJIT_SET_SIG_LESS_EQUAL)
2738
				: (cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL));
2739
	CHECK_ARGUMENT(compiler->scratches != -1 && compiler->saveds != -1);
2740
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
2741
	FUNCTION_CHECK_SRC(src1, src1w);
2742
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
2743

2744
	if (!(type & SLJIT_COMPARE_SELECT)) {
2745
		if (cond <= SLJIT_NOT_ZERO)
2746
			CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2747
		else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2748
			CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2749
			compiler->last_flags = 0;
2750
		} else
2751
			CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2752
				|| CHECK_UNORDERED(cond, compiler->last_flags));
2753
	} else
2754
		compiler->last_flags = 0;
2755
#endif /* SLJIT_ARGUMENT_CHECKS */
2756
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2757
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2758
		fprintf(compiler->verbose, "  %sselect%s %s, ",
2759
			!(type & SLJIT_COMPARE_SELECT) ? "" : "cmp_",
2760
			!(type & SLJIT_32) ? "" : "32",
2761
			jump_names[type & ~SLJIT_32]);
2762
		sljit_verbose_reg(compiler, dst_reg);
2763
		fprintf(compiler->verbose, ", ");
2764
		sljit_verbose_param(compiler, src1, src1w);
2765
		fprintf(compiler->verbose, ", ");
2766
		sljit_verbose_reg(compiler, src2_reg);
2767
		fprintf(compiler->verbose, "\n");
2768
	}
2769
#endif /* SLJIT_VERBOSE */
2770
	CHECK_RETURN_OK;
2771
}
2772

2773
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2774
	sljit_s32 dst_freg,
2775
	sljit_s32 src1, sljit_sw src1w,
2776
	sljit_s32 src2_freg)
2777
{
2778
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2779
	sljit_s32 cond = type & ~SLJIT_32;
2780

2781
	CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2782

2783
	CHECK_ARGUMENT(compiler->fscratches != -1 && compiler->fsaveds != -1);
2784
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, type & SLJIT_32));
2785
	FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2786
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src2_freg, type & SLJIT_32));
2787

2788
	if (cond <= SLJIT_NOT_ZERO)
2789
		CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2790
	else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2791
		CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2792
		compiler->last_flags = 0;
2793
	} else
2794
		CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2795
			|| CHECK_UNORDERED(cond, compiler->last_flags));
2796
#endif /* SLJIT_ARGUMENT_CHECKS */
2797
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2798
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2799
		fprintf(compiler->verbose, "  fselect%s %s, ",
2800
			!(type & SLJIT_32) ? "" : "32",
2801
			jump_names[type & ~SLJIT_32]);
2802
		sljit_verbose_freg(compiler, dst_freg);
2803
		fprintf(compiler->verbose, ", ");
2804
		sljit_verbose_fparam(compiler, src1, src1w);
2805
		fprintf(compiler->verbose, ", ");
2806
		sljit_verbose_freg(compiler, src2_freg);
2807
		fprintf(compiler->verbose, "\n");
2808
	}
2809
#endif /* SLJIT_VERBOSE */
2810
	CHECK_RETURN_OK;
2811
}
2812

2813
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2814
	sljit_s32 reg,
2815
	sljit_s32 mem, sljit_sw memw)
2816
{
2817
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2818
	sljit_s32 allowed_flags;
2819
#endif /* SLJIT_ARGUMENT_CHECKS */
2820

2821
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2822
		compiler->skip_checks = 0;
2823
		CHECK_RETURN_OK;
2824
	}
2825

2826
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2827
	if (type & SLJIT_MEM_UNALIGNED) {
2828
		CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2829
	} else if (type & SLJIT_MEM_ALIGNED_16) {
2830
		CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2831
	} else {
2832
		CHECK_ARGUMENT((reg & REG_PAIR_MASK) || (type & SLJIT_MEM_ALIGNED_32));
2833
	}
2834

2835
	allowed_flags = SLJIT_MEM_UNALIGNED;
2836

2837
	switch (type & 0xff) {
2838
	case SLJIT_MOV_P:
2839
	case SLJIT_MOV:
2840
		allowed_flags |= SLJIT_MEM_ALIGNED_32;
2841
		SLJIT_FALLTHROUGH
2842
	case SLJIT_MOV_U32:
2843
	case SLJIT_MOV_S32:
2844
	case SLJIT_MOV32:
2845
		allowed_flags |= SLJIT_MEM_ALIGNED_16;
2846
		break;
2847
	}
2848

2849
	CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | allowed_flags)) == 0);
2850

2851
	if (reg & REG_PAIR_MASK) {
2852
		CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV);
2853
		CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2854
		CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2855
		CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2856
	} else {
2857
		CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2858
		CHECK_ARGUMENT(!(type & SLJIT_32) || ((type & 0xff) >= SLJIT_MOV_U8 && (type & 0xff) <= SLJIT_MOV_S16));
2859
		CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2860
	}
2861

2862
	FUNCTION_CHECK_SRC_MEM(mem, memw);
2863
#endif /* SLJIT_ARGUMENT_CHECKS */
2864
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2865
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2866
		if ((type & 0xff) == SLJIT_MOV32)
2867
			fprintf(compiler->verbose, "  %s32",
2868
				(type & SLJIT_MEM_STORE) ? "store" : "load");
2869
		else
2870
			fprintf(compiler->verbose, "  %s%s%s",
2871
				(type & SLJIT_MEM_STORE) ? "store" : "load",
2872
				!(type & SLJIT_32) ? "" : "32", op1_types[(type & 0xff) - SLJIT_OP1_BASE]);
2873

2874
		if (type & SLJIT_MEM_UNALIGNED)
2875
			fprintf(compiler->verbose, ".unal");
2876
		else if (type & SLJIT_MEM_ALIGNED_16)
2877
			fprintf(compiler->verbose, ".al16");
2878
		else if (type & SLJIT_MEM_ALIGNED_32)
2879
			fprintf(compiler->verbose, ".al32");
2880

2881
		if (reg & REG_PAIR_MASK) {
2882
			fprintf(compiler->verbose, " {");
2883
			sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2884
			fprintf(compiler->verbose, ", ");
2885
			sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2886
			fprintf(compiler->verbose, "}, ");
2887
		} else {
2888
			fprintf(compiler->verbose, " ");
2889
			sljit_verbose_reg(compiler, reg);
2890
			fprintf(compiler->verbose, ", ");
2891
		}
2892
		sljit_verbose_param(compiler, mem, memw);
2893
		fprintf(compiler->verbose, "\n");
2894
	}
2895
#endif /* SLJIT_VERBOSE */
2896
	CHECK_RETURN_OK;
2897
}
2898

2899
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2900
	sljit_s32 reg,
2901
	sljit_s32 mem, sljit_sw memw)
2902
{
2903
	if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2904
		compiler->skip_checks = 0;
2905
		CHECK_RETURN_OK;
2906
	}
2907

2908
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2909
	CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2910
	CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2911
	CHECK_ARGUMENT((mem & REG_MASK) != 0 && (mem & REG_MASK) != reg);
2912

2913
	FUNCTION_CHECK_SRC_MEM(mem, memw);
2914
#endif /* SLJIT_ARGUMENT_CHECKS */
2915
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2916
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2917
		if (type & SLJIT_MEM_SUPP)
2918
			CHECK_RETURN_OK;
2919
		if (sljit_emit_mem_update(compiler, type | SLJIT_MEM_SUPP, reg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2920
			fprintf(compiler->verbose, "    # mem: unsupported form, no instructions are emitted\n");
2921
			CHECK_RETURN_OK;
2922
		}
2923

2924
		if ((type & 0xff) == SLJIT_MOV32)
2925
			fprintf(compiler->verbose, "  %s32.%s ",
2926
				(type & SLJIT_MEM_STORE) ? "store" : "load",
2927
				(type & SLJIT_MEM_POST) ? "post" : "pre");
2928
		else
2929
			fprintf(compiler->verbose, "  %s%s%s.%s ",
2930
				(type & SLJIT_MEM_STORE) ? "store" : "load",
2931
				!(type & SLJIT_32) ? "" : "32",
2932
				op1_types[(type & 0xff) - SLJIT_OP1_BASE],
2933
				(type & SLJIT_MEM_POST) ? "post" : "pre");
2934

2935
		sljit_verbose_reg(compiler, reg);
2936
		fprintf(compiler->verbose, ", ");
2937
		sljit_verbose_param(compiler, mem, memw);
2938
		fprintf(compiler->verbose, "\n");
2939
	}
2940
#endif /* SLJIT_VERBOSE */
2941
	CHECK_RETURN_OK;
2942
}
2943

2944
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
2945
	sljit_s32 freg,
2946
	sljit_s32 mem, sljit_sw memw)
2947
{
2948
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2949
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2950
	CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2951

2952
	if (type & SLJIT_MEM_UNALIGNED) {
2953
		CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2954
	} else if (type & SLJIT_MEM_ALIGNED_16) {
2955
		CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2956
	} else {
2957
		CHECK_ARGUMENT(type & SLJIT_MEM_ALIGNED_32);
2958
		CHECK_ARGUMENT(!(type & SLJIT_32));
2959
	}
2960

2961
	CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_UNALIGNED | SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2962
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2963
	FUNCTION_CHECK_SRC_MEM(mem, memw);
2964
#endif /* SLJIT_ARGUMENT_CHECKS */
2965
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2966
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2967
		fprintf(compiler->verbose, "  %s.%s",
2968
			(type & SLJIT_MEM_STORE) ? "store" : "load",
2969
			!(type & SLJIT_32) ? "f64" : "f32");
2970

2971
		if (type & SLJIT_MEM_UNALIGNED)
2972
			printf(".unal");
2973
		else if (type & SLJIT_MEM_ALIGNED_16)
2974
			printf(".al16");
2975
		else if (type & SLJIT_MEM_ALIGNED_32)
2976
			printf(".al32");
2977

2978
		fprintf(compiler->verbose, " ");
2979
		sljit_verbose_freg(compiler, freg);
2980
		fprintf(compiler->verbose, ", ");
2981
		sljit_verbose_param(compiler, mem, memw);
2982
		fprintf(compiler->verbose, "\n");
2983
	}
2984
#endif /* SLJIT_VERBOSE */
2985
	CHECK_RETURN_OK;
2986
}
2987

2988
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2989
	sljit_s32 freg,
2990
	sljit_s32 mem, sljit_sw memw)
2991
{
2992
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2993
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2994
	CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2995
	CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2996
	FUNCTION_CHECK_SRC_MEM(mem, memw);
2997
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2998
#endif /* SLJIT_ARGUMENT_CHECKS */
2999
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3000
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3001
		if (type & SLJIT_MEM_SUPP)
3002
			CHECK_RETURN_OK;
3003
		if (sljit_emit_fmem_update(compiler, type | SLJIT_MEM_SUPP, freg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
3004
			fprintf(compiler->verbose, "    # fmem: unsupported form, no instructions are emitted\n");
3005
			CHECK_RETURN_OK;
3006
		}
3007

3008
		fprintf(compiler->verbose, "  %s.%s.%s ",
3009
			(type & SLJIT_MEM_STORE) ? "store" : "load",
3010
			!(type & SLJIT_32) ? "f64" : "f32",
3011
			(type & SLJIT_MEM_POST) ? "post" : "pre");
3012

3013
		sljit_verbose_freg(compiler, freg);
3014
		fprintf(compiler->verbose, ", ");
3015
		sljit_verbose_param(compiler, mem, memw);
3016
		fprintf(compiler->verbose, "\n");
3017
	}
3018
#endif /* SLJIT_VERBOSE */
3019
	CHECK_RETURN_OK;
3020
}
3021

3022
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3023
	sljit_s32 vreg,
3024
	sljit_s32 srcdst, sljit_sw srcdstw)
3025
{
3026
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3027
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3028
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_STORE)) == 0);
3029
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3030
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
3031
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (srcdst & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
3032
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3033
	FUNCTION_VCHECK(srcdst, srcdstw, type);
3034
#endif /* SLJIT_ARGUMENT_CHECKS */
3035
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3036
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3037
		if (type & SLJIT_SIMD_TEST)
3038
			CHECK_RETURN_OK;
3039
		if (sljit_emit_simd_mov(compiler, type | SLJIT_SIMD_TEST, vreg, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
3040
			fprintf(compiler->verbose, "    # simd_mem: unsupported form, no instructions are emitted\n");
3041
			CHECK_RETURN_OK;
3042
		}
3043

3044
		fprintf(compiler->verbose, "  simd_%s.%d.%s%d",
3045
			(type & SLJIT_SIMD_STORE) ? "store" : "load",
3046
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3047
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3048
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3049

3050
		if ((type & 0x3f000000) == SLJIT_SIMD_MEM_UNALIGNED)
3051
			fprintf(compiler->verbose, ".unal ");
3052
		else
3053
			fprintf(compiler->verbose, ".al%d ", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
3054

3055
		sljit_verbose_vreg(compiler, vreg);
3056
		fprintf(compiler->verbose, ", ");
3057
		sljit_verbose_vparam(compiler, srcdst, srcdstw);
3058
		fprintf(compiler->verbose, "\n");
3059
	}
3060
#endif /* SLJIT_VERBOSE */
3061
	CHECK_RETURN_OK;
3062
}
3063

3064
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3065
	sljit_s32 vreg,
3066
	sljit_s32 src, sljit_sw srcw)
3067
{
3068
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3069
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3070
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
3071
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3072
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3073
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3074

3075
	if (type & SLJIT_SIMD_FLOAT) {
3076
		if (src == SLJIT_IMM) {
3077
			CHECK_ARGUMENT(srcw == 0);
3078
		} else {
3079
			FUNCTION_FCHECK(src, srcw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
3080
		}
3081
	} else if (src != SLJIT_IMM) {
3082
		FUNCTION_CHECK_DST(src, srcw);
3083
	}
3084
#endif /* SLJIT_ARGUMENT_CHECKS */
3085
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3086
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3087
		if (type & SLJIT_SIMD_TEST)
3088
			CHECK_RETURN_OK;
3089
		if (sljit_emit_simd_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
3090
			fprintf(compiler->verbose, "    # simd_dup: unsupported form, no instructions are emitted\n");
3091
			CHECK_RETURN_OK;
3092
		}
3093

3094
		fprintf(compiler->verbose, "  simd_replicate.%d.%s%d ",
3095
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3096
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3097
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3098

3099
		sljit_verbose_vreg(compiler, vreg);
3100
		fprintf(compiler->verbose, ", ");
3101
		if (type & SLJIT_SIMD_FLOAT)
3102
			sljit_verbose_fparam(compiler, src, srcw);
3103
		else
3104
			sljit_verbose_param(compiler, src, srcw);
3105
		fprintf(compiler->verbose, "\n");
3106
	}
3107
#endif /* SLJIT_VERBOSE */
3108
	CHECK_RETURN_OK;
3109
}
3110

3111
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3112
	sljit_s32 vreg, sljit_s32 lane_index,
3113
	sljit_s32 srcdst, sljit_sw srcdstw)
3114
{
3115
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3116
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3117
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO | SLJIT_SIMD_LANE_SIGNED | SLJIT_32)) == 0);
3118
	CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO)) != (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO));
3119
	CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_SIGNED)) != SLJIT_SIMD_LANE_SIGNED);
3120
	CHECK_ARGUMENT(!(type & SLJIT_SIMD_FLOAT) || !(type & (SLJIT_SIMD_LANE_SIGNED | SLJIT_32)));
3121
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3122
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3123
	CHECK_ARGUMENT(!(type & SLJIT_32) || SLJIT_SIMD_GET_ELEM_SIZE(type) <= 2);
3124
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3125
	CHECK_ARGUMENT(lane_index >= 0 && lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
3126

3127
	if (type & SLJIT_SIMD_FLOAT) {
3128
		FUNCTION_FCHECK(srcdst, srcdstw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
3129
	} else if ((type & SLJIT_SIMD_STORE) || srcdst != SLJIT_IMM) {
3130
		FUNCTION_CHECK_DST(srcdst, srcdstw);
3131
	}
3132
#endif /* SLJIT_ARGUMENT_CHECKS */
3133
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3134
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3135
		if (type & SLJIT_SIMD_TEST)
3136
			CHECK_RETURN_OK;
3137
		if (sljit_emit_simd_lane_mov(compiler, type | SLJIT_SIMD_TEST, vreg, lane_index, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
3138
			fprintf(compiler->verbose, "    # simd_move_lane: unsupported form, no instructions are emitted\n");
3139
			CHECK_RETURN_OK;
3140
		}
3141

3142
		fprintf(compiler->verbose, "  simd_%s_lane%s%s%s.%d.%s%d ",
3143
			(type & SLJIT_SIMD_STORE) ? "store" : "load",
3144
			(type & SLJIT_32) ? "32" : "",
3145
			(type & SLJIT_SIMD_LANE_ZERO) ? "_z" : "",
3146
			(type & SLJIT_SIMD_LANE_SIGNED) ? "_s" : "",
3147
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3148
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3149
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3150

3151
		sljit_verbose_vreg(compiler, vreg);
3152
		fprintf(compiler->verbose, "[%d], ", lane_index);
3153
		if (type & SLJIT_SIMD_FLOAT)
3154
			sljit_verbose_fparam(compiler, srcdst, srcdstw);
3155
		else
3156
			sljit_verbose_param(compiler, srcdst, srcdstw);
3157
		fprintf(compiler->verbose, "\n");
3158
	}
3159
#endif /* SLJIT_VERBOSE */
3160
	CHECK_RETURN_OK;
3161
}
3162

3163
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3164
	sljit_s32 vreg,
3165
	sljit_s32 src, sljit_s32 src_lane_index)
3166
{
3167
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3168
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3169
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
3170
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3171
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3172
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3173
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src, type));
3174
	CHECK_ARGUMENT(src_lane_index >= 0 && src_lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
3175
#endif /* SLJIT_ARGUMENT_CHECKS */
3176
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3177
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3178
		if (type & SLJIT_SIMD_TEST)
3179
			CHECK_RETURN_OK;
3180
		if (sljit_emit_simd_lane_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, src_lane_index) == SLJIT_ERR_UNSUPPORTED) {
3181
			fprintf(compiler->verbose, "    # simd_lane_replicate: unsupported form, no instructions are emitted\n");
3182
			CHECK_RETURN_OK;
3183
		}
3184

3185
		fprintf(compiler->verbose, "  simd_lane_replicate.%d.%s%d ",
3186
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3187
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3188
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3189

3190
		sljit_verbose_vreg(compiler, vreg);
3191
		fprintf(compiler->verbose, ", ");
3192
		sljit_verbose_vreg(compiler, src);
3193
		fprintf(compiler->verbose, "[%d]\n", src_lane_index);
3194
	}
3195
#endif /* SLJIT_VERBOSE */
3196
	CHECK_RETURN_OK;
3197
}
3198

3199
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3200
	sljit_s32 vreg,
3201
	sljit_s32 src, sljit_sw srcw)
3202
{
3203
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3204
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3205
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_EXTEND_SIGNED)) == 0);
3206
	CHECK_ARGUMENT((type & (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT)) != (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT));
3207
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3208
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3209
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_ELEM2_SIZE(type));
3210
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3211
	FUNCTION_VCHECK(src, srcw, type);
3212
#endif /* SLJIT_ARGUMENT_CHECKS */
3213
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3214
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3215
		if (type & SLJIT_SIMD_TEST)
3216
			CHECK_RETURN_OK;
3217
		if (sljit_emit_simd_extend(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
3218
			fprintf(compiler->verbose, "    # simd_extend: unsupported form, no instructions are emitted\n");
3219
			CHECK_RETURN_OK;
3220
		}
3221

3222
		fprintf(compiler->verbose, "  simd_load_extend%s.%d.%s%d.%s%d ",
3223
			(type & SLJIT_SIMD_EXTEND_SIGNED) ? "_s" : "",
3224
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3225
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3226
			(8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)),
3227
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3228
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3229

3230
		sljit_verbose_vreg(compiler, vreg);
3231
		fprintf(compiler->verbose, ", ");
3232
		sljit_verbose_vparam(compiler, src, srcw);
3233
		fprintf(compiler->verbose, "\n");
3234
	}
3235
#endif /* SLJIT_VERBOSE */
3236
	CHECK_RETURN_OK;
3237
}
3238

3239
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3240
	sljit_s32 vreg,
3241
	sljit_s32 dst, sljit_sw dstw)
3242
{
3243
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3244
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3245
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_32)) == SLJIT_SIMD_STORE);
3246
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3247
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3248
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3249
	FUNCTION_CHECK_DST(dst, dstw);
3250
#endif /* SLJIT_ARGUMENT_CHECKS */
3251
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3252
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3253
		if (type & SLJIT_SIMD_TEST)
3254
			CHECK_RETURN_OK;
3255
		if (sljit_emit_simd_sign(compiler, type | SLJIT_SIMD_TEST, vreg, dst, dstw) == SLJIT_ERR_UNSUPPORTED) {
3256
			fprintf(compiler->verbose, "    # simd_sign: unsupported form, no instructions are emitted\n");
3257
			CHECK_RETURN_OK;
3258
		}
3259

3260
		fprintf(compiler->verbose, "  simd_store_sign%s.%d.%s%d ",
3261
			(type & SLJIT_32) ? "32" : "",
3262
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3263
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3264
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3265

3266
		sljit_verbose_vreg(compiler, vreg);
3267
		fprintf(compiler->verbose, ", ");
3268
		sljit_verbose_param(compiler, dst, dstw);
3269
		fprintf(compiler->verbose, "\n");
3270
	}
3271
#endif /* SLJIT_VERBOSE */
3272
	CHECK_RETURN_OK;
3273
}
3274

3275
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3276
	sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3277
{
3278
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3279
	CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3280
	CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(0)) >= SLJIT_SIMD_OP2_AND && (type & SLJIT_SIMD_TYPE_MASK2(0)) <= SLJIT_SIMD_OP2_SHUFFLE);
3281
	CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3282
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
3283
	CHECK_ARGUMENT(SLJIT_SIMD_GET_OPCODE(type) != SLJIT_SIMD_OP2_SHUFFLE || (SLJIT_SIMD_GET_ELEM_SIZE(type) == 0 && !(type & SLJIT_SIMD_FLOAT)));
3284
	CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (src2 & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
3285
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(dst_vreg, type));
3286
	CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src1_vreg, type));
3287
	FUNCTION_VCHECK(src2, src2w, type);
3288
#endif /* SLJIT_ARGUMENT_CHECKS */
3289
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3290
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3291
		if (type & SLJIT_SIMD_TEST)
3292
			CHECK_RETURN_OK;
3293
		if (sljit_emit_simd_op2(compiler, type | SLJIT_SIMD_TEST, dst_vreg, src1_vreg, src2, src2w) == SLJIT_ERR_UNSUPPORTED) {
3294
			fprintf(compiler->verbose, "    # simd_op2: unsupported form, no instructions are emitted\n");
3295
			CHECK_RETURN_OK;
3296
		}
3297

3298
		fprintf(compiler->verbose, "  simd_%s.%d.%s%d",
3299
			simd_op2_names[SLJIT_SIMD_GET_OPCODE(type) - 1],
3300
			(8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3301
			(type & SLJIT_SIMD_FLOAT) ? "f" : "",
3302
			(8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3303

3304
		if ((type & 0x3f000000) != SLJIT_SIMD_MEM_UNALIGNED)
3305
			fprintf(compiler->verbose, ".al%d", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
3306

3307
		fprintf(compiler->verbose, " ");
3308
		sljit_verbose_vreg(compiler, dst_vreg);
3309
		fprintf(compiler->verbose, ", ");
3310
		sljit_verbose_vreg(compiler, src1_vreg);
3311
		fprintf(compiler->verbose, ", ");
3312
		sljit_verbose_vparam(compiler, src2, src2w);
3313
		fprintf(compiler->verbose, "\n");
3314
	}
3315
#endif /* SLJIT_VERBOSE */
3316
	CHECK_RETURN_OK;
3317
}
3318

3319
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3320
{
3321
	/* Any offset is allowed. */
3322
	SLJIT_UNUSED_ARG(offset);
3323

3324
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3325
	FUNCTION_CHECK_DST(dst, dstw);
3326
#endif /* SLJIT_ARGUMENT_CHECKS */
3327
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3328
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3329
		fprintf(compiler->verbose, "  local_base ");
3330
		sljit_verbose_param(compiler, dst, dstw);
3331
		fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", offset);
3332
	}
3333
#endif /* SLJIT_VERBOSE */
3334
	CHECK_RETURN_OK;
3335
}
3336

3337
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
3338
	sljit_s32 dst, sljit_sw dstw,
3339
	sljit_sw init_value)
3340
{
3341
	SLJIT_UNUSED_ARG(init_value);
3342

3343
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3344
	CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV_S32
3345
		|| op == SLJIT_MOV32 || (op | SLJIT_32) == SLJIT_MOV32_U8);
3346
	FUNCTION_CHECK_DST(dst, dstw);
3347
#endif /* SLJIT_ARGUMENT_CHECKS */
3348
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3349
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3350
		fprintf(compiler->verbose, "  const%s%s ",
3351
			!(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
3352
		sljit_verbose_param(compiler, dst, dstw);
3353
		fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", init_value);
3354
	}
3355
#endif /* SLJIT_VERBOSE */
3356
	CHECK_RETURN_OK;
3357
}
3358

3359
static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
3360
	sljit_s32 dst, sljit_sw dstw)
3361
{
3362
#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3363
	CHECK_ARGUMENT(op == SLJIT_MOV_ADDR || op == SLJIT_MOV_ABS_ADDR || op == SLJIT_ADD_ABS_ADDR);
3364
	FUNCTION_CHECK_DST(dst, dstw);
3365
#endif /* SLJIT_ARGUMENT_CHECKS */
3366
#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3367
	if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3368
		fprintf(compiler->verbose, "  %s ", op_addr_types[op]);
3369
		sljit_verbose_param(compiler, dst, dstw);
3370
		fprintf(compiler->verbose, "\n");
3371
	}
3372
#endif /* SLJIT_VERBOSE */
3373
	CHECK_RETURN_OK;
3374
}
3375

3376
#else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_VERBOSE */
3377

3378
#define SLJIT_SKIP_CHECKS(compiler)
3379

3380
#endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
3381

3382
#define SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw) \
3383
	SLJIT_COMPILE_ASSERT(!(SLJIT_CONV_SW_FROM_F64 & 0x1) && !(SLJIT_CONV_F64_FROM_SW & 0x1) && !(SLJIT_CONV_F64_FROM_UW & 0x1), \
3384
		invalid_float_opcodes); \
3385
	if (GET_OPCODE(op) >= SLJIT_CONV_SW_FROM_F64 && GET_OPCODE(op) <= SLJIT_CMP_F64) { \
3386
		if (GET_OPCODE(op) == SLJIT_CMP_F64) { \
3387
			CHECK(check_sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw)); \
3388
			ADJUST_LOCAL_OFFSET(dst, dstw); \
3389
			ADJUST_LOCAL_OFFSET(src, srcw); \
3390
			return sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw); \
3391
		} \
3392
		if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_S32_FROM_F64) { \
3393
			CHECK(check_sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw)); \
3394
			ADJUST_LOCAL_OFFSET(dst, dstw); \
3395
			ADJUST_LOCAL_OFFSET(src, srcw); \
3396
			return sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw); \
3397
		} \
3398
		if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_F64_FROM_S32) { \
3399
			CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3400
			ADJUST_LOCAL_OFFSET(dst, dstw); \
3401
			ADJUST_LOCAL_OFFSET(src, srcw); \
3402
			return sljit_emit_fop1_conv_f64_from_sw(compiler, op, dst, dstw, src, srcw); \
3403
		} \
3404
		CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3405
		ADJUST_LOCAL_OFFSET(dst, dstw); \
3406
		ADJUST_LOCAL_OFFSET(src, srcw); \
3407
		return sljit_emit_fop1_conv_f64_from_uw(compiler, op, dst, dstw, src, srcw); \
3408
	} \
3409
	CHECK(check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw)); \
3410
	ADJUST_LOCAL_OFFSET(dst, dstw); \
3411
	ADJUST_LOCAL_OFFSET(src, srcw);
3412

3413
#if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6))
3414

3415
static sljit_s32 sljit_emit_mem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3416
	sljit_s32 reg,
3417
	sljit_s32 mem, sljit_sw memw)
3418
{
3419
	SLJIT_SKIP_CHECKS(compiler);
3420

3421
	if (type & SLJIT_MEM_STORE)
3422
		return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), mem, memw, reg, 0);
3423
	return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), reg, 0, mem, memw);
3424
}
3425

3426
#endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) */
3427

3428
#if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6)) \
3429
	&& !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3430

3431
static sljit_s32 sljit_emit_fmem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3432
	sljit_s32 freg,
3433
	sljit_s32 mem, sljit_sw memw)
3434
{
3435
	SLJIT_SKIP_CHECKS(compiler);
3436

3437
	if (type & SLJIT_MEM_STORE)
3438
		return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), mem, memw, freg, 0);
3439
	return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), freg, 0, mem, memw);
3440
}
3441

3442
#endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) && !SLJIT_CONFIG_ARM */
3443

3444
static void sljit_reset_read_only_buffers(struct sljit_read_only_buffer *buffers)
3445
{
3446
	while (buffers != NULL) {
3447
		buffers->u.label = NULL;
3448
		buffers = buffers->next;
3449
	}
3450
}
3451

3452
/* CPU description section */
3453

3454
#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
3455
#define SLJIT_CPUINFO_PART1 " 32bit ("
3456
#elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3457
#define SLJIT_CPUINFO_PART1 " 64bit ("
3458
#else /* !SLJIT_32BIT_ARCHITECTURE && !SLJIT_64BIT_ARCHITECTURE */
3459
#error "Internal error: CPU type info missing"
3460
#endif /* SLJIT_32BIT_ARCHITECTURE */
3461

3462
#if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
3463
#define SLJIT_CPUINFO_PART2 "little endian + "
3464
#elif (defined SLJIT_BIG_ENDIAN && SLJIT_BIG_ENDIAN)
3465
#define SLJIT_CPUINFO_PART2 "big endian + "
3466
#else /* !SLJIT_LITTLE_ENDIAN && !SLJIT_BIG_ENDIAN */
3467
#error "Internal error: CPU type info missing"
3468
#endif /* SLJIT_LITTLE_ENDIAN */
3469

3470
#if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED)
3471
#define SLJIT_CPUINFO_PART3 "unaligned)"
3472
#else /* !SLJIT_UNALIGNED */
3473
#define SLJIT_CPUINFO_PART3 "aligned)"
3474
#endif /* SLJIT_UNALIGNED */
3475

3476
#define SLJIT_CPUINFO SLJIT_CPUINFO_PART1 SLJIT_CPUINFO_PART2 SLJIT_CPUINFO_PART3
3477

3478
#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
3479
#	include "sljitNativeX86_common.c"
3480
#elif (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
3481
#	include "sljitNativeARM_32.c"
3482
#elif (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
3483
#	include "sljitNativeARM_32.c"
3484
#elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
3485
#	include "sljitNativeARM_T2_32.c"
3486
#elif (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3487
#	include "sljitNativeARM_64.c"
3488
#elif (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3489
#	include "sljitNativePPC_common.c"
3490
#elif (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3491
#	include "sljitNativeMIPS_common.c"
3492
#elif (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
3493
#	include "sljitNativeRISCV_common.c"
3494
#elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
3495
#	include "sljitNativeS390X.c"
3496
#elif (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3497
#	include "sljitNativeLOONGARCH_64.c"
3498
#endif /* SLJIT_CONFIG_X86 */
3499

3500
#include "sljitSerialize.c"
3501

3502
static SLJIT_INLINE sljit_s32 emit_mov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3503
{
3504
#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3505
	/* At the moment the pointer size is always equal to sljit_sw. May be changed in the future. */
3506
	if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_P))
3507
		return SLJIT_SUCCESS;
3508
#else /* !SLJIT_64BIT_ARCHITECTURE */
3509
	if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P))
3510
		return SLJIT_SUCCESS;
3511
#endif /* SLJIT_64BIT_ARCHITECTURE */
3512

3513
	SLJIT_SKIP_CHECKS(compiler);
3514
	return sljit_emit_op1(compiler, op, SLJIT_RETURN_REG, 0, src, srcw);
3515
}
3516

3517
#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \
3518
	&& !((defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) && defined __SOFTFP__)
3519

3520
static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3521
{
3522
	if (src == SLJIT_FR0)
3523
		return SLJIT_SUCCESS;
3524

3525
	SLJIT_SKIP_CHECKS(compiler);
3526
	return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
3527
}
3528

3529
#endif /* !SLJIT_CONFIG_X86_32 && !(SLJIT_CONFIG_ARM_32 && __SOFTFP__) */
3530

3531
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3532
{
3533
	CHECK_ERROR();
3534
	CHECK(check_sljit_emit_return(compiler, op, src, srcw));
3535

3536
	if (GET_OPCODE(op) < SLJIT_MOV_F64) {
3537
		FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
3538
	} else {
3539
		FAIL_IF(emit_fmov_before_return(compiler, op, src, srcw));
3540
	}
3541

3542
	SLJIT_SKIP_CHECKS(compiler);
3543
	return sljit_emit_return_void(compiler);
3544
}
3545

3546
#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3547
	&& !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3548
	&& !(defined(SLJIT_CONFIG_LOONGARCH_64) && SLJIT_CONFIG_LOONGARCH_64)
3549

3550
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3551
	sljit_s32 dst_freg,
3552
	sljit_s32 src1, sljit_sw src1w,
3553
	sljit_s32 src2, sljit_sw src2w)
3554
{
3555
	CHECK_ERROR();
3556
	CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3557
	ADJUST_LOCAL_OFFSET(src1, src1w);
3558
	ADJUST_LOCAL_OFFSET(src2, src2w);
3559

3560
	SLJIT_SKIP_CHECKS(compiler);
3561
	return sljit_emit_fop2(compiler, op, dst_freg, 0, src1, src1w, src2, src2w);
3562
}
3563

3564
#endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH_64 */
3565

3566
#if !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3567
	&& !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3568
	&& !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3569

3570
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
3571
	sljit_s32 src1, sljit_sw src1w,
3572
	sljit_s32 src2, sljit_sw src2w)
3573
{
3574
	/* Default compare for most architectures. */
3575
	sljit_s32 flags, tmp_src, condition;
3576
	sljit_sw tmp_srcw;
3577

3578
	CHECK_ERROR_PTR();
3579
	CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
3580

3581
	condition = type & 0xff;
3582
#if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3583
	if ((condition == SLJIT_EQUAL || condition == SLJIT_NOT_EQUAL)) {
3584
		if (src1 == SLJIT_IMM && !src1w) {
3585
			src1 = src2;
3586
			src1w = src2w;
3587
			src2 = SLJIT_IMM;
3588
			src2w = 0;
3589
		}
3590
		if (src2 == SLJIT_IMM && !src2w)
3591
			return emit_cmp_to0(compiler, type, src1, src1w);
3592
	}
3593
#endif /* SLJIT_CONFIG_ARM_64 */
3594

3595
	if (SLJIT_UNLIKELY(src1 == SLJIT_IMM && src2 != SLJIT_IMM)) {
3596
		/* Immediate is preferred as second argument by most architectures. */
3597
		switch (condition) {
3598
		case SLJIT_LESS:
3599
			condition = SLJIT_GREATER;
3600
			break;
3601
		case SLJIT_GREATER_EQUAL:
3602
			condition = SLJIT_LESS_EQUAL;
3603
			break;
3604
		case SLJIT_GREATER:
3605
			condition = SLJIT_LESS;
3606
			break;
3607
		case SLJIT_LESS_EQUAL:
3608
			condition = SLJIT_GREATER_EQUAL;
3609
			break;
3610
		case SLJIT_SIG_LESS:
3611
			condition = SLJIT_SIG_GREATER;
3612
			break;
3613
		case SLJIT_SIG_GREATER_EQUAL:
3614
			condition = SLJIT_SIG_LESS_EQUAL;
3615
			break;
3616
		case SLJIT_SIG_GREATER:
3617
			condition = SLJIT_SIG_LESS;
3618
			break;
3619
		case SLJIT_SIG_LESS_EQUAL:
3620
			condition = SLJIT_SIG_GREATER_EQUAL;
3621
			break;
3622
		}
3623

3624
		type = condition | (type & (SLJIT_32 | SLJIT_REWRITABLE_JUMP));
3625
		tmp_src = src1;
3626
		src1 = src2;
3627
		src2 = tmp_src;
3628
		tmp_srcw = src1w;
3629
		src1w = src2w;
3630
		src2w = tmp_srcw;
3631
	}
3632

3633
	if (condition <= SLJIT_NOT_ZERO)
3634
		flags = SLJIT_SET_Z;
3635
	else
3636
		flags = (condition & 0xfe) << VARIABLE_FLAG_SHIFT;
3637

3638
	SLJIT_SKIP_CHECKS(compiler);
3639
	PTR_FAIL_IF(sljit_emit_op2u(compiler,
3640
		SLJIT_SUB | flags | (type & SLJIT_32), src1, src1w, src2, src2w));
3641

3642
	SLJIT_SKIP_CHECKS(compiler);
3643
	return sljit_emit_jump(compiler, condition | (type & (SLJIT_REWRITABLE_JUMP | SLJIT_32)));
3644
}
3645

3646
#endif /* !SLJIT_CONFIG_MIPS */
3647

3648
#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3649

3650
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
3651
{
3652
	switch (type) {
3653
	case SLJIT_UNORDERED_OR_EQUAL:
3654
	case SLJIT_ORDERED_NOT_EQUAL:
3655
		return 1;
3656
	}
3657

3658
	return 0;
3659
}
3660

3661
#endif /* SLJIT_CONFIG_ARM */
3662

3663
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
3664
	sljit_s32 src1, sljit_sw src1w,
3665
	sljit_s32 src2, sljit_sw src2w)
3666
{
3667
	CHECK_ERROR_PTR();
3668
	CHECK_PTR(check_sljit_emit_fcmp(compiler, type, src1, src1w, src2, src2w));
3669

3670
	SLJIT_SKIP_CHECKS(compiler);
3671
	sljit_emit_fop1(compiler, SLJIT_CMP_F64 | ((type & 0xfe) << VARIABLE_FLAG_SHIFT) | (type & SLJIT_32), src1, src1w, src2, src2w);
3672

3673
	SLJIT_SKIP_CHECKS(compiler);
3674
	return sljit_emit_jump(compiler, type);
3675
}
3676

3677
#if !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3678
	&& !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3679
	&& !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3680

3681
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
3682
	sljit_s32 dst, sljit_sw dstw,
3683
	sljit_s32 src1, sljit_sw src1w,
3684
	sljit_s32 src2, sljit_sw src2w)
3685
{
3686
	CHECK_ERROR_PTR();
3687
	CHECK_PTR(check_sljit_emit_op2cmpz(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3688

3689
	SLJIT_SKIP_CHECKS(compiler);
3690
	PTR_FAIL_IF(sljit_emit_op2(compiler, ((op | SLJIT_SET_Z) & ~SLJIT_REWRITABLE_JUMP), dst, dstw, src1, src1w, src2, src2w));
3691

3692
	SLJIT_SKIP_CHECKS(compiler);
3693
	return sljit_emit_jump(compiler, ((op & SLJIT_JUMP_IF_ZERO) ? SLJIT_ZERO : SLJIT_NOT_ZERO) | (op & SLJIT_REWRITABLE_JUMP));
3694
}
3695

3696
#else /* SLJIT_CONFIG_RISCV || SLJIT_CONFIG_MIPS || SLJIT_CONFIG_LOONGARCH */
3697

3698
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
3699
	sljit_s32 dst, sljit_sw dstw,
3700
	sljit_s32 src1, sljit_sw src1w,
3701
	sljit_s32 src2, sljit_sw src2w)
3702
{
3703
	sljit_s32 reg;
3704

3705
	CHECK_ERROR_PTR();
3706
	CHECK_PTR(check_sljit_emit_op2cmpz(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3707

3708
	SLJIT_SKIP_CHECKS(compiler);
3709
	PTR_FAIL_IF(sljit_emit_op2(compiler, (op & ~(SLJIT_SET_Z | SLJIT_REWRITABLE_JUMP)), dst, dstw, src1, src1w, src2, src2w));
3710

3711
	/* When dst is a memory operand, its value is stored in SLJIT_TMP_DEST_FREG. */
3712
	reg = FAST_IS_REG(dst) ? dst : SLJIT_TMP_DEST_REG;
3713

3714
	SLJIT_SKIP_CHECKS(compiler);
3715
	return sljit_emit_cmp(compiler, ((op & SLJIT_JUMP_IF_ZERO) ? SLJIT_ZERO : SLJIT_NOT_ZERO) | (op & (SLJIT_32 | SLJIT_REWRITABLE_JUMP)), reg, 0, SLJIT_IMM, 0);
3716
}
3717

3718
#endif /* !SLJIT_CONFIG_RISCV && !SLJIT_CONFIG_MIPS && !SLJIT_CONFIG_LOONGARCH */
3719

3720
#if !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3721
	&& !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3722

3723
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3724
	sljit_s32 reg,
3725
	sljit_s32 mem, sljit_sw memw)
3726
{
3727
	CHECK_ERROR();
3728
	CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3729
	SLJIT_UNUSED_ARG(type);
3730
	SLJIT_UNUSED_ARG(reg);
3731
	SLJIT_UNUSED_ARG(mem);
3732
	SLJIT_UNUSED_ARG(memw);
3733

3734
	return SLJIT_ERR_UNSUPPORTED;
3735
}
3736

3737
#endif /* !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_PPC */
3738

3739
#if !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
3740
	&& !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3741

3742
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3743
	sljit_s32 freg,
3744
	sljit_s32 mem, sljit_sw memw)
3745
{
3746
	CHECK_ERROR();
3747
	CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3748

3749
	return sljit_emit_fmem_unaligned(compiler, type, freg, mem, memw);
3750
}
3751

3752
#endif /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS */
3753

3754
#if !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \
3755
	&& !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3756

3757
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3758
	sljit_s32 freg,
3759
	sljit_s32 mem, sljit_sw memw)
3760
{
3761
	CHECK_ERROR();
3762
	CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3763
	SLJIT_UNUSED_ARG(type);
3764
	SLJIT_UNUSED_ARG(freg);
3765
	SLJIT_UNUSED_ARG(mem);
3766
	SLJIT_UNUSED_ARG(memw);
3767

3768
	return SLJIT_ERR_UNSUPPORTED;
3769
}
3770

3771
#endif /* !SLJIT_CONFIG_ARM_64 && !SLJIT_CONFIG_PPC */
3772

3773
#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3774
	&& !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3775
	&& !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3776
	&& !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3777
	&& !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3778

3779
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3780
	sljit_s32 vreg,
3781
	sljit_s32 srcdst, sljit_sw srcdstw)
3782
{
3783
	CHECK_ERROR();
3784
	CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3785
	SLJIT_UNUSED_ARG(compiler);
3786
	SLJIT_UNUSED_ARG(type);
3787
	SLJIT_UNUSED_ARG(vreg);
3788
	SLJIT_UNUSED_ARG(srcdst);
3789
	SLJIT_UNUSED_ARG(srcdstw);
3790

3791
	return SLJIT_ERR_UNSUPPORTED;
3792
}
3793

3794
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3795
	sljit_s32 vreg,
3796
	sljit_s32 src, sljit_sw srcw)
3797
{
3798
	CHECK_ERROR();
3799
	CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3800
	SLJIT_UNUSED_ARG(compiler);
3801
	SLJIT_UNUSED_ARG(type);
3802
	SLJIT_UNUSED_ARG(vreg);
3803
	SLJIT_UNUSED_ARG(src);
3804
	SLJIT_UNUSED_ARG(srcw);
3805

3806
	return SLJIT_ERR_UNSUPPORTED;
3807
}
3808

3809
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3810
	sljit_s32 vreg, sljit_s32 lane_index,
3811
	sljit_s32 srcdst, sljit_sw srcdstw)
3812
{
3813
	CHECK_ERROR();
3814
	CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3815
	SLJIT_UNUSED_ARG(compiler);
3816
	SLJIT_UNUSED_ARG(type);
3817
	SLJIT_UNUSED_ARG(vreg);
3818
	SLJIT_UNUSED_ARG(lane_index);
3819
	SLJIT_UNUSED_ARG(srcdst);
3820
	SLJIT_UNUSED_ARG(srcdstw);
3821

3822
	return SLJIT_ERR_UNSUPPORTED;
3823
}
3824

3825
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3826
	sljit_s32 vreg,
3827
	sljit_s32 src, sljit_s32 src_lane_index)
3828
{
3829
	CHECK_ERROR();
3830
	CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
3831
	SLJIT_UNUSED_ARG(compiler);
3832
	SLJIT_UNUSED_ARG(type);
3833
	SLJIT_UNUSED_ARG(vreg);
3834
	SLJIT_UNUSED_ARG(src);
3835
	SLJIT_UNUSED_ARG(src_lane_index);
3836

3837
	return SLJIT_ERR_UNSUPPORTED;
3838
}
3839

3840
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3841
	sljit_s32 vreg,
3842
	sljit_s32 src, sljit_sw srcw)
3843
{
3844
	CHECK_ERROR();
3845
	CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
3846
	SLJIT_UNUSED_ARG(compiler);
3847
	SLJIT_UNUSED_ARG(type);
3848
	SLJIT_UNUSED_ARG(vreg);
3849
	SLJIT_UNUSED_ARG(src);
3850
	SLJIT_UNUSED_ARG(srcw);
3851

3852
	return SLJIT_ERR_UNSUPPORTED;
3853
}
3854

3855
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3856
	sljit_s32 vreg,
3857
	sljit_s32 dst, sljit_sw dstw)
3858
{
3859
	CHECK_ERROR();
3860
	CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
3861
	SLJIT_UNUSED_ARG(compiler);
3862
	SLJIT_UNUSED_ARG(type);
3863
	SLJIT_UNUSED_ARG(vreg);
3864
	SLJIT_UNUSED_ARG(dst);
3865
	SLJIT_UNUSED_ARG(dstw);
3866

3867
	return SLJIT_ERR_UNSUPPORTED;
3868
}
3869

3870
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3871
	sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3872
{
3873
	CHECK_ERROR();
3874
	CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
3875
	SLJIT_UNUSED_ARG(compiler);
3876
	SLJIT_UNUSED_ARG(type);
3877
	SLJIT_UNUSED_ARG(dst_vreg);
3878
	SLJIT_UNUSED_ARG(src1_vreg);
3879
	SLJIT_UNUSED_ARG(src2);
3880
	SLJIT_UNUSED_ARG(src2w);
3881

3882
	return SLJIT_ERR_UNSUPPORTED;
3883
}
3884

3885
#endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH */
3886

3887
#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3888
	&& !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3889

3890
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3891
{
3892
	CHECK_ERROR();
3893
	CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3894

3895
	ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3896

3897
	SLJIT_SKIP_CHECKS(compiler);
3898

3899
	if (offset != 0)
3900
		return sljit_emit_op2(compiler, SLJIT_ADD, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3901
	return sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_SP, 0);
3902
}
3903

3904
#endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_64 */
3905

3906
SLJIT_API_FUNC_ATTRIBUTE void* sljit_read_only_buffer_start_writing(sljit_uw addr, sljit_uw size, sljit_sw executable_offset)
3907
{
3908
	SLJIT_UNUSED_ARG(size);
3909
	SLJIT_UNUSED_ARG(executable_offset);
3910

3911
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + size), 0);
3912
	return SLJIT_ADD_EXEC_OFFSET(addr, -executable_offset);
3913
}
3914

3915
SLJIT_API_FUNC_ATTRIBUTE void sljit_read_only_buffer_end_writing(sljit_uw addr, sljit_uw size, sljit_sw executable_offset)
3916
{
3917
	SLJIT_UNUSED_ARG(addr);
3918
	SLJIT_UNUSED_ARG(size);
3919
	SLJIT_UNUSED_ARG(executable_offset);
3920

3921
	SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + size), 1);
3922
	SLJIT_CACHE_FLUSH((void*)addr, (void*)(addr + size));
3923
}
3924

3925
#endif /* !SLJIT_CONFIG_UNSUPPORTED */
3926

3927
Product

Resources

Company
