CoCalc -- sljitNativeARM

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitNativeARM_64.c
⁹⁹¹³ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28
{
29
	return "ARM-64" SLJIT_CPUINFO;
30
}
31

32
/* Length of an instruction word */
33
typedef sljit_u32 sljit_ins;
34

35
#define TMP_ZERO	(0)
36

37
#define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
38
#define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
39
#define TMP_LR		(SLJIT_NUMBER_OF_REGISTERS + 4)
40
#define TMP_FP		(SLJIT_NUMBER_OF_REGISTERS + 5)
41

42
#define TMP_FREG1	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
43
#define TMP_FREG2	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
44

45
/* r18 - platform register, currently not used */
46
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 8] = {
47
	31, 0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 8, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 31, 9, 10, 30, 29
48
};
49

50
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
51
	0, 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 15, 14, 13, 12, 11, 10, 9, 8, 30, 31
52
};
53

54
#define W_OP ((sljit_ins)1 << 31)
55
#define RD(rd) ((sljit_ins)reg_map[rd])
56
#define RT(rt) ((sljit_ins)reg_map[rt])
57
#define RN(rn) ((sljit_ins)reg_map[rn] << 5)
58
#define RT2(rt2) ((sljit_ins)reg_map[rt2] << 10)
59
#define RM(rm) ((sljit_ins)reg_map[rm] << 16)
60
#define VD(vd) ((sljit_ins)freg_map[vd])
61
#define VT(vt) ((sljit_ins)freg_map[vt])
62
#define VT2(vt) ((sljit_ins)freg_map[vt] << 10)
63
#define VN(vn) ((sljit_ins)freg_map[vn] << 5)
64
#define VM(vm) ((sljit_ins)freg_map[vm] << 16)
65

66
/* --------------------------------------------------------------------- */
67
/*  Instrucion forms                                                     */
68
/* --------------------------------------------------------------------- */
69

70
#define ADC		0x9a000000
71
#define ADD		0x8b000000
72
#define ADDE		0x8b200000
73
#define ADDI		0x91000000
74
#define ADR		0x10000000
75
#define ADRP		0x90000000
76
#define AND		0x8a000000
77
#define ANDI		0x92000000
78
#define AND_v		0x0e201c00
79
#define ASRV		0x9ac02800
80
#define B		0x14000000
81
#define B_CC		0x54000000
82
#define BL		0x94000000
83
#define BLR		0xd63f0000
84
#define BR		0xd61f0000
85
#define BRK		0xd4200000
86
#define CAS		0xc8a07c00
87
#define CASB		0x08a07c00
88
#define CASH		0x48a07c00
89
#define CBZ		0xb4000000
90
#define CCMPI		0xfa400800
91
#define CLZ		0xdac01000
92
#define CSEL		0x9a800000
93
#define CSINC		0x9a800400
94
#define DMB_SY		0xd5033fbf
95
#define DUP_e		0x0e000400
96
#define DUP_g		0x0e000c00
97
#define EOR		0xca000000
98
#define EOR_v		0x2e201c00
99
#define EORI		0xd2000000
100
#define EXTR		0x93c00000
101
#define FABS		0x1e60c000
102
#define FADD		0x1e602800
103
#define FCMP		0x1e602000
104
#define FCSEL		0x1e600c00
105
#define FCVT		0x1e224000
106
#define FCVTL		0x0e217800
107
#define FCVTZS		0x9e780000
108
#define FDIV		0x1e601800
109
#define FMOV		0x1e604000
110
#define FMOV_R		0x9e660000
111
#define FMOV_I		0x1e601000
112
#define FMUL		0x1e600800
113
#define FNEG		0x1e614000
114
#define FSUB		0x1e603800
115
#define INS		0x4e001c00
116
#define INS_e		0x6e000400
117
#define LD1		0x0c407000
118
#define LD1_s		0x0d400000
119
#define LD1R		0x0d40c000
120
#define LDRI		0xf9400000
121
#define LDRI_F64	0xfd400000
122
#define LDRI_POST	0xf8400400
123
#define LDP		0xa9400000
124
#define LDP_F64		0x6d400000
125
#define LDP_POST	0xa8c00000
126
#define LDR_PRE		0xf8400c00
127
#define LDXR		0xc85f7c00
128
#define LDXRB		0x085f7c00
129
#define LDXRH		0x485f7c00
130
#define LSLV		0x9ac02000
131
#define LSRV		0x9ac02400
132
#define MADD		0x9b000000
133
#define MOVI		0x0f000400
134
#define MOVK		0xf2800000
135
#define MOVN		0x92800000
136
#define MOVZ		0xd2800000
137
#define NOP		0xd503201f
138
#define ORN		0xaa200000
139
#define ORR		0xaa000000
140
#define ORR_v		0x0ea01c00
141
#define ORRI		0xb2000000
142
#define RBIT		0xdac00000
143
#define RET		0xd65f0000
144
#define REV		0xdac00c00
145
#define REV16		0xdac00400
146
#define RORV		0x9ac02c00
147
#define SBC		0xda000000
148
#define SBFM		0x93400000
149
#define SCVTF		0x9e620000
150
#define SDIV		0x9ac00c00
151
#define SMADDL		0x9b200000
152
#define SMOV		0x0e002c00
153
#define SMULH		0x9b403c00
154
#define SSHLL		0x0f00a400
155
#define ST1		0x0c007000
156
#define ST1_s		0x0d000000
157
#define STP		0xa9000000
158
#define STP_F64		0x6d000000
159
#define STP_PRE		0xa9800000
160
#define STRB		0x38206800
161
#define STRBI		0x39000000
162
#define STRI		0xf9000000
163
#define STRI_F64	0xfd000000
164
#define STR_FI		0x3d000000
165
#define STR_FR		0x3c206800
166
#define STUR_FI		0x3c000000
167
#define STURBI		0x38000000
168
#define STXR		0xc8007c00
169
#define STXRB		0x8007c00
170
#define STXRH		0x48007c00
171
#define SUB		0xcb000000
172
#define SUBI		0xd1000000
173
#define SUBS		0xeb000000
174
#define TBZ		0x36000000
175
#define TBL_v		0x0e000000
176
#define UBFM		0xd3400000
177
#define UCVTF		0x9e630000
178
#define UDIV		0x9ac00800
179
#define UMOV		0x0e003c00
180
#define UMULH		0x9bc03c00
181
#define USHLL		0x2f00a400
182
#define USHR		0x2f000400
183
#define USRA		0x2f001400
184
#define XTN		0x0e212800
185

186
#define CSET		(CSINC | RM(TMP_ZERO) | RN(TMP_ZERO))
187
#define LDR		(STRI | (1 << 22))
188
#define LDRB		(STRBI | (1 << 22))
189
#define LDRH		(LDRB | (1 << 30))
190
#define MOV		(ORR | RN(TMP_ZERO))
191

192
static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
193
{
194
	sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
195
	FAIL_IF(!ptr);
196
	*ptr = ins;
197
	compiler->size++;
198
	return SLJIT_SUCCESS;
199
}
200

201
static SLJIT_INLINE sljit_s32 emit_imm64_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
202
{
203
	FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)(imm & 0xffff) << 5)));
204
	FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)(imm >> 16) & 0xffff) << 5) | (1 << 21)));
205
	FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)(imm >> 32) & 0xffff) << 5) | (2 << 21)));
206
	return push_inst(compiler, MOVK | RD(dst) | ((sljit_ins)(imm >> 48) << 5) | (3 << 21));
207
}
208

209
static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
210
{
211
	sljit_sw diff;
212
	sljit_uw target_addr;
213
	sljit_uw jump_addr = (sljit_uw)code_ptr;
214
	sljit_uw orig_addr = jump->addr;
215
	SLJIT_UNUSED_ARG(executable_offset);
216

217
	jump->addr = jump_addr;
218
	if (jump->flags & SLJIT_REWRITABLE_JUMP)
219
		goto exit;
220

221
	if (jump->flags & JUMP_ADDR)
222
		target_addr = jump->u.target;
223
	else {
224
		SLJIT_ASSERT(jump->u.label != NULL);
225
		target_addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
226

227
		if (jump->u.label->size > orig_addr)
228
			jump_addr = (sljit_uw)(code + orig_addr);
229
	}
230

231
	diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
232

233
	if (jump->flags & IS_COND) {
234
		diff += SSIZE_OF(ins);
235
		if (diff <= 0xfffff && diff >= -0x100000) {
236
			*(--code_ptr) ^= (jump->flags & IS_CBZ) ? (0x1 << 24) : 0x1;
237
			jump->flags |= PATCH_COND;
238
			jump->addr -= sizeof(sljit_ins);
239
			return code_ptr;
240
		}
241
		diff -= SSIZE_OF(ins);
242
	}
243

244
	if (diff <= 0x7ffffff && diff >= -0x8000000) {
245
		if (jump->flags & IS_COND)
246
			code_ptr[-1] -= (4 << 5);
247
		jump->flags |= PATCH_B;
248
		return code_ptr;
249
	}
250

251
	if (target_addr < 0x100000000l) {
252
		if (jump->flags & IS_COND)
253
			code_ptr[-1] -= (2 << 5);
254
		code_ptr[2] = code_ptr[0];
255
		return code_ptr + 2;
256
	}
257

258
	if (diff <= 0xfffff000l && diff >= -0x100000000l) {
259
		if (jump->flags & IS_COND)
260
			code_ptr[-1] -= (2 << 5);
261
		jump->flags |= PATCH_B32;
262
		code_ptr[2] = code_ptr[0];
263
		return code_ptr + 2;
264
	}
265

266
	if (target_addr < 0x1000000000000l) {
267
		if (jump->flags & IS_COND)
268
			code_ptr[-1] -= (1 << 5);
269
		jump->flags |= PATCH_ABS48;
270
		code_ptr[3] = code_ptr[0];
271
		return code_ptr + 3;
272
	}
273

274
exit:
275
	jump->flags |= PATCH_ABS64;
276
	code_ptr[4] = code_ptr[0];
277
	return code_ptr + 4;
278
}
279

280
static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
281
{
282
	sljit_uw addr;
283
	sljit_uw jump_addr = (sljit_uw)code_ptr;
284
	sljit_sw diff;
285
	SLJIT_UNUSED_ARG(executable_offset);
286

287
	SLJIT_ASSERT(jump->flags < ((sljit_uw)4 << JUMP_SIZE_SHIFT));
288
	if (jump->flags & JUMP_ADDR)
289
		addr = jump->u.target;
290
	else {
291
		addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
292

293
		if (jump->u.label->size > jump->addr)
294
			jump_addr = (sljit_uw)(code + jump->addr);
295
	}
296

297
	diff = (sljit_sw)addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
298

299
	if (diff <= 0xfffff && diff >= -0x100000) {
300
		jump->flags |= PATCH_B;
301
		return 0;
302
	}
303

304
	if (diff <= 0xfffff000l && diff >= -0x100000000l) {
305
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
306
		jump->flags |= PATCH_B32;
307
		return 1;
308
	}
309

310
	if (addr < 0x100000000l) {
311
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
312
		return 1;
313
	}
314

315
	if (addr < 0x1000000000000l) {
316
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)2 << JUMP_SIZE_SHIFT));
317
		jump->flags |= PATCH_ABS48;
318
		return 2;
319
	}
320

321
	SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
322
	jump->flags |= PATCH_ABS64;
323
	return 3;
324
}
325

326
static SLJIT_INLINE void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
327
{
328
	sljit_sw addr = (sljit_sw)((jump->flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr);
329
	sljit_ins* buf_ptr = (sljit_ins*)jump->addr;
330
	sljit_u32 dst;
331
	SLJIT_UNUSED_ARG(executable_offset);
332

333
	if (!(jump->flags & JUMP_MOV_ADDR)) {
334
		if (jump->flags & PATCH_COND) {
335
			addr = (addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) >> 2;
336
			SLJIT_ASSERT(addr <= 0x3ffff && addr >= -0x40000);
337
			buf_ptr[0] = (buf_ptr[0] & ~(sljit_ins)0xffffe0) | (sljit_ins)((addr & 0x7ffff) << 5);
338
			return;
339
		}
340

341
		if (jump->flags & PATCH_B) {
342
			addr = (addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) >> 2;
343
			SLJIT_ASSERT(addr <= 0x1ffffff && addr >= -0x2000000);
344
			buf_ptr[0] = ((jump->flags & IS_BL) ? BL : B) | (sljit_ins)(addr & 0x3ffffff);
345
			return;
346
		}
347

348
		dst = (buf_ptr[0] >> 5) & 0x1f;
349

350
		if (jump->flags & PATCH_B32) {
351
			addr -= (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset) & ~(sljit_sw)0xfff;
352
			SLJIT_ASSERT(addr <= 0xfffff000l && addr >= -0x100000000l);
353
			buf_ptr[0] = ADRP | (((sljit_ins)(addr >> 12) & 0x3) << 29) | (((sljit_ins)(addr >> 14) & 0x7ffff) << 5) | dst;
354
			buf_ptr[1] = ADDI | dst | (dst << 5) | ((sljit_ins)(addr & 0xfff) << 10);
355
			return;
356
		}
357
	} else {
358
		dst = *buf_ptr;
359

360
		if (jump->flags & PATCH_B) {
361
			addr -= (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset);
362
			SLJIT_ASSERT(addr <= 0xfffff && addr >= -0x100000);
363
			buf_ptr[0] = ADR | (((sljit_ins)addr & 0x3) << 29) | (((sljit_ins)(addr >> 2) & 0x7ffff) << 5) | dst;
364
			return;
365
		}
366

367
		if (jump->flags & PATCH_B32) {
368
			addr -= ((sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) & ~(sljit_sw)0xfff;
369
			SLJIT_ASSERT(addr <= 0xffffffffl && addr >= -0x100000000l);
370
			buf_ptr[0] = ADRP | (((sljit_ins)(addr >> 12) & 0x3) << 29) | (((sljit_ins)(addr >> 14) & 0x7ffff) << 5) | dst;
371
			buf_ptr[1] = ADDI | dst | (dst << 5) | ((sljit_ins)(addr & 0xfff) << 10);
372
			return;
373
		}
374
	}
375

376
	SLJIT_ASSERT((jump->flags & (PATCH_ABS48 | PATCH_ABS64)) || (sljit_uw)addr <= (sljit_uw)0xffffffff);
377
	SLJIT_ASSERT((jump->flags & PATCH_ABS64) || (sljit_uw)addr <= (sljit_uw)0xffffffffffff);
378

379
	buf_ptr[0] = MOVZ | (((sljit_ins)addr & 0xffff) << 5) | dst;
380
	buf_ptr[1] = MOVK | (((sljit_ins)(addr >> 16) & 0xffff) << 5) | (1 << 21) | dst;
381
	if (jump->flags & (PATCH_ABS48 | PATCH_ABS64))
382
		buf_ptr[2] = MOVK | (((sljit_ins)(addr >> 32) & 0xffff) << 5) | (2 << 21) | dst;
383

384
	if (jump->flags & PATCH_ABS64)
385
		buf_ptr[3] = MOVK | ((sljit_ins)((sljit_uw)addr >> 48) << 5) | (3 << 21) | dst;
386
}
387

388
static void reduce_code_size(struct sljit_compiler *compiler)
389
{
390
	struct sljit_label *label;
391
	struct sljit_jump *jump;
392
	struct sljit_const *const_;
393
	SLJIT_NEXT_DEFINE_TYPES;
394
	sljit_uw total_size;
395
	sljit_uw size_reduce = 0;
396
	sljit_sw diff;
397

398
	label = compiler->labels;
399
	jump = compiler->jumps;
400
	const_ = compiler->consts;
401
	SLJIT_NEXT_INIT_TYPES();
402

403
	while (1) {
404
		SLJIT_GET_NEXT_MIN();
405

406
		if (next_min_addr == SLJIT_MAX_ADDRESS)
407
			break;
408

409
		if (next_min_addr == next_label_size) {
410
			label->size -= size_reduce;
411

412
			label = label->next;
413
			next_label_size = SLJIT_GET_NEXT_SIZE(label);
414
		}
415

416
		if (next_min_addr == next_const_addr) {
417
			const_->addr -= size_reduce;
418
			const_ = const_->next;
419
			next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
420
			continue;
421
		}
422

423
		if (next_min_addr != next_jump_addr)
424
			continue;
425

426
		jump->addr -= size_reduce;
427
		if (!(jump->flags & JUMP_MOV_ADDR)) {
428
			total_size = JUMP_MAX_SIZE;
429

430
			if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
431
				if (jump->flags & JUMP_ADDR) {
432
					if (jump->u.target < 0x100000000l)
433
						total_size = 3;
434
					else if (jump->u.target < 0x1000000000000l)
435
						total_size = 4;
436
				} else {
437
					/* Unit size: instruction. */
438
					diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
439
					if (jump->u.label->size > jump->addr) {
440
						SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
441
						diff -= (sljit_sw)size_reduce;
442
					}
443

444
					if ((jump->flags & IS_COND) && (diff + 1) <= (0xfffff / SSIZE_OF(ins)) && (diff + 1) >= (-0x100000 / SSIZE_OF(ins)))
445
						total_size = 0;
446
					else if (diff <= (0x7ffffff / SSIZE_OF(ins)) && diff >= (-0x8000000 / SSIZE_OF(ins)))
447
						total_size = 1;
448
					else if (diff <= (0xfffff000l / SSIZE_OF(ins)) && diff >= (-0x100000000l / SSIZE_OF(ins)))
449
						total_size = 3;
450
				}
451
			}
452

453
			size_reduce += JUMP_MAX_SIZE - total_size;
454
		} else {
455
			/* Real size minus 1. Unit size: instruction. */
456
			total_size = 3;
457

458
			if (!(jump->flags & JUMP_ADDR)) {
459
				diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
460
				if (jump->u.label->size > jump->addr) {
461
					SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
462
					diff -= (sljit_sw)size_reduce;
463
				}
464

465
				if (diff <= (0xfffff / SSIZE_OF(ins)) && diff >= (-0x100000 / SSIZE_OF(ins)))
466
					total_size = 0;
467
				else if (diff <= (0xfffff000l / SSIZE_OF(ins)) && diff >= (-0x100000000l / SSIZE_OF(ins)))
468
					total_size = 1;
469
			} else if (jump->u.target < 0x100000000l)
470
				total_size = 1;
471
			else if (jump->u.target < 0x1000000000000l)
472
				total_size = 2;
473

474
			size_reduce += 3 - total_size;
475
		}
476

477
		jump->flags |= total_size << JUMP_SIZE_SHIFT;
478
		jump = jump->next;
479
		next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
480
	}
481

482
	compiler->size -= size_reduce;
483
}
484

485
SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
486
{
487
	struct sljit_memory_fragment *buf;
488
	sljit_ins *code;
489
	sljit_ins *code_ptr;
490
	sljit_ins *buf_ptr;
491
	sljit_ins *buf_end;
492
	sljit_uw word_count;
493
	SLJIT_NEXT_DEFINE_TYPES;
494
	sljit_sw executable_offset;
495
	sljit_sw addr;
496

497
	struct sljit_label *label;
498
	struct sljit_jump *jump;
499
	struct sljit_const *const_;
500

501
	CHECK_ERROR_PTR();
502
	CHECK_PTR(check_sljit_generate_code(compiler));
503

504
	reduce_code_size(compiler);
505

506
	code = (sljit_ins*)allocate_executable_memory(compiler->size * sizeof(sljit_ins), options, exec_allocator_data, &executable_offset);
507
	PTR_FAIL_WITH_EXEC_IF(code);
508

509
	reverse_buf(compiler);
510
	buf = compiler->buf;
511

512
	code_ptr = code;
513
	word_count = 0;
514
	label = compiler->labels;
515
	jump = compiler->jumps;
516
	const_ = compiler->consts;
517
	SLJIT_NEXT_INIT_TYPES();
518
	SLJIT_GET_NEXT_MIN();
519

520
	do {
521
		buf_ptr = (sljit_ins*)buf->memory;
522
		buf_end = buf_ptr + (buf->used_size >> 2);
523
		do {
524
			*code_ptr = *buf_ptr++;
525
			if (next_min_addr == word_count) {
526
				SLJIT_ASSERT(!label || label->size >= word_count);
527
				SLJIT_ASSERT(!jump || jump->addr >= word_count);
528
				SLJIT_ASSERT(!const_ || const_->addr >= word_count);
529

530
				/* These structures are ordered by their address. */
531
				if (next_min_addr == next_label_size) {
532
					label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
533
					label->size = (sljit_uw)(code_ptr - code);
534
					label = label->next;
535
					next_label_size = SLJIT_GET_NEXT_SIZE(label);
536
				}
537

538
				if (next_min_addr == next_jump_addr) {
539
					if (!(jump->flags & JUMP_MOV_ADDR)) {
540
						word_count = word_count - 1 + (jump->flags >> JUMP_SIZE_SHIFT);
541
						code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
542
						SLJIT_ASSERT((jump->flags & PATCH_COND) || ((sljit_uw)code_ptr - jump->addr < (jump->flags >> JUMP_SIZE_SHIFT) * sizeof(sljit_ins)));
543
					} else {
544
						word_count += jump->flags >> JUMP_SIZE_SHIFT;
545
						addr = (sljit_sw)code_ptr;
546
						code_ptr += mov_addr_get_length(jump, code_ptr, code, executable_offset);
547
						jump->addr = (sljit_uw)addr;
548
					}
549

550
					jump = jump->next;
551
					next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
552
				} else if (next_min_addr == next_const_addr) {
553
					const_->addr = (sljit_uw)code_ptr;
554
					const_ = const_->next;
555
					next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
556
				}
557

558
				SLJIT_GET_NEXT_MIN();
559
			}
560
			code_ptr++;
561
			word_count++;
562
		} while (buf_ptr < buf_end);
563

564
		buf = buf->next;
565
	} while (buf);
566

567
	if (label && label->size == word_count) {
568
		label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
569
		label->size = (sljit_uw)(code_ptr - code);
570
		label = label->next;
571
	}
572

573
	SLJIT_ASSERT(!label);
574
	SLJIT_ASSERT(!jump);
575
	SLJIT_ASSERT(!const_);
576
	SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
577

578
	jump = compiler->jumps;
579
	while (jump) {
580
		generate_jump_or_mov_addr(jump, executable_offset);
581
		jump = jump->next;
582
	}
583

584
	compiler->error = SLJIT_ERR_COMPILED;
585
	compiler->executable_offset = executable_offset;
586
	compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
587

588
	code = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
589
	code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
590

591
	SLJIT_CACHE_FLUSH(code, code_ptr);
592
	SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
593
	return code;
594
}
595

596
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
597
{
598
	switch (feature_type) {
599
	case SLJIT_HAS_FPU:
600
	case SLJIT_HAS_SIMD:
601
#ifdef SLJIT_IS_FPU_AVAILABLE
602
		return (SLJIT_IS_FPU_AVAILABLE) != 0;
603
#else
604
		/* Available by default. */
605
		return 1;
606
#endif
607

608
	case SLJIT_HAS_CLZ:
609
	case SLJIT_HAS_CTZ:
610
	case SLJIT_HAS_REV:
611
	case SLJIT_HAS_ROT:
612
	case SLJIT_HAS_CMOV:
613
	case SLJIT_HAS_PREFETCH:
614
	case SLJIT_HAS_COPY_F32:
615
	case SLJIT_HAS_COPY_F64:
616
	case SLJIT_HAS_ATOMIC:
617
	case SLJIT_HAS_MEMORY_BARRIER:
618
		return 1;
619

620
	default:
621
		return 0;
622
	}
623
}
624

625
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
626
{
627
	switch (type) {
628
	case SLJIT_UNORDERED_OR_EQUAL:
629
	case SLJIT_ORDERED_NOT_EQUAL:
630
		return 2;
631
	}
632

633
	return 0;
634
}
635

636
/* --------------------------------------------------------------------- */
637
/*  Core code generator functions.                                       */
638
/* --------------------------------------------------------------------- */
639

640
#define COUNT_TRAILING_ZERO(value, result) \
641
	result = 0; \
642
	if (!(value & 0xffffffff)) { \
643
		result += 32; \
644
		value >>= 32; \
645
	} \
646
	if (!(value & 0xffff)) { \
647
		result += 16; \
648
		value >>= 16; \
649
	} \
650
	if (!(value & 0xff)) { \
651
		result += 8; \
652
		value >>= 8; \
653
	} \
654
	if (!(value & 0xf)) { \
655
		result += 4; \
656
		value >>= 4; \
657
	} \
658
	if (!(value & 0x3)) { \
659
		result += 2; \
660
		value >>= 2; \
661
	} \
662
	if (!(value & 0x1)) { \
663
		result += 1; \
664
		value >>= 1; \
665
	}
666

667
#define LOGICAL_IMM_CHECK (sljit_ins)0x100
668

669
static sljit_ins logical_imm(sljit_sw imm, sljit_u32 len)
670
{
671
	sljit_s32 negated;
672
	sljit_u32 ones, right;
673
	sljit_uw mask, uimm;
674
	sljit_ins ins;
675

676
	if (len & LOGICAL_IMM_CHECK) {
677
		len &= ~LOGICAL_IMM_CHECK;
678
		if (len == 32 && (imm == 0 || imm == -1))
679
			return 0;
680
		if (len == 16 && ((sljit_s32)imm == 0 || (sljit_s32)imm == -1))
681
			return 0;
682
	}
683

684
	SLJIT_ASSERT((len == 32 && imm != 0 && imm != -1)
685
		|| (len == 16 && (sljit_s32)imm != 0 && (sljit_s32)imm != -1));
686

687
	uimm = (sljit_uw)imm;
688
	while (1) {
689
		if (len <= 0) {
690
			SLJIT_UNREACHABLE();
691
			return 0;
692
		}
693

694
		mask = ((sljit_uw)1 << len) - 1;
695
		if ((uimm & mask) != ((uimm >> len) & mask))
696
			break;
697
		len >>= 1;
698
	}
699

700
	len <<= 1;
701

702
	negated = 0;
703
	if (uimm & 0x1) {
704
		negated = 1;
705
		uimm = ~uimm;
706
	}
707

708
	if (len < 64)
709
		uimm &= ((sljit_uw)1 << len) - 1;
710

711
	/* Unsigned right shift. */
712
	COUNT_TRAILING_ZERO(uimm, right);
713

714
	/* Signed shift. We also know that the highest bit is set. */
715
	imm = (sljit_sw)~uimm;
716
	SLJIT_ASSERT(imm < 0);
717

718
	COUNT_TRAILING_ZERO(imm, ones);
719

720
	if (~imm)
721
		return 0;
722

723
	if (len == 64)
724
		ins = 1 << 22;
725
	else
726
		ins = (0x3f - ((len << 1) - 1)) << 10;
727

728
	if (negated)
729
		return ins | ((len - ones - 1) << 10) | ((len - ones - right) << 16);
730

731
	return ins | ((ones - 1) << 10) | ((len - right) << 16);
732
}
733

734
#undef COUNT_TRAILING_ZERO
735

736
static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw simm)
737
{
738
	sljit_uw imm = (sljit_uw)simm;
739
	sljit_u32 i, zeros, ones, first;
740
	sljit_ins bitmask;
741

742
	/* Handling simple immediates first. */
743
	if (imm <= 0xffff)
744
		return push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)imm << 5));
745

746
	if (simm < 0 && simm >= -0x10000)
747
		return push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5));
748

749
	if (imm <= 0xffffffffl) {
750
		if ((imm & 0xffff) == 0)
751
			return push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)(imm >> 16) << 5) | (1 << 21));
752
		if ((imm & 0xffff0000l) == 0xffff0000)
753
			return push_inst(compiler, (MOVN ^ W_OP) | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5));
754
		if ((imm & 0xffff) == 0xffff)
755
			return push_inst(compiler, (MOVN ^ W_OP) | RD(dst) | (((sljit_ins)~imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
756

757
		bitmask = logical_imm(simm, 16);
758
		if (bitmask != 0)
759
			return push_inst(compiler, (ORRI ^ W_OP) | RD(dst) | RN(TMP_ZERO) | bitmask);
760

761
		FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | (((sljit_ins)imm & 0xffff) << 5)));
762
		return push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
763
	}
764

765
	bitmask = logical_imm(simm, 32);
766
	if (bitmask != 0)
767
		return push_inst(compiler, ORRI | RD(dst) | RN(TMP_ZERO) | bitmask);
768

769
	if (simm < 0 && simm >= -0x100000000l) {
770
		if ((imm & 0xffff) == 0xffff)
771
			return push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
772

773
		FAIL_IF(push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5)));
774
		return push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
775
	}
776

777
	/* A large amount of number can be constructed from ORR and MOVx, but computing them is costly. */
778

779
	zeros = 0;
780
	ones = 0;
781
	for (i = 4; i > 0; i--) {
782
		if ((simm & 0xffff) == 0)
783
			zeros++;
784
		if ((simm & 0xffff) == 0xffff)
785
			ones++;
786
		simm >>= 16;
787
	}
788

789
	simm = (sljit_sw)imm;
790
	first = 1;
791
	if (ones > zeros) {
792
		simm = ~simm;
793
		for (i = 0; i < 4; i++) {
794
			if (!(simm & 0xffff)) {
795
				simm >>= 16;
796
				continue;
797
			}
798
			if (first) {
799
				first = 0;
800
				FAIL_IF(push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
801
			}
802
			else
803
				FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)~simm & 0xffff) << 5) | (i << 21)));
804
			simm >>= 16;
805
		}
806
		return SLJIT_SUCCESS;
807
	}
808

809
	for (i = 0; i < 4; i++) {
810
		if (!(simm & 0xffff)) {
811
			simm >>= 16;
812
			continue;
813
		}
814
		if (first) {
815
			first = 0;
816
			FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
817
		}
818
		else
819
			FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
820
		simm >>= 16;
821
	}
822
	return SLJIT_SUCCESS;
823
}
824

825
#define ARG1_IMM	0x0010000
826
#define ARG2_IMM	0x0020000
827
#define INT_OP		0x0040000
828
#define SET_FLAGS	0x0080000
829
#define UNUSED_RETURN	0x0100000
830

831
#define CHECK_FLAGS(flag_bits) \
832
	if (flags & SET_FLAGS) { \
833
		inv_bits |= flag_bits; \
834
		if (flags & UNUSED_RETURN) \
835
			dst = TMP_ZERO; \
836
	}
837

838
static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 dst, sljit_sw arg1, sljit_sw arg2)
839
{
840
	/* dst must be register, TMP_REG1
841
	   arg1 must be register, TMP_REG1, imm
842
	   arg2 must be register, TMP_REG2, imm */
843
	sljit_ins inv_bits = (flags & INT_OP) ? W_OP : 0;
844
	sljit_ins inst_bits;
845
	sljit_s32 op = (flags & 0xffff);
846
	sljit_s32 reg;
847
	sljit_sw imm, nimm;
848

849
	if (SLJIT_UNLIKELY((flags & (ARG1_IMM | ARG2_IMM)) == (ARG1_IMM | ARG2_IMM))) {
850
		/* Both are immediates. */
851
		flags &= ~ARG1_IMM;
852
		if (arg1 == 0 && op != SLJIT_ADD && op != SLJIT_SUB)
853
			arg1 = TMP_ZERO;
854
		else {
855
			FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
856
			arg1 = TMP_REG1;
857
		}
858
	}
859

860
	if (flags & (ARG1_IMM | ARG2_IMM)) {
861
		reg = (sljit_s32)((flags & ARG2_IMM) ? arg1 : arg2);
862
		imm = (flags & ARG2_IMM) ? arg2 : arg1;
863

864
		switch (op) {
865
		case SLJIT_CLZ:
866
		case SLJIT_CTZ:
867
		case SLJIT_REV:
868
		case SLJIT_REV_U16:
869
		case SLJIT_REV_S16:
870
		case SLJIT_REV_U32:
871
		case SLJIT_REV_S32:
872
		case SLJIT_ADDC:
873
		case SLJIT_SUBC:
874
		case SLJIT_MUL:
875
		case SLJIT_MULADD:
876
			/* No form with immediate operand (except imm 0, which
877
			is represented by a ZERO register). */
878
			break;
879
		case SLJIT_MOV:
880
			SLJIT_ASSERT(!(flags & SET_FLAGS) && (flags & ARG2_IMM) && arg1 == TMP_REG1);
881
			return load_immediate(compiler, dst, imm);
882
		case SLJIT_SUB:
883
			compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
884
			if (flags & ARG1_IMM)
885
				break;
886
			imm = -imm;
887
			/* Fall through. */
888
		case SLJIT_ADD:
889
			if (op != SLJIT_SUB)
890
				compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
891

892
			if (imm == 0) {
893
				CHECK_FLAGS(1 << 29);
894
				return push_inst(compiler, ((op == SLJIT_ADD ? ADDI : SUBI) ^ inv_bits) | RD(dst) | RN(reg));
895
			}
896
			if (imm > 0 && imm <= 0xfff) {
897
				CHECK_FLAGS(1 << 29);
898
				return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | ((sljit_ins)imm << 10));
899
			}
900
			nimm = -imm;
901
			if (nimm > 0 && nimm <= 0xfff) {
902
				CHECK_FLAGS(1 << 29);
903
				return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | ((sljit_ins)nimm << 10));
904
			}
905
			if (imm > 0 && imm <= 0xffffff && !(imm & 0xfff)) {
906
				CHECK_FLAGS(1 << 29);
907
				return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)imm >> 12) << 10) | (1 << 22));
908
			}
909
			if (nimm > 0 && nimm <= 0xffffff && !(nimm & 0xfff)) {
910
				CHECK_FLAGS(1 << 29);
911
				return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)nimm >> 12) << 10) | (1 << 22));
912
			}
913
			if (imm > 0 && imm <= 0xffffff && !(flags & SET_FLAGS)) {
914
				FAIL_IF(push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)imm >> 12) << 10) | (1 << 22)));
915
				return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(dst) | (((sljit_ins)imm & 0xfff) << 10));
916
			}
917
			if (nimm > 0 && nimm <= 0xffffff && !(flags & SET_FLAGS)) {
918
				FAIL_IF(push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)nimm >> 12) << 10) | (1 << 22)));
919
				return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(dst) | (((sljit_ins)nimm & 0xfff) << 10));
920
			}
921
			break;
922
		case SLJIT_AND:
923
			inst_bits = logical_imm(imm, LOGICAL_IMM_CHECK | ((flags & INT_OP) ? 16 : 32));
924
			if (!inst_bits)
925
				break;
926
			CHECK_FLAGS(3 << 29);
927
			return push_inst(compiler, (ANDI ^ inv_bits) | RD(dst) | RN(reg) | inst_bits);
928
		case SLJIT_XOR:
929
			if (imm == -1) {
930
				FAIL_IF(push_inst(compiler, (ORN ^ inv_bits) | RD(dst) | RN(TMP_ZERO) | RM(reg)));
931
				goto set_flags;
932
			}
933
			/* fallthrough */
934
		case SLJIT_OR:
935
			inst_bits = logical_imm(imm, LOGICAL_IMM_CHECK | ((flags & INT_OP) ? 16 : 32));
936
			if (!inst_bits)
937
				break;
938
			if (op == SLJIT_OR)
939
				inst_bits |= ORRI;
940
			else
941
				inst_bits |= EORI;
942
			FAIL_IF(push_inst(compiler, (inst_bits ^ inv_bits) | RD(dst) | RN(reg)));
943
			goto set_flags;
944
		case SLJIT_SHL:
945
		case SLJIT_MSHL:
946
			if (flags & ARG1_IMM)
947
				break;
948

949
			if (flags & INT_OP) {
950
				imm &= 0x1f;
951
				inst_bits = (((sljit_ins)-imm & 0x1f) << 16) | ((31 - (sljit_ins)imm) << 10);
952
			} else {
953
				imm &= 0x3f;
954
				inst_bits = ((sljit_ins)1 << 22) | (((sljit_ins)-imm & 0x3f) << 16) | ((63 - (sljit_ins)imm) << 10);
955
			}
956

957
			inv_bits |= inv_bits >> 9;
958
			FAIL_IF(push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg1) | inst_bits));
959
			goto set_flags;
960
		case SLJIT_LSHR:
961
		case SLJIT_MLSHR:
962
		case SLJIT_ASHR:
963
		case SLJIT_MASHR:
964
			if (flags & ARG1_IMM)
965
				break;
966

967
			inv_bits |= inv_bits >> 9;
968
			if (op >= SLJIT_ASHR)
969
				inv_bits |= 1 << 30;
970

971
			if (flags & INT_OP) {
972
				imm &= 0x1f;
973
				inst_bits = ((sljit_ins)imm << 16) | (31 << 10);
974
			} else {
975
				imm &= 0x3f;
976
				inst_bits = ((sljit_ins)1 << 22) | ((sljit_ins)imm << 16) | (63 << 10);
977
			}
978

979
			FAIL_IF(push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg1) | inst_bits));
980
			goto set_flags;
981
		case SLJIT_ROTL:
982
		case SLJIT_ROTR:
983
			if (flags & ARG1_IMM)
984
				break;
985

986
			if (op == SLJIT_ROTL)
987
				imm = -imm;
988

989
			imm &= (flags & INT_OP) ? 0x1f : 0x3f;
990
			return push_inst(compiler, (EXTR ^ (inv_bits | (inv_bits >> 9))) | RD(dst) | RN(arg1) | RM(arg1) | ((sljit_ins)imm << 10));
991
		default:
992
			SLJIT_UNREACHABLE();
993
			break;
994
		}
995

996
		if (flags & ARG2_IMM) {
997
			if (arg2 == 0)
998
				arg2 = TMP_ZERO;
999
			else {
1000
				FAIL_IF(load_immediate(compiler, TMP_REG2, arg2));
1001
				arg2 = TMP_REG2;
1002
			}
1003
		}
1004
		else {
1005
			if (arg1 == 0)
1006
				arg1 = TMP_ZERO;
1007
			else {
1008
				FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
1009
				arg1 = TMP_REG1;
1010
			}
1011
		}
1012
	}
1013

1014
	/* Both arguments are registers. */
1015
	switch (op) {
1016
	case SLJIT_MOV:
1017
	case SLJIT_MOV_P:
1018
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1019
		if (dst == arg2)
1020
			return SLJIT_SUCCESS;
1021
		return push_inst(compiler, MOV | RD(dst) | RM(arg2));
1022
	case SLJIT_MOV_U8:
1023
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1024
		inv_bits |= inv_bits >> 9;
1025
		return push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg2) | (7 << 10));
1026
	case SLJIT_MOV_S8:
1027
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1028
		inv_bits |= inv_bits >> 9;
1029
		return push_inst(compiler, (SBFM ^ inv_bits) | RD(dst) | RN(arg2) | (7 << 10));
1030
	case SLJIT_MOV_U16:
1031
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1032
		inv_bits |= inv_bits >> 9;
1033
		return push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg2) | (15 << 10));
1034
	case SLJIT_MOV_S16:
1035
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1036
		inv_bits |= inv_bits >> 9;
1037
		return push_inst(compiler, (SBFM ^ inv_bits) | RD(dst) | RN(arg2) | (15 << 10));
1038
	case SLJIT_MOV32:
1039
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1040
		if (dst == arg2)
1041
			return SLJIT_SUCCESS;
1042
		/* fallthrough */
1043
	case SLJIT_MOV_U32:
1044
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1045
		return push_inst(compiler, (MOV ^ W_OP) | RD(dst) | RM(arg2));
1046
	case SLJIT_MOV_S32:
1047
		SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1048
		return push_inst(compiler, SBFM | (1 << 22) | RD(dst) | RN(arg2) | (31 << 10));
1049
	case SLJIT_CLZ:
1050
		SLJIT_ASSERT(arg1 == TMP_REG1);
1051
		return push_inst(compiler, (CLZ ^ inv_bits) | RD(dst) | RN(arg2));
1052
	case SLJIT_CTZ:
1053
		SLJIT_ASSERT(arg1 == TMP_REG1);
1054
		FAIL_IF(push_inst(compiler, (RBIT ^ inv_bits) | RD(dst) | RN(arg2)));
1055
		return push_inst(compiler, (CLZ ^ inv_bits) | RD(dst) | RN(dst));
1056
	case SLJIT_REV:
1057
		SLJIT_ASSERT(arg1 == TMP_REG1);
1058
		inv_bits |= inv_bits >> 21;
1059
		return push_inst(compiler, (REV ^ inv_bits) | RD(dst) | RN(arg2));
1060
	case SLJIT_REV_U16:
1061
	case SLJIT_REV_S16:
1062
		SLJIT_ASSERT(arg1 == TMP_REG1 && dst != TMP_REG2);
1063
		FAIL_IF(push_inst(compiler, (REV16 ^ (sljit_ins)0x80000000) | RD(dst) | RN(arg2)));
1064
		if (dst == TMP_REG1 || (arg2 == TMP_REG2 && op == SLJIT_REV_U16))
1065
			return SLJIT_SUCCESS;
1066
		inv_bits |= inv_bits >> 9;
1067
		return push_inst(compiler, ((op == SLJIT_REV_U16 ? UBFM : SBFM) ^ inv_bits) | RD(dst) | RN(dst) | (15 << 10));
1068
	case SLJIT_REV_U32:
1069
	case SLJIT_REV_S32:
1070
		SLJIT_ASSERT(arg1 == TMP_REG1 && dst != TMP_REG2);
1071
		FAIL_IF(push_inst(compiler, (REV ^ (sljit_ins)0x80000400) | RD(dst) | RN(arg2)));
1072
		if (op == SLJIT_REV_U32 || dst == TMP_REG1)
1073
			return SLJIT_SUCCESS;
1074
		return push_inst(compiler, SBFM | (1 << 22) | RD(dst) | RN(dst) | (31 << 10));
1075
	case SLJIT_ADD:
1076
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1077
		CHECK_FLAGS(1 << 29);
1078
		return push_inst(compiler, (ADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1079
	case SLJIT_ADDC:
1080
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1081
		CHECK_FLAGS(1 << 29);
1082
		return push_inst(compiler, (ADC ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1083
	case SLJIT_SUB:
1084
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1085
		CHECK_FLAGS(1 << 29);
1086
		return push_inst(compiler, (SUB ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1087
	case SLJIT_SUBC:
1088
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1089
		CHECK_FLAGS(1 << 29);
1090
		return push_inst(compiler, (SBC ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1091
	case SLJIT_MUL:
1092
		compiler->status_flags_state = 0;
1093
		if (!(flags & SET_FLAGS))
1094
			return push_inst(compiler, (MADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2) | RT2(TMP_ZERO));
1095
		if (flags & INT_OP) {
1096
			FAIL_IF(push_inst(compiler, SMADDL | RD(dst) | RN(arg1) | RM(arg2) | (31 << 10)));
1097
			FAIL_IF(push_inst(compiler, ADD | RD(TMP_LR) | RN(TMP_ZERO) | RM(dst) | (2 << 22) | (31 << 10)));
1098
			return push_inst(compiler, SUBS | RD(TMP_ZERO) | RN(TMP_LR) | RM(dst) | (2 << 22) | (63 << 10));
1099
		}
1100
		FAIL_IF(push_inst(compiler, SMULH | RD(TMP_LR) | RN(arg1) | RM(arg2)));
1101
		FAIL_IF(push_inst(compiler, MADD | RD(dst) | RN(arg1) | RM(arg2) | RT2(TMP_ZERO)));
1102
		return push_inst(compiler, SUBS | RD(TMP_ZERO) | RN(TMP_LR) | RM(dst) | (2 << 22) | (63 << 10));
1103
	case SLJIT_AND:
1104
		CHECK_FLAGS(3 << 29);
1105
		return push_inst(compiler, (AND ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1106
	case SLJIT_OR:
1107
		FAIL_IF(push_inst(compiler, (ORR ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1108
		break; /* Set flags. */
1109
	case SLJIT_XOR:
1110
		FAIL_IF(push_inst(compiler, (EOR ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1111
		break; /* Set flags. */
1112
	case SLJIT_SHL:
1113
	case SLJIT_MSHL:
1114
		FAIL_IF(push_inst(compiler, (LSLV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1115
		break; /* Set flags. */
1116
	case SLJIT_LSHR:
1117
	case SLJIT_MLSHR:
1118
		FAIL_IF(push_inst(compiler, (LSRV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1119
		break; /* Set flags. */
1120
	case SLJIT_ASHR:
1121
	case SLJIT_MASHR:
1122
		FAIL_IF(push_inst(compiler, (ASRV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1123
		break; /* Set flags. */
1124
	case SLJIT_ROTL:
1125
		FAIL_IF(push_inst(compiler, (SUB ^ inv_bits) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(arg2)));
1126
		arg2 = TMP_REG2;
1127
		/* fallthrough */
1128
	case SLJIT_ROTR:
1129
		return push_inst(compiler, (RORV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1130
	case SLJIT_MULADD:
1131
		compiler->status_flags_state = 0;
1132
		return push_inst(compiler, (MADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2) | RT2(dst));
1133
	default:
1134
		SLJIT_UNREACHABLE();
1135
		return SLJIT_SUCCESS;
1136
	}
1137

1138
set_flags:
1139
	if (flags & SET_FLAGS)
1140
		return push_inst(compiler, (SUBS ^ inv_bits) | RD(TMP_ZERO) | RN(dst) | RM(TMP_ZERO));
1141
	return SLJIT_SUCCESS;
1142
}
1143

1144
#define STORE		0x10
1145
#define SIGNED		0x20
1146

1147
#define BYTE_SIZE	0x0
1148
#define HALF_SIZE	0x1
1149
#define INT_SIZE	0x2
1150
#define WORD_SIZE	0x3
1151

1152
#define MEM_SIZE_SHIFT(flags) ((sljit_ins)(flags) & 0x3)
1153

1154
static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg,
1155
	sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1156
{
1157
	sljit_u32 shift = MEM_SIZE_SHIFT(flags);
1158
	sljit_u32 type = (shift << 30);
1159

1160
	if (!(flags & STORE))
1161
		type |= (flags & SIGNED) ? 0x00800000 : 0x00400000;
1162

1163
	SLJIT_ASSERT(arg & SLJIT_MEM);
1164

1165
	if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1166
		argw &= 0x3;
1167

1168
		if (argw == 0 || argw == shift)
1169
			return push_inst(compiler, STRB | type | RT(reg)
1170
				| RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | (argw ? (1 << 12) : 0));
1171

1172
		FAIL_IF(push_inst(compiler, ADD | RD(tmp_reg) | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | ((sljit_ins)argw << 10)));
1173
		return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg));
1174
	}
1175

1176
	arg &= REG_MASK;
1177

1178
	if (!arg) {
1179
		FAIL_IF(load_immediate(compiler, tmp_reg, argw & ~(0xfff << shift)));
1180

1181
		argw = (argw >> shift) & 0xfff;
1182

1183
		return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1184
	}
1185

1186
	if ((argw & ((1 << shift) - 1)) == 0) {
1187
		if (argw >= 0) {
1188
			if ((argw >> shift) <= 0xfff)
1189
				return push_inst(compiler, STRBI | type | RT(reg) | RN(arg) | ((sljit_ins)argw << (10 - shift)));
1190

1191
			if (argw <= 0xffffff) {
1192
				FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
1193

1194
				argw = ((argw & 0xfff) >> shift);
1195
				return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1196
			}
1197
		} else if (argw < -256 && argw >= -0xfff000) {
1198
			FAIL_IF(push_inst(compiler, SUBI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)(-argw + 0xfff) >> 12) << 10)));
1199
			argw = ((0x1000 + argw) & 0xfff) >> shift;
1200
			return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1201
		}
1202
	}
1203

1204
	if (argw <= 0xff && argw >= -0x100)
1205
		return push_inst(compiler, STURBI | type | RT(reg) | RN(arg) | (((sljit_ins)argw & 0x1ff) << 12));
1206

1207
	if (((argw + 0x100) & 0xfff) <= 0x1ff && argw <= 0xfff0ff && argw >= -0xfff100) {
1208
		if (argw >= 0) {
1209
			if (argw & 0x100)
1210
				argw += 0x1000;
1211

1212
			FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
1213
			return push_inst(compiler, STURBI | type | RT(reg) | RN(tmp_reg) | (((sljit_ins)argw & 0x1ff) << 12));
1214
		} else {
1215
			if (!(argw & 0x100))
1216
				argw -= 0x1000;
1217

1218
			FAIL_IF(push_inst(compiler, SUBI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)-argw >> 12) << 10)));
1219
			return push_inst(compiler, STURBI | type | RT(reg) | RN(tmp_reg) | (((sljit_ins)argw & 0x1ff) << 12));
1220
		}
1221
	}
1222

1223
	FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1224

1225
	return push_inst(compiler, STRB | type | RT(reg) | RN(arg) | RM(tmp_reg));
1226
}
1227

1228
/* --------------------------------------------------------------------- */
1229
/*  Entry, exit                                                          */
1230
/* --------------------------------------------------------------------- */
1231

1232
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1233
	sljit_s32 options, sljit_s32 arg_types,
1234
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1235
{
1236
	sljit_s32 fscratches;
1237
	sljit_s32 fsaveds;
1238
	sljit_s32 prev, fprev, saved_regs_size, i, tmp;
1239
	sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1240
	sljit_ins offs;
1241

1242
	CHECK_ERROR();
1243
	CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
1244
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1245

1246
	scratches = ENTER_GET_REGS(scratches);
1247
	saveds = ENTER_GET_REGS(saveds);
1248
	fscratches = compiler->fscratches;
1249
	fsaveds = compiler->fsaveds;
1250

1251
	saved_regs_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 2);
1252
	saved_regs_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1253

1254
	local_size = (local_size + saved_regs_size + 0xf) & ~0xf;
1255
	compiler->local_size = local_size;
1256

1257
	if (local_size <= 512) {
1258
		FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1259
			| RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1260
		offs = (sljit_ins)(local_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1261
		local_size = 0;
1262
	} else {
1263
		saved_regs_size = ((saved_regs_size - 2 * SSIZE_OF(sw)) + 0xf) & ~0xf;
1264

1265
		FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)saved_regs_size << 10)));
1266
		offs = (sljit_ins)(saved_regs_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1267
		local_size -= saved_regs_size;
1268
		SLJIT_ASSERT(local_size > 0);
1269
	}
1270

1271
	prev = -1;
1272

1273
	tmp = SLJIT_S0 - saveds;
1274
	for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
1275
		if (prev == -1) {
1276
			prev = i;
1277
			continue;
1278
		}
1279
		FAIL_IF(push_inst(compiler, STP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1280
		offs -= (sljit_ins)2 << 15;
1281
		prev = -1;
1282
	}
1283

1284
	for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1285
		if (prev == -1) {
1286
			prev = i;
1287
			continue;
1288
		}
1289
		FAIL_IF(push_inst(compiler, STP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1290
		offs -= (sljit_ins)2 << 15;
1291
		prev = -1;
1292
	}
1293

1294
	fprev = -1;
1295

1296
	tmp = SLJIT_FS0 - fsaveds;
1297
	for (i = SLJIT_FS0; i > tmp; i--) {
1298
		if (fprev == -1) {
1299
			fprev = i;
1300
			continue;
1301
		}
1302
		FAIL_IF(push_inst(compiler, STP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1303
		offs -= (sljit_ins)2 << 15;
1304
		fprev = -1;
1305
	}
1306

1307
	for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1308
		if (fprev == -1) {
1309
			fprev = i;
1310
			continue;
1311
		}
1312
		FAIL_IF(push_inst(compiler, STP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1313
		offs -= (sljit_ins)2 << 15;
1314
		fprev = -1;
1315
	}
1316

1317
	if (fprev != -1)
1318
		FAIL_IF(push_inst(compiler, STRI_F64 | VT(fprev) | RN(SLJIT_SP) | (offs >> 5) | (1 << 10)));
1319

1320
	if (prev != -1)
1321
		FAIL_IF(push_inst(compiler, STRI | RT(prev) | RN(SLJIT_SP) | (offs >> 5) | ((fprev == -1) ? (1 << 10) : 0)));
1322

1323

1324
#ifdef _WIN32
1325
	if (local_size > 4096)
1326
		FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1327
#endif /* _WIN32 */
1328

1329
	if (!(options & SLJIT_ENTER_REG_ARG)) {
1330
		arg_types >>= SLJIT_ARG_SHIFT;
1331
		saved_arg_count = 0;
1332
		tmp = SLJIT_R0;
1333

1334
		while (arg_types) {
1335
			if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
1336
				if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1337
					FAIL_IF(push_inst(compiler, MOV | RD(SLJIT_S0 - saved_arg_count) | RM(tmp)));
1338
					saved_arg_count++;
1339
				}
1340
				tmp++;
1341
			}
1342
			arg_types >>= SLJIT_ARG_SHIFT;
1343
		}
1344
	}
1345

1346
#ifdef _WIN32
1347
	if (local_size > 4096) {
1348
		if (local_size < 4 * 4096) {
1349
			/* No need for a loop. */
1350

1351
			if (local_size >= 2 * 4096) {
1352
				if (local_size >= 3 * 4096) {
1353
					FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1354
					FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1355
				}
1356

1357
				FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1358
				FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1359
			}
1360
		}
1361
		else {
1362
			FAIL_IF(push_inst(compiler, MOVZ | RD(TMP_REG1) | ((((sljit_ins)local_size >> 12) - 1) << 5)));
1363
			FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1364
			FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1365
			FAIL_IF(push_inst(compiler, SUBI | (1 << 29) | RD(TMP_REG1) | RN(TMP_REG1) | (1 << 10)));
1366
			FAIL_IF(push_inst(compiler, B_CC | ((((sljit_ins) -3) & 0x7ffff) << 5) | 0x1 /* not-equal */));
1367
		}
1368

1369
		local_size &= 0xfff;
1370

1371
		if (local_size > 0)
1372
			FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1373
		else
1374
			FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1375
	}
1376

1377
	if (local_size > 0) {
1378
		if (local_size <= 512)
1379
			FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1380
				| RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1381
		else {
1382
			if (local_size >= 4096)
1383
				local_size = (1 << (22 - 10));
1384

1385
			FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1386
			FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1387
		}
1388
	}
1389

1390
#else /* !_WIN32 */
1391

1392
	/* The local_size does not include saved registers size. */
1393
	if (local_size != 0) {
1394
		if (local_size > 0xfff) {
1395
			FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (((sljit_ins)local_size >> 12) << 10) | (1 << 22)));
1396
			local_size &= 0xfff;
1397
		}
1398

1399
		if (local_size > 512 || local_size == 0) {
1400
			if (local_size != 0)
1401
				FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1402

1403
			FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1404
		} else
1405
			FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1406
				| RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1407
	}
1408

1409
#endif /* _WIN32 */
1410

1411
	return push_inst(compiler, ADDI | RD(TMP_FP) | RN(SLJIT_SP) | (0 << 10));
1412
}
1413

1414
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1415
	sljit_s32 options, sljit_s32 arg_types,
1416
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1417
{
1418
	sljit_s32 fscratches;
1419
	sljit_s32 fsaveds;
1420
	sljit_s32 saved_regs_size;
1421

1422
	CHECK_ERROR();
1423
	CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1424
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1425

1426
	scratches = ENTER_GET_REGS(scratches);
1427
	saveds = ENTER_GET_REGS(saveds);
1428
	fscratches = compiler->fscratches;
1429
	fsaveds = compiler->fsaveds;
1430
	saved_regs_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 2);
1431
	saved_regs_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1432

1433
	compiler->local_size = (local_size + saved_regs_size + 0xf) & ~0xf;
1434
	return SLJIT_SUCCESS;
1435
}
1436

1437
static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
1438
{
1439
	sljit_s32 local_size, prev, fprev, i, tmp;
1440
	sljit_ins offs;
1441

1442
	local_size = compiler->local_size;
1443

1444
	if (!is_return_to) {
1445
		if (local_size > 512 && local_size <= 512 + 496) {
1446
			FAIL_IF(push_inst(compiler, LDP_POST | RT(TMP_FP) | RT2(TMP_LR)
1447
				| RN(SLJIT_SP) | ((sljit_ins)(local_size - 512) << (15 - 3))));
1448
			local_size = 512;
1449
		} else
1450
			FAIL_IF(push_inst(compiler, LDP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1451
	} else {
1452
		if (local_size > 512 && local_size <= 512 + 248) {
1453
			FAIL_IF(push_inst(compiler, LDRI_POST | RT(TMP_FP) | RN(SLJIT_SP) | ((sljit_ins)(local_size - 512) << 12)));
1454
			local_size = 512;
1455
		} else
1456
			FAIL_IF(push_inst(compiler, LDRI | RT(TMP_FP) | RN(SLJIT_SP) | 0));
1457
	}
1458

1459
	if (local_size > 512) {
1460
		local_size -= 512;
1461
		if (local_size > 0xfff) {
1462
			FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP)
1463
				| (((sljit_ins)local_size >> 12) << 10) | (1 << 22)));
1464
			local_size &= 0xfff;
1465
		}
1466

1467
		FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1468
		local_size = 512;
1469
	}
1470

1471
	offs = (sljit_ins)(local_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1472
	prev = -1;
1473

1474
	tmp = SLJIT_S0 - compiler->saveds;
1475
	for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
1476
		if (prev == -1) {
1477
			prev = i;
1478
			continue;
1479
		}
1480
		FAIL_IF(push_inst(compiler, LDP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1481
		offs -= (sljit_ins)2 << 15;
1482
		prev = -1;
1483
	}
1484

1485
	for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1486
		if (prev == -1) {
1487
			prev = i;
1488
			continue;
1489
		}
1490
		FAIL_IF(push_inst(compiler, LDP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1491
		offs -= (sljit_ins)2 << 15;
1492
		prev = -1;
1493
	}
1494

1495
	fprev = -1;
1496

1497
	tmp = SLJIT_FS0 - compiler->fsaveds;
1498
	for (i = SLJIT_FS0; i > tmp; i--) {
1499
		if (fprev == -1) {
1500
			fprev = i;
1501
			continue;
1502
		}
1503
		FAIL_IF(push_inst(compiler, LDP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1504
		offs -= (sljit_ins)2 << 15;
1505
		fprev = -1;
1506
	}
1507

1508
	for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1509
		if (fprev == -1) {
1510
			fprev = i;
1511
			continue;
1512
		}
1513
		FAIL_IF(push_inst(compiler, LDP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1514
		offs -= (sljit_ins)2 << 15;
1515
		fprev = -1;
1516
	}
1517

1518
	if (fprev != -1)
1519
		FAIL_IF(push_inst(compiler, LDRI_F64 | VT(fprev) | RN(SLJIT_SP) | (offs >> 5) | (1 << 10)));
1520

1521
	if (prev != -1)
1522
		FAIL_IF(push_inst(compiler, LDRI | RT(prev) | RN(SLJIT_SP) | (offs >> 5) | ((fprev == -1) ? (1 << 10) : 0)));
1523

1524
	/* This and the next call/jump instruction can be executed parallelly. */
1525
	return push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP) | (sljit_ins)(local_size << 10));
1526
}
1527

1528
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1529
{
1530
	CHECK_ERROR();
1531
	CHECK(check_sljit_emit_return_void(compiler));
1532

1533
	FAIL_IF(emit_stack_frame_release(compiler, 0));
1534

1535
	return push_inst(compiler, RET | RN(TMP_LR));
1536
}
1537

1538
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1539
	sljit_s32 src, sljit_sw srcw)
1540
{
1541
	CHECK_ERROR();
1542
	CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1543

1544
	if (src & SLJIT_MEM) {
1545
		ADJUST_LOCAL_OFFSET(src, srcw);
1546
		FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
1547
		src = TMP_REG1;
1548
		srcw = 0;
1549
	} else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1550
		FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(src)));
1551
		src = TMP_REG1;
1552
		srcw = 0;
1553
	}
1554

1555
	FAIL_IF(emit_stack_frame_release(compiler, 1));
1556

1557
	SLJIT_SKIP_CHECKS(compiler);
1558
	return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1559
}
1560

1561
/* --------------------------------------------------------------------- */
1562
/*  Operators                                                            */
1563
/* --------------------------------------------------------------------- */
1564

1565
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1566
{
1567
	sljit_ins inv_bits = (op & SLJIT_32) ? W_OP : 0;
1568

1569
	CHECK_ERROR();
1570
	CHECK(check_sljit_emit_op0(compiler, op));
1571

1572
	op = GET_OPCODE(op);
1573
	switch (op) {
1574
	case SLJIT_BREAKPOINT:
1575
		return push_inst(compiler, BRK | (0xf000 << 5));
1576
	case SLJIT_NOP:
1577
		return push_inst(compiler, NOP);
1578
	case SLJIT_LMUL_UW:
1579
	case SLJIT_LMUL_SW:
1580
		FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(SLJIT_R0)));
1581
		FAIL_IF(push_inst(compiler, MADD | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1) | RT2(TMP_ZERO)));
1582
		return push_inst(compiler, (op == SLJIT_LMUL_UW ? UMULH : SMULH) | RD(SLJIT_R1) | RN(TMP_REG1) | RM(SLJIT_R1));
1583
	case SLJIT_DIVMOD_UW:
1584
	case SLJIT_DIVMOD_SW:
1585
		FAIL_IF(push_inst(compiler, (MOV ^ inv_bits) | RD(TMP_REG1) | RM(SLJIT_R0)));
1586
		FAIL_IF(push_inst(compiler, ((op == SLJIT_DIVMOD_UW ? UDIV : SDIV) ^ inv_bits) | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1)));
1587
		FAIL_IF(push_inst(compiler, (MADD ^ inv_bits) | RD(SLJIT_R1) | RN(SLJIT_R0) | RM(SLJIT_R1) | RT2(TMP_ZERO)));
1588
		return push_inst(compiler, (SUB ^ inv_bits) | RD(SLJIT_R1) | RN(TMP_REG1) | RM(SLJIT_R1));
1589
	case SLJIT_DIV_UW:
1590
	case SLJIT_DIV_SW:
1591
		return push_inst(compiler, ((op == SLJIT_DIV_UW ? UDIV : SDIV) ^ inv_bits) | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1));
1592
	case SLJIT_MEMORY_BARRIER:
1593
		return push_inst(compiler, DMB_SY);
1594
	case SLJIT_ENDBR:
1595
	case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1596
		return SLJIT_SUCCESS;
1597
	}
1598

1599
	return SLJIT_SUCCESS;
1600
}
1601

1602
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1603
	sljit_s32 dst, sljit_sw dstw,
1604
	sljit_s32 src, sljit_sw srcw)
1605
{
1606
	sljit_s32 dst_r, flags, mem_flags;
1607
	sljit_s32 op_flags = GET_ALL_FLAGS(op);
1608

1609
	CHECK_ERROR();
1610
	CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1611
	ADJUST_LOCAL_OFFSET(dst, dstw);
1612
	ADJUST_LOCAL_OFFSET(src, srcw);
1613

1614
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1615

1616
	op = GET_OPCODE(op);
1617
	if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1618
		/* Both operands are registers. */
1619
		if (FAST_IS_REG(dst) && FAST_IS_REG(src))
1620
			return emit_op_imm(compiler, op | ((op_flags & SLJIT_32) ? INT_OP : 0), dst_r, TMP_REG1, src);
1621

1622
		switch (op) {
1623
		case SLJIT_MOV:
1624
		case SLJIT_MOV_P:
1625
			mem_flags = WORD_SIZE;
1626
			break;
1627
		case SLJIT_MOV_U8:
1628
			mem_flags = BYTE_SIZE;
1629
			if (src == SLJIT_IMM)
1630
				srcw = (sljit_u8)srcw;
1631
			break;
1632
		case SLJIT_MOV_S8:
1633
			mem_flags = BYTE_SIZE | SIGNED;
1634
			if (src == SLJIT_IMM)
1635
				srcw = (sljit_s8)srcw;
1636
			break;
1637
		case SLJIT_MOV_U16:
1638
			mem_flags = HALF_SIZE;
1639
			if (src == SLJIT_IMM)
1640
				srcw = (sljit_u16)srcw;
1641
			break;
1642
		case SLJIT_MOV_S16:
1643
			mem_flags = HALF_SIZE | SIGNED;
1644
			if (src == SLJIT_IMM)
1645
				srcw = (sljit_s16)srcw;
1646
			break;
1647
		case SLJIT_MOV_U32:
1648
			mem_flags = INT_SIZE;
1649
			if (src == SLJIT_IMM)
1650
				srcw = (sljit_u32)srcw;
1651
			break;
1652
		case SLJIT_MOV_S32:
1653
		case SLJIT_MOV32:
1654
			mem_flags = INT_SIZE | SIGNED;
1655
			if (src == SLJIT_IMM)
1656
				srcw = (sljit_s32)srcw;
1657
			break;
1658
		default:
1659
			SLJIT_UNREACHABLE();
1660
			mem_flags = 0;
1661
			break;
1662
		}
1663

1664
		if (src == SLJIT_IMM)
1665
			FAIL_IF(emit_op_imm(compiler, SLJIT_MOV | ARG2_IMM, dst_r, TMP_REG1, srcw));
1666
		else if (!(src & SLJIT_MEM))
1667
			dst_r = src;
1668
		else
1669
			FAIL_IF(emit_op_mem(compiler, mem_flags, dst_r, src, srcw, TMP_REG2));
1670

1671
		if (dst & SLJIT_MEM)
1672
			return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1673
		return SLJIT_SUCCESS;
1674
	}
1675

1676
	flags = HAS_FLAGS(op_flags) ? SET_FLAGS : 0;
1677

1678
	switch (op) {
1679
	case SLJIT_REV_U16:
1680
	case SLJIT_REV_S16:
1681
		mem_flags = HALF_SIZE;
1682
		break;
1683
	case SLJIT_REV_U32:
1684
	case SLJIT_REV_S32:
1685
		mem_flags = INT_SIZE;
1686
		break;
1687
	default:
1688
		mem_flags = WORD_SIZE;
1689

1690
		if (op_flags & SLJIT_32) {
1691
			flags |= INT_OP;
1692
			mem_flags = INT_SIZE;
1693
		}
1694
		break;
1695
	}
1696

1697
	if (src & SLJIT_MEM) {
1698
		FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG2, src, srcw, TMP_REG2));
1699
		src = TMP_REG2;
1700
	}
1701

1702
	emit_op_imm(compiler, flags | op, dst_r, TMP_REG1, src);
1703

1704
	if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
1705
		return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1706
	return SLJIT_SUCCESS;
1707
}
1708

1709
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1710
	sljit_s32 dst, sljit_sw dstw,
1711
	sljit_s32 src1, sljit_sw src1w,
1712
	sljit_s32 src2, sljit_sw src2w)
1713
{
1714
	sljit_s32 dst_r, flags, mem_flags;
1715

1716
	CHECK_ERROR();
1717
	CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1718
	ADJUST_LOCAL_OFFSET(dst, dstw);
1719
	ADJUST_LOCAL_OFFSET(src1, src1w);
1720
	ADJUST_LOCAL_OFFSET(src2, src2w);
1721

1722
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1723
	flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
1724
	mem_flags = WORD_SIZE;
1725

1726
	if (op & SLJIT_32) {
1727
		flags |= INT_OP;
1728
		mem_flags = INT_SIZE;
1729
	}
1730

1731
	if (dst == TMP_REG2)
1732
		flags |= UNUSED_RETURN;
1733

1734
	if (src1 & SLJIT_MEM) {
1735
		FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG1, src1, src1w, TMP_REG1));
1736
		src1 = TMP_REG1;
1737
	}
1738

1739
	if (src2 & SLJIT_MEM) {
1740
		FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG2, src2, src2w, TMP_REG2));
1741
		src2 = TMP_REG2;
1742
	}
1743

1744
	if (src1 == SLJIT_IMM)
1745
		flags |= ARG1_IMM;
1746
	else
1747
		src1w = src1;
1748

1749
	if (src2 == SLJIT_IMM)
1750
		flags |= ARG2_IMM;
1751
	else
1752
		src2w = src2;
1753

1754
	emit_op_imm(compiler, flags | GET_OPCODE(op), dst_r, src1w, src2w);
1755

1756
	if (dst & SLJIT_MEM)
1757
		return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1758
	return SLJIT_SUCCESS;
1759
}
1760

1761
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
1762
	sljit_s32 src1, sljit_sw src1w,
1763
	sljit_s32 src2, sljit_sw src2w)
1764
{
1765
	CHECK_ERROR();
1766
	CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
1767

1768
	SLJIT_SKIP_CHECKS(compiler);
1769
	return sljit_emit_op2(compiler, op, TMP_REG2, 0, src1, src1w, src2, src2w);
1770
}
1771

1772
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1773
	sljit_s32 dst_reg,
1774
	sljit_s32 src1, sljit_sw src1w,
1775
	sljit_s32 src2, sljit_sw src2w)
1776
{
1777
	CHECK_ERROR();
1778
	CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
1779

1780
	switch (GET_OPCODE(op)) {
1781
	case SLJIT_MULADD:
1782
		SLJIT_SKIP_CHECKS(compiler);
1783
		return sljit_emit_op2(compiler, op, dst_reg, 0, src1, src1w, src2, src2w);
1784
	}
1785

1786
	return SLJIT_SUCCESS;
1787
}
1788

1789
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1790
	sljit_s32 dst_reg,
1791
	sljit_s32 src1_reg,
1792
	sljit_s32 src2_reg,
1793
	sljit_s32 src3, sljit_sw src3w)
1794
{
1795
	sljit_ins inv_bits, imm;
1796
	sljit_s32 is_left;
1797
	sljit_sw mask;
1798

1799
	CHECK_ERROR();
1800
	CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
1801

1802
	is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
1803

1804
	if (src1_reg == src2_reg) {
1805
		SLJIT_SKIP_CHECKS(compiler);
1806
		return sljit_emit_op2(compiler, (is_left ? SLJIT_ROTL : SLJIT_ROTR) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
1807
	}
1808

1809
	ADJUST_LOCAL_OFFSET(src3, src3w);
1810

1811
	inv_bits = (op & SLJIT_32) ? W_OP : 0;
1812

1813
	if (src3 == SLJIT_IMM) {
1814
		mask = inv_bits ? 0x1f : 0x3f;
1815
		src3w &= mask;
1816

1817
		if (src3w == 0)
1818
			return SLJIT_SUCCESS;
1819

1820
		if (is_left)
1821
			src3w = (src3w ^ mask) + 1;
1822

1823
		return push_inst(compiler, (EXTR ^ (inv_bits | (inv_bits >> 9))) | RD(dst_reg)
1824
			| RN(is_left ? src1_reg : src2_reg) | RM(is_left ? src2_reg : src1_reg) | ((sljit_ins)src3w << 10));
1825
	}
1826

1827
	if (src3 & SLJIT_MEM) {
1828
		FAIL_IF(emit_op_mem(compiler, inv_bits ? INT_SIZE : WORD_SIZE, TMP_REG2, src3, src3w, TMP_REG2));
1829
		src3 = TMP_REG2;
1830
	} else if (dst_reg == src3) {
1831
		FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG2) | RM(src3)));
1832
		src3 = TMP_REG2;
1833
	}
1834

1835
	FAIL_IF(push_inst(compiler, ((is_left ? LSLV : LSRV) ^ inv_bits) | RD(dst_reg) | RN(src1_reg) | RM(src3)));
1836

1837
	if (!(op & SLJIT_SHIFT_INTO_NON_ZERO)) {
1838
		/* Shift left/right by 1. */
1839
		if (is_left)
1840
			imm = (sljit_ins)(inv_bits ? ((1 << 16) | (31 << 10)) : ((1 << 16) | (63 << 10) | (1 << 22)));
1841
		else
1842
			imm = (sljit_ins)(inv_bits ? ((31 << 16) | (30 << 10)) : ((63 << 16) | (62 << 10) | (1 << 22)));
1843

1844
		FAIL_IF(push_inst(compiler, (UBFM ^ (inv_bits | (inv_bits >> 9))) | RD(TMP_REG1) | RN(src2_reg) | imm));
1845

1846
		/* Set imm to mask. */
1847
		imm = (sljit_ins)(inv_bits ? (4 << 10) : ((5 << 10) | (1 << 22)));
1848
		FAIL_IF(push_inst(compiler, (EORI ^ inv_bits) | RD(TMP_REG2) | RN(src3) | imm));
1849

1850
		src2_reg = TMP_REG1;
1851
	} else
1852
		FAIL_IF(push_inst(compiler, (SUB ^ inv_bits) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(src3)));
1853

1854
	FAIL_IF(push_inst(compiler, ((is_left ? LSRV : LSLV) ^ inv_bits) | RD(TMP_REG1) | RN(src2_reg) | RM(TMP_REG2)));
1855
	return push_inst(compiler, (ORR ^ inv_bits) | RD(dst_reg) | RN(dst_reg) | RM(TMP_REG1));
1856
}
1857

1858
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
1859
	sljit_s32 src, sljit_sw srcw)
1860
{
1861
	CHECK_ERROR();
1862
	CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
1863
	ADJUST_LOCAL_OFFSET(src, srcw);
1864

1865
	switch (op) {
1866
	case SLJIT_FAST_RETURN:
1867
		if (FAST_IS_REG(src))
1868
			FAIL_IF(push_inst(compiler, MOV | RD(TMP_LR) | RM(src)));
1869
		else
1870
			FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_LR, src, srcw, TMP_REG1));
1871

1872
		return push_inst(compiler, RET | RN(TMP_LR));
1873
	case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
1874
		return SLJIT_SUCCESS;
1875
	case SLJIT_PREFETCH_L1:
1876
	case SLJIT_PREFETCH_L2:
1877
	case SLJIT_PREFETCH_L3:
1878
	case SLJIT_PREFETCH_ONCE:
1879
		SLJIT_ASSERT(reg_map[1] == 0 && reg_map[3] == 2 && reg_map[5] == 4);
1880

1881
		/* The reg_map[op] should provide the appropriate constant. */
1882
		if (op == SLJIT_PREFETCH_L1)
1883
			op = 1;
1884
		else if (op == SLJIT_PREFETCH_L2)
1885
			op = 3;
1886
		else if (op == SLJIT_PREFETCH_L3)
1887
			op = 5;
1888
		else
1889
			op = 2;
1890

1891
		/* Signed word sized load is the prefetch instruction. */
1892
		return emit_op_mem(compiler, WORD_SIZE | SIGNED, op, src, srcw, TMP_REG1);
1893
	}
1894

1895
	return SLJIT_SUCCESS;
1896
}
1897

1898
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
1899
	sljit_s32 dst, sljit_sw dstw)
1900
{
1901
	sljit_s32 dst_r = TMP_LR;
1902

1903
	CHECK_ERROR();
1904
	CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
1905
	ADJUST_LOCAL_OFFSET(dst, dstw);
1906

1907
	switch (op) {
1908
	case SLJIT_FAST_ENTER:
1909
		if (FAST_IS_REG(dst))
1910
			return push_inst(compiler, MOV | RD(dst) | RM(TMP_LR));
1911
		break;
1912
	case SLJIT_GET_RETURN_ADDRESS:
1913
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1914
		FAIL_IF(emit_op_mem(compiler, WORD_SIZE, dst_r, SLJIT_MEM1(SLJIT_SP), 0x8, TMP_REG2));
1915
		break;
1916
	}
1917

1918
	if (dst & SLJIT_MEM)
1919
		return emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2);
1920

1921
	return SLJIT_SUCCESS;
1922
}
1923

1924
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
1925
{
1926
	CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
1927

1928
	if (type == SLJIT_GP_REGISTER)
1929
		return reg_map[reg];
1930

1931
	if (type != SLJIT_FLOAT_REGISTER && type != SLJIT_SIMD_REG_64 && type != SLJIT_SIMD_REG_128)
1932
		return -1;
1933

1934
	return freg_map[reg];
1935
}
1936

1937
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
1938
	void *instruction, sljit_u32 size)
1939
{
1940
	SLJIT_UNUSED_ARG(size);
1941
	CHECK_ERROR();
1942
	CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
1943

1944
	return push_inst(compiler, *(sljit_ins*)instruction);
1945
}
1946

1947
/* --------------------------------------------------------------------- */
1948
/*  Floating point operators                                             */
1949
/* --------------------------------------------------------------------- */
1950

1951
static sljit_s32 emit_fop_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1952
{
1953
	sljit_u32 shift = MEM_SIZE_SHIFT(flags);
1954
	sljit_ins type = (shift << 30);
1955

1956
	SLJIT_ASSERT(arg & SLJIT_MEM);
1957

1958
	if (!(flags & STORE))
1959
		type |= 0x00400000;
1960

1961
	if (arg & OFFS_REG_MASK) {
1962
		argw &= 3;
1963
		if (argw == 0 || argw == shift)
1964
			return push_inst(compiler, STR_FR | type | VT(reg)
1965
				| RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | (argw ? (1 << 12) : 0));
1966

1967
		FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG2) | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | ((sljit_ins)argw << 10)));
1968
		return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2));
1969
	}
1970

1971
	arg &= REG_MASK;
1972

1973
	if (!arg) {
1974
		FAIL_IF(load_immediate(compiler, TMP_REG2, argw & ~(0xfff << shift)));
1975

1976
		argw = (argw >> shift) & 0xfff;
1977

1978
		return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2) | ((sljit_ins)argw << 10));
1979
	}
1980

1981
	if (argw >= 0 && (argw & ((1 << shift) - 1)) == 0) {
1982
		if ((argw >> shift) <= 0xfff)
1983
			return push_inst(compiler, STR_FI | type | VT(reg) | RN(arg) | ((sljit_ins)argw << (10 - shift)));
1984

1985
		if (argw <= 0xffffff) {
1986
			FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(TMP_REG2) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
1987

1988
			argw = ((argw & 0xfff) >> shift);
1989
			return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2) | ((sljit_ins)argw << 10));
1990
		}
1991
	}
1992

1993
	if (argw <= 255 && argw >= -256)
1994
		return push_inst(compiler, STUR_FI | type | VT(reg) | RN(arg) | (((sljit_ins)argw & 0x1ff) << 12));
1995

1996
	FAIL_IF(load_immediate(compiler, TMP_REG2, argw));
1997
	return push_inst(compiler, STR_FR | type | VT(reg) | RN(arg) | RM(TMP_REG2));
1998
}
1999

2000
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2001
	sljit_s32 dst, sljit_sw dstw,
2002
	sljit_s32 src, sljit_sw srcw)
2003
{
2004
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2005
	sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2006

2007
	if (GET_OPCODE(op) == SLJIT_CONV_S32_FROM_F64)
2008
		inv_bits |= W_OP;
2009

2010
	if (src & SLJIT_MEM) {
2011
		FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) ? INT_SIZE : WORD_SIZE, TMP_FREG1, src, srcw));
2012
		src = TMP_FREG1;
2013
	}
2014

2015
	FAIL_IF(push_inst(compiler, (FCVTZS ^ inv_bits) | RD(dst_r) | VN(src)));
2016

2017
	if (dst & SLJIT_MEM)
2018
		return emit_op_mem(compiler, ((GET_OPCODE(op) == SLJIT_CONV_S32_FROM_F64) ? INT_SIZE : WORD_SIZE) | STORE, TMP_REG1, dst, dstw, TMP_REG2);
2019
	return SLJIT_SUCCESS;
2020
}
2021

2022
static sljit_s32 sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_ins ins,
2023
	sljit_s32 dst, sljit_sw dstw,
2024
	sljit_s32 src, sljit_sw srcw)
2025
{
2026
	sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2027

2028
	if (src & SLJIT_MEM) {
2029
		emit_op_mem(compiler, (ins & W_OP) ? WORD_SIZE : INT_SIZE, TMP_REG1, src, srcw, TMP_REG1);
2030
		src = TMP_REG1;
2031
	} else if (src == SLJIT_IMM) {
2032
		FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2033
		src = TMP_REG1;
2034
	}
2035

2036
	FAIL_IF(push_inst(compiler, ins | VD(dst_r) | RN(src)));
2037

2038
	if (dst & SLJIT_MEM)
2039
		return emit_fop_mem(compiler, ((ins & (1 << 22)) ? WORD_SIZE : INT_SIZE) | STORE, TMP_FREG1, dst, dstw);
2040
	return SLJIT_SUCCESS;
2041
}
2042

2043
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2044
	sljit_s32 dst, sljit_sw dstw,
2045
	sljit_s32 src, sljit_sw srcw)
2046
{
2047
	sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2048

2049
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32) {
2050
		inv_bits |= W_OP;
2051

2052
		if (src == SLJIT_IMM)
2053
			srcw = (sljit_s32)srcw;
2054
	}
2055

2056
	return sljit_emit_fop1_conv_f64_from_w(compiler, SCVTF ^ inv_bits, dst, dstw, src, srcw);
2057
}
2058

2059
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
2060
	sljit_s32 dst, sljit_sw dstw,
2061
	sljit_s32 src, sljit_sw srcw)
2062
{
2063
	sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2064

2065
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_U32) {
2066
		inv_bits |= W_OP;
2067

2068
		if (src == SLJIT_IMM)
2069
			srcw = (sljit_u32)srcw;
2070
	}
2071

2072
	return sljit_emit_fop1_conv_f64_from_w(compiler, UCVTF ^ inv_bits, dst, dstw, src, srcw);
2073
}
2074

2075
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2076
	sljit_s32 src1, sljit_sw src1w,
2077
	sljit_s32 src2, sljit_sw src2w)
2078
{
2079
	sljit_s32 mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2080
	sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2081

2082
	if (src1 & SLJIT_MEM) {
2083
		FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG1, src1, src1w));
2084
		src1 = TMP_FREG1;
2085
	}
2086

2087
	if (src2 & SLJIT_MEM) {
2088
		FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG2, src2, src2w));
2089
		src2 = TMP_FREG2;
2090
	}
2091

2092
	FAIL_IF(push_inst(compiler, (FCMP ^ inv_bits) | VN(src1) | VM(src2)));
2093

2094
	if (GET_FLAG_TYPE(op) != SLJIT_UNORDERED_OR_EQUAL)
2095
		return SLJIT_SUCCESS;
2096

2097
	FAIL_IF(push_inst(compiler, CSINC | (0x0 << 12) | RD(TMP_REG1) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2098
	return push_inst(compiler, CCMPI | (0x0 << 16) | (0x7 << 12) | RN(TMP_REG1) | 0x4);
2099
}
2100

2101
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2102
	sljit_s32 dst, sljit_sw dstw,
2103
	sljit_s32 src, sljit_sw srcw)
2104
{
2105
	sljit_s32 dst_r, mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2106
	sljit_ins inv_bits;
2107

2108
	CHECK_ERROR();
2109

2110
	SLJIT_COMPILE_ASSERT((INT_SIZE ^ 0x1) == WORD_SIZE, must_be_one_bit_difference);
2111
	SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2112

2113
	inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2114
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2115

2116
	if (src & SLJIT_MEM) {
2117
		FAIL_IF(emit_fop_mem(compiler, (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) ? (mem_flags ^ 0x1) : mem_flags, dst_r, src, srcw));
2118
		src = dst_r;
2119
	}
2120

2121
	switch (GET_OPCODE(op)) {
2122
	case SLJIT_MOV_F64:
2123
		if (src != dst_r) {
2124
			if (!(dst & SLJIT_MEM))
2125
				FAIL_IF(push_inst(compiler, (FMOV ^ inv_bits) | VD(dst_r) | VN(src)));
2126
			else
2127
				dst_r = src;
2128
		}
2129
		break;
2130
	case SLJIT_NEG_F64:
2131
		FAIL_IF(push_inst(compiler, (FNEG ^ inv_bits) | VD(dst_r) | VN(src)));
2132
		break;
2133
	case SLJIT_ABS_F64:
2134
		FAIL_IF(push_inst(compiler, (FABS ^ inv_bits) | VD(dst_r) | VN(src)));
2135
		break;
2136
	case SLJIT_CONV_F64_FROM_F32:
2137
		FAIL_IF(push_inst(compiler, FCVT | (sljit_ins)((op & SLJIT_32) ? (1 << 22) : (1 << 15)) | VD(dst_r) | VN(src)));
2138
		break;
2139
	}
2140

2141
	if (dst & SLJIT_MEM)
2142
		return emit_fop_mem(compiler, mem_flags | STORE, dst_r, dst, dstw);
2143
	return SLJIT_SUCCESS;
2144
}
2145

2146
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2147
	sljit_s32 dst, sljit_sw dstw,
2148
	sljit_s32 src1, sljit_sw src1w,
2149
	sljit_s32 src2, sljit_sw src2w)
2150
{
2151
	sljit_s32 dst_r, mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2152
	sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2153

2154
	CHECK_ERROR();
2155
	CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2156
	ADJUST_LOCAL_OFFSET(dst, dstw);
2157
	ADJUST_LOCAL_OFFSET(src1, src1w);
2158
	ADJUST_LOCAL_OFFSET(src2, src2w);
2159

2160
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2161
	if (src1 & SLJIT_MEM) {
2162
		FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG1, src1, src1w));
2163
		src1 = TMP_FREG1;
2164
	}
2165
	if (src2 & SLJIT_MEM) {
2166
		FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG2, src2, src2w));
2167
		src2 = TMP_FREG2;
2168
	}
2169

2170
	switch (GET_OPCODE(op)) {
2171
	case SLJIT_ADD_F64:
2172
		FAIL_IF(push_inst(compiler, (FADD ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2173
		break;
2174
	case SLJIT_SUB_F64:
2175
		FAIL_IF(push_inst(compiler, (FSUB ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2176
		break;
2177
	case SLJIT_MUL_F64:
2178
		FAIL_IF(push_inst(compiler, (FMUL ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2179
		break;
2180
	case SLJIT_DIV_F64:
2181
		FAIL_IF(push_inst(compiler, (FDIV ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2182
		break;
2183
	case SLJIT_COPYSIGN_F64:
2184
		FAIL_IF(push_inst(compiler, (FMOV_R ^ ((op & SLJIT_32) ? (W_OP | (1 << 22)) : 0)) | VN(src2) | RD(TMP_REG1)));
2185
		FAIL_IF(push_inst(compiler, (FABS ^ inv_bits) | VD(dst_r) | VN(src1)));
2186
		FAIL_IF(push_inst(compiler, TBZ | ((op & SLJIT_32) ? 0 : ((sljit_ins)1 << 31)) | (0x1f << 19) | (2 << 5) | RT(TMP_REG1)));
2187
		return push_inst(compiler, (FNEG ^ inv_bits) | VD(dst_r) | VN(dst_r));
2188
	}
2189

2190
	if (!(dst & SLJIT_MEM))
2191
		return SLJIT_SUCCESS;
2192
	return emit_fop_mem(compiler, mem_flags | STORE, TMP_FREG1, dst, dstw);
2193
}
2194

2195
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2196
	sljit_s32 freg, sljit_f32 value)
2197
{
2198
	sljit_u32 exp;
2199
	union {
2200
		sljit_u32 imm;
2201
		sljit_f32 value;
2202
	} u;
2203

2204
	CHECK_ERROR();
2205
	CHECK(check_sljit_emit_fset32(compiler, freg, value));
2206

2207
	u.value = value;
2208

2209
	if (u.imm == 0)
2210
		return push_inst(compiler, (FMOV_R ^ (W_OP | (1 << 22))) | RN(TMP_ZERO) | VD(freg) | (1 << 16));
2211

2212
	if ((u.imm << (32 - 19)) == 0) {
2213
		exp = (u.imm >> (23 + 2)) & 0x3f;
2214

2215
		if (exp == 0x20 || exp == 0x1f)
2216
			return push_inst(compiler, (FMOV_I ^ (1 << 22)) | (sljit_ins)((((u.imm >> 24) & 0x80) | ((u.imm >> 19) & 0x7f)) << 13) | VD(freg));
2217
	}
2218

2219
	FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_s32)u.imm));
2220
	return push_inst(compiler, (FMOV_R ^ (W_OP | (1 << 22))) | RN(TMP_REG1) | VD(freg) | (1 << 16));
2221
}
2222

2223
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset64(struct sljit_compiler *compiler,
2224
	sljit_s32 freg, sljit_f64 value)
2225
{
2226
	sljit_uw exp;
2227
	union {
2228
		sljit_uw imm;
2229
		sljit_f64 value;
2230
	} u;
2231

2232
	CHECK_ERROR();
2233
	CHECK(check_sljit_emit_fset64(compiler, freg, value));
2234

2235
	u.value = value;
2236

2237
	if (u.imm == 0)
2238
		return push_inst(compiler, FMOV_R | RN(TMP_ZERO) | VD(freg) | (sljit_ins)1 << 16);
2239

2240
	if ((u.imm << (64 - 48)) == 0) {
2241
		exp = (u.imm >> (52 + 2)) & 0x1ff;
2242

2243
		if (exp == 0x100 || exp == 0xff)
2244
			return push_inst(compiler, FMOV_I | (sljit_ins)((((u.imm >> 56) & 0x80) | ((u.imm >> 48) & 0x7f)) << 13) | VD(freg));
2245
	}
2246

2247
	FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_sw)u.imm));
2248
	return push_inst(compiler, FMOV_R | RN(TMP_REG1) | VD(freg) | (1 << 16));
2249
}
2250

2251
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2252
	sljit_s32 freg, sljit_s32 reg)
2253
{
2254
	sljit_ins inst;
2255

2256
	CHECK_ERROR();
2257
	CHECK(check_sljit_emit_fcopy(compiler, op, freg, reg));
2258

2259
	if (GET_OPCODE(op) == SLJIT_COPY_TO_F64)
2260
		inst = FMOV_R | RN(reg) | VD(freg) | (1 << 16);
2261
	else
2262
		inst = FMOV_R | VN(freg) | RD(reg);
2263

2264
	if (op & SLJIT_32)
2265
		inst ^= W_OP | (1 << 22);
2266

2267
	return push_inst(compiler, inst);
2268
}
2269

2270
/* --------------------------------------------------------------------- */
2271
/*  Conditional instructions                                             */
2272
/* --------------------------------------------------------------------- */
2273

2274
static sljit_ins get_cc(struct sljit_compiler *compiler, sljit_s32 type)
2275
{
2276
	switch (type) {
2277
	case SLJIT_EQUAL:
2278
	case SLJIT_ATOMIC_STORED:
2279
	case SLJIT_F_EQUAL:
2280
	case SLJIT_ORDERED_EQUAL:
2281
	case SLJIT_UNORDERED_OR_EQUAL:
2282
		return 0x1;
2283

2284
	case SLJIT_NOT_EQUAL:
2285
	case SLJIT_ATOMIC_NOT_STORED:
2286
	case SLJIT_F_NOT_EQUAL:
2287
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
2288
	case SLJIT_ORDERED_NOT_EQUAL:
2289
		return 0x0;
2290

2291
	case SLJIT_CARRY:
2292
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2293
			return 0x3;
2294
		/* fallthrough */
2295

2296
	case SLJIT_LESS:
2297
		return 0x2;
2298

2299
	case SLJIT_NOT_CARRY:
2300
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2301
			return 0x2;
2302
		/* fallthrough */
2303

2304
	case SLJIT_GREATER_EQUAL:
2305
		return 0x3;
2306

2307
	case SLJIT_GREATER:
2308
	case SLJIT_UNORDERED_OR_GREATER:
2309
		return 0x9;
2310

2311
	case SLJIT_LESS_EQUAL:
2312
	case SLJIT_F_LESS_EQUAL:
2313
	case SLJIT_ORDERED_LESS_EQUAL:
2314
		return 0x8;
2315

2316
	case SLJIT_SIG_LESS:
2317
	case SLJIT_UNORDERED_OR_LESS:
2318
		return 0xa;
2319

2320
	case SLJIT_SIG_GREATER_EQUAL:
2321
	case SLJIT_F_GREATER_EQUAL:
2322
	case SLJIT_ORDERED_GREATER_EQUAL:
2323
		return 0xb;
2324

2325
	case SLJIT_SIG_GREATER:
2326
	case SLJIT_F_GREATER:
2327
	case SLJIT_ORDERED_GREATER:
2328
		return 0xd;
2329

2330
	case SLJIT_SIG_LESS_EQUAL:
2331
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
2332
		return 0xc;
2333

2334
	case SLJIT_OVERFLOW:
2335
		if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2336
			return 0x0;
2337
		/* fallthrough */
2338

2339
	case SLJIT_UNORDERED:
2340
		return 0x7;
2341

2342
	case SLJIT_NOT_OVERFLOW:
2343
		if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2344
			return 0x1;
2345
		/* fallthrough */
2346

2347
	case SLJIT_ORDERED:
2348
		return 0x6;
2349

2350
	case SLJIT_F_LESS:
2351
	case SLJIT_ORDERED_LESS:
2352
		return 0x5;
2353

2354
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2355
		return 0x4;
2356

2357
	default:
2358
		SLJIT_UNREACHABLE();
2359
		return 0xe;
2360
	}
2361
}
2362

2363
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2364
{
2365
	struct sljit_label *label;
2366

2367
	CHECK_ERROR_PTR();
2368
	CHECK_PTR(check_sljit_emit_label(compiler));
2369

2370
	if (compiler->last_label && compiler->last_label->size == compiler->size)
2371
		return compiler->last_label;
2372

2373
	label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2374
	PTR_FAIL_IF(!label);
2375
	set_label(label, compiler);
2376
	return label;
2377
}
2378

2379
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2380
{
2381
	struct sljit_jump *jump;
2382

2383
	CHECK_ERROR_PTR();
2384
	CHECK_PTR(check_sljit_emit_jump(compiler, type));
2385

2386
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2387
	PTR_FAIL_IF(!jump);
2388
	set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2389
	type &= 0xff;
2390

2391
	if (type < SLJIT_JUMP) {
2392
		jump->flags |= IS_COND;
2393
		PTR_FAIL_IF(push_inst(compiler, B_CC | (6 << 5) | get_cc(compiler, type)));
2394
	} else if (type >= SLJIT_FAST_CALL)
2395
		jump->flags |= IS_BL;
2396

2397
	jump->addr = compiler->size;
2398
	PTR_FAIL_IF(push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(TMP_REG2)));
2399

2400
	/* Maximum number of instructions required for generating a constant. */
2401
	compiler->size += JUMP_MAX_SIZE - 1;
2402
	return jump;
2403
}
2404

2405
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2406
	sljit_s32 arg_types)
2407
{
2408
	SLJIT_UNUSED_ARG(arg_types);
2409
	CHECK_ERROR_PTR();
2410
	CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2411

2412
	if (type & SLJIT_CALL_RETURN) {
2413
		PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
2414
		type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2415
	}
2416

2417
	SLJIT_SKIP_CHECKS(compiler);
2418
	return sljit_emit_jump(compiler, type);
2419
}
2420

2421
static SLJIT_INLINE struct sljit_jump* emit_cmp_to0(struct sljit_compiler *compiler, sljit_s32 type,
2422
	sljit_s32 src, sljit_sw srcw)
2423
{
2424
	struct sljit_jump *jump;
2425
	sljit_ins inv_bits = (type & SLJIT_32) ? W_OP : 0;
2426

2427
	SLJIT_ASSERT((type & 0xff) == SLJIT_EQUAL || (type & 0xff) == SLJIT_NOT_EQUAL);
2428
	ADJUST_LOCAL_OFFSET(src, srcw);
2429

2430
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2431
	PTR_FAIL_IF(!jump);
2432
	set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2433
	jump->flags |= IS_CBZ | IS_COND;
2434

2435
	if (src & SLJIT_MEM) {
2436
		PTR_FAIL_IF(emit_op_mem(compiler, inv_bits ? INT_SIZE : WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
2437
		src = TMP_REG1;
2438
	}
2439
	else if (src == SLJIT_IMM) {
2440
		PTR_FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2441
		src = TMP_REG1;
2442
	}
2443

2444
	SLJIT_ASSERT(FAST_IS_REG(src));
2445

2446
	if ((type & 0xff) == SLJIT_EQUAL)
2447
		inv_bits |= 1 << 24;
2448

2449
	PTR_FAIL_IF(push_inst(compiler, (CBZ ^ inv_bits) | (6 << 5) | RT(src)));
2450
	jump->addr = compiler->size;
2451
	PTR_FAIL_IF(push_inst(compiler, BR | RN(TMP_REG2)));
2452

2453
	/* Maximum number of instructions required for generating a constant. */
2454
	compiler->size += JUMP_MAX_SIZE - 1;
2455
	return jump;
2456
}
2457

2458
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2459
{
2460
	struct sljit_jump *jump;
2461

2462
	CHECK_ERROR();
2463
	CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2464

2465
	if (src != SLJIT_IMM) {
2466
		if (src & SLJIT_MEM) {
2467
			ADJUST_LOCAL_OFFSET(src, srcw);
2468
			FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2));
2469
			src = TMP_REG2;
2470
		}
2471
		return push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(src));
2472
	}
2473

2474
	/* These jumps are converted to jump/call instructions when possible. */
2475
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2476
	FAIL_IF(!jump);
2477
	set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_BL : 0));
2478
	jump->u.target = (sljit_uw)srcw;
2479

2480
	jump->addr = compiler->size;
2481
	/* Maximum number of instructions required for generating a constant. */
2482
	compiler->size += JUMP_MAX_SIZE - 1;
2483
	return push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(TMP_REG2));
2484
}
2485

2486
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2487
	sljit_s32 arg_types,
2488
	sljit_s32 src, sljit_sw srcw)
2489
{
2490
	SLJIT_UNUSED_ARG(arg_types);
2491
	CHECK_ERROR();
2492
	CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2493

2494
	if (src & SLJIT_MEM) {
2495
		ADJUST_LOCAL_OFFSET(src, srcw);
2496
		FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
2497
		src = TMP_REG1;
2498
	}
2499

2500
	if (type & SLJIT_CALL_RETURN) {
2501
		if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
2502
			FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(src)));
2503
			src = TMP_REG1;
2504
		}
2505

2506
		FAIL_IF(emit_stack_frame_release(compiler, 0));
2507
		type = SLJIT_JUMP;
2508
	}
2509

2510
	SLJIT_SKIP_CHECKS(compiler);
2511
	return sljit_emit_ijump(compiler, type, src, srcw);
2512
}
2513

2514
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2515
	sljit_s32 dst, sljit_sw dstw,
2516
	sljit_s32 type)
2517
{
2518
	sljit_s32 dst_r, src_r, flags, mem_flags;
2519
	sljit_ins cc;
2520

2521
	CHECK_ERROR();
2522
	CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2523
	ADJUST_LOCAL_OFFSET(dst, dstw);
2524

2525
	cc = get_cc(compiler, type);
2526
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2527

2528
	if (GET_OPCODE(op) < SLJIT_ADD) {
2529
		FAIL_IF(push_inst(compiler, CSINC | (cc << 12) | RD(dst_r) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2530

2531
		if (dst & SLJIT_MEM) {
2532
			mem_flags = (GET_OPCODE(op) == SLJIT_MOV ? WORD_SIZE : INT_SIZE) | STORE;
2533
			return emit_op_mem(compiler, mem_flags, TMP_REG1, dst, dstw, TMP_REG2);
2534
		}
2535

2536
		return SLJIT_SUCCESS;
2537
	}
2538

2539
	flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
2540
	mem_flags = WORD_SIZE;
2541

2542
	if (op & SLJIT_32) {
2543
		flags |= INT_OP;
2544
		mem_flags = INT_SIZE;
2545
	}
2546

2547
	src_r = dst;
2548

2549
	if (dst & SLJIT_MEM) {
2550
		FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG1, dst, dstw, TMP_REG1));
2551
		src_r = TMP_REG1;
2552
	}
2553

2554
	FAIL_IF(push_inst(compiler, CSINC | (cc << 12) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2555
	emit_op_imm(compiler, flags | GET_OPCODE(op), dst_r, src_r, TMP_REG2);
2556

2557
	if (dst & SLJIT_MEM)
2558
		return emit_op_mem(compiler, mem_flags | STORE, TMP_REG1, dst, dstw, TMP_REG2);
2559
	return SLJIT_SUCCESS;
2560
}
2561

2562
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2563
	sljit_s32 dst_reg,
2564
	sljit_s32 src1, sljit_sw src1w,
2565
	sljit_s32 src2_reg)
2566
{
2567
	sljit_ins inv_bits = (type & SLJIT_32) ? W_OP : 0;
2568
	sljit_ins cc;
2569

2570
	CHECK_ERROR();
2571
	CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
2572

2573
	ADJUST_LOCAL_OFFSET(src1, src1w);
2574

2575
	if (src1 == SLJIT_IMM) {
2576
		if (type & SLJIT_32)
2577
			src1w = (sljit_s32)src1w;
2578
		FAIL_IF(load_immediate(compiler, TMP_REG2, src1w));
2579
		src1 = TMP_REG2;
2580
	} else if (src1 & SLJIT_MEM) {
2581
		FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src1, src1w, TMP_REG2));
2582
		src1 = TMP_REG2;
2583
	}
2584

2585
	cc = get_cc(compiler, type & ~SLJIT_32);
2586
	return push_inst(compiler, (CSEL ^ inv_bits) | (cc << 12) | RD(dst_reg) | RN(src2_reg) | RM(src1));
2587
}
2588

2589
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2590
	sljit_s32 dst_freg,
2591
	sljit_s32 src1, sljit_sw src1w,
2592
	sljit_s32 src2_freg)
2593
{
2594
	sljit_ins inv_bits = (type & SLJIT_32) ? (1 << 22) : 0;
2595
	sljit_ins cc;
2596

2597
	CHECK_ERROR();
2598
	CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
2599

2600
	ADJUST_LOCAL_OFFSET(src1, src1w);
2601

2602
	if (src1 & SLJIT_MEM) {
2603
		FAIL_IF(emit_fop_mem(compiler, (type & SLJIT_32) ? INT_SIZE : WORD_SIZE, TMP_FREG2, src1, src1w));
2604
		src1 = TMP_FREG2;
2605
	}
2606

2607
	cc = get_cc(compiler, type & ~SLJIT_32);
2608
	return push_inst(compiler, (FCSEL ^ inv_bits) | (cc << 12) | VD(dst_freg) | VN(src2_freg) | VM(src1));
2609
}
2610

2611
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2612
	sljit_s32 reg,
2613
	sljit_s32 mem, sljit_sw memw)
2614
{
2615
	sljit_u32 inst;
2616

2617
	CHECK_ERROR();
2618
	CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
2619

2620
	if (!(reg & REG_PAIR_MASK))
2621
		return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
2622

2623
	ADJUST_LOCAL_OFFSET(mem, memw);
2624

2625
	if (!(mem & REG_MASK)) {
2626
		FAIL_IF(load_immediate(compiler, TMP_REG1, memw & ~0x1f8));
2627

2628
		mem = SLJIT_MEM1(TMP_REG1);
2629
		memw &= 0x1f8;
2630
	} else if (mem & OFFS_REG_MASK) {
2631
		FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RN(mem & REG_MASK) | RM(OFFS_REG(mem)) | ((sljit_ins)(memw & 0x3) << 10)));
2632

2633
		mem = SLJIT_MEM1(TMP_REG1);
2634
		memw = 0;
2635
	} else if ((memw & 0x7) != 0 || memw > 0x1f8 || memw < -0x200) {
2636
		inst = ADDI;
2637

2638
		if (memw < 0) {
2639
			/* Remains negative for integer min. */
2640
			memw = -memw;
2641
			inst = SUBI;
2642
		} else if ((memw & 0x7) == 0 && memw <= 0x7ff0) {
2643
			if (!(type & SLJIT_MEM_STORE) && (mem & REG_MASK) == REG_PAIR_FIRST(reg)) {
2644
				FAIL_IF(push_inst(compiler, LDRI | RD(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | ((sljit_ins)memw << 7)));
2645
				return push_inst(compiler, LDRI | RD(REG_PAIR_FIRST(reg)) | RN(mem & REG_MASK) | ((sljit_ins)(memw + 0x8) << 7));
2646
			}
2647

2648
			inst = (type & SLJIT_MEM_STORE) ? STRI : LDRI;
2649

2650
			FAIL_IF(push_inst(compiler, inst | RD(REG_PAIR_FIRST(reg)) | RN(mem & REG_MASK) | ((sljit_ins)memw << 7)));
2651
			return push_inst(compiler, inst | RD(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | ((sljit_ins)(memw + 0x8) << 7));
2652
		}
2653

2654
		if ((sljit_uw)memw <= 0xfff) {
2655
			FAIL_IF(push_inst(compiler, inst | RD(TMP_REG1) | RN(mem & REG_MASK) | ((sljit_ins)memw << 10)));
2656
			memw = 0;
2657
		} else if ((sljit_uw)memw <= 0xffffff) {
2658
			FAIL_IF(push_inst(compiler, inst | (1 << 22) | RD(TMP_REG1) | RN(mem & REG_MASK) | (((sljit_ins)memw >> 12) << 10)));
2659

2660
			if ((memw & 0xe07) != 0) {
2661
				FAIL_IF(push_inst(compiler, inst | RD(TMP_REG1) | RN(TMP_REG1) | (((sljit_ins)memw & 0xfff) << 10)));
2662
				memw = 0;
2663
			} else {
2664
				memw &= 0xfff;
2665
			}
2666
		} else {
2667
			FAIL_IF(load_immediate(compiler, TMP_REG1, memw));
2668
			FAIL_IF(push_inst(compiler, (inst == ADDI ? ADD : SUB) | RD(TMP_REG1) | RN(mem & REG_MASK) | RM(TMP_REG1)));
2669
			memw = 0;
2670
		}
2671

2672
		mem = SLJIT_MEM1(TMP_REG1);
2673

2674
		if (inst == SUBI)
2675
			memw = -memw;
2676
	}
2677

2678
	SLJIT_ASSERT((memw & 0x7) == 0 && memw <= 0x1f8 && memw >= -0x200);
2679
	return push_inst(compiler, ((type & SLJIT_MEM_STORE) ? STP : LDP) | RT(REG_PAIR_FIRST(reg)) | RT2(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x3f8) << 12));
2680
}
2681

2682
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2683
	sljit_s32 reg,
2684
	sljit_s32 mem, sljit_sw memw)
2685
{
2686
	sljit_u32 sign = 0, inst;
2687

2688
	CHECK_ERROR();
2689
	CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
2690

2691
	if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256))
2692
		return SLJIT_ERR_UNSUPPORTED;
2693

2694
	if (type & SLJIT_MEM_SUPP)
2695
		return SLJIT_SUCCESS;
2696

2697
	switch (type & 0xff) {
2698
	case SLJIT_MOV:
2699
	case SLJIT_MOV_P:
2700
		inst = STURBI | (MEM_SIZE_SHIFT(WORD_SIZE) << 30) | 0x400;
2701
		break;
2702
	case SLJIT_MOV_S8:
2703
		sign = 1;
2704
		/* fallthrough */
2705
	case SLJIT_MOV_U8:
2706
		inst = STURBI | (MEM_SIZE_SHIFT(BYTE_SIZE) << 30) | 0x400;
2707
		break;
2708
	case SLJIT_MOV_S16:
2709
		sign = 1;
2710
		/* fallthrough */
2711
	case SLJIT_MOV_U16:
2712
		inst = STURBI | (MEM_SIZE_SHIFT(HALF_SIZE) << 30) | 0x400;
2713
		break;
2714
	case SLJIT_MOV_S32:
2715
		sign = 1;
2716
		/* fallthrough */
2717
	case SLJIT_MOV_U32:
2718
	case SLJIT_MOV32:
2719
		inst = STURBI | (MEM_SIZE_SHIFT(INT_SIZE) << 30) | 0x400;
2720
		break;
2721
	default:
2722
		SLJIT_UNREACHABLE();
2723
		inst = STURBI | (MEM_SIZE_SHIFT(WORD_SIZE) << 30) | 0x400;
2724
		break;
2725
	}
2726

2727
	if (!(type & SLJIT_MEM_STORE))
2728
		inst |= sign ? 0x00800000 : 0x00400000;
2729

2730
	if (!(type & SLJIT_MEM_POST))
2731
		inst |= 0x800;
2732

2733
	return push_inst(compiler, inst | RT(reg) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x1ff) << 12));
2734
}
2735

2736
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2737
	sljit_s32 freg,
2738
	sljit_s32 mem, sljit_sw memw)
2739
{
2740
	sljit_u32 inst;
2741

2742
	CHECK_ERROR();
2743
	CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
2744

2745
	if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256))
2746
		return SLJIT_ERR_UNSUPPORTED;
2747

2748
	if (type & SLJIT_MEM_SUPP)
2749
		return SLJIT_SUCCESS;
2750

2751
	inst = STUR_FI | 0x80000400;
2752

2753
	if (!(type & SLJIT_32))
2754
		inst |= 0x40000000;
2755

2756
	if (!(type & SLJIT_MEM_STORE))
2757
		inst |= 0x00400000;
2758

2759
	if (!(type & SLJIT_MEM_POST))
2760
		inst |= 0x800;
2761

2762
	return push_inst(compiler, inst | VT(freg) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x1ff) << 12));
2763
}
2764

2765
static sljit_s32 sljit_emit_simd_mem_offset(struct sljit_compiler *compiler, sljit_s32 *mem_ptr, sljit_sw memw)
2766
{
2767
	sljit_ins ins;
2768
	sljit_s32 mem = *mem_ptr;
2769

2770
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
2771
		*mem_ptr = TMP_REG2;
2772
		return push_inst(compiler, ADD | RD(TMP_REG2) | RN(mem & REG_MASK) | RM(OFFS_REG(mem)) | ((sljit_ins)(memw & 0x3) << 10));
2773
	}
2774

2775
	if (!(mem & REG_MASK)) {
2776
		*mem_ptr = TMP_REG2;
2777
		return load_immediate(compiler, TMP_REG2, memw);
2778
	}
2779

2780
	mem &= REG_MASK;
2781

2782
	if (memw == 0) {
2783
		*mem_ptr = mem;
2784
		return SLJIT_SUCCESS;
2785
	}
2786

2787
	*mem_ptr = TMP_REG2;
2788

2789
	if (memw < -0xffffff || memw > 0xffffff) {
2790
		FAIL_IF(load_immediate(compiler, TMP_REG2, memw));
2791
		return push_inst(compiler, ADD | RD(TMP_REG2) | RN(TMP_REG2) | RM(mem));
2792
	}
2793

2794
	ins = ADDI;
2795

2796
	if (memw < 0) {
2797
		memw = -memw;
2798
		ins = SUBI;
2799
	}
2800

2801
	if (memw > 0xfff) {
2802
		FAIL_IF(push_inst(compiler, ins | (1 << 22) | RD(TMP_REG2) | RN(mem) | ((sljit_ins)(memw >> 12) << 10)));
2803

2804
		memw &= 0xfff;
2805
		if (memw == 0)
2806
			return SLJIT_SUCCESS;
2807

2808
		mem = TMP_REG2;
2809
	}
2810

2811
	return push_inst(compiler, ins | RD(TMP_REG2) | RN(mem) | ((sljit_ins)memw << 10));
2812
}
2813

2814
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
2815
	sljit_s32 vreg,
2816
	sljit_s32 srcdst, sljit_sw srcdstw)
2817
{
2818
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
2819
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
2820
	sljit_ins ins;
2821

2822
	CHECK_ERROR();
2823
	CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
2824

2825
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
2826

2827
	if (reg_size != 3 && reg_size != 4)
2828
		return SLJIT_ERR_UNSUPPORTED;
2829

2830
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
2831
		return SLJIT_ERR_UNSUPPORTED;
2832

2833
	if (type & SLJIT_SIMD_TEST)
2834
		return SLJIT_SUCCESS;
2835

2836
	if (!(srcdst & SLJIT_MEM)) {
2837
		if (type & SLJIT_SIMD_STORE)
2838
			ins = VD(srcdst) | VN(vreg) | VM(vreg);
2839
		else
2840
			ins = VD(vreg) | VN(srcdst) | VM(srcdst);
2841

2842
		if (reg_size == 4)
2843
			ins |= (1 << 30);
2844

2845
		return push_inst(compiler, ORR_v | ins);
2846
	}
2847

2848
	FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
2849

2850
	if (elem_size > 3)
2851
		elem_size = 3;
2852

2853
	ins = (type & SLJIT_SIMD_STORE) ? ST1 : LD1;
2854

2855
	if (reg_size == 4)
2856
		ins |= (1 << 30);
2857

2858
	return push_inst(compiler, ins | ((sljit_ins)elem_size << 10) | RN(srcdst) | VT(vreg));
2859
}
2860

2861
static sljit_ins simd_get_imm(sljit_s32 elem_size, sljit_uw value)
2862
{
2863
	sljit_ins result;
2864

2865
	if (elem_size > 2 && (sljit_u32)value == (value >> 32)) {
2866
		elem_size = 2;
2867
		value = (sljit_u32)value;
2868
	}
2869

2870
	if (elem_size == 2 && (sljit_u16)value == (value >> 16)) {
2871
		elem_size = 1;
2872
		value = (sljit_u16)value;
2873
	}
2874

2875
	if (elem_size == 1 && (sljit_u8)value == (value >> 8)) {
2876
		elem_size = 0;
2877
		value = (sljit_u8)value;
2878
	}
2879

2880
	switch (elem_size) {
2881
	case 0:
2882
		SLJIT_ASSERT(value <= 0xff);
2883
		result = 0xe000;
2884
		break;
2885
	case 1:
2886
		SLJIT_ASSERT(value <= 0xffff);
2887
		result = 0;
2888

2889
		while (1) {
2890
			if (value <= 0xff) {
2891
				result |= 0x8000;
2892
				break;
2893
			}
2894

2895
			if ((value & 0xff) == 0) {
2896
				value >>= 8;
2897
				result |= 0xa000;
2898
				break;
2899
			}
2900

2901
			if (result != 0)
2902
				return ~(sljit_ins)0;
2903

2904
			value ^= (sljit_uw)0xffff;
2905
			result = (1 << 29);
2906
		}
2907
		break;
2908
	case 2:
2909
		SLJIT_ASSERT(value <= 0xffffffff);
2910
		result = 0;
2911

2912
		while (1) {
2913
			if (value <= 0xff) {
2914
				result |= 0x0000;
2915
				break;
2916
			}
2917

2918
			if ((value & ~(sljit_uw)0xff00) == 0) {
2919
				value >>= 8;
2920
				result |= 0x2000;
2921
				break;
2922
			}
2923

2924
			if ((value & ~(sljit_uw)0xff0000) == 0) {
2925
				value >>= 16;
2926
				result |= 0x4000;
2927
				break;
2928
			}
2929

2930
			if ((value & ~(sljit_uw)0xff000000) == 0) {
2931
				value >>= 24;
2932
				result |= 0x6000;
2933
				break;
2934
			}
2935

2936
			if ((value & (sljit_uw)0xff) == 0xff && (value >> 16) == 0) {
2937
				value >>= 8;
2938
				result |= 0xc000;
2939
				break;
2940
			}
2941

2942
			if ((value & (sljit_uw)0xffff) == 0xffff && (value >> 24) == 0) {
2943
				value >>= 16;
2944
				result |= 0xd000;
2945
				break;
2946
			}
2947

2948
			if (result != 0)
2949
				return ~(sljit_ins)0;
2950

2951
			value ^= (sljit_uw)0xffffffff;
2952
			result = (1 << 29);
2953
		}
2954
		break;
2955
	default:
2956
		return ~(sljit_ins)0;
2957
	}
2958

2959
	return (((sljit_ins)value & 0x1f) << 5) | (((sljit_ins)value & 0xe0) << 11) | result;
2960
}
2961

2962
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
2963
	sljit_s32 vreg,
2964
	sljit_s32 src, sljit_sw srcw)
2965
{
2966
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
2967
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
2968
	sljit_ins ins, imm;
2969

2970
	CHECK_ERROR();
2971
	CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
2972

2973
	ADJUST_LOCAL_OFFSET(src, srcw);
2974

2975
	if (reg_size != 3 && reg_size != 4)
2976
		return SLJIT_ERR_UNSUPPORTED;
2977

2978
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
2979
		return SLJIT_ERR_UNSUPPORTED;
2980

2981
	if (type & SLJIT_SIMD_TEST)
2982
		return SLJIT_SUCCESS;
2983

2984
	if (src & SLJIT_MEM) {
2985
		FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
2986

2987
		ins = (sljit_ins)elem_size << 10;
2988

2989
		if (reg_size == 4)
2990
			ins |= (sljit_ins)1 << 30;
2991

2992
		return push_inst(compiler, LD1R | ins | RN(src) | VT(vreg));
2993
	}
2994

2995
	ins = (sljit_ins)1 << (16 + elem_size);
2996

2997
	if (reg_size == 4)
2998
		ins |= (sljit_ins)1 << 30;
2999

3000
	if (type & SLJIT_SIMD_FLOAT) {
3001
		if (src == SLJIT_IMM)
3002
			return push_inst(compiler, MOVI | (ins & ((sljit_ins)1 << 30)) | VD(vreg));
3003

3004
		return push_inst(compiler, DUP_e | ins | VD(vreg) | VN(src));
3005
	}
3006

3007
	if (src == SLJIT_IMM) {
3008
		if (elem_size < 3)
3009
			srcw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3010

3011
		imm = simd_get_imm(elem_size, (sljit_uw)srcw);
3012

3013
		if (imm != ~(sljit_ins)0) {
3014
			imm |= ins & ((sljit_ins)1 << 30);
3015

3016
			return push_inst(compiler, MOVI | imm | VD(vreg));
3017
		}
3018

3019
		FAIL_IF(load_immediate(compiler, TMP_REG2, srcw));
3020
		src = TMP_REG2;
3021
	}
3022

3023
	return push_inst(compiler, DUP_g | ins | VD(vreg) | RN(src));
3024
}
3025

3026
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3027
	sljit_s32 vreg, sljit_s32 lane_index,
3028
	sljit_s32 srcdst, sljit_sw srcdstw)
3029
{
3030
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3031
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3032
	sljit_ins ins;
3033

3034
	CHECK_ERROR();
3035
	CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3036

3037
	ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3038

3039
	if (reg_size != 3 && reg_size != 4)
3040
		return SLJIT_ERR_UNSUPPORTED;
3041

3042
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3043
		return SLJIT_ERR_UNSUPPORTED;
3044

3045
	if (type & SLJIT_SIMD_TEST)
3046
		return SLJIT_SUCCESS;
3047

3048
	if (type & SLJIT_SIMD_LANE_ZERO) {
3049
		ins = (reg_size == 3) ? 0 : ((sljit_ins)1 << 30);
3050

3051
		if ((type & SLJIT_SIMD_FLOAT) && vreg == srcdst) {
3052
			FAIL_IF(push_inst(compiler, ORR_v | ins | VD(TMP_FREG1) | VN(vreg) | VM(vreg)));
3053
			srcdst = TMP_FREG1;
3054
			srcdstw = 0;
3055
		}
3056

3057
		FAIL_IF(push_inst(compiler, MOVI | ins | VD(vreg)));
3058
	}
3059

3060
	if (srcdst & SLJIT_MEM) {
3061
		FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
3062

3063
		if (elem_size == 3)
3064
			ins = 0x8400;
3065
		else if (elem_size == 0)
3066
			ins = 0;
3067
		else
3068
			ins = (sljit_ins)0x2000 << elem_size;
3069

3070
		lane_index = lane_index << elem_size;
3071
		ins |= (sljit_ins)(((lane_index & 0x8) << 27) | ((lane_index & 0x7) << 10));
3072

3073
		return push_inst(compiler, ((type & SLJIT_SIMD_STORE) ? ST1_s : LD1_s) | ins | RN(srcdst) | VT(vreg));
3074
	}
3075

3076
	if (type & SLJIT_SIMD_FLOAT) {
3077
		if (type & SLJIT_SIMD_STORE)
3078
			ins = INS_e | ((sljit_ins)1 << (16 + elem_size)) | ((sljit_ins)lane_index << (11 + elem_size)) | VD(srcdst) | VN(vreg);
3079
		else
3080
			ins = INS_e | ((((sljit_ins)lane_index << 1) | 1) << (16 + elem_size)) | VD(vreg) | VN(srcdst);
3081

3082
		return push_inst(compiler, ins);
3083
	}
3084

3085
	if (srcdst == SLJIT_IMM) {
3086
		if (elem_size < 3)
3087
			srcdstw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3088

3089
		FAIL_IF(load_immediate(compiler, TMP_REG2, srcdstw));
3090
		srcdst = TMP_REG2;
3091
	}
3092

3093
	if (type & SLJIT_SIMD_STORE) {
3094
		ins = RD(srcdst) | VN(vreg);
3095

3096
		if ((type & SLJIT_SIMD_LANE_SIGNED) && (elem_size < 2 || (elem_size == 2 && !(type & SLJIT_32)))) {
3097
			ins |= SMOV;
3098

3099
			if (!(type & SLJIT_32))
3100
				ins |= (sljit_ins)1 << 30;
3101
		} else
3102
			ins |= UMOV;
3103
	} else
3104
		ins = INS | VD(vreg) | RN(srcdst);
3105

3106
	if (elem_size == 3)
3107
		ins |= (sljit_ins)1 << 30;
3108

3109
	return push_inst(compiler, ins | ((((sljit_ins)lane_index << 1) | 1) << (16 + elem_size)));
3110
}
3111

3112
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3113
	sljit_s32 vreg,
3114
	sljit_s32 src, sljit_s32 src_lane_index)
3115
{
3116
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3117
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3118
	sljit_ins ins;
3119

3120
	CHECK_ERROR();
3121
	CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
3122

3123
	if (reg_size != 3 && reg_size != 4)
3124
		return SLJIT_ERR_UNSUPPORTED;
3125

3126
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3127
		return SLJIT_ERR_UNSUPPORTED;
3128

3129
	if (type & SLJIT_SIMD_TEST)
3130
		return SLJIT_SUCCESS;
3131

3132
	ins = (((sljit_ins)src_lane_index << 1) | 1) << (16 + elem_size);
3133

3134
	if (reg_size == 4)
3135
		ins |= (sljit_ins)1 << 30;
3136

3137
	return push_inst(compiler, DUP_e | ins | VD(vreg) | VN(src));
3138
}
3139

3140
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3141
	sljit_s32 vreg,
3142
	sljit_s32 src, sljit_sw srcw)
3143
{
3144
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3145
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3146
	sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
3147

3148
	CHECK_ERROR();
3149
	CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
3150

3151
	ADJUST_LOCAL_OFFSET(src, srcw);
3152

3153
	if (reg_size != 3 && reg_size != 4)
3154
		return SLJIT_ERR_UNSUPPORTED;
3155

3156
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size != 2 || elem2_size != 3))
3157
		return SLJIT_ERR_UNSUPPORTED;
3158

3159
	if (type & SLJIT_SIMD_TEST)
3160
		return SLJIT_SUCCESS;
3161

3162
	if (src & SLJIT_MEM) {
3163
		FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
3164

3165
		if (reg_size == 4 && elem2_size - elem_size == 1)
3166
			FAIL_IF(push_inst(compiler, LD1 | ((sljit_ins)elem_size << 10) | RN(src) | VT(vreg)));
3167
		else
3168
			FAIL_IF(push_inst(compiler, LD1_s | ((sljit_ins)0x2000 << (reg_size - elem2_size + elem_size)) | RN(src) | VT(vreg)));
3169
		src = vreg;
3170
	}
3171

3172
	if (type & SLJIT_SIMD_FLOAT) {
3173
		SLJIT_ASSERT(reg_size == 4);
3174
		return push_inst(compiler, FCVTL | (1 << 22) | VD(vreg) | VN(src));
3175
	}
3176

3177
	do {
3178
		FAIL_IF(push_inst(compiler, ((type & SLJIT_SIMD_EXTEND_SIGNED) ? SSHLL : USHLL)
3179
			| ((sljit_ins)1 << (19 + elem_size)) | VD(vreg) | VN(src)));
3180
		src = vreg;
3181
	} while (++elem_size < elem2_size);
3182

3183
	return SLJIT_SUCCESS;
3184
}
3185

3186
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3187
	sljit_s32 vreg,
3188
	sljit_s32 dst, sljit_sw dstw)
3189
{
3190
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3191
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3192
	sljit_ins ins, imms;
3193
	sljit_s32 dst_r;
3194

3195
	CHECK_ERROR();
3196
	CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
3197

3198
	ADJUST_LOCAL_OFFSET(dst, dstw);
3199

3200
	if (reg_size != 3 && reg_size != 4)
3201
		return SLJIT_ERR_UNSUPPORTED;
3202

3203
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3204
		return SLJIT_ERR_UNSUPPORTED;
3205

3206
	if (type & SLJIT_SIMD_TEST)
3207
		return SLJIT_SUCCESS;
3208

3209
	switch (elem_size) {
3210
	case 0:
3211
		imms = 0x643219;
3212
		ins = USHR | (0x9 << 16);
3213
		break;
3214
	case 1:
3215
		imms = (reg_size == 4) ? 0x643219 : 0x6231;
3216
		ins = USHR | (0x11 << 16);
3217
		break;
3218
	case 2:
3219
		imms = (reg_size == 4) ? 0x6231 : 0x61;
3220
		ins = USHR | (0x21 << 16);
3221
		break;
3222
	default:
3223
		imms = 0x61;
3224
		ins = USHR | (0x41 << 16);
3225
		break;
3226
	}
3227

3228
	if (reg_size == 4)
3229
		ins |= (1 << 30);
3230

3231
	FAIL_IF(push_inst(compiler, ins | VD(TMP_FREG1) | VN(vreg)));
3232

3233
	if (reg_size == 4 && elem_size > 0)
3234
		FAIL_IF(push_inst(compiler, XTN | ((sljit_ins)(elem_size - 1) << 22) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3235

3236
	if (imms >= 0x100) {
3237
		ins = (reg_size == 4 && elem_size == 0) ? (1 << 30) : 0;
3238

3239
		do {
3240
			FAIL_IF(push_inst(compiler, USRA | ins | ((imms & 0xff) << 16) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3241
			imms >>= 8;
3242
		} while (imms >= 0x100);
3243
	}
3244

3245
	FAIL_IF(push_inst(compiler, USRA | (1 << 30) | (imms << 16) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3246

3247
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3248
	ins = (0x1 << 16);
3249

3250
	if (reg_size == 4 && elem_size == 0) {
3251
		FAIL_IF(push_inst(compiler, INS_e | (0x3 << 16) | (0x8 << 11) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3252
		ins = (0x2 << 16);
3253
	}
3254

3255
	FAIL_IF(push_inst(compiler, UMOV | ins | RD(dst_r) | VN(TMP_FREG1)));
3256

3257
	if (dst_r == TMP_REG2)
3258
		return emit_op_mem(compiler, STORE | ((type & SLJIT_32) ? INT_SIZE : WORD_SIZE), TMP_REG2, dst, dstw, TMP_REG1);
3259

3260
	return SLJIT_SUCCESS;
3261
}
3262

3263
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3264
	sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3265
{
3266
	sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3267
	sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3268
	sljit_ins ins = 0;
3269

3270
	CHECK_ERROR();
3271
	CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
3272
	ADJUST_LOCAL_OFFSET(src2, src2w);
3273

3274
	if (reg_size != 3 && reg_size != 4)
3275
		return SLJIT_ERR_UNSUPPORTED;
3276

3277
	if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3278
		return SLJIT_ERR_UNSUPPORTED;
3279

3280
	if (type & SLJIT_SIMD_TEST)
3281
		return SLJIT_SUCCESS;
3282

3283
	switch (SLJIT_SIMD_GET_OPCODE(type)) {
3284
	case SLJIT_SIMD_OP2_AND:
3285
		ins = AND_v;
3286
		break;
3287
	case SLJIT_SIMD_OP2_OR:
3288
		ins = ORR_v;
3289
		break;
3290
	case SLJIT_SIMD_OP2_XOR:
3291
		ins = EOR_v;
3292
		break;
3293
	case SLJIT_SIMD_OP2_SHUFFLE:
3294
		ins = TBL_v;
3295
		break;
3296
	}
3297

3298
	if (src2 & SLJIT_MEM) {
3299
		if (elem_size > 3)
3300
			elem_size = 3;
3301

3302
		FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src2, src2w));
3303
		push_inst(compiler,  LD1 | (reg_size == 4 ? (1 << 30) : 0) | ((sljit_ins)elem_size << 10) | RN(src2) | VT(TMP_FREG1));
3304
		src2 = TMP_FREG1;
3305
	}
3306

3307
	if (reg_size == 4)
3308
		ins |= (sljit_ins)1 << 30;
3309

3310
	return push_inst(compiler, ins | VD(dst_vreg) | VN(src1_vreg) | VM(src2));
3311
}
3312

3313
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
3314
	sljit_s32 dst_reg,
3315
	sljit_s32 mem_reg)
3316
{
3317
	sljit_ins ins;
3318

3319
	CHECK_ERROR();
3320
	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
3321

3322
#ifndef __ARM_FEATURE_ATOMICS
3323
	if (op & SLJIT_ATOMIC_USE_CAS)
3324
		return SLJIT_ERR_UNSUPPORTED;
3325
#endif /* ARM_FEATURE_ATOMICS */
3326

3327
	switch (GET_OPCODE(op)) {
3328
	case SLJIT_MOV_S8:
3329
	case SLJIT_MOV_S16:
3330
	case SLJIT_MOV_S32:
3331
		return SLJIT_ERR_UNSUPPORTED;
3332

3333
	case SLJIT_MOV32:
3334
	case SLJIT_MOV_U32:
3335
#ifdef __ARM_FEATURE_ATOMICS
3336
		if (!(op & SLJIT_ATOMIC_USE_LS))
3337
			ins = LDR ^ (1 << 30);
3338
		else
3339
#endif /* ARM_FEATURE_ATOMICS */
3340
			ins = LDXR ^ (1 << 30);
3341
		break;
3342
	case SLJIT_MOV_U8:
3343
#ifdef __ARM_FEATURE_ATOMICS
3344
		if (!(op & SLJIT_ATOMIC_USE_LS))
3345
			ins = LDRB;
3346
		else
3347
#endif /* ARM_FEATURE_ATOMICS */
3348
			ins = LDXRB;
3349
		break;
3350
	case SLJIT_MOV_U16:
3351
#ifdef __ARM_FEATURE_ATOMICS
3352
		if (!(op & SLJIT_ATOMIC_USE_LS))
3353
			ins = LDRH;
3354
		else
3355
#endif /* ARM_FEATURE_ATOMICS */
3356
			ins = LDXRH;
3357
		break;
3358
	default:
3359
#ifdef __ARM_FEATURE_ATOMICS
3360
		if (!(op & SLJIT_ATOMIC_USE_LS))
3361
			ins = LDR;
3362
		else
3363
#endif /* ARM_FEATURE_ATOMICS */
3364
			ins = LDXR;
3365
		break;
3366
	}
3367

3368
	if (op & SLJIT_ATOMIC_TEST)
3369
		return SLJIT_SUCCESS;
3370

3371
	return push_inst(compiler, ins | RN(mem_reg) | RT(dst_reg));
3372
}
3373

3374
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
3375
	sljit_s32 src_reg,
3376
	sljit_s32 mem_reg,
3377
	sljit_s32 temp_reg)
3378
{
3379
	sljit_ins ins;
3380
	sljit_ins cmp = 0;
3381

3382
	CHECK_ERROR();
3383
	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
3384

3385
#ifdef __ARM_FEATURE_ATOMICS
3386
	if (!(op & SLJIT_ATOMIC_USE_LS)) {
3387
		if (op & SLJIT_SET_ATOMIC_STORED)
3388
			cmp = (SUBS ^ W_OP) | RD(TMP_ZERO);
3389

3390
		switch (GET_OPCODE(op)) {
3391
		case SLJIT_MOV_S8:
3392
		case SLJIT_MOV_S16:
3393
		case SLJIT_MOV_S32:
3394
			return SLJIT_ERR_UNSUPPORTED;
3395

3396
		case SLJIT_MOV32:
3397
		case SLJIT_MOV_U32:
3398
			ins = CAS ^ (1 << 30);
3399
			break;
3400
		case SLJIT_MOV_U16:
3401
			ins = CASH;
3402
			break;
3403
		case SLJIT_MOV_U8:
3404
			ins = CASB;
3405
			break;
3406
		default:
3407
			ins = CAS;
3408
			if (cmp)
3409
				cmp ^= W_OP;
3410
			break;
3411
		}
3412

3413
		if (op & SLJIT_ATOMIC_TEST)
3414
			return SLJIT_SUCCESS;
3415

3416
		if (cmp)
3417
			FAIL_IF(push_inst(compiler, ((MOV ^ W_OP) ^ (cmp & W_OP)) | RM(temp_reg) | RD(TMP_REG2)));
3418

3419
		FAIL_IF(push_inst(compiler, ins | RM(temp_reg) | RN(mem_reg) | RD(src_reg)));
3420
		if (!cmp)
3421
			return SLJIT_SUCCESS;
3422

3423
		return push_inst(compiler, cmp | RM(TMP_REG2) | RN(temp_reg));
3424
	}
3425
#else /* !__ARM_FEATURE_ATOMICS */
3426
	if (op & SLJIT_ATOMIC_USE_CAS)
3427
		return SLJIT_ERR_UNSUPPORTED;
3428
#endif /* __ARM_FEATURE_ATOMICS */
3429

3430
	if (op & SLJIT_SET_ATOMIC_STORED)
3431
		cmp = (SUBI ^ W_OP) | (1 << 29);
3432

3433
	switch (GET_OPCODE(op)) {
3434
	case SLJIT_MOV_S8:
3435
	case SLJIT_MOV_S16:
3436
	case SLJIT_MOV_S32:
3437
		return SLJIT_ERR_UNSUPPORTED;
3438

3439
	case SLJIT_MOV32:
3440
	case SLJIT_MOV_U32:
3441
		ins = STXR ^ (1 << 30);
3442
		break;
3443
	case SLJIT_MOV_U8:
3444
		ins = STXRB;
3445
		break;
3446
	case SLJIT_MOV_U16:
3447
		ins = STXRH;
3448
		break;
3449
	default:
3450
		ins = STXR;
3451
		break;
3452
	}
3453

3454
	if (op & SLJIT_ATOMIC_TEST)
3455
		return SLJIT_SUCCESS;
3456

3457
	FAIL_IF(push_inst(compiler, ins | RM(TMP_REG2) | RN(mem_reg) | RT(src_reg)));
3458
	if (!cmp)
3459
		return SLJIT_SUCCESS;
3460
	return push_inst(compiler, cmp | RD(TMP_ZERO) | RN(TMP_REG2));
3461
}
3462

3463
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3464
{
3465
	sljit_s32 dst_reg;
3466
	sljit_ins ins;
3467

3468
	CHECK_ERROR();
3469
	CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3470
	ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3471

3472
	dst_reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3473

3474
	/* Not all instruction forms support accessing SP register. */
3475
	if (offset <= 0xffffff && offset >= -0xffffff) {
3476
		ins = ADDI;
3477
		if (offset < 0) {
3478
			offset = -offset;
3479
			ins = SUBI;
3480
		}
3481

3482
		if (offset <= 0xfff)
3483
			FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(SLJIT_SP) | (sljit_ins)(offset << 10)));
3484
		else {
3485
			FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(SLJIT_SP) | (sljit_ins)((offset & 0xfff000) >> (12 - 10)) | (1 << 22)));
3486

3487
			offset &= 0xfff;
3488
			if (offset != 0)
3489
				FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(dst_reg) | (sljit_ins)(offset << 10)));
3490
		}
3491
	}
3492
	else {
3493
		FAIL_IF(load_immediate (compiler, dst_reg, offset));
3494
		/* Add extended register form. */
3495
		FAIL_IF(push_inst(compiler, ADDE | (0x3 << 13) | RD(dst_reg) | RN(SLJIT_SP) | RM(dst_reg)));
3496
	}
3497

3498
	if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
3499
		return emit_op_mem(compiler, WORD_SIZE | STORE, dst_reg, dst, dstw, TMP_REG1);
3500
	return SLJIT_SUCCESS;
3501
}
3502

3503
SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3504
{
3505
	struct sljit_const *const_;
3506
	sljit_s32 dst_r;
3507

3508
	CHECK_ERROR_PTR();
3509
	CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3510
	ADJUST_LOCAL_OFFSET(dst, dstw);
3511

3512
	const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3513
	PTR_FAIL_IF(!const_);
3514
	set_const(const_, compiler);
3515

3516
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3517
	PTR_FAIL_IF(emit_imm64_const(compiler, dst_r, (sljit_uw)init_value));
3518

3519
	if (dst & SLJIT_MEM)
3520
		PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
3521
	return const_;
3522
}
3523

3524
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3525
{
3526
	struct sljit_jump *jump;
3527
	sljit_s32 dst_r;
3528

3529
	CHECK_ERROR_PTR();
3530
	CHECK_PTR(check_sljit_emit_mov_addr(compiler, dst, dstw));
3531
	ADJUST_LOCAL_OFFSET(dst, dstw);
3532

3533
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3534
	PTR_FAIL_IF(push_inst(compiler, RD(dst_r)));
3535

3536
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3537
	PTR_FAIL_IF(!jump);
3538
	set_mov_addr(jump, compiler, 1);
3539

3540
	compiler->size += 3;
3541

3542
	if (dst & SLJIT_MEM)
3543
		PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
3544

3545
	return jump;
3546
}
3547

3548
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3549
{
3550
	sljit_ins* inst = (sljit_ins*)addr;
3551
	sljit_u32 dst;
3552
	SLJIT_UNUSED_ARG(executable_offset);
3553

3554
	SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0);
3555

3556
	dst = inst[0] & 0x1f;
3557
	SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21)));
3558
	inst[0] = MOVZ | dst | (((sljit_u32)new_target & 0xffff) << 5);
3559
	inst[1] = MOVK | dst | (((sljit_u32)(new_target >> 16) & 0xffff) << 5) | (1 << 21);
3560
	inst[2] = MOVK | dst | (((sljit_u32)(new_target >> 32) & 0xffff) << 5) | (2 << 21);
3561
	inst[3] = MOVK | dst | ((sljit_u32)(new_target >> 48) << 5) | (3 << 21);
3562

3563
	SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1);
3564
	inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3565
	SLJIT_CACHE_FLUSH(inst, inst + 4);
3566
}
3567

3568
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3569
{
3570
	sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3571
}
3572

3573
Product

Resources

Company
