CoCalc -- sljitNativePPC

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitNativePPC_common.c
⁹⁹¹³ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28
{
29
	return "PowerPC" SLJIT_CPUINFO;
30
}
31

32
/* Length of an instruction word.
33
   Both for ppc-32 and ppc-64. */
34
typedef sljit_u32 sljit_ins;
35

36
#if ((defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) && (defined _AIX)) \
37
	|| (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
38
#define SLJIT_PPC_STACK_FRAME_V2 1
39
#endif
40

41
#ifdef _AIX
42
#include <sys/cache.h>
43
#endif
44

45
#if (defined _CALL_ELF && _CALL_ELF == 2)
46
#define SLJIT_PASS_ENTRY_ADDR_TO_CALL 1
47
#endif
48

49
#if (defined SLJIT_CACHE_FLUSH_OWN_IMPL && SLJIT_CACHE_FLUSH_OWN_IMPL)
50

51
static void ppc_cache_flush(sljit_ins *from, sljit_ins *to)
52
{
53
#ifdef _AIX
54
	_sync_cache_range((caddr_t)from, (int)((size_t)to - (size_t)from));
55
#elif defined(__GNUC__) || (defined(__IBM_GCC_ASM) && __IBM_GCC_ASM)
56
#	if defined(_ARCH_PWR) || defined(_ARCH_PWR2)
57
	/* Cache flush for POWER architecture. */
58
	while (from < to) {
59
		__asm__ volatile (
60
			"clf 0, %0\n"
61
			"dcs\n"
62
			: : "r"(from)
63
		);
64
		from++;
65
	}
66
	__asm__ volatile ( "ics" );
67
#	elif defined(_ARCH_COM) && !defined(_ARCH_PPC)
68
#	error "Cache flush is not implemented for PowerPC/POWER common mode."
69
#	else
70
	/* Cache flush for PowerPC architecture. */
71
	while (from < to) {
72
		__asm__ volatile (
73
			"dcbf 0, %0\n"
74
			"sync\n"
75
			"icbi 0, %0\n"
76
			: : "r"(from)
77
		);
78
		from++;
79
	}
80
	__asm__ volatile ( "isync" );
81
#	endif
82
#	ifdef __xlc__
83
#	warning "This file may fail to compile if -qfuncsect is used"
84
#	endif
85
#elif defined(__xlc__)
86
#error "Please enable GCC syntax for inline assembly statements with -qasm=gcc"
87
#else
88
#error "This platform requires a cache flush implementation."
89
#endif /* _AIX */
90
}
91

92
#endif /* (defined SLJIT_CACHE_FLUSH_OWN_IMPL && SLJIT_CACHE_FLUSH_OWN_IMPL) */
93

94
#define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
95
#define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
96
#define TMP_ZERO	(SLJIT_NUMBER_OF_REGISTERS + 4)
97

98
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
99
#define TMP_CALL_REG	(SLJIT_NUMBER_OF_REGISTERS + 5)
100
#else
101
#define TMP_CALL_REG	TMP_REG1
102
#endif
103

104
#define TMP_FREG1	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
105
#define TMP_FREG2	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
106

107
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 7] = {
108
	0, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 1, 9, 10, 31, 12
109
};
110

111
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
112
	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 0, 13
113
};
114

115
/* --------------------------------------------------------------------- */
116
/*  Instrucion forms                                                     */
117
/* --------------------------------------------------------------------- */
118
#define D(d)		((sljit_ins)reg_map[d] << 21)
119
#define S(s)		((sljit_ins)reg_map[s] << 21)
120
#define A(a)		((sljit_ins)reg_map[a] << 16)
121
#define B(b)		((sljit_ins)reg_map[b] << 11)
122
#define C(c)		((sljit_ins)reg_map[c] << 6)
123
#define FD(fd)		((sljit_ins)freg_map[fd] << 21)
124
#define FS(fs)		((sljit_ins)freg_map[fs] << 21)
125
#define FA(fa)		((sljit_ins)freg_map[fa] << 16)
126
#define FB(fb)		((sljit_ins)freg_map[fb] << 11)
127
#define FC(fc)		((sljit_ins)freg_map[fc] << 6)
128
#define IMM(imm)	((sljit_ins)(imm) & 0xffff)
129
#define CRD(d)		((sljit_ins)(d) << 21)
130

131
/* Instruction bit sections.
132
   OE and Rc flag (see ALT_SET_FLAGS). */
133
#define OE(flags)	((flags) & ALT_SET_FLAGS)
134
/* Rc flag (see ALT_SET_FLAGS). */
135
#define RC(flags)	((sljit_ins)((flags) & ALT_SET_FLAGS) >> 10)
136
#define HI(opcode)	((sljit_ins)(opcode) << 26)
137
#define LO(opcode)	((sljit_ins)(opcode) << 1)
138

139
#define ADD		(HI(31) | LO(266))
140
#define ADDC		(HI(31) | LO(10))
141
#define ADDE		(HI(31) | LO(138))
142
#define ADDI		(HI(14))
143
#define ADDIC		(HI(13))
144
#define ADDIS		(HI(15))
145
#define ADDME		(HI(31) | LO(234))
146
#define AND		(HI(31) | LO(28))
147
#define ANDI		(HI(28))
148
#define ANDIS		(HI(29))
149
#define Bx		(HI(18))
150
#define BCx		(HI(16))
151
#define BCCTR		(HI(19) | LO(528) | (3 << 11))
152
#define BLR		(HI(19) | LO(16) | (0x14 << 21))
153
#if defined(_ARCH_PWR10) && _ARCH_PWR10
154
#define BRD		(HI(31) | LO(187))
155
#endif /* POWER10 */
156
#define CNTLZD		(HI(31) | LO(58))
157
#define CNTLZW		(HI(31) | LO(26))
158
#define CMP		(HI(31) | LO(0))
159
#define CMPI		(HI(11))
160
#define CMPL		(HI(31) | LO(32))
161
#define CMPLI		(HI(10))
162
#define CROR		(HI(19) | LO(449))
163
#define DCBT		(HI(31) | LO(278))
164
#define DIVD		(HI(31) | LO(489))
165
#define DIVDU		(HI(31) | LO(457))
166
#define DIVW		(HI(31) | LO(491))
167
#define DIVWU		(HI(31) | LO(459))
168
#define EXTSB		(HI(31) | LO(954))
169
#define EXTSH		(HI(31) | LO(922))
170
#define EXTSW		(HI(31) | LO(986))
171
#define FABS		(HI(63) | LO(264))
172
#define FADD		(HI(63) | LO(21))
173
#define FADDS		(HI(59) | LO(21))
174
#define FCFID		(HI(63) | LO(846))
175
#define FCMPU		(HI(63) | LO(0))
176
#define FCTIDZ		(HI(63) | LO(815))
177
#define FCTIWZ		(HI(63) | LO(15))
178
#define FDIV		(HI(63) | LO(18))
179
#define FDIVS		(HI(59) | LO(18))
180
#define FMR		(HI(63) | LO(72))
181
#define FMUL		(HI(63) | LO(25))
182
#define FMULS		(HI(59) | LO(25))
183
#define FNEG		(HI(63) | LO(40))
184
#define FRSP		(HI(63) | LO(12))
185
#define FSUB		(HI(63) | LO(20))
186
#define FSUBS		(HI(59) | LO(20))
187
#define LD		(HI(58) | 0)
188
#define LFD		(HI(50))
189
#define LFS		(HI(48))
190
#define LDARX		(HI(31) | LO(84))
191
#if defined(_ARCH_PWR7) && _ARCH_PWR7
192
#define LDBRX		(HI(31) | LO(532))
193
#endif /* POWER7 */
194
#define LHBRX		(HI(31) | LO(790))
195
#define LWARX		(HI(31) | LO(20))
196
#define LWBRX		(HI(31) | LO(534))
197
#define LWZ		(HI(32))
198
#define MFCR		(HI(31) | LO(19))
199
#define MFLR		(HI(31) | LO(339) | 0x80000)
200
#define MFXER		(HI(31) | LO(339) | 0x10000)
201
#define MTCTR		(HI(31) | LO(467) | 0x90000)
202
#define MTLR		(HI(31) | LO(467) | 0x80000)
203
#define MTXER		(HI(31) | LO(467) | 0x10000)
204
#define MULHD		(HI(31) | LO(73))
205
#define MULHDU		(HI(31) | LO(9))
206
#define MULHW		(HI(31) | LO(75))
207
#define MULHWU		(HI(31) | LO(11))
208
#define MULLD		(HI(31) | LO(233))
209
#define MULLI		(HI(7))
210
#define MULLW		(HI(31) | LO(235))
211
#define NEG		(HI(31) | LO(104))
212
#define NOP		(HI(24))
213
#define NOR		(HI(31) | LO(124))
214
#define OR		(HI(31) | LO(444))
215
#define ORI		(HI(24))
216
#define ORIS		(HI(25))
217
#define RLDCL		(HI(30) | LO(8))
218
#define RLDICL		(HI(30) | LO(0 << 1))
219
#define RLDICR		(HI(30) | LO(1 << 1))
220
#define RLDIMI		(HI(30) | LO(3 << 1))
221
#define RLWIMI		(HI(20))
222
#define RLWINM		(HI(21))
223
#define RLWNM		(HI(23))
224
#define SLD		(HI(31) | LO(27))
225
#define SLW		(HI(31) | LO(24))
226
#define SRAD		(HI(31) | LO(794))
227
#define SRADI		(HI(31) | LO(413 << 1))
228
#define SRAW		(HI(31) | LO(792))
229
#define SRAWI		(HI(31) | LO(824))
230
#define SRD		(HI(31) | LO(539))
231
#define SRW		(HI(31) | LO(536))
232
#define STD		(HI(62) | 0)
233
#if defined(_ARCH_PWR7) && _ARCH_PWR7
234
#define STDBRX		(HI(31) | LO(660))
235
#endif /* POWER7 */
236
#define STDCX		(HI(31) | LO(214))
237
#define STDU		(HI(62) | 1)
238
#define STDUX		(HI(31) | LO(181))
239
#define STFD		(HI(54))
240
#define STFIWX		(HI(31) | LO(983))
241
#define STFS		(HI(52))
242
#define STHBRX		(HI(31) | LO(918))
243
#define STW		(HI(36))
244
#define STWBRX		(HI(31) | LO(662))
245
#define STWCX		(HI(31) | LO(150))
246
#define STWU		(HI(37))
247
#define STWUX		(HI(31) | LO(183))
248
#define SUBF		(HI(31) | LO(40))
249
#define SUBFC		(HI(31) | LO(8))
250
#define SUBFE		(HI(31) | LO(136))
251
#define SUBFIC		(HI(8))
252
#define SYNC		(HI(31) | LO(598))
253
#define XOR		(HI(31) | LO(316))
254
#define XORI		(HI(26))
255
#define XORIS		(HI(27))
256

257
#define SIMM_MAX	(0x7fff)
258
#define SIMM_MIN	(-0x8000)
259
#define UIMM_MAX	(0xffff)
260

261
/* Shift helpers. */
262
#define RLWI_SH(sh) ((sljit_ins)(sh) << 11)
263
#define RLWI_MBE(mb, me) (((sljit_ins)(mb) << 6) | ((sljit_ins)(me) << 1))
264
#define RLDI_SH(sh) ((((sljit_ins)(sh) & 0x1f) << 11) | (((sljit_ins)(sh) & 0x20) >> 4))
265
#define RLDI_MB(mb) ((((sljit_ins)(mb) & 0x1f) << 6) | ((sljit_ins)(mb) & 0x20))
266
#define RLDI_ME(me) RLDI_MB(me)
267

268
#define SLWI(shift) (RLWINM | RLWI_SH(shift) | RLWI_MBE(0, 31 - (shift)))
269
#define SLDI(shift) (RLDICR | RLDI_SH(shift) | RLDI_ME(63 - (shift)))
270
/* shift > 0 */
271
#define SRWI(shift) (RLWINM | RLWI_SH(32 - (shift)) | RLWI_MBE((shift), 31))
272
#define SRDI(shift) (RLDICL | RLDI_SH(64 - (shift)) | RLDI_MB(shift))
273

274
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
275
#define SLWI_W(shift) SLWI(shift)
276
#define TMP_MEM_OFFSET (2 * sizeof(sljit_sw))
277
#else /* !SLJIT_CONFIG_PPC_32 */
278
#define SLWI_W(shift) SLDI(shift)
279
#define TMP_MEM_OFFSET (6 * sizeof(sljit_sw))
280
#endif /* SLJIT_CONFIG_PPC_32 */
281

282
#if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
283
#define TMP_MEM_OFFSET_LO	(TMP_MEM_OFFSET)
284
#define TMP_MEM_OFFSET_HI	(TMP_MEM_OFFSET + sizeof(sljit_s32))
285
#define LWBRX_FIRST_REG		S(TMP_REG1)
286
#define LWBRX_SECOND_REG	S(dst)
287
#else /* !SLJIT_LITTLE_ENDIAN */
288
#define TMP_MEM_OFFSET_LO	(TMP_MEM_OFFSET + sizeof(sljit_s32))
289
#define TMP_MEM_OFFSET_HI	(TMP_MEM_OFFSET)
290
#define LWBRX_FIRST_REG		S(dst)
291
#define LWBRX_SECOND_REG	S(TMP_REG1)
292
#endif /* SLJIT_LITTLE_ENDIAN */
293

294
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
295
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_function_context(void** func_ptr, struct sljit_function_context* context, sljit_uw addr, void* func)
296
{
297
	sljit_uw* ptrs;
298

299
	if (func_ptr)
300
		*func_ptr = (void*)context;
301

302
	ptrs = (sljit_uw*)func;
303
	context->addr = addr ? addr : ptrs[0];
304
	context->r2 = ptrs[1];
305
	context->r11 = ptrs[2];
306
}
307
#endif
308

309
static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
310
{
311
	sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
312
	FAIL_IF(!ptr);
313
	*ptr = ins;
314
	compiler->size++;
315
	return SLJIT_SUCCESS;
316
}
317

318
static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
319
{
320
	sljit_sw diff;
321
	sljit_uw target_addr;
322
	sljit_uw jump_addr = (sljit_uw)code_ptr;
323
	sljit_uw orig_addr = jump->addr;
324
	SLJIT_UNUSED_ARG(executable_offset);
325

326
	jump->addr = jump_addr;
327
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL) && (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
328
	if (jump->flags & (SLJIT_REWRITABLE_JUMP | IS_CALL))
329
		goto exit;
330
#else
331
	if (jump->flags & SLJIT_REWRITABLE_JUMP)
332
		goto exit;
333
#endif
334

335
	if (jump->flags & JUMP_ADDR)
336
		target_addr = jump->u.target;
337
	else {
338
		SLJIT_ASSERT(jump->u.label != NULL);
339
		target_addr = (sljit_uw)(code + jump->u.label->size) + (sljit_uw)executable_offset;
340

341
		if (jump->u.label->size > orig_addr)
342
			jump_addr = (sljit_uw)(code + orig_addr);
343
	}
344

345
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL) && (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
346
	if (jump->flags & IS_CALL)
347
		goto keep_address;
348
#endif
349

350
	diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
351

352
	if (jump->flags & IS_COND) {
353
		if (diff <= 0x7fff && diff >= -0x8000) {
354
			jump->flags |= PATCH_B;
355
			return code_ptr;
356
		}
357
		if (target_addr <= 0xffff) {
358
			jump->flags |= PATCH_B | PATCH_ABS_B;
359
			return code_ptr;
360
		}
361

362
		diff -= SSIZE_OF(ins);
363
	}
364

365
	if (diff <= 0x01ffffff && diff >= -0x02000000) {
366
		jump->flags |= PATCH_B;
367
	} else if (target_addr <= 0x01ffffff) {
368
		jump->flags |= PATCH_B | PATCH_ABS_B;
369
	}
370

371
	if (jump->flags & PATCH_B) {
372
		if (!(jump->flags & IS_COND))
373
			return code_ptr;
374

375
		code_ptr[0] = BCx | (2 << 2) | ((code_ptr[0] ^ (8 << 21)) & 0x03ff0001);
376
		code_ptr[1] = Bx;
377
		jump->addr += sizeof(sljit_ins);
378
		jump->flags -= IS_COND;
379
		return code_ptr + 1;
380
	}
381

382
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
383
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
384
keep_address:
385
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
386
	if (target_addr < 0x80000000l) {
387
		jump->flags |= PATCH_ABS32;
388
		code_ptr[2] = MTCTR | S(TMP_CALL_REG);
389
		code_ptr[3] = code_ptr[0];
390
		return code_ptr + 3;
391
	}
392

393
	if (target_addr < 0x800000000000l) {
394
		jump->flags |= PATCH_ABS48;
395
		code_ptr[4] = MTCTR | S(TMP_CALL_REG);
396
		code_ptr[5] = code_ptr[0];
397
		return code_ptr + 5;
398
	}
399
#endif /* SLJIT_CONFIG_PPC_64 */
400

401
exit:
402
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
403
	code_ptr[2] = MTCTR | S(TMP_CALL_REG);
404
	code_ptr[3] = code_ptr[0];
405
#else /* !SLJIT_CONFIG_PPC_32 */
406
	code_ptr[5] = MTCTR | S(TMP_CALL_REG);
407
	code_ptr[6] = code_ptr[0];
408
#endif /* SLJIT_CONFIG_PPC_32 */
409
	return code_ptr + JUMP_MAX_SIZE - 1;
410
}
411

412
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
413

414
static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_ins *code, sljit_sw executable_offset)
415
{
416
	sljit_uw addr;
417
	SLJIT_UNUSED_ARG(executable_offset);
418

419
	SLJIT_ASSERT(jump->flags < ((sljit_uw)5 << JUMP_SIZE_SHIFT));
420
	if (jump->flags & JUMP_ADDR)
421
		addr = jump->u.target;
422
	else
423
		addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
424

425
	if (addr < 0x80000000l) {
426
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
427
		jump->flags |= PATCH_ABS32;
428
		return 1;
429
	}
430

431
	if (addr < 0x800000000000l) {
432
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
433
		jump->flags |= PATCH_ABS48;
434
		return 3;
435
	}
436

437
	SLJIT_ASSERT(jump->flags >= ((sljit_uw)4 << JUMP_SIZE_SHIFT));
438
	return 4;
439
}
440

441
#endif /* SLJIT_CONFIG_PPC_64 */
442

443
static void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
444
{
445
	sljit_uw flags = jump->flags;
446
	sljit_uw addr = (flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
447
	sljit_ins *ins = (sljit_ins*)jump->addr;
448
	sljit_s32 reg;
449
	SLJIT_UNUSED_ARG(executable_offset);
450

451
	if (flags & PATCH_B) {
452
		if (flags & IS_COND) {
453
			if (!(flags & PATCH_ABS_B)) {
454
				addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(ins, executable_offset);
455
				SLJIT_ASSERT((sljit_sw)addr <= 0x7fff && (sljit_sw)addr >= -0x8000);
456
				ins[0] = BCx | ((sljit_ins)addr & 0xfffc) | (ins[0] & 0x03ff0001);
457
			} else {
458
				SLJIT_ASSERT(addr <= 0xffff);
459
				ins[0] = BCx | ((sljit_ins)addr & 0xfffc) | 0x2 | ((*ins) & 0x03ff0001);
460
			}
461
			return;
462
		}
463

464
		if (!(flags & PATCH_ABS_B)) {
465
			addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(ins, executable_offset);
466
			SLJIT_ASSERT((sljit_sw)addr <= 0x01ffffff && (sljit_sw)addr >= -0x02000000);
467
			ins[0] = Bx | ((sljit_ins)addr & 0x03fffffc) | (ins[0] & 0x1);
468
		} else {
469
			SLJIT_ASSERT(addr <= 0x03ffffff);
470
			ins[0] = Bx | ((sljit_ins)addr & 0x03fffffc) | 0x2 | (ins[0] & 0x1);
471
		}
472
		return;
473
	}
474

475
	reg = (flags & JUMP_MOV_ADDR) ? (sljit_s32)ins[0] : TMP_CALL_REG;
476

477
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
478
	ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 16);
479
	ins[1] = ORI | S(reg) | A(reg) | IMM(addr);
480
#else /* !SLJIT_CONFIG_PPC_32 */
481

482
	/* The TMP_ZERO cannot be used because it is restored for tail calls. */
483
	if (flags & PATCH_ABS32) {
484
		SLJIT_ASSERT(addr < 0x80000000l);
485
		ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 16);
486
		ins[1] = ORI | S(reg) | A(reg) | IMM(addr);
487
		return;
488
	}
489

490
	if (flags & PATCH_ABS48) {
491
		SLJIT_ASSERT(addr < 0x800000000000l);
492
		ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 32);
493
		ins[1] = ORI | S(reg) | A(reg) | IMM(addr >> 16);
494
		ins[2] = SLDI(16) | S(reg) | A(reg);
495
		ins[3] = ORI | S(reg) | A(reg) | IMM(addr);
496
		return;
497
	}
498

499
	ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 48);
500
	ins[1] = ORI | S(reg) | A(reg) | IMM(addr >> 32);
501
	ins[2] = SLDI(32) | S(reg) | A(reg);
502
	ins[3] = ORIS | S(reg) | A(reg) | IMM(addr >> 16);
503
	ins[4] = ORI | S(reg) | A(reg) | IMM(addr);
504
#endif /* SLJIT_CONFIG_PPC_32 */
505
}
506

507
static void reduce_code_size(struct sljit_compiler *compiler)
508
{
509
	struct sljit_label *label;
510
	struct sljit_jump *jump;
511
	struct sljit_const *const_;
512
	SLJIT_NEXT_DEFINE_TYPES;
513
	sljit_uw total_size;
514
	sljit_uw size_reduce = 0;
515
	sljit_sw diff;
516

517
	label = compiler->labels;
518
	jump = compiler->jumps;
519
	const_ = compiler->consts;
520
	SLJIT_NEXT_INIT_TYPES();
521

522
	while (1) {
523
		SLJIT_GET_NEXT_MIN();
524

525
		if (next_min_addr == SLJIT_MAX_ADDRESS)
526
			break;
527

528
		if (next_min_addr == next_label_size) {
529
			label->size -= size_reduce;
530

531
			label = label->next;
532
			next_label_size = SLJIT_GET_NEXT_SIZE(label);
533
		}
534

535
		if (next_min_addr == next_const_addr) {
536
			const_->addr -= size_reduce;
537
			const_ = const_->next;
538
			next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
539
			continue;
540
		}
541

542
		if (next_min_addr != next_jump_addr)
543
			continue;
544

545
		jump->addr -= size_reduce;
546
		if (!(jump->flags & JUMP_MOV_ADDR)) {
547
			total_size = JUMP_MAX_SIZE - 1;
548

549
			if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
550
				if (jump->flags & JUMP_ADDR) {
551
					if (jump->u.target <= 0x01ffffff)
552
						total_size = 1 - 1;
553
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
554
					else if (jump->u.target < 0x80000000l)
555
						total_size = 4 - 1;
556
					else if (jump->u.target < 0x800000000000l)
557
						total_size = 6 - 1;
558
#endif /* SLJIT_CONFIG_PPC_64 */
559
				} else {
560
					/* Unit size: instruction. */
561
					diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
562
					if (jump->u.label->size > jump->addr) {
563
						SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
564
						diff -= (sljit_sw)size_reduce;
565
					}
566

567
					if (jump->flags & IS_COND) {
568
						if (diff <= (0x7fff / SSIZE_OF(ins)) && diff >= (-0x8000 / SSIZE_OF(ins)))
569
							total_size = 1 - 1;
570
						else if ((diff - 1) <= (0x01ffffff / SSIZE_OF(ins)) && (diff - 1) >= (-0x02000000 / SSIZE_OF(ins)))
571
							total_size = 2 - 1;
572
					} else if (diff <= (0x01ffffff / SSIZE_OF(ins)) && diff >= (-0x02000000 / SSIZE_OF(ins)))
573
						total_size = 1 - 1;
574
				}
575
			}
576

577
			size_reduce += (JUMP_MAX_SIZE - 1) - total_size;
578
			jump->flags |= total_size << JUMP_SIZE_SHIFT;
579
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
580
		} else {
581
			total_size = (sljit_uw)4 << JUMP_SIZE_SHIFT;
582

583
			if (jump->flags & JUMP_ADDR) {
584
				if (jump->u.target < 0x80000000l) {
585
					total_size = (sljit_uw)1 << JUMP_SIZE_SHIFT;
586
					size_reduce += 3;
587
				} else if (jump->u.target < 0x800000000000l) {
588
					total_size = (sljit_uw)3 << JUMP_SIZE_SHIFT;
589
					size_reduce += 1;
590
				}
591
			}
592
			jump->flags |= total_size;
593
#endif /* SLJIT_CONFIG_PPC_64 */
594
		}
595

596
		jump = jump->next;
597
		next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
598
	}
599

600
	compiler->size -= size_reduce;
601
}
602

603
SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
604
{
605
	struct sljit_memory_fragment *buf;
606
	sljit_ins *code;
607
	sljit_ins *code_ptr;
608
	sljit_ins *buf_ptr;
609
	sljit_ins *buf_end;
610
	sljit_uw word_count;
611
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
612
	sljit_uw jump_addr;
613
#endif
614
	SLJIT_NEXT_DEFINE_TYPES;
615
	sljit_sw executable_offset;
616

617
	struct sljit_label *label;
618
	struct sljit_jump *jump;
619
	struct sljit_const *const_;
620

621
	CHECK_ERROR_PTR();
622
	CHECK_PTR(check_sljit_generate_code(compiler));
623

624
	reduce_code_size(compiler);
625

626
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
627
	/* add to compiler->size additional instruction space to hold the trampoline and padding */
628
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
629
	compiler->size += (compiler->size & 0x1) + (sizeof(struct sljit_function_context) / sizeof(sljit_ins));
630
#else
631
	compiler->size += (sizeof(struct sljit_function_context) / sizeof(sljit_ins));
632
#endif
633
#endif
634
	code = (sljit_ins*)allocate_executable_memory(compiler->size * sizeof(sljit_ins), options, exec_allocator_data, &executable_offset);
635
	PTR_FAIL_WITH_EXEC_IF(code);
636

637
	reverse_buf(compiler);
638
	buf = compiler->buf;
639

640
	code_ptr = code;
641
	word_count = 0;
642
	label = compiler->labels;
643
	jump = compiler->jumps;
644
	const_ = compiler->consts;
645
	SLJIT_NEXT_INIT_TYPES();
646
	SLJIT_GET_NEXT_MIN();
647

648
	do {
649
		buf_ptr = (sljit_ins*)buf->memory;
650
		buf_end = buf_ptr + (buf->used_size >> 2);
651
		do {
652
			*code_ptr = *buf_ptr++;
653
			if (next_min_addr == word_count) {
654
				SLJIT_ASSERT(!label || label->size >= word_count);
655
				SLJIT_ASSERT(!jump || jump->addr >= word_count);
656
				SLJIT_ASSERT(!const_ || const_->addr >= word_count);
657

658
				/* These structures are ordered by their address. */
659
				if (next_min_addr == next_label_size) {
660
					/* Just recording the address. */
661
					label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
662
					label->size = (sljit_uw)(code_ptr - code);
663
					label = label->next;
664
					next_label_size = SLJIT_GET_NEXT_SIZE(label);
665
				}
666

667
				if (next_min_addr == next_jump_addr) {
668
					if (!(jump->flags & JUMP_MOV_ADDR)) {
669
						word_count += jump->flags >> JUMP_SIZE_SHIFT;
670
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
671
						jump_addr = (sljit_uw)code_ptr;
672
#endif
673
						code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
674
						SLJIT_ASSERT(((sljit_uw)code_ptr - jump_addr <= (jump->flags >> JUMP_SIZE_SHIFT) * sizeof(sljit_ins)));
675
					} else {
676
						jump->addr = (sljit_uw)code_ptr;
677
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
678
						word_count += jump->flags >> JUMP_SIZE_SHIFT;
679
						code_ptr += mov_addr_get_length(jump, code, executable_offset);
680
#else /* !SLJIT_CONFIG_PPC_64 */
681
						word_count++;
682
						code_ptr++;
683
#endif /* SLJIT_CONFIG_PPC_64 */
684
					}
685
					jump = jump->next;
686
					next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
687
				} else if (next_min_addr == next_const_addr) {
688
					const_->addr = (sljit_uw)code_ptr;
689
					const_ = const_->next;
690
					next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
691
				}
692

693
				SLJIT_GET_NEXT_MIN();
694
			}
695
			code_ptr++;
696
			word_count++;
697
		} while (buf_ptr < buf_end);
698

699
		buf = buf->next;
700
	} while (buf);
701

702
	if (label && label->size == word_count) {
703
		label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
704
		label->size = (sljit_uw)(code_ptr - code);
705
		label = label->next;
706
	}
707

708
	SLJIT_ASSERT(!label);
709
	SLJIT_ASSERT(!jump);
710
	SLJIT_ASSERT(!const_);
711

712
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
713
	SLJIT_ASSERT(code_ptr - code <= (sljit_sw)(compiler->size - (sizeof(struct sljit_function_context) / sizeof(sljit_ins))));
714
#else
715
	SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
716
#endif
717

718
	jump = compiler->jumps;
719
	while (jump) {
720
		generate_jump_or_mov_addr(jump, executable_offset);
721
		jump = jump->next;
722
	}
723

724
	compiler->error = SLJIT_ERR_COMPILED;
725
	compiler->executable_offset = executable_offset;
726

727
	code = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
728

729
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
730
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
731
	if (((sljit_sw)code_ptr) & 0x4)
732
		code_ptr++;
733
#endif
734
	sljit_set_function_context(NULL, (struct sljit_function_context*)code_ptr, (sljit_uw)code, (void*)sljit_generate_code);
735
#endif
736

737
	code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
738

739
	SLJIT_CACHE_FLUSH(code, code_ptr);
740
	SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
741

742
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
743
	compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins) + sizeof(struct sljit_function_context);
744
	return code_ptr;
745
#else
746
	compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
747
	return code;
748
#endif
749
}
750

751
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
752
{
753
	switch (feature_type) {
754
	case SLJIT_HAS_FPU:
755
#ifdef SLJIT_IS_FPU_AVAILABLE
756
		return (SLJIT_IS_FPU_AVAILABLE) != 0;
757
#else
758
		/* Available by default. */
759
		return 1;
760
#endif
761
	case SLJIT_HAS_REV:
762
#if defined(_ARCH_PWR10) && _ARCH_PWR10
763
		return 1;
764
#else /* !POWER10 */
765
		return 2;
766
#endif /* POWER10 */
767
	/* A saved register is set to a zero value. */
768
	case SLJIT_HAS_ZERO_REGISTER:
769
	case SLJIT_HAS_CLZ:
770
	case SLJIT_HAS_ROT:
771
	case SLJIT_HAS_PREFETCH:
772
	case SLJIT_HAS_ATOMIC:
773
	case SLJIT_HAS_MEMORY_BARRIER:
774
		return 1;
775

776
	case SLJIT_HAS_CTZ:
777
		return 2;
778

779
	default:
780
		return 0;
781
	}
782
}
783

784
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
785
{
786
	switch (type) {
787
	case SLJIT_UNORDERED_OR_EQUAL:
788
	case SLJIT_ORDERED_NOT_EQUAL:
789
	case SLJIT_UNORDERED_OR_LESS:
790
	case SLJIT_ORDERED_GREATER_EQUAL:
791
	case SLJIT_UNORDERED_OR_GREATER:
792
	case SLJIT_ORDERED_LESS_EQUAL:
793
		return 1;
794
	}
795

796
	return 0;
797
}
798

799
/* --------------------------------------------------------------------- */
800
/*  Entry, exit                                                          */
801
/* --------------------------------------------------------------------- */
802

803
/* inp_flags: */
804

805
/* Creates an index in data_transfer_insts array. */
806
#define LOAD_DATA	0x01
807
#define INDEXED		0x02
808
#define SIGNED_DATA	0x04
809

810
#define WORD_DATA	0x00
811
#define BYTE_DATA	0x08
812
#define HALF_DATA	0x10
813
#define INT_DATA	0x18
814
/* Separates integer and floating point registers */
815
#define GPR_REG		0x1f
816
#define DOUBLE_DATA	0x20
817

818
#define MEM_MASK	0x7f
819

820
#define FLOAT_DATA(op) (DOUBLE_DATA | ((op & SLJIT_32) >> 6))
821

822
/* Other inp_flags. */
823

824
/* Integer opertion and set flags -> requires exts on 64 bit systems. */
825
#define ALT_SIGN_EXT	0x000100
826
/* This flag affects the RC() and OERC() macros. */
827
#define ALT_SET_FLAGS	0x000400
828
#define ALT_FORM1	0x001000
829
#define ALT_FORM2	0x002000
830
#define ALT_FORM3	0x004000
831
#define ALT_FORM4	0x008000
832
#define ALT_FORM5	0x010000
833

834
/* Source and destination is register. */
835
#define REG_DEST	0x000001
836
#define REG1_SOURCE	0x000002
837
#define REG2_SOURCE	0x000004
838
/*
839
ALT_SIGN_EXT		0x000100
840
ALT_SET_FLAGS		0x000200
841
ALT_FORM1		0x001000
842
...
843
ALT_FORM5		0x010000 */
844

845
static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 inp_flags, sljit_s32 reg,
846
	sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg);
847

848
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
849
#include "sljitNativePPC_32.c"
850
#else
851
#include "sljitNativePPC_64.c"
852
#endif
853

854
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
855
#define STACK_STORE	STW
856
#define STACK_LOAD	LWZ
857
#else
858
#define STACK_STORE	STD
859
#define STACK_LOAD	LD
860
#endif
861

862
#if (defined SLJIT_PPC_STACK_FRAME_V2 && SLJIT_PPC_STACK_FRAME_V2)
863
#define LR_SAVE_OFFSET		(2 * SSIZE_OF(sw))
864
#else
865
#define LR_SAVE_OFFSET		SSIZE_OF(sw)
866
#endif
867

868
#define STACK_MAX_DISTANCE	(0x8000 - SSIZE_OF(sw) - LR_SAVE_OFFSET)
869

870
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
871
	sljit_s32 options, sljit_s32 arg_types,
872
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
873
{
874
	sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
875
	sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
876
	sljit_s32 i, tmp, base, offset;
877
	sljit_s32 word_arg_count = 0;
878
	sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
879
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
880
	sljit_s32 arg_count = 0;
881
#endif
882

883
	CHECK_ERROR();
884
	CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
885
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
886

887
	scratches = ENTER_GET_REGS(scratches);
888
	saveds = ENTER_GET_REGS(saveds);
889
	local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 0)
890
		+ GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
891

892
	if (!(options & SLJIT_ENTER_REG_ARG))
893
		local_size += SSIZE_OF(sw);
894

895
	local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
896
	compiler->local_size = local_size;
897

898
	FAIL_IF(push_inst(compiler, MFLR | D(0)));
899

900
	base = SLJIT_SP;
901
	offset = local_size;
902

903
	if (local_size <= STACK_MAX_DISTANCE) {
904
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
905
		FAIL_IF(push_inst(compiler, STWU | S(SLJIT_SP) | A(SLJIT_SP) | IMM(-local_size)));
906
#else
907
		FAIL_IF(push_inst(compiler, STDU | S(SLJIT_SP) | A(SLJIT_SP) | IMM(-local_size)));
908
#endif
909
	} else {
910
		base = TMP_REG1;
911
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_SP) | A(TMP_REG1) | B(SLJIT_SP)));
912
		FAIL_IF(load_immediate(compiler, TMP_REG2, -local_size));
913
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
914
		FAIL_IF(push_inst(compiler, STWUX | S(SLJIT_SP) | A(SLJIT_SP) | B(TMP_REG2)));
915
#else
916
		FAIL_IF(push_inst(compiler, STDUX | S(SLJIT_SP) | A(SLJIT_SP) | B(TMP_REG2)));
917
#endif
918
		local_size = 0;
919
		offset = 0;
920
	}
921

922
	tmp = SLJIT_FS0 - fsaveds;
923
	for (i = SLJIT_FS0; i > tmp; i--) {
924
		offset -= SSIZE_OF(f64);
925
		FAIL_IF(push_inst(compiler, STFD | FS(i) | A(base) | IMM(offset)));
926
	}
927

928
	for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
929
		offset -= SSIZE_OF(f64);
930
		FAIL_IF(push_inst(compiler, STFD | FS(i) | A(base) | IMM(offset)));
931
	}
932

933
	if (!(options & SLJIT_ENTER_REG_ARG)) {
934
		offset -= SSIZE_OF(sw);
935
		FAIL_IF(push_inst(compiler, STACK_STORE | S(TMP_ZERO) | A(base) | IMM(offset)));
936
	}
937

938
	tmp = SLJIT_S0 - saveds;
939
	for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
940
		offset -= SSIZE_OF(sw);
941
		FAIL_IF(push_inst(compiler, STACK_STORE | S(i) | A(base) | IMM(offset)));
942
	}
943

944
	for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
945
		offset -= SSIZE_OF(sw);
946
		FAIL_IF(push_inst(compiler, STACK_STORE | S(i) | A(base) | IMM(offset)));
947
	}
948

949
	FAIL_IF(push_inst(compiler, STACK_STORE | S(0) | A(base) | IMM(local_size + LR_SAVE_OFFSET)));
950

951
	if (options & SLJIT_ENTER_REG_ARG)
952
		return SLJIT_SUCCESS;
953

954
	FAIL_IF(push_inst(compiler, ADDI | D(TMP_ZERO) | A(0) | 0));
955

956
	arg_types >>= SLJIT_ARG_SHIFT;
957
	saved_arg_count = 0;
958

959
	while (arg_types > 0) {
960
		if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
961
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
962
			do {
963
				if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
964
					tmp = SLJIT_S0 - saved_arg_count;
965
					saved_arg_count++;
966
				} else if (arg_count != word_arg_count)
967
					tmp = SLJIT_R0 + word_arg_count;
968
				else
969
					break;
970

971
				FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0 + arg_count) | A(tmp) | B(SLJIT_R0 + arg_count)));
972
			} while (0);
973
#else
974
			if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
975
				FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0 + word_arg_count) | A(SLJIT_S0 - saved_arg_count) | B(SLJIT_R0 + word_arg_count)));
976
				saved_arg_count++;
977
			}
978
#endif
979
			word_arg_count++;
980
		}
981

982
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
983
		arg_count++;
984
#endif
985
		arg_types >>= SLJIT_ARG_SHIFT;
986
	}
987

988
	return SLJIT_SUCCESS;
989
}
990

991
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
992
	sljit_s32 options, sljit_s32 arg_types,
993
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
994
{
995
	sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
996
	sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
997

998
	CHECK_ERROR();
999
	CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1000
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1001

1002
	scratches = ENTER_GET_REGS(scratches);
1003
	saveds = ENTER_GET_REGS(saveds);
1004
	local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 0)
1005
		+ GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1006

1007
	if (!(options & SLJIT_ENTER_REG_ARG))
1008
		local_size += SSIZE_OF(sw);
1009

1010
	compiler->local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
1011
	return SLJIT_SUCCESS;
1012
}
1013

1014
static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
1015
{
1016
	sljit_s32 i, tmp, base, offset;
1017
	sljit_s32 local_size = compiler->local_size;
1018

1019
	SLJIT_ASSERT(TMP_CALL_REG != TMP_REG2);
1020

1021
	base = SLJIT_SP;
1022
	if (local_size > STACK_MAX_DISTANCE) {
1023
		base = TMP_REG2;
1024
		if (local_size > 2 * STACK_MAX_DISTANCE + LR_SAVE_OFFSET) {
1025
			FAIL_IF(push_inst(compiler, STACK_LOAD | D(base) | A(SLJIT_SP) | IMM(0)));
1026
			local_size = 0;
1027
		} else {
1028
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(SLJIT_SP) | IMM(local_size - STACK_MAX_DISTANCE)));
1029
			local_size = STACK_MAX_DISTANCE;
1030
		}
1031
	}
1032

1033
	offset = local_size;
1034
	if (!is_return_to)
1035
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(0) | A(base) | IMM(offset + LR_SAVE_OFFSET)));
1036

1037
	tmp = SLJIT_FS0 - compiler->fsaveds;
1038
	for (i = SLJIT_FS0; i > tmp; i--) {
1039
		offset -= SSIZE_OF(f64);
1040
		FAIL_IF(push_inst(compiler, LFD | FS(i) | A(base) | IMM(offset)));
1041
	}
1042

1043
	for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1044
		offset -= SSIZE_OF(f64);
1045
		FAIL_IF(push_inst(compiler, LFD | FS(i) | A(base) | IMM(offset)));
1046
	}
1047

1048
	if (!(compiler->options & SLJIT_ENTER_REG_ARG)) {
1049
		offset -= SSIZE_OF(sw);
1050
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(TMP_ZERO) | A(base) | IMM(offset)));
1051
	}
1052

1053
	tmp = SLJIT_S0 - compiler->saveds;
1054
	for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
1055
		offset -= SSIZE_OF(sw);
1056
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(i) | A(base) | IMM(offset)));
1057
	}
1058

1059
	for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1060
		offset -= SSIZE_OF(sw);
1061
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(i) | A(base) | IMM(offset)));
1062
	}
1063

1064
	if (!is_return_to)
1065
		push_inst(compiler, MTLR | S(0));
1066

1067
	if (local_size > 0)
1068
		return push_inst(compiler, ADDI | D(SLJIT_SP) | A(base) | IMM(local_size));
1069

1070
	SLJIT_ASSERT(base == TMP_REG2);
1071
	return push_inst(compiler, OR | S(base) | A(SLJIT_SP) | B(base));
1072
}
1073

1074
#undef STACK_STORE
1075
#undef STACK_LOAD
1076

1077
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1078
{
1079
	CHECK_ERROR();
1080
	CHECK(check_sljit_emit_return_void(compiler));
1081

1082
	FAIL_IF(emit_stack_frame_release(compiler, 0));
1083
	return push_inst(compiler, BLR);
1084
}
1085

1086
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1087
	sljit_s32 src, sljit_sw srcw)
1088
{
1089
	CHECK_ERROR();
1090
	CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1091

1092
	if (src & SLJIT_MEM) {
1093
		ADJUST_LOCAL_OFFSET(src, srcw);
1094
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
1095
		src = TMP_CALL_REG;
1096
		srcw = 0;
1097
	} else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1098
		FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
1099
		src = TMP_CALL_REG;
1100
		srcw = 0;
1101
	}
1102

1103
	FAIL_IF(emit_stack_frame_release(compiler, 1));
1104

1105
	SLJIT_SKIP_CHECKS(compiler);
1106
	return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1107
}
1108

1109
/* --------------------------------------------------------------------- */
1110
/*  Operators                                                            */
1111
/* --------------------------------------------------------------------- */
1112

1113
/* s/l - store/load (1 bit)
1114
   i/x - immediate/indexed form
1115
   u/s - signed/unsigned (1 bit)
1116
   w/b/h/i - word/byte/half/int allowed (2 bit)
1117

1118
   Some opcodes are repeated (e.g. store signed / unsigned byte is the same instruction). */
1119

1120
/* 64 bit only: [reg+imm] must be aligned to 4 bytes. */
1121
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1122
#define INT_ALIGNED	0x10000
1123
#endif
1124

1125
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
1126
#define ARCH_32_64(a, b)	a
1127
#define INST_CODE_AND_DST(inst, flags, reg) \
1128
	((sljit_ins)(inst) | (sljit_ins)(((flags) & MEM_MASK) <= GPR_REG ? D(reg) : FD(reg)))
1129
#else
1130
#define ARCH_32_64(a, b)	b
1131
#define INST_CODE_AND_DST(inst, flags, reg) \
1132
	(((sljit_ins)(inst) & ~(sljit_ins)INT_ALIGNED) | (sljit_ins)(((flags) & MEM_MASK) <= GPR_REG ? D(reg) : FD(reg)))
1133
#endif
1134

1135
static const sljit_ins data_transfer_insts[64 + 16] = {
1136

1137
/* -------- Integer -------- */
1138

1139
/* Word. */
1140

1141
/* w u i s */ ARCH_32_64(HI(36) /* stw */, HI(62) | INT_ALIGNED | 0x0 /* std */),
1142
/* w u i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x0 /* ld */),
1143
/* w u x s */ ARCH_32_64(HI(31) | LO(151) /* stwx */, HI(31) | LO(149) /* stdx */),
1144
/* w u x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(21) /* ldx */),
1145

1146
/* w s i s */ ARCH_32_64(HI(36) /* stw */, HI(62) | INT_ALIGNED | 0x0 /* std */),
1147
/* w s i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x0 /* ld */),
1148
/* w s x s */ ARCH_32_64(HI(31) | LO(151) /* stwx */, HI(31) | LO(149) /* stdx */),
1149
/* w s x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(21) /* ldx */),
1150

1151
/* Byte. */
1152

1153
/* b u i s */ HI(38) /* stb */,
1154
/* b u i l */ HI(34) /* lbz */,
1155
/* b u x s */ HI(31) | LO(215) /* stbx */,
1156
/* b u x l */ HI(31) | LO(87) /* lbzx */,
1157

1158
/* b s i s */ HI(38) /* stb */,
1159
/* b s i l */ HI(34) /* lbz */ /* EXTS_REQ */,
1160
/* b s x s */ HI(31) | LO(215) /* stbx */,
1161
/* b s x l */ HI(31) | LO(87) /* lbzx */ /* EXTS_REQ */,
1162

1163
/* Half. */
1164

1165
/* h u i s */ HI(44) /* sth */,
1166
/* h u i l */ HI(40) /* lhz */,
1167
/* h u x s */ HI(31) | LO(407) /* sthx */,
1168
/* h u x l */ HI(31) | LO(279) /* lhzx */,
1169

1170
/* h s i s */ HI(44) /* sth */,
1171
/* h s i l */ HI(42) /* lha */,
1172
/* h s x s */ HI(31) | LO(407) /* sthx */,
1173
/* h s x l */ HI(31) | LO(343) /* lhax */,
1174

1175
/* Int. */
1176

1177
/* i u i s */ HI(36) /* stw */,
1178
/* i u i l */ HI(32) /* lwz */,
1179
/* i u x s */ HI(31) | LO(151) /* stwx */,
1180
/* i u x l */ HI(31) | LO(23) /* lwzx */,
1181

1182
/* i s i s */ HI(36) /* stw */,
1183
/* i s i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x2 /* lwa */),
1184
/* i s x s */ HI(31) | LO(151) /* stwx */,
1185
/* i s x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(341) /* lwax */),
1186

1187
/* -------- Floating point -------- */
1188

1189
/* d   i s */ HI(54) /* stfd */,
1190
/* d   i l */ HI(50) /* lfd */,
1191
/* d   x s */ HI(31) | LO(727) /* stfdx */,
1192
/* d   x l */ HI(31) | LO(599) /* lfdx */,
1193

1194
/* s   i s */ HI(52) /* stfs */,
1195
/* s   i l */ HI(48) /* lfs */,
1196
/* s   x s */ HI(31) | LO(663) /* stfsx */,
1197
/* s   x l */ HI(31) | LO(535) /* lfsx */,
1198
};
1199

1200
static const sljit_ins updated_data_transfer_insts[64] = {
1201

1202
/* -------- Integer -------- */
1203

1204
/* Word. */
1205

1206
/* w u i s */ ARCH_32_64(HI(37) /* stwu */, HI(62) | INT_ALIGNED | 0x1 /* stdu */),
1207
/* w u i l */ ARCH_32_64(HI(33) /* lwzu */, HI(58) | INT_ALIGNED | 0x1 /* ldu */),
1208
/* w u x s */ ARCH_32_64(HI(31) | LO(183) /* stwux */, HI(31) | LO(181) /* stdux */),
1209
/* w u x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(53) /* ldux */),
1210

1211
/* w s i s */ ARCH_32_64(HI(37) /* stwu */, HI(62) | INT_ALIGNED | 0x1 /* stdu */),
1212
/* w s i l */ ARCH_32_64(HI(33) /* lwzu */, HI(58) | INT_ALIGNED | 0x1 /* ldu */),
1213
/* w s x s */ ARCH_32_64(HI(31) | LO(183) /* stwux */, HI(31) | LO(181) /* stdux */),
1214
/* w s x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(53) /* ldux */),
1215

1216
/* Byte. */
1217

1218
/* b u i s */ HI(39) /* stbu */,
1219
/* b u i l */ HI(35) /* lbzu */,
1220
/* b u x s */ HI(31) | LO(247) /* stbux */,
1221
/* b u x l */ HI(31) | LO(119) /* lbzux */,
1222

1223
/* b s i s */ HI(39) /* stbu */,
1224
/* b s i l */ 0 /* no such instruction */,
1225
/* b s x s */ HI(31) | LO(247) /* stbux */,
1226
/* b s x l */ 0 /* no such instruction */,
1227

1228
/* Half. */
1229

1230
/* h u i s */ HI(45) /* sthu */,
1231
/* h u i l */ HI(41) /* lhzu */,
1232
/* h u x s */ HI(31) | LO(439) /* sthux */,
1233
/* h u x l */ HI(31) | LO(311) /* lhzux */,
1234

1235
/* h s i s */ HI(45) /* sthu */,
1236
/* h s i l */ HI(43) /* lhau */,
1237
/* h s x s */ HI(31) | LO(439) /* sthux */,
1238
/* h s x l */ HI(31) | LO(375) /* lhaux */,
1239

1240
/* Int. */
1241

1242
/* i u i s */ HI(37) /* stwu */,
1243
/* i u i l */ HI(33) /* lwzu */,
1244
/* i u x s */ HI(31) | LO(183) /* stwux */,
1245
/* i u x l */ HI(31) | LO(55) /* lwzux */,
1246

1247
/* i s i s */ HI(37) /* stwu */,
1248
/* i s i l */ ARCH_32_64(HI(33) /* lwzu */, 0 /* no such instruction */),
1249
/* i s x s */ HI(31) | LO(183) /* stwux */,
1250
/* i s x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(373) /* lwaux */),
1251

1252
/* -------- Floating point -------- */
1253

1254
/* d   i s */ HI(55) /* stfdu */,
1255
/* d   i l */ HI(51) /* lfdu */,
1256
/* d   x s */ HI(31) | LO(759) /* stfdux */,
1257
/* d   x l */ HI(31) | LO(631) /* lfdux */,
1258

1259
/* s   i s */ HI(53) /* stfsu */,
1260
/* s   i l */ HI(49) /* lfsu */,
1261
/* s   x s */ HI(31) | LO(695) /* stfsux */,
1262
/* s   x l */ HI(31) | LO(567) /* lfsux */,
1263
};
1264

1265
#undef ARCH_32_64
1266

1267
/* Simple cases, (no caching is required). */
1268
static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 inp_flags, sljit_s32 reg,
1269
	sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1270
{
1271
	sljit_ins inst;
1272
	sljit_s32 offs_reg;
1273

1274
	/* Should work when (arg & REG_MASK) == 0. */
1275
	SLJIT_ASSERT(A(0) == 0);
1276
	SLJIT_ASSERT(arg & SLJIT_MEM);
1277

1278
	if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1279
		argw &= 0x3;
1280
		offs_reg = OFFS_REG(arg);
1281

1282
		if (argw != 0) {
1283
			FAIL_IF(push_inst(compiler, SLWI_W(argw) | S(OFFS_REG(arg)) | A(tmp_reg)));
1284
			offs_reg = tmp_reg;
1285
		}
1286

1287
		inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1288

1289
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1290
		SLJIT_ASSERT(!(inst & INT_ALIGNED));
1291
#endif /* SLJIT_CONFIG_PPC_64 */
1292

1293
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg & REG_MASK) | B(offs_reg));
1294
	}
1295

1296
	inst = data_transfer_insts[inp_flags & MEM_MASK];
1297
	arg &= REG_MASK;
1298

1299
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1300
	if ((inst & INT_ALIGNED) && (argw & 0x3) != 0) {
1301
		FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1302

1303
		inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1304
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | B(tmp_reg));
1305
	}
1306
#endif /* SLJIT_CONFIG_PPC_64 */
1307

1308
	if (argw <= SIMM_MAX && argw >= SIMM_MIN)
1309
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | IMM(argw));
1310

1311
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1312
	if (argw <= 0x7fff7fffl && argw >= -0x80000000l) {
1313
#endif /* SLJIT_CONFIG_PPC_64 */
1314
		FAIL_IF(push_inst(compiler, ADDIS | D(tmp_reg) | A(arg) | IMM((argw + 0x8000) >> 16)));
1315
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(tmp_reg) | IMM(argw));
1316
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1317
	}
1318

1319
	FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1320

1321
	inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1322
	return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | B(tmp_reg));
1323
#endif /* SLJIT_CONFIG_PPC_64 */
1324
}
1325

1326
static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 input_flags,
1327
	sljit_s32 dst, sljit_sw dstw,
1328
	sljit_s32 src1, sljit_sw src1w,
1329
	sljit_s32 src2, sljit_sw src2w)
1330
{
1331
	/* arg1 goes to TMP_REG1 or src reg
1332
	   arg2 goes to TMP_REG2, imm or src reg
1333
	   result goes to TMP_REG2, so put result can use TMP_REG1. */
1334
	sljit_s32 dst_r = TMP_REG2;
1335
	sljit_s32 src1_r;
1336
	sljit_s32 src2_r;
1337
	sljit_s32 src2_tmp_reg = (!(input_flags & ALT_SIGN_EXT) && GET_OPCODE(op) >= SLJIT_OP2_BASE && FAST_IS_REG(src1)) ? TMP_REG1 : TMP_REG2;
1338
	sljit_s32 flags = input_flags & (ALT_FORM1 | ALT_FORM2 | ALT_FORM3 | ALT_FORM4 | ALT_FORM5 | ALT_SIGN_EXT | ALT_SET_FLAGS);
1339

1340
	/* Destination check. */
1341
	if (FAST_IS_REG(dst)) {
1342
		dst_r = dst;
1343
		/* The REG_DEST is only used by SLJIT_MOV operations, although
1344
		 * it is set for op2 operations with unset destination. */
1345
		flags |= REG_DEST;
1346

1347
		if (op >= SLJIT_MOV && op <= SLJIT_MOV_P)
1348
			src2_tmp_reg = dst_r;
1349
	}
1350

1351
	/* Source 2. */
1352
	if (FAST_IS_REG(src2)) {
1353
		src2_r = src2;
1354
		flags |= REG2_SOURCE;
1355

1356
		if (!(flags & REG_DEST) && op >= SLJIT_MOV && op <= SLJIT_MOV_P)
1357
			dst_r = src2_r;
1358
	} else if (src2 == SLJIT_IMM) {
1359
		src2_r = TMP_ZERO;
1360
		if (src2w != 0) {
1361
			FAIL_IF(load_immediate(compiler, src2_tmp_reg, src2w));
1362
			src2_r = src2_tmp_reg;
1363
		}
1364
	} else {
1365
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, src2_tmp_reg, src2, src2w, TMP_REG1));
1366
		src2_r = src2_tmp_reg;
1367
	}
1368

1369
	/* Source 1. */
1370
	if (FAST_IS_REG(src1)) {
1371
		src1_r = src1;
1372
		flags |= REG1_SOURCE;
1373
	} else if (src1 == SLJIT_IMM) {
1374
		src1_r = TMP_ZERO;
1375
		if (src1w != 0) {
1376
			FAIL_IF(load_immediate(compiler, TMP_REG1, src1w));
1377
			src1_r = TMP_REG1;
1378
		}
1379
	} else {
1380
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, TMP_REG1, src1, src1w, TMP_REG1));
1381
		src1_r = TMP_REG1;
1382
	}
1383

1384
	FAIL_IF(emit_single_op(compiler, op, flags, dst_r, src1_r, src2_r));
1385

1386
	if (!(dst & SLJIT_MEM))
1387
		return SLJIT_SUCCESS;
1388

1389
	return emit_op_mem(compiler, input_flags, dst_r, dst, dstw, TMP_REG1);
1390
}
1391

1392
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1393
{
1394
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1395
	sljit_s32 int_op = op & SLJIT_32;
1396
#endif
1397

1398
	CHECK_ERROR();
1399
	CHECK(check_sljit_emit_op0(compiler, op));
1400

1401
	op = GET_OPCODE(op);
1402
	switch (op) {
1403
	case SLJIT_BREAKPOINT:
1404
	case SLJIT_NOP:
1405
		return push_inst(compiler, NOP);
1406
	case SLJIT_LMUL_UW:
1407
	case SLJIT_LMUL_SW:
1408
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R0)));
1409
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1410
		FAIL_IF(push_inst(compiler, MULLD | D(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R1)));
1411
		return push_inst(compiler, (op == SLJIT_LMUL_UW ? MULHDU : MULHD) | D(SLJIT_R1) | A(TMP_REG1) | B(SLJIT_R1));
1412
#else
1413
		FAIL_IF(push_inst(compiler, MULLW | D(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R1)));
1414
		return push_inst(compiler, (op == SLJIT_LMUL_UW ? MULHWU : MULHW) | D(SLJIT_R1) | A(TMP_REG1) | B(SLJIT_R1));
1415
#endif
1416
	case SLJIT_DIVMOD_UW:
1417
	case SLJIT_DIVMOD_SW:
1418
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R0)));
1419
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1420
		FAIL_IF(push_inst(compiler, (int_op ? (op == SLJIT_DIVMOD_UW ? DIVWU : DIVW) : (op == SLJIT_DIVMOD_UW ? DIVDU : DIVD)) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1)));
1421
		FAIL_IF(push_inst(compiler, (int_op ? MULLW : MULLD) | D(SLJIT_R1) | A(SLJIT_R0) | B(SLJIT_R1)));
1422
#else
1423
		FAIL_IF(push_inst(compiler, (op == SLJIT_DIVMOD_UW ? DIVWU : DIVW) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1)));
1424
		FAIL_IF(push_inst(compiler, MULLW | D(SLJIT_R1) | A(SLJIT_R0) | B(SLJIT_R1)));
1425
#endif
1426
		return push_inst(compiler, SUBF | D(SLJIT_R1) | A(SLJIT_R1) | B(TMP_REG1));
1427
	case SLJIT_DIV_UW:
1428
	case SLJIT_DIV_SW:
1429
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1430
		return push_inst(compiler, (int_op ? (op == SLJIT_DIV_UW ? DIVWU : DIVW) : (op == SLJIT_DIV_UW ? DIVDU : DIVD)) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1));
1431
#else
1432
		return push_inst(compiler, (op == SLJIT_DIV_UW ? DIVWU : DIVW) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1));
1433
#endif
1434
	case SLJIT_MEMORY_BARRIER:
1435
		return push_inst(compiler, SYNC);
1436
	case SLJIT_ENDBR:
1437
	case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1438
		return SLJIT_SUCCESS;
1439
	}
1440

1441
	return SLJIT_SUCCESS;
1442
}
1443

1444
static sljit_s32 emit_rev(struct sljit_compiler *compiler, sljit_s32 op,
1445
	sljit_s32 dst, sljit_sw dstw,
1446
	sljit_s32 src, sljit_sw srcw)
1447
{
1448
	sljit_s32 mem, offs_reg, inp_flags;
1449
	sljit_sw memw;
1450
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1451
	sljit_s32 is_32 = op & SLJIT_32;
1452

1453
	op = GET_OPCODE(op);
1454
#endif /* SLJIT_CONFIG_PPC_64 */
1455

1456
	if (!((dst | src) & SLJIT_MEM)) {
1457
		/* Both are registers. */
1458
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1459
			if (src == dst) {
1460
				FAIL_IF(push_inst(compiler, RLWIMI | S(dst) | A(dst) | RLWI_SH(16) | RLWI_MBE(8, 15)));
1461
				FAIL_IF(push_inst(compiler, RLWINM | S(dst) | A(dst) | RLWI_SH(24) | RLWI_MBE(16, 31)));
1462
			} else {
1463
				FAIL_IF(push_inst(compiler, RLWINM | S(src) | A(dst) | RLWI_SH(8) | RLWI_MBE(16, 23)));
1464
				FAIL_IF(push_inst(compiler, RLWIMI | S(src) | A(dst) | RLWI_SH(24) | RLWI_MBE(24, 31)));
1465
			}
1466

1467
			if (op == SLJIT_REV_U16)
1468
				return SLJIT_SUCCESS;
1469
			return push_inst(compiler, EXTSH | S(dst) | A(dst));
1470
		}
1471

1472
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1473
		if (!is_32) {
1474
#if defined(_ARCH_PWR10) && _ARCH_PWR10
1475
			return push_inst(compiler, BRD | S(src) | A(dst));
1476
#else /* !POWER10 */
1477
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET_HI)));
1478
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1479
			FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(SLJIT_SP) | B(TMP_REG2)));
1480
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET_LO)));
1481
			FAIL_IF(push_inst(compiler, STWBRX | S(TMP_REG1) | A(SLJIT_SP) | B(TMP_REG2)));
1482
			return push_inst(compiler, LD | D(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
1483
#endif /* POWER10 */
1484
		}
1485
#endif /* SLJIT_CONFIG_PPC_64 */
1486

1487
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET)));
1488
		FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(SLJIT_SP) | B(TMP_REG2)));
1489
		FAIL_IF(push_inst(compiler, LWZ | D(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET));
1490

1491
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1492
		if (op == SLJIT_REV_S32)
1493
			return push_inst(compiler, EXTSW | S(dst) | A(dst));
1494
#endif /* SLJIT_CONFIG_PPC_64 */
1495
		return SLJIT_SUCCESS;
1496
	}
1497

1498
	mem = src;
1499
	memw = srcw;
1500

1501
	if (dst & SLJIT_MEM) {
1502
		mem = dst;
1503
		memw = dstw;
1504

1505
		if (src & SLJIT_MEM) {
1506
			inp_flags = HALF_DATA | LOAD_DATA;
1507

1508
			if (op != SLJIT_REV_U16 && op != SLJIT_REV_S16) {
1509
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1510
				inp_flags = (is_32 ? INT_DATA : WORD_DATA) | LOAD_DATA;
1511
#else /* !SLJIT_CONFIG_PPC_64 */
1512
				inp_flags = WORD_DATA | LOAD_DATA;
1513
#endif /* SLJIT_CONFIG_PPC_64 */
1514
			}
1515

1516
			FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG1, src, srcw, TMP_REG2));
1517
			src = TMP_REG1;
1518
		}
1519
	}
1520

1521
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
1522
		offs_reg = OFFS_REG(mem);
1523
		mem &= REG_MASK;
1524
		memw &= 0x3;
1525

1526
		if (memw != 0) {
1527
			FAIL_IF(push_inst(compiler, SLWI_W(memw) | S(offs_reg) | A(TMP_REG2)));
1528
			offs_reg = TMP_REG2;
1529
		}
1530
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1531
	} else if (memw > 0x7fff7fffl || memw < -0x80000000l) {
1532
		FAIL_IF(load_immediate(compiler, TMP_REG2, memw));
1533
		offs_reg = TMP_REG2;
1534
		mem &= REG_MASK;
1535
#endif /* SLJIT_CONFIG_PPC_64 */
1536
	} else {
1537
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(mem & REG_MASK) | IMM(memw)));
1538
		if (memw > SIMM_MAX || memw < SIMM_MIN)
1539
			FAIL_IF(push_inst(compiler, ADDIS | D(TMP_REG2) | A(TMP_REG2) | IMM((memw + 0x8000) >> 16)));
1540

1541
		mem = 0;
1542
		offs_reg = TMP_REG2;
1543
	}
1544

1545
	if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1546
		if (dst & SLJIT_MEM)
1547
			return push_inst(compiler, STHBRX | S(src) | A(mem) | B(offs_reg));
1548

1549
		FAIL_IF(push_inst(compiler, LHBRX | S(dst) | A(mem) | B(offs_reg)));
1550

1551
		if (op == SLJIT_REV_U16)
1552
			return SLJIT_SUCCESS;
1553
		return push_inst(compiler, EXTSH | S(dst) | A(dst));
1554
	}
1555

1556
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1557
	if (!is_32) {
1558
		if (dst & SLJIT_MEM) {
1559
#if defined(_ARCH_PWR7) && _ARCH_PWR7
1560
			return push_inst(compiler, STDBRX | S(src) | A(mem) | B(offs_reg));
1561
#else /* !POWER7 */
1562
#if defined(SLJIT_LITTLE_ENDIAN) && SLJIT_LITTLE_ENDIAN
1563
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1564
			FAIL_IF(push_inst(compiler, STWBRX | S(TMP_REG1) | A(mem) | B(offs_reg)));
1565
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1566
			return push_inst(compiler, STWBRX | S(src) | A(mem) | B(TMP_REG2));
1567
#else /* !SLJIT_LITTLE_ENDIAN */
1568
			FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(mem) | B(offs_reg)));
1569
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1570
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1571
			return push_inst(compiler, STWBRX | S(TMP_REG1) | A(mem) | B(TMP_REG2));
1572
#endif /* SLJIT_LITTLE_ENDIAN */
1573
#endif /* POWER7 */
1574
		}
1575
#if defined(_ARCH_PWR7) && _ARCH_PWR7
1576
		return push_inst(compiler, LDBRX | S(dst) | A(mem) | B(offs_reg));
1577
#else /* !POWER7 */
1578
		FAIL_IF(push_inst(compiler, LWBRX | LWBRX_FIRST_REG | A(mem) | B(offs_reg)));
1579
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1580
		FAIL_IF(push_inst(compiler, LWBRX | LWBRX_SECOND_REG | A(mem) | B(TMP_REG2)));
1581
		return push_inst(compiler, RLDIMI | S(TMP_REG1) | A(dst) | RLDI_SH(32) | RLDI_MB(0));
1582
#endif /* POWER7 */
1583
	}
1584
#endif /* SLJIT_CONFIG_PPC_64 */
1585

1586
	if (dst & SLJIT_MEM)
1587
		return push_inst(compiler, STWBRX | S(src) | A(mem) | B(offs_reg));
1588

1589
	FAIL_IF(push_inst(compiler, LWBRX | S(dst) | A(mem) | B(offs_reg)));
1590
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1591
	if (op == SLJIT_REV_S32)
1592
		return push_inst(compiler, EXTSW | S(dst) | A(dst));
1593
#endif /* SLJIT_CONFIG_PPC_64 */
1594
	return SLJIT_SUCCESS;
1595
}
1596

1597
#define EMIT_MOV(type, type_flags, type_cast) \
1598
	emit_op(compiler, (src == SLJIT_IMM) ? SLJIT_MOV : type, flags | (type_flags), dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? type_cast srcw : srcw)
1599

1600
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1601
	sljit_s32 dst, sljit_sw dstw,
1602
	sljit_s32 src, sljit_sw srcw)
1603
{
1604
	sljit_s32 flags = HAS_FLAGS(op) ? ALT_SET_FLAGS : 0;
1605
	sljit_s32 op_flags = GET_ALL_FLAGS(op);
1606

1607
	CHECK_ERROR();
1608
	CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1609
	ADJUST_LOCAL_OFFSET(dst, dstw);
1610
	ADJUST_LOCAL_OFFSET(src, srcw);
1611

1612
	op = GET_OPCODE(op);
1613

1614
	if (GET_FLAG_TYPE(op_flags) == SLJIT_OVERFLOW)
1615
		FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1616

1617
	if (op <= SLJIT_MOV_P && FAST_IS_REG(src) && src == dst) {
1618
		if (!TYPE_CAST_NEEDED(op))
1619
			return SLJIT_SUCCESS;
1620
	}
1621

1622
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1623
	if (op_flags & SLJIT_32) {
1624
		if (op <= SLJIT_MOV_P) {
1625
			if (src & SLJIT_MEM) {
1626
				if (op == SLJIT_MOV_S32)
1627
					op = SLJIT_MOV_U32;
1628
			}
1629
			else if (src == SLJIT_IMM) {
1630
				if (op == SLJIT_MOV_U32)
1631
					op = SLJIT_MOV_S32;
1632
			}
1633
		}
1634
		else {
1635
			/* Most operations expect sign extended arguments. */
1636
			flags |= INT_DATA | SIGNED_DATA;
1637
			if (HAS_FLAGS(op_flags))
1638
				flags |= ALT_SIGN_EXT;
1639
		}
1640
	}
1641
#endif
1642

1643
	switch (op) {
1644
	case SLJIT_MOV:
1645
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
1646
	case SLJIT_MOV_U32:
1647
	case SLJIT_MOV_S32:
1648
	case SLJIT_MOV32:
1649
#endif
1650
	case SLJIT_MOV_P:
1651
		return emit_op(compiler, SLJIT_MOV, flags | WORD_DATA, dst, dstw, TMP_REG1, 0, src, srcw);
1652

1653
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1654
	case SLJIT_MOV_U32:
1655
		return EMIT_MOV(SLJIT_MOV_U32, INT_DATA, (sljit_u32));
1656

1657
	case SLJIT_MOV_S32:
1658
	case SLJIT_MOV32:
1659
		return EMIT_MOV(SLJIT_MOV_S32, INT_DATA | SIGNED_DATA, (sljit_s32));
1660
#endif
1661

1662
	case SLJIT_MOV_U8:
1663
		return EMIT_MOV(SLJIT_MOV_U8, BYTE_DATA, (sljit_u8));
1664

1665
	case SLJIT_MOV_S8:
1666
		return EMIT_MOV(SLJIT_MOV_S8, BYTE_DATA | SIGNED_DATA, (sljit_s8));
1667

1668
	case SLJIT_MOV_U16:
1669
		return EMIT_MOV(SLJIT_MOV_U16, HALF_DATA, (sljit_u16));
1670

1671
	case SLJIT_MOV_S16:
1672
		return EMIT_MOV(SLJIT_MOV_S16, HALF_DATA | SIGNED_DATA, (sljit_s16));
1673

1674
	case SLJIT_CLZ:
1675
	case SLJIT_CTZ:
1676
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1677
		if (op_flags & SLJIT_32)
1678
			flags |= ALT_FORM1;
1679
#endif /* SLJIT_CONFIG_PPC_64 */
1680
		return emit_op(compiler, op, flags, dst, dstw, TMP_REG1, 0, src, srcw);
1681
	case SLJIT_REV_U32:
1682
	case SLJIT_REV_S32:
1683
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1684
		op |= SLJIT_32;
1685
#endif /* SLJIT_CONFIG_PPC_64 */
1686
		/* fallthrough */
1687
	case SLJIT_REV:
1688
	case SLJIT_REV_U16:
1689
	case SLJIT_REV_S16:
1690
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1691
		op |= (op_flags & SLJIT_32);
1692
#endif /* SLJIT_CONFIG_PPC_64 */
1693
		return emit_rev(compiler, op, dst, dstw, src, srcw);
1694
	}
1695

1696
	return SLJIT_SUCCESS;
1697
}
1698

1699
#undef EMIT_MOV
1700

1701
/* Macros for checking different operand types / values. */
1702
#define TEST_SL_IMM(src, srcw) \
1703
	((src) == SLJIT_IMM && (srcw) <= SIMM_MAX && (srcw) >= SIMM_MIN)
1704
#define TEST_UL_IMM(src, srcw) \
1705
	((src) == SLJIT_IMM && !((srcw) & ~0xffff))
1706
#define TEST_UH_IMM(src, srcw) \
1707
	((src) == SLJIT_IMM && !((srcw) & ~(sljit_sw)0xffff0000))
1708

1709
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1710
#define TEST_SH_IMM(src, srcw) \
1711
	((src) == SLJIT_IMM && !((srcw) & 0xffff) && (srcw) <= 0x7fffffffl && (srcw) >= -0x80000000l)
1712
#define TEST_ADD_IMM(src, srcw) \
1713
	((src) == SLJIT_IMM && (srcw) <= 0x7fff7fffl && (srcw) >= -0x80000000l)
1714
#define TEST_UI_IMM(src, srcw) \
1715
	((src) == SLJIT_IMM && !((srcw) & ~0xffffffff))
1716

1717
#define TEST_ADD_FORM1(op) \
1718
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW \
1719
		|| (op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_32 | SLJIT_SET_Z | SLJIT_SET_CARRY))
1720
#define TEST_SUB_FORM2(op) \
1721
	((GET_FLAG_TYPE(op) >= SLJIT_SIG_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) \
1722
		|| (op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_32 | SLJIT_SET_Z))
1723
#define TEST_SUB_FORM3(op) \
1724
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW \
1725
		|| (op & (SLJIT_32 | SLJIT_SET_Z)) == (SLJIT_32 | SLJIT_SET_Z))
1726

1727
#else /* !SLJIT_CONFIG_PPC_64 */
1728
#define TEST_SH_IMM(src, srcw) \
1729
	((src) == SLJIT_IMM && !((srcw) & 0xffff))
1730
#define TEST_ADD_IMM(src, srcw) \
1731
	((src) == SLJIT_IMM)
1732
#define TEST_UI_IMM(src, srcw) \
1733
	((src) == SLJIT_IMM)
1734

1735
#define TEST_ADD_FORM1(op) \
1736
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1737
#define TEST_SUB_FORM2(op) \
1738
	(GET_FLAG_TYPE(op) >= SLJIT_SIG_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL)
1739
#define TEST_SUB_FORM3(op) \
1740
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1741
#endif /* SLJIT_CONFIG_PPC_64 */
1742

1743
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1744
	sljit_s32 dst, sljit_sw dstw,
1745
	sljit_s32 src1, sljit_sw src1w,
1746
	sljit_s32 src2, sljit_sw src2w)
1747
{
1748
	sljit_s32 flags = HAS_FLAGS(op) ? ALT_SET_FLAGS : 0;
1749

1750
	CHECK_ERROR();
1751
	CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1752
	ADJUST_LOCAL_OFFSET(dst, dstw);
1753
	ADJUST_LOCAL_OFFSET(src1, src1w);
1754
	ADJUST_LOCAL_OFFSET(src2, src2w);
1755

1756
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1757
	if (op & SLJIT_32) {
1758
		/* Most operations expect sign extended arguments. */
1759
		flags |= INT_DATA | SIGNED_DATA;
1760
		if (src1 == SLJIT_IMM)
1761
			src1w = (sljit_s32)(src1w);
1762
		if (src2 == SLJIT_IMM)
1763
			src2w = (sljit_s32)(src2w);
1764
		if (HAS_FLAGS(op))
1765
			flags |= ALT_SIGN_EXT;
1766
	}
1767
#endif
1768
	if (GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1769
		FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1770

1771
	switch (GET_OPCODE(op)) {
1772
	case SLJIT_ADD:
1773
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1774

1775
		if (TEST_ADD_FORM1(op))
1776
			return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM1, dst, dstw, src1, src1w, src2, src2w);
1777

1778
		if (!HAS_FLAGS(op) && (src1 == SLJIT_IMM || src2 == SLJIT_IMM)) {
1779
			if (TEST_SL_IMM(src2, src2w)) {
1780
				compiler->imm = (sljit_ins)src2w & 0xffff;
1781
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1782
			}
1783
			if (TEST_SL_IMM(src1, src1w)) {
1784
				compiler->imm = (sljit_ins)src1w & 0xffff;
1785
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2, dst, dstw, src2, src2w, TMP_REG2, 0);
1786
			}
1787
			if (TEST_SH_IMM(src2, src2w)) {
1788
				compiler->imm = (sljit_ins)(src2w >> 16) & 0xffff;
1789
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1790
			}
1791
			if (TEST_SH_IMM(src1, src1w)) {
1792
				compiler->imm = (sljit_ins)(src1w >> 16) & 0xffff;
1793
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1794
			}
1795
			/* Range between -1 and -32768 is covered above. */
1796
			if (TEST_ADD_IMM(src2, src2w)) {
1797
				compiler->imm = (sljit_ins)src2w & 0xffffffff;
1798
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1799
			}
1800
			if (TEST_ADD_IMM(src1, src1w)) {
1801
				compiler->imm = (sljit_ins)src1w & 0xffffffff;
1802
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src2, src2w, TMP_REG2, 0);
1803
			}
1804
		}
1805

1806
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1807
		if ((op & (SLJIT_32 | SLJIT_SET_Z)) == (SLJIT_32 | SLJIT_SET_Z)) {
1808
			if (TEST_SL_IMM(src2, src2w)) {
1809
				compiler->imm = (sljit_ins)src2w & 0xffff;
1810
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4 | ALT_FORM5, dst, dstw, src1, src1w, TMP_REG2, 0);
1811
			}
1812
			if (TEST_SL_IMM(src1, src1w)) {
1813
				compiler->imm = (sljit_ins)src1w & 0xffff;
1814
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4 | ALT_FORM5, dst, dstw, src2, src2w, TMP_REG2, 0);
1815
			}
1816
			return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4, dst, dstw, src1, src1w, src2, src2w);
1817
		}
1818
#endif
1819
		if (HAS_FLAGS(op)) {
1820
			if (TEST_SL_IMM(src2, src2w)) {
1821
				compiler->imm = (sljit_ins)src2w & 0xffff;
1822
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1823
			}
1824
			if (TEST_SL_IMM(src1, src1w)) {
1825
				compiler->imm = (sljit_ins)src1w & 0xffff;
1826
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1827
			}
1828
		}
1829
		return emit_op(compiler, SLJIT_ADD, flags | ((GET_FLAG_TYPE(op) == SLJIT_CARRY) ? ALT_FORM5 : 0), dst, dstw, src1, src1w, src2, src2w);
1830

1831
	case SLJIT_ADDC:
1832
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1833
		return emit_op(compiler, SLJIT_ADDC, flags, dst, dstw, src1, src1w, src2, src2w);
1834

1835
	case SLJIT_SUB:
1836
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1837

1838
		if (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_LESS_EQUAL) {
1839
			if (dst == TMP_REG1) {
1840
				if (TEST_UL_IMM(src2, src2w)) {
1841
					compiler->imm = (sljit_ins)src2w & 0xffff;
1842
					return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1843
				}
1844
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1, dst, dstw, src1, src1w, src2, src2w);
1845
			}
1846

1847
			if (src2 == SLJIT_IMM && src2w >= 0 && src2w <= (SIMM_MAX + 1)) {
1848
				compiler->imm = (sljit_ins)src2w;
1849
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1850
			}
1851
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM3, dst, dstw, src1, src1w, src2, src2w);
1852
		}
1853

1854
		if (dst == TMP_REG1 && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) {
1855
			if (TEST_SL_IMM(src2, src2w)) {
1856
				compiler->imm = (sljit_ins)src2w & 0xffff;
1857
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1858
			}
1859
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2, dst, dstw, src1, src1w, src2, src2w);
1860
		}
1861

1862
		if (TEST_SUB_FORM2(op)) {
1863
			if (src2 == SLJIT_IMM && src2w >= -SIMM_MAX && src2w <= SIMM_MAX) {
1864
				compiler->imm = (sljit_ins)src2w & 0xffff;
1865
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM3 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1866
			}
1867
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, src2, src2w);
1868
		}
1869

1870
		if (TEST_SUB_FORM3(op))
1871
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM3, dst, dstw, src1, src1w, src2, src2w);
1872

1873
		if (TEST_SL_IMM(src2, -src2w)) {
1874
			compiler->imm = (sljit_ins)(-src2w) & 0xffff;
1875
			return emit_op(compiler, SLJIT_ADD, flags | (!HAS_FLAGS(op) ? ALT_FORM2 : ALT_FORM3), dst, dstw, src1, src1w, TMP_REG2, 0);
1876
		}
1877

1878
		if (TEST_SL_IMM(src1, src1w) && !(op & SLJIT_SET_Z)) {
1879
			compiler->imm = (sljit_ins)src1w & 0xffff;
1880
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM4, dst, dstw, src2, src2w, TMP_REG2, 0);
1881
		}
1882

1883
		if (!HAS_FLAGS(op)) {
1884
			if (TEST_SH_IMM(src2, -src2w)) {
1885
				compiler->imm = (sljit_ins)((-src2w) >> 16) & 0xffff;
1886
				return emit_op(compiler, SLJIT_ADD, flags |  ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1887
			}
1888
			/* Range between -1 and -32768 is covered above. */
1889
			if (TEST_ADD_IMM(src2, -src2w)) {
1890
				compiler->imm = (sljit_ins)-src2w;
1891
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1892
			}
1893
		}
1894

1895
		/* We know ALT_SIGN_EXT is set if it is an SLJIT_32 on 64 bit systems. */
1896
		return emit_op(compiler, SLJIT_SUB, flags | ((GET_FLAG_TYPE(op) == SLJIT_CARRY) ? ALT_FORM5 : 0), dst, dstw, src1, src1w, src2, src2w);
1897

1898
	case SLJIT_SUBC:
1899
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1900
		return emit_op(compiler, SLJIT_SUBC, flags, dst, dstw, src1, src1w, src2, src2w);
1901

1902
	case SLJIT_MUL:
1903
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1904
		if (op & SLJIT_32)
1905
			flags |= ALT_FORM2;
1906
#endif
1907
		if (!HAS_FLAGS(op)) {
1908
			if (TEST_SL_IMM(src2, src2w)) {
1909
				compiler->imm = (sljit_ins)src2w & 0xffff;
1910
				return emit_op(compiler, SLJIT_MUL, flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1911
			}
1912
			if (TEST_SL_IMM(src1, src1w)) {
1913
				compiler->imm = (sljit_ins)src1w & 0xffff;
1914
				return emit_op(compiler, SLJIT_MUL, flags | ALT_FORM1, dst, dstw, src2, src2w, TMP_REG2, 0);
1915
			}
1916
		}
1917
		else
1918
			FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1919
		return emit_op(compiler, SLJIT_MUL, flags, dst, dstw, src1, src1w, src2, src2w);
1920

1921
	case SLJIT_XOR:
1922
		if (src2 == SLJIT_IMM && src2w == -1) {
1923
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM4, dst, dstw, TMP_REG1, 0, src1, src1w);
1924
		}
1925
		if (src1 == SLJIT_IMM && src1w == -1) {
1926
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM4, dst, dstw, TMP_REG1, 0, src2, src2w);
1927
		}
1928
		/* fallthrough */
1929
	case SLJIT_AND:
1930
	case SLJIT_OR:
1931
		/* Commutative unsigned operations. */
1932
		if (!HAS_FLAGS(op) || GET_OPCODE(op) == SLJIT_AND) {
1933
			if (TEST_UL_IMM(src2, src2w)) {
1934
				compiler->imm = (sljit_ins)src2w;
1935
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1936
			}
1937
			if (TEST_UL_IMM(src1, src1w)) {
1938
				compiler->imm = (sljit_ins)src1w;
1939
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src2, src2w, TMP_REG2, 0);
1940
			}
1941
			if (TEST_UH_IMM(src2, src2w)) {
1942
				compiler->imm = (sljit_ins)(src2w >> 16) & 0xffff;
1943
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1944
			}
1945
			if (TEST_UH_IMM(src1, src1w)) {
1946
				compiler->imm = (sljit_ins)(src1w >> 16) & 0xffff;
1947
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM2, dst, dstw, src2, src2w, TMP_REG2, 0);
1948
			}
1949
		}
1950
		if (!HAS_FLAGS(op) && GET_OPCODE(op) != SLJIT_AND) {
1951
			/* Unlike or and xor, the and resets unwanted bits as well. */
1952
			if (TEST_UI_IMM(src2, src2w)) {
1953
				compiler->imm = (sljit_ins)src2w;
1954
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1955
			}
1956
			if (TEST_UI_IMM(src1, src1w)) {
1957
				compiler->imm = (sljit_ins)src1w;
1958
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1959
			}
1960
		}
1961
		return emit_op(compiler, GET_OPCODE(op), flags, dst, dstw, src1, src1w, src2, src2w);
1962

1963
	case SLJIT_SHL:
1964
	case SLJIT_MSHL:
1965
	case SLJIT_LSHR:
1966
	case SLJIT_MLSHR:
1967
	case SLJIT_ASHR:
1968
	case SLJIT_MASHR:
1969
	case SLJIT_ROTL:
1970
	case SLJIT_ROTR:
1971
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1972
		if (op & SLJIT_32)
1973
			flags |= ALT_FORM2;
1974
#endif
1975
		if (src2 == SLJIT_IMM) {
1976
			compiler->imm = (sljit_ins)src2w;
1977
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1978
		}
1979
		return emit_op(compiler, GET_OPCODE(op), flags, dst, dstw, src1, src1w, src2, src2w);
1980
	}
1981

1982
	return SLJIT_SUCCESS;
1983
}
1984

1985
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
1986
	sljit_s32 src1, sljit_sw src1w,
1987
	sljit_s32 src2, sljit_sw src2w)
1988
{
1989
	CHECK_ERROR();
1990
	CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
1991

1992
	SLJIT_SKIP_CHECKS(compiler);
1993
	return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
1994
}
1995

1996
#undef TEST_ADD_FORM1
1997
#undef TEST_SUB_FORM2
1998
#undef TEST_SUB_FORM3
1999

2000
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2001
	sljit_s32 dst_reg,
2002
	sljit_s32 src1, sljit_sw src1w,
2003
	sljit_s32 src2, sljit_sw src2w)
2004
{
2005
	CHECK_ERROR();
2006
	CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2007

2008
	switch (GET_OPCODE(op)) {
2009
	case SLJIT_MULADD:
2010
		SLJIT_SKIP_CHECKS(compiler);
2011
		FAIL_IF(sljit_emit_op2(compiler, SLJIT_MUL | (op & SLJIT_32), TMP_REG2, 0, src1, src1w, src2, src2w));
2012
		return push_inst(compiler, ADD | D(dst_reg) | A(dst_reg) | B(TMP_REG2));
2013
	}
2014

2015
	return SLJIT_SUCCESS;
2016
}
2017

2018
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2019
	sljit_s32 dst_reg,
2020
	sljit_s32 src1_reg,
2021
	sljit_s32 src2_reg,
2022
	sljit_s32 src3, sljit_sw src3w)
2023
{
2024
	sljit_s32 is_right;
2025
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2026
	sljit_s32 inp_flags = ((op & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2027
	sljit_sw bit_length = (op & SLJIT_32) ? 32 : 64;
2028
#else /* !SLJIT_CONFIG_PPC_64 */
2029
	sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2030
	sljit_sw bit_length = 32;
2031
#endif /* SLJIT_CONFIG_PPC_64 */
2032

2033
	CHECK_ERROR();
2034
	CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2035

2036
	is_right = (GET_OPCODE(op) == SLJIT_LSHR || GET_OPCODE(op) == SLJIT_MLSHR);
2037

2038
	if (src1_reg == src2_reg) {
2039
		SLJIT_SKIP_CHECKS(compiler);
2040
		return sljit_emit_op2(compiler, (is_right ? SLJIT_ROTR : SLJIT_ROTL) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
2041
	}
2042

2043
	ADJUST_LOCAL_OFFSET(src3, src3w);
2044

2045
	if (src3 == SLJIT_IMM) {
2046
		src3w &= bit_length - 1;
2047

2048
		if (src3w == 0)
2049
			return SLJIT_SUCCESS;
2050

2051
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2052
		if (!(op & SLJIT_32)) {
2053
			if (is_right) {
2054
				FAIL_IF(push_inst(compiler, SRDI(src3w) | S(src1_reg) | A(dst_reg)));
2055
				return push_inst(compiler, RLDIMI | S(src2_reg) | A(dst_reg) | RLDI_SH(64 - src3w) | RLDI_MB(0));
2056
			}
2057

2058
			FAIL_IF(push_inst(compiler, SLDI(src3w) | S(src1_reg) | A(dst_reg)));
2059
			/* Computes SRDI(64 - src2w). */
2060
			FAIL_IF(push_inst(compiler, RLDICL | S(src2_reg) | A(TMP_REG1) | RLDI_SH(src3w) | RLDI_MB(64 - src3w)));
2061
			return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2062
		}
2063
#endif /* SLJIT_CONFIG_PPC_64 */
2064

2065
		if (is_right) {
2066
			FAIL_IF(push_inst(compiler, SRWI(src3w) | S(src1_reg) | A(dst_reg)));
2067
			return push_inst(compiler, RLWIMI | S(src2_reg) | A(dst_reg) | RLWI_SH(32 - src3w) | RLWI_MBE(0, src3w - 1));
2068
		}
2069

2070
		FAIL_IF(push_inst(compiler, SLWI(src3w) | S(src1_reg) | A(dst_reg)));
2071
		return push_inst(compiler, RLWIMI | S(src2_reg) | A(dst_reg) | RLWI_SH(src3w) | RLWI_MBE(32 - src3w, 31));
2072
	}
2073

2074
	if (src3 & SLJIT_MEM) {
2075
		FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG2, src3, src3w, TMP_REG2));
2076
		src3 = TMP_REG2;
2077
	}
2078

2079
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2080
	if (!(op & SLJIT_32)) {
2081
		if (GET_OPCODE(op) == SLJIT_MSHL || GET_OPCODE(op) == SLJIT_MLSHR || dst_reg == src3) {
2082
			FAIL_IF(push_inst(compiler, ANDI | S(src3) | A(TMP_REG2) | 0x3f));
2083
			src3 = TMP_REG2;
2084
		}
2085

2086
		FAIL_IF(push_inst(compiler, (is_right ? SRD : SLD) | S(src1_reg) | A(dst_reg) | B(src3)));
2087
		FAIL_IF(push_inst(compiler, (is_right ? SLDI(1) : SRDI(1)) | S(src2_reg) | A(TMP_REG1)));
2088
		FAIL_IF(push_inst(compiler, XORI | S(src3) | A(TMP_REG2) | 0x3f));
2089
		FAIL_IF(push_inst(compiler, (is_right ? SLD : SRD) | S(TMP_REG1) | A(TMP_REG1) | B(TMP_REG2)));
2090
		return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2091
	}
2092
#endif /* SLJIT_CONFIG_PPC_64 */
2093

2094
	if (GET_OPCODE(op) == SLJIT_MSHL || GET_OPCODE(op) == SLJIT_MLSHR || dst_reg == src3) {
2095
		FAIL_IF(push_inst(compiler, ANDI | S(src3) | A(TMP_REG2) | 0x1f));
2096
		src3 = TMP_REG2;
2097
	}
2098

2099
	FAIL_IF(push_inst(compiler, (is_right ? SRW : SLW) | S(src1_reg) | A(dst_reg) | B(src3)));
2100
	FAIL_IF(push_inst(compiler, (is_right ? SLWI(1) : SRWI(1)) | S(src2_reg) | A(TMP_REG1)));
2101
	FAIL_IF(push_inst(compiler, XORI | S(src3) | A(TMP_REG2) | 0x1f));
2102
	FAIL_IF(push_inst(compiler, (is_right ? SLW : SRW) | S(TMP_REG1) | A(TMP_REG1) | B(TMP_REG2)));
2103
	return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2104
}
2105

2106
static sljit_s32 emit_prefetch(struct sljit_compiler *compiler,
2107
        sljit_s32 src, sljit_sw srcw)
2108
{
2109
	if (!(src & OFFS_REG_MASK)) {
2110
		if (srcw == 0 && (src & REG_MASK))
2111
			return push_inst(compiler, DCBT | A(0) | B(src & REG_MASK));
2112

2113
		FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2114
		/* Works with SLJIT_MEM0() case as well. */
2115
		return push_inst(compiler, DCBT | A(src & REG_MASK) | B(TMP_REG1));
2116
	}
2117

2118
	srcw &= 0x3;
2119

2120
	if (srcw == 0)
2121
		return push_inst(compiler, DCBT | A(src & REG_MASK) | B(OFFS_REG(src)));
2122

2123
	FAIL_IF(push_inst(compiler, SLWI_W(srcw) | S(OFFS_REG(src)) | A(TMP_REG1)));
2124
	return push_inst(compiler, DCBT | A(src & REG_MASK) | B(TMP_REG1));
2125
}
2126

2127
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2128
	sljit_s32 src, sljit_sw srcw)
2129
{
2130
	CHECK_ERROR();
2131
	CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2132
	ADJUST_LOCAL_OFFSET(src, srcw);
2133

2134
	switch (op) {
2135
	case SLJIT_FAST_RETURN:
2136
		if (FAST_IS_REG(src))
2137
			FAIL_IF(push_inst(compiler, MTLR | S(src)));
2138
		else {
2139
			FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG2, src, srcw, TMP_REG2));
2140
			FAIL_IF(push_inst(compiler, MTLR | S(TMP_REG2)));
2141
		}
2142

2143
		return push_inst(compiler, BLR);
2144
	case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2145
		return SLJIT_SUCCESS;
2146
	case SLJIT_PREFETCH_L1:
2147
	case SLJIT_PREFETCH_L2:
2148
	case SLJIT_PREFETCH_L3:
2149
	case SLJIT_PREFETCH_ONCE:
2150
		return emit_prefetch(compiler, src, srcw);
2151
	}
2152

2153
	return SLJIT_SUCCESS;
2154
}
2155

2156
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2157
	sljit_s32 dst, sljit_sw dstw)
2158
{
2159
	sljit_s32 dst_r;
2160

2161
	CHECK_ERROR();
2162
	CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
2163
	ADJUST_LOCAL_OFFSET(dst, dstw);
2164

2165
	switch (op) {
2166
	case SLJIT_FAST_ENTER:
2167
		if (FAST_IS_REG(dst))
2168
			return push_inst(compiler, MFLR | D(dst));
2169

2170
		FAIL_IF(push_inst(compiler, MFLR | D(TMP_REG1)));
2171
		break;
2172
	case SLJIT_GET_RETURN_ADDRESS:
2173
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2174
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, dst_r, SLJIT_MEM1(SLJIT_SP), compiler->local_size + LR_SAVE_OFFSET, TMP_REG2));
2175
		break;
2176
	}
2177

2178
	if (dst & SLJIT_MEM)
2179
		return emit_op_mem(compiler, WORD_DATA, TMP_REG1, dst, dstw, TMP_REG2);
2180

2181
	return SLJIT_SUCCESS;
2182
}
2183

2184
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2185
{
2186
	CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
2187

2188
	if (type == SLJIT_GP_REGISTER)
2189
		return reg_map[reg];
2190

2191
	if (type != SLJIT_FLOAT_REGISTER)
2192
		return -1;
2193

2194
	return freg_map[reg];
2195
}
2196

2197
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2198
	void *instruction, sljit_u32 size)
2199
{
2200
	SLJIT_UNUSED_ARG(size);
2201

2202
	CHECK_ERROR();
2203
	CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2204

2205
	return push_inst(compiler, *(sljit_ins*)instruction);
2206
}
2207

2208
/* --------------------------------------------------------------------- */
2209
/*  Floating point operators                                             */
2210
/* --------------------------------------------------------------------- */
2211

2212
#define SELECT_FOP(op, single, double) ((sljit_ins)((op & SLJIT_32) ? single : double))
2213

2214
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2215
	sljit_s32 dst, sljit_sw dstw,
2216
	sljit_s32 src, sljit_sw srcw)
2217
{
2218
	if (src & SLJIT_MEM) {
2219
		/* We can ignore the temporary data store on the stack from caching point of view. */
2220
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src, srcw, TMP_REG1));
2221
		src = TMP_FREG1;
2222
	}
2223

2224
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2225
	op = GET_OPCODE(op);
2226
	FAIL_IF(push_inst(compiler, (op == SLJIT_CONV_S32_FROM_F64 ? FCTIWZ : FCTIDZ) | FD(TMP_FREG1) | FB(src)));
2227

2228
	if (op == SLJIT_CONV_SW_FROM_F64) {
2229
		if (FAST_IS_REG(dst)) {
2230
			FAIL_IF(push_inst(compiler, STFD | FS(TMP_FREG1) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2231
			return push_inst(compiler, LD | S(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2232
		}
2233
		return emit_op_mem(compiler, DOUBLE_DATA, TMP_FREG1, dst, dstw, TMP_REG1);
2234
	}
2235
#else /* !SLJIT_CONFIG_PPC_64 */
2236
	FAIL_IF(push_inst(compiler, FCTIWZ | FD(TMP_FREG1) | FB(src)));
2237
#endif /* SLJIT_CONFIG_PPC_64 */
2238

2239
	if (FAST_IS_REG(dst)) {
2240
		FAIL_IF(load_immediate(compiler, TMP_REG1, TMP_MEM_OFFSET));
2241
		FAIL_IF(push_inst(compiler, STFIWX | FS(TMP_FREG1) | A(SLJIT_SP) | B(TMP_REG1)));
2242
		return push_inst(compiler, LWZ | S(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2243
	}
2244

2245
	SLJIT_ASSERT(dst & SLJIT_MEM);
2246

2247
	if (dst & OFFS_REG_MASK) {
2248
		dstw &= 0x3;
2249
		if (dstw) {
2250
			FAIL_IF(push_inst(compiler, SLWI_W(dstw) | S(OFFS_REG(dst)) | A(TMP_REG1)));
2251
			dstw = TMP_REG1;
2252
		} else
2253
			dstw = OFFS_REG(dst);
2254
	}
2255
	else {
2256
		if ((dst & REG_MASK) && !dstw) {
2257
			dstw = dst & REG_MASK;
2258
			dst = 0;
2259
		} else {
2260
			/* This works regardless we have SLJIT_MEM1 or SLJIT_MEM0. */
2261
			FAIL_IF(load_immediate(compiler, TMP_REG1, dstw));
2262
			dstw = TMP_REG1;
2263
		}
2264
	}
2265

2266
	return push_inst(compiler, STFIWX | FS(TMP_FREG1) | A(dst & REG_MASK) | B(dstw));
2267
}
2268

2269
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2270
	sljit_s32 src1, sljit_sw src1w,
2271
	sljit_s32 src2, sljit_sw src2w)
2272
{
2273
	if (src1 & SLJIT_MEM) {
2274
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, TMP_REG1));
2275
		src1 = TMP_FREG1;
2276
	}
2277

2278
	if (src2 & SLJIT_MEM) {
2279
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, TMP_REG2));
2280
		src2 = TMP_FREG2;
2281
	}
2282

2283
	FAIL_IF(push_inst(compiler, FCMPU | CRD(4) | FA(src1) | FB(src2)));
2284

2285
	switch (GET_FLAG_TYPE(op)) {
2286
	case SLJIT_UNORDERED_OR_EQUAL:
2287
		return push_inst(compiler, CROR | ((4 + 2) << 21) | ((4 + 2) << 16) | ((4 + 3) << 11));
2288
	case SLJIT_UNORDERED_OR_LESS:
2289
		return push_inst(compiler, CROR | ((4 + 0) << 21) | ((4 + 0) << 16) | ((4 + 3) << 11));
2290
	case SLJIT_UNORDERED_OR_GREATER:
2291
		return push_inst(compiler, CROR | ((4 + 1) << 21) | ((4 + 1) << 16) | ((4 + 3) << 11));
2292
	}
2293

2294
	return SLJIT_SUCCESS;
2295
}
2296

2297
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2298
	sljit_s32 dst, sljit_sw dstw,
2299
	sljit_s32 src, sljit_sw srcw)
2300
{
2301
	sljit_s32 dst_r;
2302

2303
	CHECK_ERROR();
2304

2305
	SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100) && !(DOUBLE_DATA & 0x4), float_transfer_bit_error);
2306
	SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2307

2308
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2309
		op ^= SLJIT_32;
2310

2311
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2312

2313
	if (src & SLJIT_MEM) {
2314
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, dst_r, src, srcw, TMP_REG1));
2315
		src = dst_r;
2316
	}
2317

2318
	switch (GET_OPCODE(op)) {
2319
	case SLJIT_CONV_F64_FROM_F32:
2320
		op ^= SLJIT_32;
2321
		if (op & SLJIT_32) {
2322
			FAIL_IF(push_inst(compiler, FRSP | FD(dst_r) | FB(src)));
2323
			break;
2324
		}
2325
		/* Fall through. */
2326
	case SLJIT_MOV_F64:
2327
		if (src != dst_r) {
2328
			if (!(dst & SLJIT_MEM))
2329
				FAIL_IF(push_inst(compiler, FMR | FD(dst_r) | FB(src)));
2330
			else
2331
				dst_r = src;
2332
		}
2333
		break;
2334
	case SLJIT_NEG_F64:
2335
		FAIL_IF(push_inst(compiler, FNEG | FD(dst_r) | FB(src)));
2336
		break;
2337
	case SLJIT_ABS_F64:
2338
		FAIL_IF(push_inst(compiler, FABS | FD(dst_r) | FB(src)));
2339
		break;
2340
	}
2341

2342
	if (dst & SLJIT_MEM)
2343
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op), dst_r, dst, dstw, TMP_REG1));
2344
	return SLJIT_SUCCESS;
2345
}
2346

2347
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2348
	sljit_s32 dst, sljit_sw dstw,
2349
	sljit_s32 src1, sljit_sw src1w,
2350
	sljit_s32 src2, sljit_sw src2w)
2351
{
2352
	sljit_s32 dst_r;
2353

2354
	CHECK_ERROR();
2355
	CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2356
	ADJUST_LOCAL_OFFSET(dst, dstw);
2357
	ADJUST_LOCAL_OFFSET(src1, src1w);
2358
	ADJUST_LOCAL_OFFSET(src2, src2w);
2359

2360
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG2;
2361

2362
	if (src1 & SLJIT_MEM) {
2363
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, TMP_REG1));
2364
		src1 = TMP_FREG1;
2365
	}
2366

2367
	if (src2 & SLJIT_MEM) {
2368
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, TMP_REG1));
2369
		src2 = TMP_FREG2;
2370
	}
2371

2372
	switch (GET_OPCODE(op)) {
2373
	case SLJIT_ADD_F64:
2374
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FADDS, FADD) | FD(dst_r) | FA(src1) | FB(src2)));
2375
		break;
2376
	case SLJIT_SUB_F64:
2377
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FSUBS, FSUB) | FD(dst_r) | FA(src1) | FB(src2)));
2378
		break;
2379
	case SLJIT_MUL_F64:
2380
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FMULS, FMUL) | FD(dst_r) | FA(src1) | FC(src2) /* FMUL use FC as src2 */));
2381
		break;
2382
	case SLJIT_DIV_F64:
2383
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FDIVS, FDIV) | FD(dst_r) | FA(src1) | FB(src2)));
2384
		break;
2385
	case SLJIT_COPYSIGN_F64:
2386
		FAIL_IF(push_inst(compiler, ((op & SLJIT_32) ? STFS : STFD) | FS(src2) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2387
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
2388
		FAIL_IF(push_inst(compiler, LWZ | S(TMP_REG1) | A(SLJIT_SP) | ((op & SLJIT_32) ? TMP_MEM_OFFSET : TMP_MEM_OFFSET_HI)));
2389
#else /* !SLJIT_CONFIG_PPC_32 */
2390
		FAIL_IF(push_inst(compiler, ((op & SLJIT_32) ? LWZ : LD) | S(TMP_REG1) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2391
#endif /* SLJIT_CONFIG_PPC_32 */
2392
		FAIL_IF(push_inst(compiler, FABS | FD(dst_r) | FB(src1)));
2393
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
2394
		FAIL_IF(push_inst(compiler, CMPI | CRD(0) | A(TMP_REG1) | 0));
2395
#else /* !SLJIT_CONFIG_PPC_32 */
2396
		FAIL_IF(push_inst(compiler, CMPI | CRD(0 | ((op & SLJIT_32) ? 0 : 1)) | A(TMP_REG1) | 0));
2397
#endif /* SLJIT_CONFIG_PPC_32 */
2398
		FAIL_IF(push_inst(compiler, BCx | (4 << 21) | (0 << 16) | 8));
2399
		return push_inst(compiler, FNEG | FD(dst_r) | FB(dst_r));
2400
	}
2401

2402
	if (dst & SLJIT_MEM)
2403
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op), TMP_FREG2, dst, dstw, TMP_REG1));
2404

2405
	return SLJIT_SUCCESS;
2406
}
2407

2408
#undef SELECT_FOP
2409

2410
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2411
	sljit_s32 freg, sljit_f32 value)
2412
{
2413
	union {
2414
		sljit_s32 imm;
2415
		sljit_f32 value;
2416
	} u;
2417

2418
	CHECK_ERROR();
2419
	CHECK(check_sljit_emit_fset32(compiler, freg, value));
2420

2421
	u.value = value;
2422

2423
	if (u.imm != 0)
2424
		FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm));
2425

2426
	FAIL_IF(push_inst(compiler, STW | S(u.imm != 0 ? TMP_REG1 : TMP_ZERO) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2427
	return push_inst(compiler, LFS | FS(freg) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2428
}
2429

2430
/* --------------------------------------------------------------------- */
2431
/*  Conditional instructions                                             */
2432
/* --------------------------------------------------------------------- */
2433

2434
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2435
{
2436
	struct sljit_label *label;
2437

2438
	CHECK_ERROR_PTR();
2439
	CHECK_PTR(check_sljit_emit_label(compiler));
2440

2441
	if (compiler->last_label && compiler->last_label->size == compiler->size)
2442
		return compiler->last_label;
2443

2444
	label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2445
	PTR_FAIL_IF(!label);
2446
	set_label(label, compiler);
2447
	return label;
2448
}
2449

2450
static sljit_ins get_bo_bi_flags(struct sljit_compiler *compiler, sljit_s32 type)
2451
{
2452
	switch (type) {
2453
	case SLJIT_NOT_CARRY:
2454
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
2455
			return (4 << 21) | (2 << 16);
2456
		/* fallthrough */
2457

2458
	case SLJIT_EQUAL:
2459
	case SLJIT_ATOMIC_STORED:
2460
		return (12 << 21) | (2 << 16);
2461

2462
	case SLJIT_CARRY:
2463
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
2464
			return (12 << 21) | (2 << 16);
2465
		/* fallthrough */
2466

2467
	case SLJIT_NOT_EQUAL:
2468
	case SLJIT_ATOMIC_NOT_STORED:
2469
		return (4 << 21) | (2 << 16);
2470

2471
	case SLJIT_LESS:
2472
	case SLJIT_SIG_LESS:
2473
		return (12 << 21) | (0 << 16);
2474

2475
	case SLJIT_GREATER_EQUAL:
2476
	case SLJIT_SIG_GREATER_EQUAL:
2477
		return (4 << 21) | (0 << 16);
2478

2479
	case SLJIT_GREATER:
2480
	case SLJIT_SIG_GREATER:
2481
		return (12 << 21) | (1 << 16);
2482

2483
	case SLJIT_LESS_EQUAL:
2484
	case SLJIT_SIG_LESS_EQUAL:
2485
		return (4 << 21) | (1 << 16);
2486

2487
	case SLJIT_OVERFLOW:
2488
		return (12 << 21) | (3 << 16);
2489

2490
	case SLJIT_NOT_OVERFLOW:
2491
		return (4 << 21) | (3 << 16);
2492

2493
	case SLJIT_F_LESS:
2494
	case SLJIT_ORDERED_LESS:
2495
	case SLJIT_UNORDERED_OR_LESS:
2496
		return (12 << 21) | ((4 + 0) << 16);
2497

2498
	case SLJIT_F_GREATER_EQUAL:
2499
	case SLJIT_ORDERED_GREATER_EQUAL:
2500
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2501
		return (4 << 21) | ((4 + 0) << 16);
2502

2503
	case SLJIT_F_GREATER:
2504
	case SLJIT_ORDERED_GREATER:
2505
	case SLJIT_UNORDERED_OR_GREATER:
2506
		return (12 << 21) | ((4 + 1) << 16);
2507

2508
	case SLJIT_F_LESS_EQUAL:
2509
	case SLJIT_ORDERED_LESS_EQUAL:
2510
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
2511
		return (4 << 21) | ((4 + 1) << 16);
2512

2513
	case SLJIT_F_EQUAL:
2514
	case SLJIT_ORDERED_EQUAL:
2515
	case SLJIT_UNORDERED_OR_EQUAL:
2516
		return (12 << 21) | ((4 + 2) << 16);
2517

2518
	case SLJIT_F_NOT_EQUAL:
2519
	case SLJIT_ORDERED_NOT_EQUAL:
2520
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
2521
		return (4 << 21) | ((4 + 2) << 16);
2522

2523
	case SLJIT_UNORDERED:
2524
		return (12 << 21) | ((4 + 3) << 16);
2525

2526
	case SLJIT_ORDERED:
2527
		return (4 << 21) | ((4 + 3) << 16);
2528

2529
	default:
2530
		SLJIT_ASSERT(type >= SLJIT_JUMP && type <= SLJIT_CALL_REG_ARG);
2531
		return (20 << 21);
2532
	}
2533
}
2534

2535
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2536
{
2537
	struct sljit_jump *jump;
2538
	sljit_ins bo_bi_flags;
2539

2540
	CHECK_ERROR_PTR();
2541
	CHECK_PTR(check_sljit_emit_jump(compiler, type));
2542

2543
	bo_bi_flags = get_bo_bi_flags(compiler, type & 0xff);
2544
	if (!bo_bi_flags)
2545
		return NULL;
2546

2547
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2548
	PTR_FAIL_IF(!jump);
2549
	set_jump(jump, compiler, (sljit_u32)type & SLJIT_REWRITABLE_JUMP);
2550
	type &= 0xff;
2551

2552
	if ((type | 0x1) == SLJIT_NOT_CARRY)
2553
		PTR_FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG2) | A(TMP_ZERO) | B(TMP_ZERO)));
2554

2555
	/* In PPC, we don't need to touch the arguments. */
2556
	if (type < SLJIT_JUMP)
2557
		jump->flags |= IS_COND;
2558
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2559
	if (type >= SLJIT_CALL)
2560
		jump->flags |= IS_CALL;
2561
#endif
2562

2563
	jump->addr = compiler->size;
2564
	PTR_FAIL_IF(push_inst(compiler, BCCTR | bo_bi_flags | (type >= SLJIT_FAST_CALL ? 1 : 0)));
2565

2566
	/* Maximum number of instructions required for generating a constant. */
2567
	compiler->size += JUMP_MAX_SIZE - 1;
2568
	return jump;
2569
}
2570

2571
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2572
	sljit_s32 arg_types)
2573
{
2574
	SLJIT_UNUSED_ARG(arg_types);
2575

2576
	CHECK_ERROR_PTR();
2577
	CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2578

2579
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2580
	if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2581
		PTR_FAIL_IF(call_with_args(compiler, arg_types, NULL));
2582
#endif
2583

2584
	if (type & SLJIT_CALL_RETURN) {
2585
		PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
2586
		type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2587
	}
2588

2589
	SLJIT_SKIP_CHECKS(compiler);
2590
	return sljit_emit_jump(compiler, type);
2591
}
2592

2593
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2594
{
2595
	struct sljit_jump *jump = NULL;
2596
	sljit_s32 src_r;
2597

2598
	CHECK_ERROR();
2599
	CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2600

2601
	if (src == SLJIT_IMM) {
2602
		/* These jumps are converted to jump/call instructions when possible. */
2603
		jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2604
		FAIL_IF(!jump);
2605
		set_jump(jump, compiler, JUMP_ADDR);
2606
		jump->u.target = (sljit_uw)srcw;
2607

2608
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2609
		if (type >= SLJIT_CALL)
2610
			jump->flags |= IS_CALL;
2611
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2612

2613
		jump->addr = compiler->size;
2614
		FAIL_IF(push_inst(compiler, BCCTR | (20 << 21) | (type >= SLJIT_FAST_CALL ? 1 : 0)));
2615

2616
		/* Maximum number of instructions required for generating a constant. */
2617
		compiler->size += JUMP_MAX_SIZE - 1;
2618
		return SLJIT_SUCCESS;
2619
	}
2620

2621
	if (FAST_IS_REG(src)) {
2622
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2623
		if (type >= SLJIT_CALL && src != TMP_CALL_REG) {
2624
			FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
2625
			src_r = TMP_CALL_REG;
2626
		} else
2627
			src_r = src;
2628
#else /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2629
		src_r = src;
2630
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2631
	} else {
2632
		ADJUST_LOCAL_OFFSET(src, srcw);
2633
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
2634
		src_r = TMP_CALL_REG;
2635
	}
2636

2637
	FAIL_IF(push_inst(compiler, MTCTR | S(src_r)));
2638
	return push_inst(compiler, BCCTR | (20 << 21) | (type >= SLJIT_FAST_CALL ? 1 : 0));
2639
}
2640

2641
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2642
	sljit_s32 arg_types,
2643
	sljit_s32 src, sljit_sw srcw)
2644
{
2645
	SLJIT_UNUSED_ARG(arg_types);
2646

2647
	CHECK_ERROR();
2648
	CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2649

2650
	if (src & SLJIT_MEM) {
2651
		ADJUST_LOCAL_OFFSET(src, srcw);
2652
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
2653
		src = TMP_CALL_REG;
2654
	}
2655

2656
	if (type & SLJIT_CALL_RETURN) {
2657
		if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
2658
			FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
2659
			src = TMP_CALL_REG;
2660
		}
2661

2662
		FAIL_IF(emit_stack_frame_release(compiler, 0));
2663
		type = SLJIT_JUMP;
2664
	}
2665

2666
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2667
	if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2668
		FAIL_IF(call_with_args(compiler, arg_types, &src));
2669
#endif
2670

2671
	SLJIT_SKIP_CHECKS(compiler);
2672
	return sljit_emit_ijump(compiler, type, src, srcw);
2673
}
2674

2675
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2676
	sljit_s32 dst, sljit_sw dstw,
2677
	sljit_s32 type)
2678
{
2679
	sljit_s32 reg, invert;
2680
	sljit_u32 bit, from_xer;
2681
	sljit_s32 saved_op = op;
2682
	sljit_sw saved_dstw = dstw;
2683
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2684
	sljit_s32 input_flags = ((op & SLJIT_32) || op == SLJIT_MOV32) ? INT_DATA : WORD_DATA;
2685
#else
2686
	sljit_s32 input_flags = WORD_DATA;
2687
#endif
2688

2689
	CHECK_ERROR();
2690
	CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2691
	ADJUST_LOCAL_OFFSET(dst, dstw);
2692

2693
	op = GET_OPCODE(op);
2694
	reg = (op < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG2;
2695

2696
	if (op >= SLJIT_ADD && (dst & SLJIT_MEM))
2697
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, TMP_REG1, dst, dstw, TMP_REG1));
2698

2699
	invert = 0;
2700
	bit = 0;
2701
	from_xer = 0;
2702

2703
	switch (type) {
2704
	case SLJIT_LESS:
2705
	case SLJIT_SIG_LESS:
2706
		break;
2707

2708
	case SLJIT_GREATER_EQUAL:
2709
	case SLJIT_SIG_GREATER_EQUAL:
2710
		invert = 1;
2711
		break;
2712

2713
	case SLJIT_GREATER:
2714
	case SLJIT_SIG_GREATER:
2715
		bit = 1;
2716
		break;
2717

2718
	case SLJIT_LESS_EQUAL:
2719
	case SLJIT_SIG_LESS_EQUAL:
2720
		bit = 1;
2721
		invert = 1;
2722
		break;
2723

2724
	case SLJIT_EQUAL:
2725
	case SLJIT_ATOMIC_STORED:
2726
		bit = 2;
2727
		break;
2728

2729
	case SLJIT_NOT_EQUAL:
2730
	case SLJIT_ATOMIC_NOT_STORED:
2731
		bit = 2;
2732
		invert = 1;
2733
		break;
2734

2735
	case SLJIT_OVERFLOW:
2736
		from_xer = 1;
2737
		bit = 1;
2738
		break;
2739

2740
	case SLJIT_NOT_OVERFLOW:
2741
		from_xer = 1;
2742
		bit = 1;
2743
		invert = 1;
2744
		break;
2745

2746
	case SLJIT_CARRY:
2747
		from_xer = 1;
2748
		bit = 2;
2749
		invert = (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB) != 0;
2750
		break;
2751

2752
	case SLJIT_NOT_CARRY:
2753
		from_xer = 1;
2754
		bit = 2;
2755
		invert = (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD) != 0;
2756
		break;
2757

2758
	case SLJIT_F_LESS:
2759
	case SLJIT_ORDERED_LESS:
2760
	case SLJIT_UNORDERED_OR_LESS:
2761
		bit = 4 + 0;
2762
		break;
2763

2764
	case SLJIT_F_GREATER_EQUAL:
2765
	case SLJIT_ORDERED_GREATER_EQUAL:
2766
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2767
		bit = 4 + 0;
2768
		invert = 1;
2769
		break;
2770

2771
	case SLJIT_F_GREATER:
2772
	case SLJIT_ORDERED_GREATER:
2773
	case SLJIT_UNORDERED_OR_GREATER:
2774
		bit = 4 + 1;
2775
		break;
2776

2777
	case SLJIT_F_LESS_EQUAL:
2778
	case SLJIT_ORDERED_LESS_EQUAL:
2779
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
2780
		bit = 4 + 1;
2781
		invert = 1;
2782
		break;
2783

2784
	case SLJIT_F_EQUAL:
2785
	case SLJIT_ORDERED_EQUAL:
2786
	case SLJIT_UNORDERED_OR_EQUAL:
2787
		bit = 4 + 2;
2788
		break;
2789

2790
	case SLJIT_F_NOT_EQUAL:
2791
	case SLJIT_ORDERED_NOT_EQUAL:
2792
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
2793
		bit = 4 + 2;
2794
		invert = 1;
2795
		break;
2796

2797
	case SLJIT_UNORDERED:
2798
		bit = 4 + 3;
2799
		break;
2800

2801
	case SLJIT_ORDERED:
2802
		bit = 4 + 3;
2803
		invert = 1;
2804
		break;
2805

2806
	default:
2807
		SLJIT_UNREACHABLE();
2808
		break;
2809
	}
2810

2811
	FAIL_IF(push_inst(compiler, (from_xer ? MFXER : MFCR) | D(reg)));
2812
	/* Simplified mnemonics: extrwi. */
2813
	FAIL_IF(push_inst(compiler, RLWINM | S(reg) | A(reg) | RLWI_SH(1 + bit) | RLWI_MBE(31, 31)));
2814

2815
	if (invert)
2816
		FAIL_IF(push_inst(compiler, XORI | S(reg) | A(reg) | 0x1));
2817

2818
	if (op < SLJIT_ADD) {
2819
		if (!(dst & SLJIT_MEM))
2820
			return SLJIT_SUCCESS;
2821
		return emit_op_mem(compiler, input_flags, reg, dst, dstw, TMP_REG1);
2822
	}
2823

2824
	SLJIT_SKIP_CHECKS(compiler);
2825

2826
	if (dst & SLJIT_MEM)
2827
		return sljit_emit_op2(compiler, saved_op, dst, saved_dstw, TMP_REG1, 0, TMP_REG2, 0);
2828
	return sljit_emit_op2(compiler, saved_op, dst, 0, dst, 0, TMP_REG2, 0);
2829
}
2830

2831
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2832
	sljit_s32 dst_reg,
2833
	sljit_s32 src1, sljit_sw src1w,
2834
	sljit_s32 src2_reg)
2835
{
2836
	sljit_ins *ptr;
2837
	sljit_uw size;
2838
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2839
	sljit_s32 inp_flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2840
#else /* !SLJIT_CONFIG_PPC_64 */
2841
        sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2842
#endif /* SLJIT_CONFIG_PPC_64 */
2843

2844
	CHECK_ERROR();
2845
	CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
2846

2847
	ADJUST_LOCAL_OFFSET(src1, src1w);
2848

2849
	if (dst_reg != src2_reg) {
2850
		if (dst_reg == src1) {
2851
				src1 = src2_reg;
2852
				src1w = 0;
2853
				type ^= 0x1;
2854
		} else {
2855
			if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
2856
				FAIL_IF(push_inst(compiler, OR | S(dst_reg) | A(TMP_REG1) | B(dst_reg)));
2857

2858
				if ((src1 & REG_MASK) == dst_reg)
2859
					src1 = (src1 & ~REG_MASK) | TMP_REG1;
2860

2861
				if (OFFS_REG(src1) == dst_reg)
2862
					src1 = (src1 & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG1);
2863
			}
2864

2865
			FAIL_IF(push_inst(compiler, OR | S(src2_reg) | A(dst_reg) | B(src2_reg)));
2866
		}
2867
	}
2868

2869
	if (((type & ~SLJIT_32) | 0x1) == SLJIT_NOT_CARRY)
2870
		FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG1) | A(TMP_ZERO) | B(TMP_ZERO)));
2871

2872
	size = compiler->size;
2873

2874
	ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
2875
	FAIL_IF(!ptr);
2876
	compiler->size++;
2877

2878
	if (src1 & SLJIT_MEM) {
2879
		FAIL_IF(emit_op_mem(compiler, inp_flags, dst_reg, src1, src1w, TMP_REG1));
2880
	} else if (src1 == SLJIT_IMM) {
2881
#if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2882
		if (type & SLJIT_32)
2883
			src1w = (sljit_s32)src1w;
2884
#endif /* SLJIT_CONFIG_RISCV_64 */
2885
		FAIL_IF(load_immediate(compiler, dst_reg, src1w));
2886
	} else
2887
		FAIL_IF(push_inst(compiler, OR | S(src1) | A(dst_reg) | B(src1)));
2888

2889
	*ptr = BCx | get_bo_bi_flags(compiler, (type ^ 0x1) & ~SLJIT_32) | (sljit_ins)((compiler->size - size) << 2);
2890
	return SLJIT_SUCCESS;
2891
}
2892

2893
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2894
	sljit_s32 dst_freg,
2895
	sljit_s32 src1, sljit_sw src1w,
2896
	sljit_s32 src2_freg)
2897
{
2898
	sljit_ins *ptr;
2899
	sljit_uw size;
2900

2901
	CHECK_ERROR();
2902
	CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
2903

2904
	ADJUST_LOCAL_OFFSET(src1, src1w);
2905

2906
	if (dst_freg != src2_freg) {
2907
		if (dst_freg == src1) {
2908
			src1 = src2_freg;
2909
			src1w = 0;
2910
			type ^= 0x1;
2911
		} else
2912
			FAIL_IF(push_inst(compiler, FMR | FD(dst_freg) | FB(src2_freg)));
2913
	}
2914

2915
	if (((type & ~SLJIT_32) | 0x1) == SLJIT_NOT_CARRY)
2916
		FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG1) | A(TMP_ZERO) | B(TMP_ZERO)));
2917

2918
	size = compiler->size;
2919

2920
	ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
2921
	FAIL_IF(!ptr);
2922
	compiler->size++;
2923

2924
	if (src1 & SLJIT_MEM)
2925
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(type) | LOAD_DATA, dst_freg, src1, src1w, TMP_REG1));
2926
	else
2927
		FAIL_IF(push_inst(compiler, FMR | FD(dst_freg) | FB(src1)));
2928

2929
	*ptr = BCx | get_bo_bi_flags(compiler, (type ^ 0x1) & ~SLJIT_32) | (sljit_ins)((compiler->size - size) << 2);
2930
	return SLJIT_SUCCESS;
2931
}
2932

2933
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
2934

2935
#define EMIT_MEM_LOAD_IMM(inst, mem, memw) \
2936
	((sljit_s16)(memw) > SIMM_MAX - SSIZE_OF(sw))
2937

2938
#else /* !SLJIT_CONFIG_PPC_32 */
2939

2940
#define EMIT_MEM_LOAD_IMM(inst, mem, memw) \
2941
	((((inst) & INT_ALIGNED) && ((memw) & 0x3) != 0) \
2942
		|| ((sljit_s16)(memw) > SIMM_MAX - SSIZE_OF(sw)) \
2943
		|| ((memw) > 0x7fff7fffl || (memw) < -0x80000000l)) \
2944

2945
#endif /* SLJIT_CONFIG_PPC_32 */
2946

2947
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2948
	sljit_s32 reg,
2949
	sljit_s32 mem, sljit_sw memw)
2950
{
2951
	sljit_ins inst;
2952

2953
	CHECK_ERROR();
2954
	CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
2955

2956
	if (!(reg & REG_PAIR_MASK))
2957
		return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
2958

2959
	ADJUST_LOCAL_OFFSET(mem, memw);
2960

2961
	inst = data_transfer_insts[WORD_DATA | ((type & SLJIT_MEM_STORE) ? 0 : LOAD_DATA)];
2962

2963
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
2964
		memw &= 0x3;
2965

2966
		if (memw != 0) {
2967
			FAIL_IF(push_inst(compiler, SLWI_W(memw) | S(OFFS_REG(mem)) | A(TMP_REG1)));
2968
			FAIL_IF(push_inst(compiler, ADD | D(TMP_REG1) | A(TMP_REG1) | B(mem & REG_MASK)));
2969
		} else
2970
			FAIL_IF(push_inst(compiler, ADD | D(TMP_REG1) | A(mem & REG_MASK) | B(OFFS_REG(mem))));
2971

2972
		mem = TMP_REG1;
2973
		memw = 0;
2974
	} else {
2975
		if (EMIT_MEM_LOAD_IMM(inst, mem, memw)) {
2976
			if ((mem & REG_MASK) != 0) {
2977
				SLJIT_SKIP_CHECKS(compiler);
2978
				FAIL_IF(sljit_emit_op2(compiler, SLJIT_ADD, TMP_REG1, 0, mem & REG_MASK, 0, SLJIT_IMM, memw));
2979
			} else
2980
				FAIL_IF(load_immediate(compiler, TMP_REG1, memw));
2981

2982
			memw = 0;
2983
			mem = TMP_REG1;
2984
		} else if (memw > SIMM_MAX || memw < SIMM_MIN) {
2985
			FAIL_IF(push_inst(compiler, ADDIS | D(TMP_REG1) | A(mem & REG_MASK) | IMM((memw + 0x8000) >> 16)));
2986

2987
			memw &= 0xffff;
2988
			mem = TMP_REG1;
2989
		} else {
2990
			memw &= 0xffff;
2991
			mem &= REG_MASK;
2992
		}
2993
	}
2994

2995
	SLJIT_ASSERT((memw >= 0 && memw <= SIMM_MAX - SSIZE_OF(sw)) || (memw >= 0x8000 && memw <= 0xffff));
2996

2997
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2998
	inst &= (sljit_ins)~INT_ALIGNED;
2999
#endif /* SLJIT_CONFIG_PPC_64 */
3000

3001
	if (!(type & SLJIT_MEM_STORE) && mem == REG_PAIR_FIRST(reg)) {
3002
		FAIL_IF(push_inst(compiler, inst | D(REG_PAIR_SECOND(reg)) | A(mem) | IMM(memw + SSIZE_OF(sw))));
3003
		return push_inst(compiler, inst | D(REG_PAIR_FIRST(reg)) | A(mem) | IMM(memw));
3004
	}
3005

3006
	FAIL_IF(push_inst(compiler, inst | D(REG_PAIR_FIRST(reg)) | A(mem) | IMM(memw)));
3007
	return push_inst(compiler, inst | D(REG_PAIR_SECOND(reg)) | A(mem) | IMM(memw + SSIZE_OF(sw)));
3008
}
3009

3010
#undef EMIT_MEM_LOAD_IMM
3011

3012
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3013
	sljit_s32 reg,
3014
	sljit_s32 mem, sljit_sw memw)
3015
{
3016
	sljit_s32 mem_flags;
3017
	sljit_ins inst;
3018

3019
	CHECK_ERROR();
3020
	CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3021

3022
	if (type & SLJIT_MEM_POST)
3023
		return SLJIT_ERR_UNSUPPORTED;
3024

3025
	switch (type & 0xff) {
3026
	case SLJIT_MOV:
3027
	case SLJIT_MOV_P:
3028
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
3029
	case SLJIT_MOV_U32:
3030
	case SLJIT_MOV_S32:
3031
	case SLJIT_MOV32:
3032
#endif
3033
		mem_flags = WORD_DATA;
3034
		break;
3035

3036
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3037
	case SLJIT_MOV_U32:
3038
	case SLJIT_MOV32:
3039
		mem_flags = INT_DATA;
3040
		break;
3041

3042
	case SLJIT_MOV_S32:
3043
		mem_flags = INT_DATA;
3044

3045
		if (!(type & SLJIT_MEM_STORE) && !(type & SLJIT_32)) {
3046
			if (mem & OFFS_REG_MASK)
3047
				mem_flags |= SIGNED_DATA;
3048
			else
3049
				return SLJIT_ERR_UNSUPPORTED;
3050
		}
3051
		break;
3052
#endif
3053

3054
	case SLJIT_MOV_U8:
3055
	case SLJIT_MOV_S8:
3056
		mem_flags = BYTE_DATA;
3057
		break;
3058

3059
	case SLJIT_MOV_U16:
3060
		mem_flags = HALF_DATA;
3061
		break;
3062

3063
	case SLJIT_MOV_S16:
3064
		mem_flags = HALF_DATA | SIGNED_DATA;
3065
		break;
3066

3067
	default:
3068
		SLJIT_UNREACHABLE();
3069
		mem_flags = WORD_DATA;
3070
		break;
3071
	}
3072

3073
	if (!(type & SLJIT_MEM_STORE))
3074
		mem_flags |= LOAD_DATA;
3075

3076
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3077
		if (memw != 0)
3078
			return SLJIT_ERR_UNSUPPORTED;
3079

3080
		if (type & SLJIT_MEM_SUPP)
3081
			return SLJIT_SUCCESS;
3082

3083
		inst = updated_data_transfer_insts[mem_flags | INDEXED];
3084
		FAIL_IF(push_inst(compiler, INST_CODE_AND_DST(inst, 0, reg) | A(mem & REG_MASK) | B(OFFS_REG(mem))));
3085
	}
3086
	else {
3087
		if (memw > SIMM_MAX || memw < SIMM_MIN)
3088
			return SLJIT_ERR_UNSUPPORTED;
3089

3090
		inst = updated_data_transfer_insts[mem_flags];
3091

3092
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3093
		if ((inst & INT_ALIGNED) && (memw & 0x3) != 0)
3094
			return SLJIT_ERR_UNSUPPORTED;
3095
#endif
3096

3097
		if (type & SLJIT_MEM_SUPP)
3098
			return SLJIT_SUCCESS;
3099

3100
		FAIL_IF(push_inst(compiler, INST_CODE_AND_DST(inst, 0, reg) | A(mem & REG_MASK) | IMM(memw)));
3101
	}
3102

3103
	if ((mem_flags & LOAD_DATA) && (type & 0xff) == SLJIT_MOV_S8)
3104
		return push_inst(compiler, EXTSB | S(reg) | A(reg));
3105
	return SLJIT_SUCCESS;
3106
}
3107

3108
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3109
	sljit_s32 freg,
3110
	sljit_s32 mem, sljit_sw memw)
3111
{
3112
	sljit_s32 mem_flags;
3113
	sljit_ins inst;
3114

3115
	CHECK_ERROR();
3116
	CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3117

3118
	if (type & SLJIT_MEM_POST)
3119
		return SLJIT_ERR_UNSUPPORTED;
3120

3121
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3122
		if (memw != 0)
3123
			return SLJIT_ERR_UNSUPPORTED;
3124
	}
3125
	else {
3126
		if (memw > SIMM_MAX || memw < SIMM_MIN)
3127
			return SLJIT_ERR_UNSUPPORTED;
3128
	}
3129

3130
	if (type & SLJIT_MEM_SUPP)
3131
		return SLJIT_SUCCESS;
3132

3133
	mem_flags = FLOAT_DATA(type);
3134

3135
	if (!(type & SLJIT_MEM_STORE))
3136
		mem_flags |= LOAD_DATA;
3137

3138
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3139
		inst = updated_data_transfer_insts[mem_flags | INDEXED];
3140
		return push_inst(compiler, INST_CODE_AND_DST(inst, DOUBLE_DATA, freg) | A(mem & REG_MASK) | B(OFFS_REG(mem)));
3141
	}
3142

3143
	inst = updated_data_transfer_insts[mem_flags];
3144
	return push_inst(compiler, INST_CODE_AND_DST(inst, DOUBLE_DATA, freg) | A(mem & REG_MASK) | IMM(memw));
3145
}
3146

3147
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
3148
	sljit_s32 dst_reg,
3149
	sljit_s32 mem_reg)
3150
{
3151
	sljit_ins ins;
3152

3153
	CHECK_ERROR();
3154
	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
3155

3156
	if (op & SLJIT_ATOMIC_USE_CAS)
3157
		return SLJIT_ERR_UNSUPPORTED;
3158

3159
	switch (GET_OPCODE(op)) {
3160
	case SLJIT_MOV:
3161
	case SLJIT_MOV_P:
3162
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3163
		ins = LDARX;
3164
		break;
3165
#endif /* SLJIT_CONFIG_RISCV_64 */
3166
	case SLJIT_MOV_U32:
3167
	case SLJIT_MOV32:
3168
		ins = LWARX;
3169
		break;
3170

3171
	default:
3172
		return SLJIT_ERR_UNSUPPORTED;
3173
	}
3174

3175
	if (op & SLJIT_ATOMIC_TEST)
3176
		return SLJIT_SUCCESS;
3177

3178
	return push_inst(compiler, ins | D(dst_reg) | B(mem_reg));
3179
}
3180

3181
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
3182
	sljit_s32 src_reg,
3183
	sljit_s32 mem_reg,
3184
	sljit_s32 temp_reg)
3185
{
3186
	sljit_ins ins;
3187

3188
	/* temp_reg == mem_reg is undefined so use another temp register */
3189
	SLJIT_UNUSED_ARG(temp_reg);
3190

3191
	CHECK_ERROR();
3192
	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
3193

3194
	if (op & SLJIT_ATOMIC_USE_CAS)
3195
		return SLJIT_ERR_UNSUPPORTED;
3196

3197
	switch (GET_OPCODE(op)) {
3198
	case SLJIT_MOV:
3199
	case SLJIT_MOV_P:
3200
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3201
		ins = STDCX | 0x1;
3202
		break;
3203
#endif /* SLJIT_CONFIG_RISCV_64 */
3204
	case SLJIT_MOV_U32:
3205
	case SLJIT_MOV32:
3206
		ins = STWCX | 0x1;
3207
		break;
3208

3209
	default:
3210
		return SLJIT_ERR_UNSUPPORTED;
3211
	}
3212

3213
	if (op & SLJIT_ATOMIC_TEST)
3214
		return SLJIT_SUCCESS;
3215

3216
	return push_inst(compiler, ins | D(src_reg) | B(mem_reg));
3217
}
3218

3219
SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3220
{
3221
	struct sljit_const *const_;
3222
	sljit_s32 dst_r;
3223

3224
	CHECK_ERROR_PTR();
3225
	CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3226
	ADJUST_LOCAL_OFFSET(dst, dstw);
3227

3228
	const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3229
	PTR_FAIL_IF(!const_);
3230
	set_const(const_, compiler);
3231

3232
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3233
	PTR_FAIL_IF(emit_const(compiler, dst_r, init_value));
3234

3235
	if (dst & SLJIT_MEM)
3236
		PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, dst_r, dst, dstw, TMP_REG1));
3237

3238
	return const_;
3239
}
3240

3241
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3242
{
3243
	struct sljit_jump *jump;
3244
	sljit_s32 dst_r;
3245

3246
	CHECK_ERROR_PTR();
3247
	CHECK_PTR(check_sljit_emit_mov_addr(compiler, dst, dstw));
3248
	ADJUST_LOCAL_OFFSET(dst, dstw);
3249

3250
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3251
	PTR_FAIL_IF(!jump);
3252
	set_mov_addr(jump, compiler, 0);
3253

3254
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3255
	PTR_FAIL_IF(push_inst(compiler, (sljit_ins)dst_r));
3256
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
3257
	compiler->size++;
3258
#else
3259
	compiler->size += 4;
3260
#endif
3261

3262
	if (dst & SLJIT_MEM)
3263
		PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0));
3264

3265
	return jump;
3266
}
3267

3268
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3269
{
3270
	sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3271
}
3272

3273
Product

Resources

Company
