CoCalc -- sljitNativePPC

GitHub Repository: godotengine/godot
Path: blob/master/thirdparty/pcre2/deps/sljit/sljit_src/sljitNativePPC_common.c
²¹⁸²⁷ views
1
/*
2
 *    Stack-less Just-In-Time compiler
3
 *
4
 *    Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without modification, are
7
 * permitted provided that the following conditions are met:
8
 *
9
 *   1. Redistributions of source code must retain the above copyright notice, this list of
10
 *      conditions and the following disclaimer.
11
 *
12
 *   2. Redistributions in binary form must reproduce the above copyright notice, this list
13
 *      of conditions and the following disclaimer in the documentation and/or other materials
14
 *      provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19
 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
 */
26

27
SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28
{
29
	return "PowerPC" SLJIT_CPUINFO;
30
}
31

32
/* Length of an instruction word.
33
   Both for ppc-32 and ppc-64. */
34
typedef sljit_u32 sljit_ins;
35

36
#if ((defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) && (defined _AIX)) \
37
	|| (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
38
#define SLJIT_PPC_STACK_FRAME_V2 1
39
#endif
40

41
#ifdef _AIX
42
#include <sys/cache.h>
43
#endif
44

45
#if (defined _CALL_ELF && _CALL_ELF == 2)
46
#define SLJIT_PASS_ENTRY_ADDR_TO_CALL 1
47
#endif
48

49
#if (defined SLJIT_CACHE_FLUSH_OWN_IMPL && SLJIT_CACHE_FLUSH_OWN_IMPL)
50

51
static void ppc_cache_flush(sljit_ins *from, sljit_ins *to)
52
{
53
#ifdef _AIX
54
	_sync_cache_range((caddr_t)from, (int)((size_t)to - (size_t)from));
55
#elif defined(__GNUC__) || (defined(__IBM_GCC_ASM) && __IBM_GCC_ASM)
56
#	if defined(_ARCH_PWR) || defined(_ARCH_PWR2)
57
	/* Cache flush for POWER architecture. */
58
	while (from < to) {
59
		__asm__ volatile (
60
			"clf 0, %0\n"
61
			"dcs\n"
62
			: : "r"(from)
63
		);
64
		from++;
65
	}
66
	__asm__ volatile ( "ics" );
67
#	elif defined(_ARCH_COM) && !defined(_ARCH_PPC)
68
#	error "Cache flush is not implemented for PowerPC/POWER common mode."
69
#	else
70
	/* Cache flush for PowerPC architecture. */
71
	while (from < to) {
72
		__asm__ volatile (
73
			"dcbf 0, %0\n"
74
			"sync\n"
75
			"icbi 0, %0\n"
76
			: : "r"(from)
77
		);
78
		from++;
79
	}
80
	__asm__ volatile ( "isync" );
81
#	endif
82
#	ifdef __xlc__
83
#	warning "This file may fail to compile if -qfuncsect is used"
84
#	endif
85
#elif defined(__xlc__)
86
#error "Please enable GCC syntax for inline assembly statements with -qasm=gcc"
87
#else
88
#error "This platform requires a cache flush implementation."
89
#endif /* _AIX */
90
}
91

92
#endif /* (defined SLJIT_CACHE_FLUSH_OWN_IMPL && SLJIT_CACHE_FLUSH_OWN_IMPL) */
93

94
#define TMP_REG1	(SLJIT_NUMBER_OF_REGISTERS + 2)
95
#define TMP_REG2	(SLJIT_NUMBER_OF_REGISTERS + 3)
96
#define TMP_ZERO	(SLJIT_NUMBER_OF_REGISTERS + 4)
97

98
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
99
#define TMP_CALL_REG	(SLJIT_NUMBER_OF_REGISTERS + 5)
100
#else
101
#define TMP_CALL_REG	TMP_REG1
102
#endif
103

104
#define TMP_FREG1	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
105
#define TMP_FREG2	(SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
106

107
static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 7] = {
108
	0, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 1, 9, 10, 31, 12
109
};
110

111
static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
112
	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 0, 13
113
};
114

115
/* --------------------------------------------------------------------- */
116
/*  Instruction forms                                                     */
117
/* --------------------------------------------------------------------- */
118
#define D(d)		((sljit_ins)reg_map[d] << 21)
119
#define S(s)		((sljit_ins)reg_map[s] << 21)
120
#define A(a)		((sljit_ins)reg_map[a] << 16)
121
#define B(b)		((sljit_ins)reg_map[b] << 11)
122
#define C(c)		((sljit_ins)reg_map[c] << 6)
123
#define FD(fd)		((sljit_ins)freg_map[fd] << 21)
124
#define FS(fs)		((sljit_ins)freg_map[fs] << 21)
125
#define FA(fa)		((sljit_ins)freg_map[fa] << 16)
126
#define FB(fb)		((sljit_ins)freg_map[fb] << 11)
127
#define FC(fc)		((sljit_ins)freg_map[fc] << 6)
128
#define IMM(imm)	((sljit_ins)(imm) & 0xffff)
129
#define CRD(d)		((sljit_ins)(d) << 21)
130

131
/* Instruction bit sections.
132
   OE and Rc flag (see ALT_SET_FLAGS). */
133
#define OE(flags)	((flags) & ALT_SET_FLAGS)
134
/* Rc flag (see ALT_SET_FLAGS). */
135
#define RC(flags)	((sljit_ins)((flags) & ALT_SET_FLAGS) >> 10)
136
#define HI(opcode)	((sljit_ins)(opcode) << 26)
137
#define LO(opcode)	((sljit_ins)(opcode) << 1)
138

139
#define ADD		(HI(31) | LO(266))
140
#define ADDC		(HI(31) | LO(10))
141
#define ADDE		(HI(31) | LO(138))
142
#define ADDI		(HI(14))
143
#define ADDIC		(HI(13))
144
#define ADDIS		(HI(15))
145
#define ADDME		(HI(31) | LO(234))
146
#define AND		(HI(31) | LO(28))
147
#define ANDI		(HI(28))
148
#define ANDIS		(HI(29))
149
#define Bx		(HI(18))
150
#define BCx		(HI(16))
151
#define BCCTR		(HI(19) | LO(528) | (3 << 11))
152
#define BLR		(HI(19) | LO(16) | (0x14 << 21))
153
#if defined(_ARCH_PWR10) && _ARCH_PWR10
154
#define BRD		(HI(31) | LO(187))
155
#endif /* POWER10 */
156
#define CNTLZD		(HI(31) | LO(58))
157
#define CNTLZW		(HI(31) | LO(26))
158
#define CMP		(HI(31) | LO(0))
159
#define CMPI		(HI(11))
160
#define CMPL		(HI(31) | LO(32))
161
#define CMPLI		(HI(10))
162
#define CROR		(HI(19) | LO(449))
163
#define DCBT		(HI(31) | LO(278))
164
#define DIVD		(HI(31) | LO(489))
165
#define DIVDU		(HI(31) | LO(457))
166
#define DIVW		(HI(31) | LO(491))
167
#define DIVWU		(HI(31) | LO(459))
168
#define EXTSB		(HI(31) | LO(954))
169
#define EXTSH		(HI(31) | LO(922))
170
#define EXTSW		(HI(31) | LO(986))
171
#define FABS		(HI(63) | LO(264))
172
#define FADD		(HI(63) | LO(21))
173
#define FADDS		(HI(59) | LO(21))
174
#define FCFID		(HI(63) | LO(846))
175
#define FCMPU		(HI(63) | LO(0))
176
#define FCTIDZ		(HI(63) | LO(815))
177
#define FCTIWZ		(HI(63) | LO(15))
178
#define FDIV		(HI(63) | LO(18))
179
#define FDIVS		(HI(59) | LO(18))
180
#define FMR		(HI(63) | LO(72))
181
#define FMUL		(HI(63) | LO(25))
182
#define FMULS		(HI(59) | LO(25))
183
#define FNEG		(HI(63) | LO(40))
184
#define FRSP		(HI(63) | LO(12))
185
#define FSUB		(HI(63) | LO(20))
186
#define FSUBS		(HI(59) | LO(20))
187
#define LD		(HI(58) | 0)
188
#define LFD		(HI(50))
189
#define LFS		(HI(48))
190
#define LDARX		(HI(31) | LO(84))
191
#if defined(_ARCH_PWR7) && _ARCH_PWR7
192
#define LDBRX		(HI(31) | LO(532))
193
#endif /* POWER7 */
194
#define LHBRX		(HI(31) | LO(790))
195
#define LWARX		(HI(31) | LO(20))
196
#define LWBRX		(HI(31) | LO(534))
197
#define LWZ		(HI(32))
198
#define MFCR		(HI(31) | LO(19))
199
#define MFLR		(HI(31) | LO(339) | 0x80000)
200
#define MFXER		(HI(31) | LO(339) | 0x10000)
201
#define MTCTR		(HI(31) | LO(467) | 0x90000)
202
#define MTLR		(HI(31) | LO(467) | 0x80000)
203
#define MTXER		(HI(31) | LO(467) | 0x10000)
204
#define MULHD		(HI(31) | LO(73))
205
#define MULHDU		(HI(31) | LO(9))
206
#define MULHW		(HI(31) | LO(75))
207
#define MULHWU		(HI(31) | LO(11))
208
#define MULLD		(HI(31) | LO(233))
209
#define MULLI		(HI(7))
210
#define MULLW		(HI(31) | LO(235))
211
#define NEG		(HI(31) | LO(104))
212
#define NOP		(HI(24))
213
#define NOR		(HI(31) | LO(124))
214
#define OR		(HI(31) | LO(444))
215
#define ORI		(HI(24))
216
#define ORIS		(HI(25))
217
#define RLDCL		(HI(30) | LO(8))
218
#define RLDICL		(HI(30) | LO(0 << 1))
219
#define RLDICR		(HI(30) | LO(1 << 1))
220
#define RLDIMI		(HI(30) | LO(3 << 1))
221
#define RLWIMI		(HI(20))
222
#define RLWINM		(HI(21))
223
#define RLWNM		(HI(23))
224
#define SLD		(HI(31) | LO(27))
225
#define SLW		(HI(31) | LO(24))
226
#define SRAD		(HI(31) | LO(794))
227
#define SRADI		(HI(31) | LO(413 << 1))
228
#define SRAW		(HI(31) | LO(792))
229
#define SRAWI		(HI(31) | LO(824))
230
#define SRD		(HI(31) | LO(539))
231
#define SRW		(HI(31) | LO(536))
232
#define STD		(HI(62) | 0)
233
#if defined(_ARCH_PWR7) && _ARCH_PWR7
234
#define STDBRX		(HI(31) | LO(660))
235
#endif /* POWER7 */
236
#define STDCX		(HI(31) | LO(214))
237
#define STDU		(HI(62) | 1)
238
#define STDUX		(HI(31) | LO(181))
239
#define STFD		(HI(54))
240
#define STFIWX		(HI(31) | LO(983))
241
#define STFS		(HI(52))
242
#define STHBRX		(HI(31) | LO(918))
243
#define STW		(HI(36))
244
#define STWBRX		(HI(31) | LO(662))
245
#define STWCX		(HI(31) | LO(150))
246
#define STWU		(HI(37))
247
#define STWUX		(HI(31) | LO(183))
248
#define SUBF		(HI(31) | LO(40))
249
#define SUBFC		(HI(31) | LO(8))
250
#define SUBFE		(HI(31) | LO(136))
251
#define SUBFIC		(HI(8))
252
#define SYNC		(HI(31) | LO(598))
253
#define XOR		(HI(31) | LO(316))
254
#define XORI		(HI(26))
255
#define XORIS		(HI(27))
256

257
#define SIMM_MAX	(0x7fff)
258
#define SIMM_MIN	(-0x8000)
259
#define UIMM_MAX	(0xffff)
260

261
/* Shift helpers. */
262
#define RLWI_SH(sh) ((sljit_ins)(sh) << 11)
263
#define RLWI_MBE(mb, me) (((sljit_ins)(mb) << 6) | ((sljit_ins)(me) << 1))
264
#define RLDI_SH(sh) ((((sljit_ins)(sh) & 0x1f) << 11) | (((sljit_ins)(sh) & 0x20) >> 4))
265
#define RLDI_MB(mb) ((((sljit_ins)(mb) & 0x1f) << 6) | ((sljit_ins)(mb) & 0x20))
266
#define RLDI_ME(me) RLDI_MB(me)
267

268
#define SLWI(shift) (RLWINM | RLWI_SH(shift) | RLWI_MBE(0, 31 - (shift)))
269
#define SLDI(shift) (RLDICR | RLDI_SH(shift) | RLDI_ME(63 - (shift)))
270
/* shift > 0 */
271
#define SRWI(shift) (RLWINM | RLWI_SH(32 - (shift)) | RLWI_MBE((shift), 31))
272
#define SRDI(shift) (RLDICL | RLDI_SH(64 - (shift)) | RLDI_MB(shift))
273

274
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
275
#define SLWI_W(shift) SLWI(shift)
276
#define TMP_MEM_OFFSET (2 * sizeof(sljit_sw))
277
#else /* !SLJIT_CONFIG_PPC_32 */
278
#define SLWI_W(shift) SLDI(shift)
279
#define TMP_MEM_OFFSET (6 * sizeof(sljit_sw))
280
#endif /* SLJIT_CONFIG_PPC_32 */
281

282
#if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
283
#define TMP_MEM_OFFSET_LO	(TMP_MEM_OFFSET)
284
#define TMP_MEM_OFFSET_HI	(TMP_MEM_OFFSET + sizeof(sljit_s32))
285
#define LWBRX_FIRST_REG		S(TMP_REG1)
286
#define LWBRX_SECOND_REG	S(dst)
287
#else /* !SLJIT_LITTLE_ENDIAN */
288
#define TMP_MEM_OFFSET_LO	(TMP_MEM_OFFSET + sizeof(sljit_s32))
289
#define TMP_MEM_OFFSET_HI	(TMP_MEM_OFFSET)
290
#define LWBRX_FIRST_REG		S(dst)
291
#define LWBRX_SECOND_REG	S(TMP_REG1)
292
#endif /* SLJIT_LITTLE_ENDIAN */
293

294
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
295
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_function_context(void** func_ptr, struct sljit_function_context* context, sljit_uw addr, void* func)
296
{
297
	sljit_uw* ptrs;
298

299
	if (func_ptr)
300
		*func_ptr = (void*)context;
301

302
	ptrs = (sljit_uw*)func;
303
	context->addr = addr ? addr : ptrs[0];
304
	context->r2 = ptrs[1];
305
	context->r11 = ptrs[2];
306
}
307
#endif
308

309
static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
310
{
311
	sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
312
	FAIL_IF(!ptr);
313
	*ptr = ins;
314
	compiler->size++;
315
	return SLJIT_SUCCESS;
316
}
317

318
static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
319
{
320
	sljit_sw diff;
321
	sljit_uw target_addr;
322
	sljit_uw jump_addr = (sljit_uw)code_ptr;
323
	sljit_uw orig_addr = jump->addr;
324
	SLJIT_UNUSED_ARG(executable_offset);
325

326
	jump->addr = jump_addr;
327
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL) && (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
328
	if (jump->flags & (SLJIT_REWRITABLE_JUMP | IS_CALL))
329
		goto exit;
330
#else
331
	if (jump->flags & SLJIT_REWRITABLE_JUMP)
332
		goto exit;
333
#endif
334

335
	if (jump->flags & JUMP_ADDR)
336
		target_addr = jump->u.target;
337
	else {
338
		SLJIT_ASSERT(jump->u.label != NULL);
339
		target_addr = (sljit_uw)(code + jump->u.label->size) + (sljit_uw)executable_offset;
340

341
		if (jump->u.label->size > orig_addr)
342
			jump_addr = (sljit_uw)(code + orig_addr);
343
	}
344

345
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL) && (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
346
	if (jump->flags & IS_CALL)
347
		goto keep_address;
348
#endif
349

350
	diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
351

352
	if (jump->flags & IS_COND) {
353
		if (diff <= 0x7fff && diff >= -0x8000) {
354
			jump->flags |= PATCH_B;
355
			return code_ptr;
356
		}
357
		if (target_addr <= 0xffff) {
358
			jump->flags |= PATCH_B | PATCH_ABS_B;
359
			return code_ptr;
360
		}
361

362
		diff -= SSIZE_OF(ins);
363
	}
364

365
	if (diff <= 0x01ffffff && diff >= -0x02000000) {
366
		jump->flags |= PATCH_B;
367
	} else if (target_addr <= 0x01ffffff) {
368
		jump->flags |= PATCH_B | PATCH_ABS_B;
369
	}
370

371
	if (jump->flags & PATCH_B) {
372
		if (!(jump->flags & IS_COND))
373
			return code_ptr;
374

375
		code_ptr[0] = BCx | (2 << 2) | ((code_ptr[0] ^ (8 << 21)) & 0x03ff0001);
376
		code_ptr[1] = Bx;
377
		jump->addr += sizeof(sljit_ins);
378
		jump->flags -= IS_COND;
379
		return code_ptr + 1;
380
	}
381

382
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
383
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
384
keep_address:
385
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
386
	if (target_addr < 0x80000000l) {
387
		jump->flags |= PATCH_ABS32;
388
		code_ptr[2] = MTCTR | S(TMP_CALL_REG);
389
		code_ptr[3] = code_ptr[0];
390
		return code_ptr + 3;
391
	}
392

393
	if (target_addr < 0x800000000000l) {
394
		jump->flags |= PATCH_ABS48;
395
		code_ptr[4] = MTCTR | S(TMP_CALL_REG);
396
		code_ptr[5] = code_ptr[0];
397
		return code_ptr + 5;
398
	}
399
#endif /* SLJIT_CONFIG_PPC_64 */
400

401
exit:
402
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
403
	code_ptr[2] = MTCTR | S(TMP_CALL_REG);
404
	code_ptr[3] = code_ptr[0];
405
#else /* !SLJIT_CONFIG_PPC_32 */
406
	code_ptr[5] = MTCTR | S(TMP_CALL_REG);
407
	code_ptr[6] = code_ptr[0];
408
#endif /* SLJIT_CONFIG_PPC_32 */
409
	return code_ptr + JUMP_MAX_SIZE - 1;
410
}
411

412
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
413

414
static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_ins *code, sljit_sw executable_offset)
415
{
416
	sljit_uw addr;
417
	SLJIT_UNUSED_ARG(executable_offset);
418

419
	SLJIT_ASSERT(jump->flags < ((sljit_uw)5 << JUMP_SIZE_SHIFT));
420
	if (jump->flags & JUMP_ADDR)
421
		addr = jump->u.target;
422
	else
423
		addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
424

425
	if (addr < 0x80000000l) {
426
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
427
		jump->flags |= PATCH_ABS32;
428
		return 1;
429
	}
430

431
	if (addr < 0x800000000000l) {
432
		SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
433
		jump->flags |= PATCH_ABS48;
434
		return 3;
435
	}
436

437
	SLJIT_ASSERT(jump->flags >= ((sljit_uw)4 << JUMP_SIZE_SHIFT));
438
	return 4;
439
}
440

441
#endif /* SLJIT_CONFIG_PPC_64 */
442

443
static void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
444
{
445
	sljit_uw flags = jump->flags;
446
	sljit_uw addr = (flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
447
	sljit_ins *ins = (sljit_ins*)jump->addr;
448
	sljit_s32 reg;
449
	SLJIT_UNUSED_ARG(executable_offset);
450

451
	if (flags & PATCH_B) {
452
		if (flags & IS_COND) {
453
			if (!(flags & PATCH_ABS_B)) {
454
				addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(ins, executable_offset);
455
				SLJIT_ASSERT((sljit_sw)addr <= 0x7fff && (sljit_sw)addr >= -0x8000);
456
				ins[0] = BCx | ((sljit_ins)addr & 0xfffc) | (ins[0] & 0x03ff0001);
457
			} else {
458
				SLJIT_ASSERT(addr <= 0xffff);
459
				ins[0] = BCx | ((sljit_ins)addr & 0xfffc) | 0x2 | ((*ins) & 0x03ff0001);
460
			}
461
			return;
462
		}
463

464
		if (!(flags & PATCH_ABS_B)) {
465
			addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(ins, executable_offset);
466
			SLJIT_ASSERT((sljit_sw)addr <= 0x01ffffff && (sljit_sw)addr >= -0x02000000);
467
			ins[0] = Bx | ((sljit_ins)addr & 0x03fffffc) | (ins[0] & 0x1);
468
		} else {
469
			SLJIT_ASSERT(addr <= 0x03ffffff);
470
			ins[0] = Bx | ((sljit_ins)addr & 0x03fffffc) | 0x2 | (ins[0] & 0x1);
471
		}
472
		return;
473
	}
474

475
	reg = (flags & JUMP_MOV_ADDR) ? (sljit_s32)ins[0] : TMP_CALL_REG;
476

477
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
478
	ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 16);
479
	ins[1] = ORI | S(reg) | A(reg) | IMM(addr);
480
#else /* !SLJIT_CONFIG_PPC_32 */
481

482
	/* The TMP_ZERO cannot be used because it is restored for tail calls. */
483
	if (flags & PATCH_ABS32) {
484
		SLJIT_ASSERT(addr < 0x80000000l);
485
		ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 16);
486
		ins[1] = ORI | S(reg) | A(reg) | IMM(addr);
487
		return;
488
	}
489

490
	if (flags & PATCH_ABS48) {
491
		SLJIT_ASSERT(addr < 0x800000000000l);
492
		ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 32);
493
		ins[1] = ORI | S(reg) | A(reg) | IMM(addr >> 16);
494
		ins[2] = SLDI(16) | S(reg) | A(reg);
495
		ins[3] = ORI | S(reg) | A(reg) | IMM(addr);
496
		return;
497
	}
498

499
	ins[0] = ADDIS | D(reg) | A(0) | IMM(addr >> 48);
500
	ins[1] = ORI | S(reg) | A(reg) | IMM(addr >> 32);
501
	ins[2] = SLDI(32) | S(reg) | A(reg);
502
	ins[3] = ORIS | S(reg) | A(reg) | IMM(addr >> 16);
503
	ins[4] = ORI | S(reg) | A(reg) | IMM(addr);
504
#endif /* SLJIT_CONFIG_PPC_32 */
505
}
506

507
static SLJIT_INLINE sljit_ins *process_extended_label(sljit_ins *code_ptr, struct sljit_extended_label *ext_label)
508
{
509
	SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
510
	return (sljit_ins*)((sljit_uw)code_ptr & ~(ext_label->data));
511
}
512

513
static void reduce_code_size(struct sljit_compiler *compiler)
514
{
515
	struct sljit_label *label;
516
	struct sljit_jump *jump;
517
	struct sljit_const *const_;
518
	SLJIT_NEXT_DEFINE_TYPES;
519
	sljit_uw total_size;
520
	sljit_uw size_reduce = 0;
521
	sljit_sw diff;
522

523
	label = compiler->labels;
524
	jump = compiler->jumps;
525
	const_ = compiler->consts;
526
	SLJIT_NEXT_INIT_TYPES();
527

528
	while (1) {
529
		SLJIT_GET_NEXT_MIN();
530

531
		if (next_min_addr == SLJIT_MAX_ADDRESS)
532
			break;
533

534
		if (next_min_addr == next_label_size) {
535
			label->size -= size_reduce;
536

537
			label = label->next;
538
			next_label_size = SLJIT_GET_NEXT_SIZE(label);
539
		}
540

541
		if (next_min_addr == next_const_addr) {
542
			const_->addr -= size_reduce;
543
			const_ = const_->next;
544
			next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
545
			continue;
546
		}
547

548
		if (next_min_addr != next_jump_addr)
549
			continue;
550

551
		jump->addr -= size_reduce;
552
		if (!(jump->flags & JUMP_MOV_ADDR)) {
553
			total_size = JUMP_MAX_SIZE - 1;
554

555
			if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
556
				if (jump->flags & JUMP_ADDR) {
557
					if (jump->u.target <= 0x01ffffff)
558
						total_size = 1 - 1;
559
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
560
					else if (jump->u.target < 0x80000000l)
561
						total_size = 4 - 1;
562
					else if (jump->u.target < 0x800000000000l)
563
						total_size = 6 - 1;
564
#endif /* SLJIT_CONFIG_PPC_64 */
565
				} else {
566
					/* Unit size: instruction. */
567
					diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
568
					if (jump->u.label->size > jump->addr) {
569
						SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
570
						diff -= (sljit_sw)size_reduce;
571
					}
572

573
					if (jump->flags & IS_COND) {
574
						if (diff <= (0x7fff / SSIZE_OF(ins)) && diff >= (-0x8000 / SSIZE_OF(ins)))
575
							total_size = 1 - 1;
576
						else if ((diff - 1) <= (0x01ffffff / SSIZE_OF(ins)) && (diff - 1) >= (-0x02000000 / SSIZE_OF(ins)))
577
							total_size = 2 - 1;
578
					} else if (diff <= (0x01ffffff / SSIZE_OF(ins)) && diff >= (-0x02000000 / SSIZE_OF(ins)))
579
						total_size = 1 - 1;
580
				}
581
			}
582

583
			size_reduce += (JUMP_MAX_SIZE - 1) - total_size;
584
			jump->flags |= total_size << JUMP_SIZE_SHIFT;
585
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
586
		} else {
587
			total_size = (sljit_uw)4 << JUMP_SIZE_SHIFT;
588

589
			if (jump->flags & JUMP_ADDR) {
590
				if (jump->u.target < 0x80000000l) {
591
					total_size = (sljit_uw)1 << JUMP_SIZE_SHIFT;
592
					size_reduce += 3;
593
				} else if (jump->u.target < 0x800000000000l) {
594
					total_size = (sljit_uw)3 << JUMP_SIZE_SHIFT;
595
					size_reduce += 1;
596
				}
597
			}
598
			jump->flags |= total_size;
599
#endif /* SLJIT_CONFIG_PPC_64 */
600
		}
601

602
		jump = jump->next;
603
		next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
604
	}
605

606
	compiler->size -= size_reduce;
607
}
608

609
SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
610
{
611
	struct sljit_memory_fragment *buf;
612
	sljit_ins *code;
613
	sljit_ins *code_ptr;
614
	sljit_ins *buf_ptr;
615
	sljit_ins *buf_end;
616
	sljit_uw word_count;
617
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
618
	sljit_uw jump_addr;
619
#endif
620
	SLJIT_NEXT_DEFINE_TYPES;
621
	sljit_sw executable_offset;
622

623
	struct sljit_label *label;
624
	struct sljit_jump *jump;
625
	struct sljit_const *const_;
626

627
	CHECK_ERROR_PTR();
628
	CHECK_PTR(check_sljit_generate_code(compiler, options));
629

630
	reduce_code_size(compiler);
631

632
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
633
	if (!(options & SLJIT_GENERATE_CODE_NO_CONTEXT)) {
634
	/* add to compiler->size additional instruction space to hold the trampoline and padding */
635
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
636
		compiler->size += (compiler->size & 0x1) + (sizeof(struct sljit_function_context) / sizeof(sljit_ins));
637
#else /* !SLJIT_CONFIG_PPC_64 */
638
		compiler->size += (sizeof(struct sljit_function_context) / sizeof(sljit_ins));
639
#endif /* SLJIT_CONFIG_PPC_64 */
640
	}
641
#endif /* SLJIT_INDIRECT_CALL */
642
	code = (sljit_ins*)allocate_executable_memory(compiler->size * sizeof(sljit_ins), options, exec_allocator_data, &executable_offset);
643
	PTR_FAIL_WITH_EXEC_IF(code);
644

645
	reverse_buf(compiler);
646
	buf = compiler->buf;
647

648
	code_ptr = code;
649
	word_count = 0;
650
	label = compiler->labels;
651
	jump = compiler->jumps;
652
	const_ = compiler->consts;
653
	SLJIT_NEXT_INIT_TYPES();
654
	SLJIT_GET_NEXT_MIN();
655

656
	do {
657
		buf_ptr = (sljit_ins*)buf->memory;
658
		buf_end = buf_ptr + (buf->used_size >> 2);
659
		do {
660
			*code_ptr = *buf_ptr++;
661
			if (next_min_addr == word_count) {
662
				SLJIT_ASSERT(!label || label->size >= word_count);
663
				SLJIT_ASSERT(!jump || jump->addr >= word_count);
664
				SLJIT_ASSERT(!const_ || const_->addr >= word_count);
665

666
				/* These structures are ordered by their address. */
667
				if (next_min_addr == next_label_size) {
668
					if (label->u.index >= SLJIT_LABEL_ALIGNED) {
669
						code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
670
						*code_ptr = buf_ptr[-1];
671
					}
672

673
					/* Just recording the address. */
674
					label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
675
					label->size = (sljit_uw)(code_ptr - code);
676
					label = label->next;
677
					next_label_size = SLJIT_GET_NEXT_SIZE(label);
678
				}
679

680
				if (next_min_addr == next_jump_addr) {
681
					if (!(jump->flags & JUMP_MOV_ADDR)) {
682
						word_count += jump->flags >> JUMP_SIZE_SHIFT;
683
#if (defined SLJIT_DEBUG && SLJIT_DEBUG)
684
						jump_addr = (sljit_uw)code_ptr;
685
#endif
686
						code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
687
						SLJIT_ASSERT(((sljit_uw)code_ptr - jump_addr <= (jump->flags >> JUMP_SIZE_SHIFT) * sizeof(sljit_ins)));
688
					} else {
689
						jump->addr = (sljit_uw)code_ptr;
690
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
691
						word_count += jump->flags >> JUMP_SIZE_SHIFT;
692
						code_ptr += mov_addr_get_length(jump, code, executable_offset);
693
#else /* !SLJIT_CONFIG_PPC_64 */
694
						word_count++;
695
						code_ptr++;
696
#endif /* SLJIT_CONFIG_PPC_64 */
697
					}
698
					jump = jump->next;
699
					next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
700
				} else if (next_min_addr == next_const_addr) {
701
					const_->addr = (sljit_uw)code_ptr;
702
					const_ = const_->next;
703
					next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
704
				}
705

706
				SLJIT_GET_NEXT_MIN();
707
			}
708
			code_ptr++;
709
			word_count++;
710
		} while (buf_ptr < buf_end);
711

712
		buf = buf->next;
713
	} while (buf);
714

715
	if (label && label->size == word_count) {
716
		if (label->u.index >= SLJIT_LABEL_ALIGNED)
717
			code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
718

719
		label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
720
		label->size = (sljit_uw)(code_ptr - code);
721
		label = label->next;
722
	}
723

724
	SLJIT_ASSERT(!label);
725
	SLJIT_ASSERT(!jump);
726
	SLJIT_ASSERT(!const_);
727

728
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
729
	SLJIT_ASSERT(code_ptr - code <= (sljit_sw)(compiler->size -
730
		((options & SLJIT_GENERATE_CODE_NO_CONTEXT) ? 0 : (sizeof(struct sljit_function_context) / sizeof(sljit_ins)))));
731
#else
732
	SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
733
#endif
734

735
	jump = compiler->jumps;
736
	while (jump) {
737
		generate_jump_or_mov_addr(jump, executable_offset);
738
		jump = jump->next;
739
	}
740

741
	compiler->error = SLJIT_ERR_COMPILED;
742
	compiler->executable_offset = executable_offset;
743

744
	code = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
745

746
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
747
	if (!(options & SLJIT_GENERATE_CODE_NO_CONTEXT)) {
748
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
749
		if (((sljit_sw)code_ptr) & 0x4)
750
			code_ptr++;
751
#endif /* SLJIT_CONFIG_PPC_64 */
752
		sljit_set_function_context(NULL, (struct sljit_function_context*)code_ptr, (sljit_uw)code, (void*)sljit_generate_code);
753
	}
754
#endif /* SLJIT_INDIRECT_CALL */
755

756
	code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
757

758
	SLJIT_CACHE_FLUSH(code, code_ptr);
759
	SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
760

761
#if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
762
	if (!(options & SLJIT_GENERATE_CODE_NO_CONTEXT)) {
763
		compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins) + sizeof(struct sljit_function_context);
764
		return code_ptr;
765
	}
766
#endif /* SLJIT_INDIRECT_CALL */
767

768
	compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
769
	return code;
770
}
771

772
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
773
{
774
	switch (feature_type) {
775
	case SLJIT_HAS_FPU:
776
#ifdef SLJIT_IS_FPU_AVAILABLE
777
		return (SLJIT_IS_FPU_AVAILABLE) != 0;
778
#else
779
		/* Available by default. */
780
		return 1;
781
#endif
782
	case SLJIT_HAS_REV:
783
#if defined(_ARCH_PWR10) && _ARCH_PWR10
784
		return 1;
785
#else /* !POWER10 */
786
		return 2;
787
#endif /* POWER10 */
788
	/* A saved register is set to a zero value. */
789
	case SLJIT_HAS_ZERO_REGISTER:
790
	case SLJIT_HAS_CLZ:
791
	case SLJIT_HAS_ROT:
792
	case SLJIT_HAS_PREFETCH:
793
	case SLJIT_HAS_ATOMIC:
794
	case SLJIT_HAS_MEMORY_BARRIER:
795
		return 1;
796

797
	case SLJIT_HAS_CTZ:
798
		return 2;
799

800
	default:
801
		return 0;
802
	}
803
}
804

805
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
806
{
807
	switch (type) {
808
	case SLJIT_UNORDERED_OR_EQUAL:
809
	case SLJIT_ORDERED_NOT_EQUAL:
810
	case SLJIT_UNORDERED_OR_LESS:
811
	case SLJIT_ORDERED_GREATER_EQUAL:
812
	case SLJIT_UNORDERED_OR_GREATER:
813
	case SLJIT_ORDERED_LESS_EQUAL:
814
		return 1;
815
	}
816

817
	return 0;
818
}
819

820
/* --------------------------------------------------------------------- */
821
/*  Entry, exit                                                          */
822
/* --------------------------------------------------------------------- */
823

824
/* inp_flags: */
825

826
/* Creates an index in data_transfer_insts array. */
827
#define LOAD_DATA	0x01
828
#define INDEXED		0x02
829
#define SIGNED_DATA	0x04
830

831
#define WORD_DATA	0x00
832
#define BYTE_DATA	0x08
833
#define HALF_DATA	0x10
834
#define INT_DATA	0x18
835
/* Separates integer and floating point registers */
836
#define GPR_REG		0x1f
837
#define DOUBLE_DATA	0x20
838

839
#define MEM_MASK	0x7f
840

841
#define FLOAT_DATA(op) (DOUBLE_DATA | ((op & SLJIT_32) >> 6))
842

843
/* Other inp_flags. */
844

845
/* Integer opertion and set flags -> requires exts on 64 bit systems. */
846
#define ALT_SIGN_EXT	0x000100
847
/* This flag affects the RC() and OERC() macros. */
848
#define ALT_SET_FLAGS	0x000400
849
#define ALT_FORM1	0x001000
850
#define ALT_FORM2	0x002000
851
#define ALT_FORM3	0x004000
852
#define ALT_FORM4	0x008000
853
#define ALT_FORM5	0x010000
854

855
/* Source and destination is register. */
856
#define REG_DEST	0x000001
857
#define REG1_SOURCE	0x000002
858
#define REG2_SOURCE	0x000004
859
/*
860
ALT_SIGN_EXT		0x000100
861
ALT_SET_FLAGS		0x000200
862
ALT_FORM1		0x001000
863
...
864
ALT_FORM5		0x010000 */
865

866
static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 inp_flags, sljit_s32 reg,
867
	sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg);
868

869
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
870
#include "sljitNativePPC_32.c"
871
#else
872
#include "sljitNativePPC_64.c"
873
#endif
874

875
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
876
#define STACK_STORE	STW
877
#define STACK_LOAD	LWZ
878
#else
879
#define STACK_STORE	STD
880
#define STACK_LOAD	LD
881
#endif
882

883
#if (defined SLJIT_PPC_STACK_FRAME_V2 && SLJIT_PPC_STACK_FRAME_V2)
884
#define LR_SAVE_OFFSET		(2 * SSIZE_OF(sw))
885
#else
886
#define LR_SAVE_OFFSET		SSIZE_OF(sw)
887
#endif
888

889
#define STACK_MAX_DISTANCE	(0x8000 - SSIZE_OF(sw) - LR_SAVE_OFFSET)
890

891
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
892
	sljit_s32 options, sljit_s32 arg_types,
893
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
894
{
895
	sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
896
	sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
897
	sljit_s32 i, tmp, base, offset;
898
	sljit_s32 word_arg_count = 0;
899
	sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
900
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
901
	sljit_s32 arg_count = 0;
902
#endif
903

904
	CHECK_ERROR();
905
	CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
906
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
907

908
	scratches = ENTER_GET_REGS(scratches);
909
	saveds = ENTER_GET_REGS(saveds);
910
	local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 0)
911
		+ GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
912

913
	if (!(options & SLJIT_ENTER_REG_ARG))
914
		local_size += SSIZE_OF(sw);
915

916
	local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
917
	compiler->local_size = local_size;
918

919
	FAIL_IF(push_inst(compiler, MFLR | D(0)));
920

921
	base = SLJIT_SP;
922
	offset = local_size;
923

924
	if (local_size <= STACK_MAX_DISTANCE) {
925
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
926
		FAIL_IF(push_inst(compiler, STWU | S(SLJIT_SP) | A(SLJIT_SP) | IMM(-local_size)));
927
#else
928
		FAIL_IF(push_inst(compiler, STDU | S(SLJIT_SP) | A(SLJIT_SP) | IMM(-local_size)));
929
#endif
930
	} else {
931
		base = TMP_REG1;
932
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_SP) | A(TMP_REG1) | B(SLJIT_SP)));
933
		FAIL_IF(load_immediate(compiler, TMP_REG2, -local_size));
934
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
935
		FAIL_IF(push_inst(compiler, STWUX | S(SLJIT_SP) | A(SLJIT_SP) | B(TMP_REG2)));
936
#else
937
		FAIL_IF(push_inst(compiler, STDUX | S(SLJIT_SP) | A(SLJIT_SP) | B(TMP_REG2)));
938
#endif
939
		local_size = 0;
940
		offset = 0;
941
	}
942

943
	tmp = SLJIT_FS0 - fsaveds;
944
	for (i = SLJIT_FS0; i > tmp; i--) {
945
		offset -= SSIZE_OF(f64);
946
		FAIL_IF(push_inst(compiler, STFD | FS(i) | A(base) | IMM(offset)));
947
	}
948

949
	for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
950
		offset -= SSIZE_OF(f64);
951
		FAIL_IF(push_inst(compiler, STFD | FS(i) | A(base) | IMM(offset)));
952
	}
953

954
	if (!(options & SLJIT_ENTER_REG_ARG)) {
955
		offset -= SSIZE_OF(sw);
956
		FAIL_IF(push_inst(compiler, STACK_STORE | S(TMP_ZERO) | A(base) | IMM(offset)));
957
	}
958

959
	tmp = SLJIT_S0 - saveds;
960
	for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
961
		offset -= SSIZE_OF(sw);
962
		FAIL_IF(push_inst(compiler, STACK_STORE | S(i) | A(base) | IMM(offset)));
963
	}
964

965
	for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
966
		offset -= SSIZE_OF(sw);
967
		FAIL_IF(push_inst(compiler, STACK_STORE | S(i) | A(base) | IMM(offset)));
968
	}
969

970
	FAIL_IF(push_inst(compiler, STACK_STORE | S(0) | A(base) | IMM(local_size + LR_SAVE_OFFSET)));
971

972
	if (options & SLJIT_ENTER_REG_ARG)
973
		return SLJIT_SUCCESS;
974

975
	FAIL_IF(push_inst(compiler, ADDI | D(TMP_ZERO) | A(0) | 0));
976

977
	arg_types >>= SLJIT_ARG_SHIFT;
978
	saved_arg_count = 0;
979

980
	while (arg_types > 0) {
981
		if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
982
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
983
			do {
984
				if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
985
					tmp = SLJIT_S0 - saved_arg_count;
986
					saved_arg_count++;
987
				} else if (arg_count != word_arg_count)
988
					tmp = SLJIT_R0 + word_arg_count;
989
				else
990
					break;
991

992
				FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0 + arg_count) | A(tmp) | B(SLJIT_R0 + arg_count)));
993
			} while (0);
994
#else
995
			if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
996
				FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0 + word_arg_count) | A(SLJIT_S0 - saved_arg_count) | B(SLJIT_R0 + word_arg_count)));
997
				saved_arg_count++;
998
			}
999
#endif
1000
			word_arg_count++;
1001
		}
1002

1003
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1004
		arg_count++;
1005
#endif
1006
		arg_types >>= SLJIT_ARG_SHIFT;
1007
	}
1008

1009
	return SLJIT_SUCCESS;
1010
}
1011

1012
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1013
	sljit_s32 options, sljit_s32 arg_types,
1014
	sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1015
{
1016
	sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
1017
	sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1018

1019
	CHECK_ERROR();
1020
	CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1021
	set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1022

1023
	scratches = ENTER_GET_REGS(scratches);
1024
	saveds = ENTER_GET_REGS(saveds);
1025
	local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 0)
1026
		+ GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1027

1028
	if (!(options & SLJIT_ENTER_REG_ARG))
1029
		local_size += SSIZE_OF(sw);
1030

1031
	compiler->local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
1032
	return SLJIT_SUCCESS;
1033
}
1034

1035
static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
1036
{
1037
	sljit_s32 i, tmp, base, offset;
1038
	sljit_s32 local_size = compiler->local_size;
1039

1040
	SLJIT_ASSERT(TMP_CALL_REG != TMP_REG2);
1041

1042
	base = SLJIT_SP;
1043
	if (local_size > STACK_MAX_DISTANCE) {
1044
		base = TMP_REG2;
1045
		if (local_size > 2 * STACK_MAX_DISTANCE + LR_SAVE_OFFSET) {
1046
			FAIL_IF(push_inst(compiler, STACK_LOAD | D(base) | A(SLJIT_SP) | IMM(0)));
1047
			local_size = 0;
1048
		} else {
1049
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(SLJIT_SP) | IMM(local_size - STACK_MAX_DISTANCE)));
1050
			local_size = STACK_MAX_DISTANCE;
1051
		}
1052
	}
1053

1054
	offset = local_size;
1055
	if (!is_return_to)
1056
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(0) | A(base) | IMM(offset + LR_SAVE_OFFSET)));
1057

1058
	tmp = SLJIT_FS0 - compiler->fsaveds;
1059
	for (i = SLJIT_FS0; i > tmp; i--) {
1060
		offset -= SSIZE_OF(f64);
1061
		FAIL_IF(push_inst(compiler, LFD | FS(i) | A(base) | IMM(offset)));
1062
	}
1063

1064
	for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1065
		offset -= SSIZE_OF(f64);
1066
		FAIL_IF(push_inst(compiler, LFD | FS(i) | A(base) | IMM(offset)));
1067
	}
1068

1069
	if (!(compiler->options & SLJIT_ENTER_REG_ARG)) {
1070
		offset -= SSIZE_OF(sw);
1071
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(TMP_ZERO) | A(base) | IMM(offset)));
1072
	}
1073

1074
	tmp = SLJIT_S0 - compiler->saveds;
1075
	for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
1076
		offset -= SSIZE_OF(sw);
1077
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(i) | A(base) | IMM(offset)));
1078
	}
1079

1080
	for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1081
		offset -= SSIZE_OF(sw);
1082
		FAIL_IF(push_inst(compiler, STACK_LOAD | S(i) | A(base) | IMM(offset)));
1083
	}
1084

1085
	if (!is_return_to)
1086
		push_inst(compiler, MTLR | S(0));
1087

1088
	if (local_size > 0)
1089
		return push_inst(compiler, ADDI | D(SLJIT_SP) | A(base) | IMM(local_size));
1090

1091
	SLJIT_ASSERT(base == TMP_REG2);
1092
	return push_inst(compiler, OR | S(base) | A(SLJIT_SP) | B(base));
1093
}
1094

1095
#undef STACK_STORE
1096
#undef STACK_LOAD
1097

1098
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1099
{
1100
	CHECK_ERROR();
1101
	CHECK(check_sljit_emit_return_void(compiler));
1102

1103
	FAIL_IF(emit_stack_frame_release(compiler, 0));
1104
	return push_inst(compiler, BLR);
1105
}
1106

1107
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1108
	sljit_s32 src, sljit_sw srcw)
1109
{
1110
	CHECK_ERROR();
1111
	CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1112

1113
	if (src & SLJIT_MEM) {
1114
		ADJUST_LOCAL_OFFSET(src, srcw);
1115
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
1116
		src = TMP_CALL_REG;
1117
		srcw = 0;
1118
	} else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1119
		FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
1120
		src = TMP_CALL_REG;
1121
		srcw = 0;
1122
	}
1123

1124
	FAIL_IF(emit_stack_frame_release(compiler, 1));
1125

1126
	SLJIT_SKIP_CHECKS(compiler);
1127
	return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1128
}
1129

1130
/* --------------------------------------------------------------------- */
1131
/*  Operators                                                            */
1132
/* --------------------------------------------------------------------- */
1133

1134
/* s/l - store/load (1 bit)
1135
   i/x - immediate/indexed form
1136
   u/s - signed/unsigned (1 bit)
1137
   w/b/h/i - word/byte/half/int allowed (2 bit)
1138

1139
   Some opcodes are repeated (e.g. store signed / unsigned byte is the same instruction). */
1140

1141
/* 64 bit only: [reg+imm] must be aligned to 4 bytes. */
1142
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1143
#define INT_ALIGNED	0x10000
1144
#endif
1145

1146
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
1147
#define ARCH_32_64(a, b)	a
1148
#define INST_CODE_AND_DST(inst, flags, reg) \
1149
	((sljit_ins)(inst) | (sljit_ins)(((flags) & MEM_MASK) <= GPR_REG ? D(reg) : FD(reg)))
1150
#else
1151
#define ARCH_32_64(a, b)	b
1152
#define INST_CODE_AND_DST(inst, flags, reg) \
1153
	(((sljit_ins)(inst) & ~(sljit_ins)INT_ALIGNED) | (sljit_ins)(((flags) & MEM_MASK) <= GPR_REG ? D(reg) : FD(reg)))
1154
#endif
1155

1156
static const sljit_ins data_transfer_insts[64 + 16] = {
1157

1158
/* -------- Integer -------- */
1159

1160
/* Word. */
1161

1162
/* w u i s */ ARCH_32_64(HI(36) /* stw */, HI(62) | INT_ALIGNED | 0x0 /* std */),
1163
/* w u i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x0 /* ld */),
1164
/* w u x s */ ARCH_32_64(HI(31) | LO(151) /* stwx */, HI(31) | LO(149) /* stdx */),
1165
/* w u x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(21) /* ldx */),
1166

1167
/* w s i s */ ARCH_32_64(HI(36) /* stw */, HI(62) | INT_ALIGNED | 0x0 /* std */),
1168
/* w s i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x0 /* ld */),
1169
/* w s x s */ ARCH_32_64(HI(31) | LO(151) /* stwx */, HI(31) | LO(149) /* stdx */),
1170
/* w s x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(21) /* ldx */),
1171

1172
/* Byte. */
1173

1174
/* b u i s */ HI(38) /* stb */,
1175
/* b u i l */ HI(34) /* lbz */,
1176
/* b u x s */ HI(31) | LO(215) /* stbx */,
1177
/* b u x l */ HI(31) | LO(87) /* lbzx */,
1178

1179
/* b s i s */ HI(38) /* stb */,
1180
/* b s i l */ HI(34) /* lbz */ /* EXTS_REQ */,
1181
/* b s x s */ HI(31) | LO(215) /* stbx */,
1182
/* b s x l */ HI(31) | LO(87) /* lbzx */ /* EXTS_REQ */,
1183

1184
/* Half. */
1185

1186
/* h u i s */ HI(44) /* sth */,
1187
/* h u i l */ HI(40) /* lhz */,
1188
/* h u x s */ HI(31) | LO(407) /* sthx */,
1189
/* h u x l */ HI(31) | LO(279) /* lhzx */,
1190

1191
/* h s i s */ HI(44) /* sth */,
1192
/* h s i l */ HI(42) /* lha */,
1193
/* h s x s */ HI(31) | LO(407) /* sthx */,
1194
/* h s x l */ HI(31) | LO(343) /* lhax */,
1195

1196
/* Int. */
1197

1198
/* i u i s */ HI(36) /* stw */,
1199
/* i u i l */ HI(32) /* lwz */,
1200
/* i u x s */ HI(31) | LO(151) /* stwx */,
1201
/* i u x l */ HI(31) | LO(23) /* lwzx */,
1202

1203
/* i s i s */ HI(36) /* stw */,
1204
/* i s i l */ ARCH_32_64(HI(32) /* lwz */, HI(58) | INT_ALIGNED | 0x2 /* lwa */),
1205
/* i s x s */ HI(31) | LO(151) /* stwx */,
1206
/* i s x l */ ARCH_32_64(HI(31) | LO(23) /* lwzx */, HI(31) | LO(341) /* lwax */),
1207

1208
/* -------- Floating point -------- */
1209

1210
/* d   i s */ HI(54) /* stfd */,
1211
/* d   i l */ HI(50) /* lfd */,
1212
/* d   x s */ HI(31) | LO(727) /* stfdx */,
1213
/* d   x l */ HI(31) | LO(599) /* lfdx */,
1214

1215
/* s   i s */ HI(52) /* stfs */,
1216
/* s   i l */ HI(48) /* lfs */,
1217
/* s   x s */ HI(31) | LO(663) /* stfsx */,
1218
/* s   x l */ HI(31) | LO(535) /* lfsx */,
1219
};
1220

1221
static const sljit_ins updated_data_transfer_insts[64] = {
1222

1223
/* -------- Integer -------- */
1224

1225
/* Word. */
1226

1227
/* w u i s */ ARCH_32_64(HI(37) /* stwu */, HI(62) | INT_ALIGNED | 0x1 /* stdu */),
1228
/* w u i l */ ARCH_32_64(HI(33) /* lwzu */, HI(58) | INT_ALIGNED | 0x1 /* ldu */),
1229
/* w u x s */ ARCH_32_64(HI(31) | LO(183) /* stwux */, HI(31) | LO(181) /* stdux */),
1230
/* w u x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(53) /* ldux */),
1231

1232
/* w s i s */ ARCH_32_64(HI(37) /* stwu */, HI(62) | INT_ALIGNED | 0x1 /* stdu */),
1233
/* w s i l */ ARCH_32_64(HI(33) /* lwzu */, HI(58) | INT_ALIGNED | 0x1 /* ldu */),
1234
/* w s x s */ ARCH_32_64(HI(31) | LO(183) /* stwux */, HI(31) | LO(181) /* stdux */),
1235
/* w s x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(53) /* ldux */),
1236

1237
/* Byte. */
1238

1239
/* b u i s */ HI(39) /* stbu */,
1240
/* b u i l */ HI(35) /* lbzu */,
1241
/* b u x s */ HI(31) | LO(247) /* stbux */,
1242
/* b u x l */ HI(31) | LO(119) /* lbzux */,
1243

1244
/* b s i s */ HI(39) /* stbu */,
1245
/* b s i l */ 0 /* no such instruction */,
1246
/* b s x s */ HI(31) | LO(247) /* stbux */,
1247
/* b s x l */ 0 /* no such instruction */,
1248

1249
/* Half. */
1250

1251
/* h u i s */ HI(45) /* sthu */,
1252
/* h u i l */ HI(41) /* lhzu */,
1253
/* h u x s */ HI(31) | LO(439) /* sthux */,
1254
/* h u x l */ HI(31) | LO(311) /* lhzux */,
1255

1256
/* h s i s */ HI(45) /* sthu */,
1257
/* h s i l */ HI(43) /* lhau */,
1258
/* h s x s */ HI(31) | LO(439) /* sthux */,
1259
/* h s x l */ HI(31) | LO(375) /* lhaux */,
1260

1261
/* Int. */
1262

1263
/* i u i s */ HI(37) /* stwu */,
1264
/* i u i l */ HI(33) /* lwzu */,
1265
/* i u x s */ HI(31) | LO(183) /* stwux */,
1266
/* i u x l */ HI(31) | LO(55) /* lwzux */,
1267

1268
/* i s i s */ HI(37) /* stwu */,
1269
/* i s i l */ ARCH_32_64(HI(33) /* lwzu */, 0 /* no such instruction */),
1270
/* i s x s */ HI(31) | LO(183) /* stwux */,
1271
/* i s x l */ ARCH_32_64(HI(31) | LO(55) /* lwzux */, HI(31) | LO(373) /* lwaux */),
1272

1273
/* -------- Floating point -------- */
1274

1275
/* d   i s */ HI(55) /* stfdu */,
1276
/* d   i l */ HI(51) /* lfdu */,
1277
/* d   x s */ HI(31) | LO(759) /* stfdux */,
1278
/* d   x l */ HI(31) | LO(631) /* lfdux */,
1279

1280
/* s   i s */ HI(53) /* stfsu */,
1281
/* s   i l */ HI(49) /* lfsu */,
1282
/* s   x s */ HI(31) | LO(695) /* stfsux */,
1283
/* s   x l */ HI(31) | LO(567) /* lfsux */,
1284
};
1285

1286
#undef ARCH_32_64
1287

1288
/* Simple cases, (no caching is required). */
1289
static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 inp_flags, sljit_s32 reg,
1290
	sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1291
{
1292
	sljit_ins inst;
1293
	sljit_s32 offs_reg;
1294

1295
	/* Should work when (arg & REG_MASK) == 0. */
1296
	SLJIT_ASSERT(A(0) == 0);
1297
	SLJIT_ASSERT(arg & SLJIT_MEM);
1298

1299
	if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1300
		argw &= 0x3;
1301
		offs_reg = OFFS_REG(arg);
1302

1303
		if (argw != 0) {
1304
			FAIL_IF(push_inst(compiler, SLWI_W(argw) | S(OFFS_REG(arg)) | A(tmp_reg)));
1305
			offs_reg = tmp_reg;
1306
		}
1307

1308
		inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1309

1310
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1311
		SLJIT_ASSERT(!(inst & INT_ALIGNED));
1312
#endif /* SLJIT_CONFIG_PPC_64 */
1313

1314
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg & REG_MASK) | B(offs_reg));
1315
	}
1316

1317
	inst = data_transfer_insts[inp_flags & MEM_MASK];
1318
	arg &= REG_MASK;
1319

1320
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1321
	if ((inst & INT_ALIGNED) && (argw & 0x3) != 0) {
1322
		FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1323

1324
		inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1325
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | B(tmp_reg));
1326
	}
1327
#endif /* SLJIT_CONFIG_PPC_64 */
1328

1329
	if (argw <= SIMM_MAX && argw >= SIMM_MIN)
1330
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | IMM(argw));
1331

1332
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1333
	if (argw <= 0x7fff7fffl && argw >= -0x80000000l) {
1334
#endif /* SLJIT_CONFIG_PPC_64 */
1335
		FAIL_IF(push_inst(compiler, ADDIS | D(tmp_reg) | A(arg) | IMM((argw + 0x8000) >> 16)));
1336
		return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(tmp_reg) | IMM(argw));
1337
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1338
	}
1339

1340
	FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1341

1342
	inst = data_transfer_insts[(inp_flags | INDEXED) & MEM_MASK];
1343
	return push_inst(compiler, INST_CODE_AND_DST(inst, inp_flags, reg) | A(arg) | B(tmp_reg));
1344
#endif /* SLJIT_CONFIG_PPC_64 */
1345
}
1346

1347
static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 input_flags,
1348
	sljit_s32 dst, sljit_sw dstw,
1349
	sljit_s32 src1, sljit_sw src1w,
1350
	sljit_s32 src2, sljit_sw src2w)
1351
{
1352
	/* arg1 goes to TMP_REG1 or src reg
1353
	   arg2 goes to TMP_REG2, imm or src reg
1354
	   result goes to TMP_REG2, so put result can use TMP_REG1. */
1355
	sljit_s32 dst_r = TMP_REG2;
1356
	sljit_s32 src1_r;
1357
	sljit_s32 src2_r;
1358
	sljit_s32 src2_tmp_reg = (!(input_flags & ALT_SIGN_EXT) && GET_OPCODE(op) >= SLJIT_OP2_BASE && FAST_IS_REG(src1)) ? TMP_REG1 : TMP_REG2;
1359
	sljit_s32 flags = input_flags & (ALT_FORM1 | ALT_FORM2 | ALT_FORM3 | ALT_FORM4 | ALT_FORM5 | ALT_SIGN_EXT | ALT_SET_FLAGS);
1360

1361
	/* Destination check. */
1362
	if (FAST_IS_REG(dst)) {
1363
		dst_r = dst;
1364
		/* The REG_DEST is only used by SLJIT_MOV operations, although
1365
		 * it is set for op2 operations with unset destination. */
1366
		flags |= REG_DEST;
1367

1368
		if (op >= SLJIT_MOV && op <= SLJIT_MOV_P)
1369
			src2_tmp_reg = dst_r;
1370
	}
1371

1372
	/* Source 2. */
1373
	if (FAST_IS_REG(src2)) {
1374
		src2_r = src2;
1375
		flags |= REG2_SOURCE;
1376

1377
		if (!(flags & REG_DEST) && op >= SLJIT_MOV && op <= SLJIT_MOV_P)
1378
			dst_r = src2_r;
1379
	} else if (src2 == SLJIT_IMM) {
1380
		src2_r = TMP_ZERO;
1381
		if (src2w != 0) {
1382
			FAIL_IF(load_immediate(compiler, src2_tmp_reg, src2w));
1383
			src2_r = src2_tmp_reg;
1384
		}
1385
	} else {
1386
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, src2_tmp_reg, src2, src2w, TMP_REG1));
1387
		src2_r = src2_tmp_reg;
1388
	}
1389

1390
	/* Source 1. */
1391
	if (FAST_IS_REG(src1)) {
1392
		src1_r = src1;
1393
		flags |= REG1_SOURCE;
1394
	} else if (src1 == SLJIT_IMM) {
1395
		src1_r = TMP_ZERO;
1396
		if (src1w != 0) {
1397
			FAIL_IF(load_immediate(compiler, TMP_REG1, src1w));
1398
			src1_r = TMP_REG1;
1399
		}
1400
	} else {
1401
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, TMP_REG1, src1, src1w, TMP_REG1));
1402
		src1_r = TMP_REG1;
1403
	}
1404

1405
	FAIL_IF(emit_single_op(compiler, op, flags, dst_r, src1_r, src2_r));
1406

1407
	if (!(dst & SLJIT_MEM))
1408
		return SLJIT_SUCCESS;
1409

1410
	return emit_op_mem(compiler, input_flags, dst_r, dst, dstw, TMP_REG1);
1411
}
1412

1413
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1414
{
1415
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1416
	sljit_s32 int_op = op & SLJIT_32;
1417
#endif
1418

1419
	CHECK_ERROR();
1420
	CHECK(check_sljit_emit_op0(compiler, op));
1421

1422
	op = GET_OPCODE(op);
1423
	switch (op) {
1424
	case SLJIT_BREAKPOINT:
1425
	case SLJIT_NOP:
1426
		return push_inst(compiler, NOP);
1427
	case SLJIT_LMUL_UW:
1428
	case SLJIT_LMUL_SW:
1429
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R0)));
1430
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1431
		FAIL_IF(push_inst(compiler, MULLD | D(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R1)));
1432
		return push_inst(compiler, (op == SLJIT_LMUL_UW ? MULHDU : MULHD) | D(SLJIT_R1) | A(TMP_REG1) | B(SLJIT_R1));
1433
#else
1434
		FAIL_IF(push_inst(compiler, MULLW | D(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R1)));
1435
		return push_inst(compiler, (op == SLJIT_LMUL_UW ? MULHWU : MULHW) | D(SLJIT_R1) | A(TMP_REG1) | B(SLJIT_R1));
1436
#endif
1437
	case SLJIT_DIVMOD_UW:
1438
	case SLJIT_DIVMOD_SW:
1439
		FAIL_IF(push_inst(compiler, OR | S(SLJIT_R0) | A(TMP_REG1) | B(SLJIT_R0)));
1440
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1441
		FAIL_IF(push_inst(compiler, (int_op ? (op == SLJIT_DIVMOD_UW ? DIVWU : DIVW) : (op == SLJIT_DIVMOD_UW ? DIVDU : DIVD)) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1)));
1442
		FAIL_IF(push_inst(compiler, (int_op ? MULLW : MULLD) | D(SLJIT_R1) | A(SLJIT_R0) | B(SLJIT_R1)));
1443
#else
1444
		FAIL_IF(push_inst(compiler, (op == SLJIT_DIVMOD_UW ? DIVWU : DIVW) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1)));
1445
		FAIL_IF(push_inst(compiler, MULLW | D(SLJIT_R1) | A(SLJIT_R0) | B(SLJIT_R1)));
1446
#endif
1447
		return push_inst(compiler, SUBF | D(SLJIT_R1) | A(SLJIT_R1) | B(TMP_REG1));
1448
	case SLJIT_DIV_UW:
1449
	case SLJIT_DIV_SW:
1450
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1451
		return push_inst(compiler, (int_op ? (op == SLJIT_DIV_UW ? DIVWU : DIVW) : (op == SLJIT_DIV_UW ? DIVDU : DIVD)) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1));
1452
#else
1453
		return push_inst(compiler, (op == SLJIT_DIV_UW ? DIVWU : DIVW) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1));
1454
#endif
1455
	case SLJIT_MEMORY_BARRIER:
1456
		return push_inst(compiler, SYNC);
1457
	case SLJIT_ENDBR:
1458
	case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1459
		return SLJIT_SUCCESS;
1460
	}
1461

1462
	return SLJIT_SUCCESS;
1463
}
1464

1465
static sljit_s32 emit_rev(struct sljit_compiler *compiler, sljit_s32 op,
1466
	sljit_s32 dst, sljit_sw dstw,
1467
	sljit_s32 src, sljit_sw srcw)
1468
{
1469
	sljit_s32 mem, offs_reg, inp_flags;
1470
	sljit_sw memw;
1471
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1472
	sljit_s32 is_32 = op & SLJIT_32;
1473

1474
	op = GET_OPCODE(op);
1475
#endif /* SLJIT_CONFIG_PPC_64 */
1476

1477
	if (!((dst | src) & SLJIT_MEM)) {
1478
		/* Both are registers. */
1479
		if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1480
			if (src == dst) {
1481
				FAIL_IF(push_inst(compiler, RLWIMI | S(dst) | A(dst) | RLWI_SH(16) | RLWI_MBE(8, 15)));
1482
				FAIL_IF(push_inst(compiler, RLWINM | S(dst) | A(dst) | RLWI_SH(24) | RLWI_MBE(16, 31)));
1483
			} else {
1484
				FAIL_IF(push_inst(compiler, RLWINM | S(src) | A(dst) | RLWI_SH(8) | RLWI_MBE(16, 23)));
1485
				FAIL_IF(push_inst(compiler, RLWIMI | S(src) | A(dst) | RLWI_SH(24) | RLWI_MBE(24, 31)));
1486
			}
1487

1488
			if (op == SLJIT_REV_U16)
1489
				return SLJIT_SUCCESS;
1490
			return push_inst(compiler, EXTSH | S(dst) | A(dst));
1491
		}
1492

1493
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1494
		if (!is_32) {
1495
#if defined(_ARCH_PWR10) && _ARCH_PWR10
1496
			return push_inst(compiler, BRD | S(src) | A(dst));
1497
#else /* !POWER10 */
1498
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET_HI)));
1499
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1500
			FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(SLJIT_SP) | B(TMP_REG2)));
1501
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET_LO)));
1502
			FAIL_IF(push_inst(compiler, STWBRX | S(TMP_REG1) | A(SLJIT_SP) | B(TMP_REG2)));
1503
			return push_inst(compiler, LD | D(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
1504
#endif /* POWER10 */
1505
		}
1506
#endif /* SLJIT_CONFIG_PPC_64 */
1507

1508
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(0) | IMM(TMP_MEM_OFFSET)));
1509
		FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(SLJIT_SP) | B(TMP_REG2)));
1510
		FAIL_IF(push_inst(compiler, LWZ | D(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET));
1511

1512
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1513
		if (op == SLJIT_REV_S32)
1514
			return push_inst(compiler, EXTSW | S(dst) | A(dst));
1515
#endif /* SLJIT_CONFIG_PPC_64 */
1516
		return SLJIT_SUCCESS;
1517
	}
1518

1519
	mem = src;
1520
	memw = srcw;
1521

1522
	if (dst & SLJIT_MEM) {
1523
		mem = dst;
1524
		memw = dstw;
1525

1526
		if (src & SLJIT_MEM) {
1527
			inp_flags = HALF_DATA | LOAD_DATA;
1528

1529
			if (op != SLJIT_REV_U16 && op != SLJIT_REV_S16) {
1530
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1531
				inp_flags = (is_32 ? INT_DATA : WORD_DATA) | LOAD_DATA;
1532
#else /* !SLJIT_CONFIG_PPC_64 */
1533
				inp_flags = WORD_DATA | LOAD_DATA;
1534
#endif /* SLJIT_CONFIG_PPC_64 */
1535
			}
1536

1537
			FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG1, src, srcw, TMP_REG2));
1538
			src = TMP_REG1;
1539
		}
1540
	}
1541

1542
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
1543
		offs_reg = OFFS_REG(mem);
1544
		mem &= REG_MASK;
1545
		memw &= 0x3;
1546

1547
		if (memw != 0) {
1548
			FAIL_IF(push_inst(compiler, SLWI_W(memw) | S(offs_reg) | A(TMP_REG2)));
1549
			offs_reg = TMP_REG2;
1550
		}
1551
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1552
	} else if (memw > 0x7fff7fffl || memw < -0x80000000l) {
1553
		FAIL_IF(load_immediate(compiler, TMP_REG2, memw));
1554
		offs_reg = TMP_REG2;
1555
		mem &= REG_MASK;
1556
#endif /* SLJIT_CONFIG_PPC_64 */
1557
	} else {
1558
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(mem & REG_MASK) | IMM(memw)));
1559
		if (memw > SIMM_MAX || memw < SIMM_MIN)
1560
			FAIL_IF(push_inst(compiler, ADDIS | D(TMP_REG2) | A(TMP_REG2) | IMM((memw + 0x8000) >> 16)));
1561

1562
		mem = 0;
1563
		offs_reg = TMP_REG2;
1564
	}
1565

1566
	if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
1567
		if (dst & SLJIT_MEM)
1568
			return push_inst(compiler, STHBRX | S(src) | A(mem) | B(offs_reg));
1569

1570
		FAIL_IF(push_inst(compiler, LHBRX | S(dst) | A(mem) | B(offs_reg)));
1571

1572
		if (op == SLJIT_REV_U16)
1573
			return SLJIT_SUCCESS;
1574
		return push_inst(compiler, EXTSH | S(dst) | A(dst));
1575
	}
1576

1577
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1578
	if (!is_32) {
1579
		if (dst & SLJIT_MEM) {
1580
#if defined(_ARCH_PWR7) && _ARCH_PWR7
1581
			return push_inst(compiler, STDBRX | S(src) | A(mem) | B(offs_reg));
1582
#else /* !POWER7 */
1583
#if defined(SLJIT_LITTLE_ENDIAN) && SLJIT_LITTLE_ENDIAN
1584
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1585
			FAIL_IF(push_inst(compiler, STWBRX | S(TMP_REG1) | A(mem) | B(offs_reg)));
1586
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1587
			return push_inst(compiler, STWBRX | S(src) | A(mem) | B(TMP_REG2));
1588
#else /* !SLJIT_LITTLE_ENDIAN */
1589
			FAIL_IF(push_inst(compiler, STWBRX | S(src) | A(mem) | B(offs_reg)));
1590
			FAIL_IF(push_inst(compiler, RLDICL | S(src) | A(TMP_REG1) | RLDI_SH(32) | RLDI_MB(32)));
1591
			FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1592
			return push_inst(compiler, STWBRX | S(TMP_REG1) | A(mem) | B(TMP_REG2));
1593
#endif /* SLJIT_LITTLE_ENDIAN */
1594
#endif /* POWER7 */
1595
		}
1596
#if defined(_ARCH_PWR7) && _ARCH_PWR7
1597
		return push_inst(compiler, LDBRX | S(dst) | A(mem) | B(offs_reg));
1598
#else /* !POWER7 */
1599
		FAIL_IF(push_inst(compiler, LWBRX | LWBRX_FIRST_REG | A(mem) | B(offs_reg)));
1600
		FAIL_IF(push_inst(compiler, ADDI | D(TMP_REG2) | A(offs_reg) | IMM(SSIZE_OF(s32))));
1601
		FAIL_IF(push_inst(compiler, LWBRX | LWBRX_SECOND_REG | A(mem) | B(TMP_REG2)));
1602
		return push_inst(compiler, RLDIMI | S(TMP_REG1) | A(dst) | RLDI_SH(32) | RLDI_MB(0));
1603
#endif /* POWER7 */
1604
	}
1605
#endif /* SLJIT_CONFIG_PPC_64 */
1606

1607
	if (dst & SLJIT_MEM)
1608
		return push_inst(compiler, STWBRX | S(src) | A(mem) | B(offs_reg));
1609

1610
	FAIL_IF(push_inst(compiler, LWBRX | S(dst) | A(mem) | B(offs_reg)));
1611
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1612
	if (op == SLJIT_REV_S32)
1613
		return push_inst(compiler, EXTSW | S(dst) | A(dst));
1614
#endif /* SLJIT_CONFIG_PPC_64 */
1615
	return SLJIT_SUCCESS;
1616
}
1617

1618
#define EMIT_MOV(type, type_flags, type_cast) \
1619
	emit_op(compiler, (src == SLJIT_IMM) ? SLJIT_MOV : type, flags | (type_flags), dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? type_cast srcw : srcw)
1620

1621
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1622
	sljit_s32 dst, sljit_sw dstw,
1623
	sljit_s32 src, sljit_sw srcw)
1624
{
1625
	sljit_s32 flags = HAS_FLAGS(op) ? ALT_SET_FLAGS : 0;
1626
	sljit_s32 op_flags = GET_ALL_FLAGS(op);
1627

1628
	CHECK_ERROR();
1629
	CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1630
	ADJUST_LOCAL_OFFSET(dst, dstw);
1631
	ADJUST_LOCAL_OFFSET(src, srcw);
1632

1633
	op = GET_OPCODE(op);
1634

1635
	if (GET_FLAG_TYPE(op_flags) == SLJIT_OVERFLOW)
1636
		FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1637

1638
	if (op <= SLJIT_MOV_P && FAST_IS_REG(src) && src == dst) {
1639
		if (!TYPE_CAST_NEEDED(op))
1640
			return SLJIT_SUCCESS;
1641
	}
1642

1643
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1644
	if (op_flags & SLJIT_32) {
1645
		if (op <= SLJIT_MOV_P) {
1646
			if (src & SLJIT_MEM) {
1647
				if (op == SLJIT_MOV_S32)
1648
					op = SLJIT_MOV_U32;
1649
			}
1650
			else if (src == SLJIT_IMM) {
1651
				if (op == SLJIT_MOV_U32)
1652
					op = SLJIT_MOV_S32;
1653
			}
1654
		}
1655
		else {
1656
			/* Most operations expect sign extended arguments. */
1657
			flags |= INT_DATA | SIGNED_DATA;
1658
			if (HAS_FLAGS(op_flags))
1659
				flags |= ALT_SIGN_EXT;
1660
		}
1661
	}
1662
#endif
1663

1664
	switch (op) {
1665
	case SLJIT_MOV:
1666
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
1667
	case SLJIT_MOV_U32:
1668
	case SLJIT_MOV_S32:
1669
	case SLJIT_MOV32:
1670
#endif
1671
	case SLJIT_MOV_P:
1672
		return emit_op(compiler, SLJIT_MOV, flags | WORD_DATA, dst, dstw, TMP_REG1, 0, src, srcw);
1673

1674
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1675
	case SLJIT_MOV_U32:
1676
		return EMIT_MOV(SLJIT_MOV_U32, INT_DATA, (sljit_u32));
1677

1678
	case SLJIT_MOV_S32:
1679
	case SLJIT_MOV32:
1680
		return EMIT_MOV(SLJIT_MOV_S32, INT_DATA | SIGNED_DATA, (sljit_s32));
1681
#endif
1682

1683
	case SLJIT_MOV_U8:
1684
		return EMIT_MOV(SLJIT_MOV_U8, BYTE_DATA, (sljit_u8));
1685

1686
	case SLJIT_MOV_S8:
1687
		return EMIT_MOV(SLJIT_MOV_S8, BYTE_DATA | SIGNED_DATA, (sljit_s8));
1688

1689
	case SLJIT_MOV_U16:
1690
		return EMIT_MOV(SLJIT_MOV_U16, HALF_DATA, (sljit_u16));
1691

1692
	case SLJIT_MOV_S16:
1693
		return EMIT_MOV(SLJIT_MOV_S16, HALF_DATA | SIGNED_DATA, (sljit_s16));
1694

1695
	case SLJIT_CLZ:
1696
	case SLJIT_CTZ:
1697
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1698
		if (op_flags & SLJIT_32)
1699
			flags |= ALT_FORM1;
1700
#endif /* SLJIT_CONFIG_PPC_64 */
1701
		return emit_op(compiler, op, flags, dst, dstw, TMP_REG1, 0, src, srcw);
1702
	case SLJIT_REV_U32:
1703
	case SLJIT_REV_S32:
1704
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1705
		op |= SLJIT_32;
1706
#endif /* SLJIT_CONFIG_PPC_64 */
1707
		SLJIT_FALLTHROUGH
1708
	case SLJIT_REV:
1709
	case SLJIT_REV_U16:
1710
	case SLJIT_REV_S16:
1711
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1712
		op |= (op_flags & SLJIT_32);
1713
#endif /* SLJIT_CONFIG_PPC_64 */
1714
		return emit_rev(compiler, op, dst, dstw, src, srcw);
1715
	}
1716

1717
	return SLJIT_SUCCESS;
1718
}
1719

1720
#undef EMIT_MOV
1721

1722
/* Macros for checking different operand types / values. */
1723
#define TEST_SL_IMM(src, srcw) \
1724
	((src) == SLJIT_IMM && (srcw) <= SIMM_MAX && (srcw) >= SIMM_MIN)
1725
#define TEST_UL_IMM(src, srcw) \
1726
	((src) == SLJIT_IMM && !((srcw) & ~0xffff))
1727
#define TEST_UH_IMM(src, srcw) \
1728
	((src) == SLJIT_IMM && !((srcw) & ~(sljit_sw)0xffff0000))
1729

1730
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1731
#define TEST_SH_IMM(src, srcw) \
1732
	((src) == SLJIT_IMM && !((srcw) & 0xffff) && (srcw) <= 0x7fffffffl && (srcw) >= -0x80000000l)
1733
#define TEST_ADD_IMM(src, srcw) \
1734
	((src) == SLJIT_IMM && (srcw) <= 0x7fff7fffl && (srcw) >= -0x80000000l)
1735
#define TEST_UI_IMM(src, srcw) \
1736
	((src) == SLJIT_IMM && !((srcw) & ~0xffffffff))
1737

1738
#define TEST_ADD_FORM1(op) \
1739
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW \
1740
		|| (op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_32 | SLJIT_SET_Z | SLJIT_SET_CARRY))
1741
#define TEST_SUB_FORM2(op) \
1742
	((GET_FLAG_TYPE(op) >= SLJIT_SIG_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) \
1743
		|| (op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)) == (SLJIT_32 | SLJIT_SET_Z))
1744
#define TEST_SUB_FORM3(op) \
1745
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW \
1746
		|| (op & (SLJIT_32 | SLJIT_SET_Z)) == (SLJIT_32 | SLJIT_SET_Z))
1747

1748
#else /* !SLJIT_CONFIG_PPC_64 */
1749
#define TEST_SH_IMM(src, srcw) \
1750
	((src) == SLJIT_IMM && !((srcw) & 0xffff))
1751
#define TEST_ADD_IMM(src, srcw) \
1752
	((src) == SLJIT_IMM)
1753
#define TEST_UI_IMM(src, srcw) \
1754
	((src) == SLJIT_IMM)
1755

1756
#define TEST_ADD_FORM1(op) \
1757
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1758
#define TEST_SUB_FORM2(op) \
1759
	(GET_FLAG_TYPE(op) >= SLJIT_SIG_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL)
1760
#define TEST_SUB_FORM3(op) \
1761
	(GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1762
#endif /* SLJIT_CONFIG_PPC_64 */
1763

1764
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1765
	sljit_s32 dst, sljit_sw dstw,
1766
	sljit_s32 src1, sljit_sw src1w,
1767
	sljit_s32 src2, sljit_sw src2w)
1768
{
1769
	sljit_s32 flags = HAS_FLAGS(op) ? ALT_SET_FLAGS : 0;
1770

1771
	CHECK_ERROR();
1772
	CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1773
	ADJUST_LOCAL_OFFSET(dst, dstw);
1774
	ADJUST_LOCAL_OFFSET(src1, src1w);
1775
	ADJUST_LOCAL_OFFSET(src2, src2w);
1776

1777
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1778
	if (op & SLJIT_32) {
1779
		/* Most operations expect sign extended arguments. */
1780
		flags |= INT_DATA | SIGNED_DATA;
1781
		if (src1 == SLJIT_IMM)
1782
			src1w = (sljit_s32)(src1w);
1783
		if (src2 == SLJIT_IMM)
1784
			src2w = (sljit_s32)(src2w);
1785
		if (HAS_FLAGS(op))
1786
			flags |= ALT_SIGN_EXT;
1787
	}
1788
#endif
1789
	if (GET_FLAG_TYPE(op) == SLJIT_OVERFLOW)
1790
		FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1791

1792
	switch (GET_OPCODE(op)) {
1793
	case SLJIT_ADD:
1794
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1795

1796
		if (TEST_ADD_FORM1(op))
1797
			return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM1, dst, dstw, src1, src1w, src2, src2w);
1798

1799
		if (!HAS_FLAGS(op) && (src1 == SLJIT_IMM || src2 == SLJIT_IMM)) {
1800
			if (TEST_SL_IMM(src2, src2w)) {
1801
				compiler->imm = (sljit_ins)src2w & 0xffff;
1802
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1803
			}
1804
			if (TEST_SL_IMM(src1, src1w)) {
1805
				compiler->imm = (sljit_ins)src1w & 0xffff;
1806
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2, dst, dstw, src2, src2w, TMP_REG2, 0);
1807
			}
1808
			if (TEST_SH_IMM(src2, src2w)) {
1809
				compiler->imm = (sljit_ins)(src2w >> 16) & 0xffff;
1810
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1811
			}
1812
			if (TEST_SH_IMM(src1, src1w)) {
1813
				compiler->imm = (sljit_ins)(src1w >> 16) & 0xffff;
1814
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1815
			}
1816
			/* Range between -1 and -32768 is covered above. */
1817
			if (TEST_ADD_IMM(src2, src2w)) {
1818
				compiler->imm = (sljit_ins)src2w & 0xffffffff;
1819
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1820
			}
1821
			if (TEST_ADD_IMM(src1, src1w)) {
1822
				compiler->imm = (sljit_ins)src1w & 0xffffffff;
1823
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src2, src2w, TMP_REG2, 0);
1824
			}
1825
		}
1826

1827
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1828
		if ((op & (SLJIT_32 | SLJIT_SET_Z)) == (SLJIT_32 | SLJIT_SET_Z)) {
1829
			if (TEST_SL_IMM(src2, src2w)) {
1830
				compiler->imm = (sljit_ins)src2w & 0xffff;
1831
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4 | ALT_FORM5, dst, dstw, src1, src1w, TMP_REG2, 0);
1832
			}
1833
			if (TEST_SL_IMM(src1, src1w)) {
1834
				compiler->imm = (sljit_ins)src1w & 0xffff;
1835
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4 | ALT_FORM5, dst, dstw, src2, src2w, TMP_REG2, 0);
1836
			}
1837
			return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM4, dst, dstw, src1, src1w, src2, src2w);
1838
		}
1839
#endif
1840
		if (HAS_FLAGS(op)) {
1841
			if (TEST_SL_IMM(src2, src2w)) {
1842
				compiler->imm = (sljit_ins)src2w & 0xffff;
1843
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1844
			}
1845
			if (TEST_SL_IMM(src1, src1w)) {
1846
				compiler->imm = (sljit_ins)src1w & 0xffff;
1847
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1848
			}
1849
		}
1850
		return emit_op(compiler, SLJIT_ADD, flags | ((GET_FLAG_TYPE(op) == SLJIT_CARRY) ? ALT_FORM5 : 0), dst, dstw, src1, src1w, src2, src2w);
1851

1852
	case SLJIT_ADDC:
1853
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1854
		return emit_op(compiler, SLJIT_ADDC, flags, dst, dstw, src1, src1w, src2, src2w);
1855

1856
	case SLJIT_SUB:
1857
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1858

1859
		if (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_LESS_EQUAL) {
1860
			if (dst == TMP_REG1) {
1861
				if (TEST_UL_IMM(src2, src2w)) {
1862
					compiler->imm = (sljit_ins)src2w & 0xffff;
1863
					return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1864
				}
1865
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1, dst, dstw, src1, src1w, src2, src2w);
1866
			}
1867

1868
			if (src2 == SLJIT_IMM && src2w >= 0 && src2w <= (SIMM_MAX + 1)) {
1869
				compiler->imm = (sljit_ins)src2w;
1870
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1871
			}
1872
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM1 | ALT_FORM3, dst, dstw, src1, src1w, src2, src2w);
1873
		}
1874

1875
		if (dst == TMP_REG1 && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) {
1876
			if (TEST_SL_IMM(src2, src2w)) {
1877
				compiler->imm = (sljit_ins)src2w & 0xffff;
1878
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1879
			}
1880
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2, dst, dstw, src1, src1w, src2, src2w);
1881
		}
1882

1883
		if (TEST_SUB_FORM2(op)) {
1884
			if (src2 == SLJIT_IMM && src2w >= -SIMM_MAX && src2w <= SIMM_MAX) {
1885
				compiler->imm = (sljit_ins)src2w & 0xffff;
1886
				return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM3 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1887
			}
1888
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, src2, src2w);
1889
		}
1890

1891
		if (TEST_SUB_FORM3(op))
1892
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM3, dst, dstw, src1, src1w, src2, src2w);
1893

1894
		if (TEST_SL_IMM(src2, -src2w)) {
1895
			compiler->imm = (sljit_ins)(-src2w) & 0xffff;
1896
			return emit_op(compiler, SLJIT_ADD, flags | (!HAS_FLAGS(op) ? ALT_FORM2 : ALT_FORM3), dst, dstw, src1, src1w, TMP_REG2, 0);
1897
		}
1898

1899
		if (TEST_SL_IMM(src1, src1w) && !(op & SLJIT_SET_Z)) {
1900
			compiler->imm = (sljit_ins)src1w & 0xffff;
1901
			return emit_op(compiler, SLJIT_SUB, flags | ALT_FORM4, dst, dstw, src2, src2w, TMP_REG2, 0);
1902
		}
1903

1904
		if (!HAS_FLAGS(op)) {
1905
			if (TEST_SH_IMM(src2, -src2w)) {
1906
				compiler->imm = (sljit_ins)((-src2w) >> 16) & 0xffff;
1907
				return emit_op(compiler, SLJIT_ADD, flags |  ALT_FORM2 | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1908
			}
1909
			/* Range between -1 and -32768 is covered above. */
1910
			if (TEST_ADD_IMM(src2, -src2w)) {
1911
				compiler->imm = (sljit_ins)-src2w;
1912
				return emit_op(compiler, SLJIT_ADD, flags | ALT_FORM2 | ALT_FORM4, dst, dstw, src1, src1w, TMP_REG2, 0);
1913
			}
1914
		}
1915

1916
		/* We know ALT_SIGN_EXT is set if it is an SLJIT_32 on 64 bit systems. */
1917
		return emit_op(compiler, SLJIT_SUB, flags | ((GET_FLAG_TYPE(op) == SLJIT_CARRY) ? ALT_FORM5 : 0), dst, dstw, src1, src1w, src2, src2w);
1918

1919
	case SLJIT_SUBC:
1920
		compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1921
		return emit_op(compiler, SLJIT_SUBC, flags, dst, dstw, src1, src1w, src2, src2w);
1922

1923
	case SLJIT_MUL:
1924
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1925
		if (op & SLJIT_32)
1926
			flags |= ALT_FORM2;
1927
#endif
1928
		if (!HAS_FLAGS(op)) {
1929
			if (TEST_SL_IMM(src2, src2w)) {
1930
				compiler->imm = (sljit_ins)src2w & 0xffff;
1931
				return emit_op(compiler, SLJIT_MUL, flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1932
			}
1933
			if (TEST_SL_IMM(src1, src1w)) {
1934
				compiler->imm = (sljit_ins)src1w & 0xffff;
1935
				return emit_op(compiler, SLJIT_MUL, flags | ALT_FORM1, dst, dstw, src2, src2w, TMP_REG2, 0);
1936
			}
1937
		}
1938
		else
1939
			FAIL_IF(push_inst(compiler, MTXER | S(TMP_ZERO)));
1940
		return emit_op(compiler, SLJIT_MUL, flags, dst, dstw, src1, src1w, src2, src2w);
1941

1942
	case SLJIT_XOR:
1943
		if (src2 == SLJIT_IMM && src2w == -1) {
1944
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM4, dst, dstw, TMP_REG1, 0, src1, src1w);
1945
		}
1946
		if (src1 == SLJIT_IMM && src1w == -1) {
1947
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM4, dst, dstw, TMP_REG1, 0, src2, src2w);
1948
		}
1949
		SLJIT_FALLTHROUGH
1950
	case SLJIT_AND:
1951
	case SLJIT_OR:
1952
		/* Commutative unsigned operations. */
1953
		if (!HAS_FLAGS(op) || GET_OPCODE(op) == SLJIT_AND) {
1954
			if (TEST_UL_IMM(src2, src2w)) {
1955
				compiler->imm = (sljit_ins)src2w;
1956
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1957
			}
1958
			if (TEST_UL_IMM(src1, src1w)) {
1959
				compiler->imm = (sljit_ins)src1w;
1960
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src2, src2w, TMP_REG2, 0);
1961
			}
1962
			if (TEST_UH_IMM(src2, src2w)) {
1963
				compiler->imm = (sljit_ins)(src2w >> 16) & 0xffff;
1964
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM2, dst, dstw, src1, src1w, TMP_REG2, 0);
1965
			}
1966
			if (TEST_UH_IMM(src1, src1w)) {
1967
				compiler->imm = (sljit_ins)(src1w >> 16) & 0xffff;
1968
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM2, dst, dstw, src2, src2w, TMP_REG2, 0);
1969
			}
1970
		}
1971
		if (!HAS_FLAGS(op) && GET_OPCODE(op) != SLJIT_AND) {
1972
			/* Unlike or and xor, the and resets unwanted bits as well. */
1973
			if (TEST_UI_IMM(src2, src2w)) {
1974
				compiler->imm = (sljit_ins)src2w;
1975
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM3, dst, dstw, src1, src1w, TMP_REG2, 0);
1976
			}
1977
			if (TEST_UI_IMM(src1, src1w)) {
1978
				compiler->imm = (sljit_ins)src1w;
1979
				return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM3, dst, dstw, src2, src2w, TMP_REG2, 0);
1980
			}
1981
		}
1982
		return emit_op(compiler, GET_OPCODE(op), flags, dst, dstw, src1, src1w, src2, src2w);
1983

1984
	case SLJIT_SHL:
1985
	case SLJIT_MSHL:
1986
	case SLJIT_LSHR:
1987
	case SLJIT_MLSHR:
1988
	case SLJIT_ASHR:
1989
	case SLJIT_MASHR:
1990
	case SLJIT_ROTL:
1991
	case SLJIT_ROTR:
1992
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
1993
		if (op & SLJIT_32)
1994
			flags |= ALT_FORM2;
1995
#endif
1996
		if (src2 == SLJIT_IMM) {
1997
			compiler->imm = (sljit_ins)src2w;
1998
			return emit_op(compiler, GET_OPCODE(op), flags | ALT_FORM1, dst, dstw, src1, src1w, TMP_REG2, 0);
1999
		}
2000
		return emit_op(compiler, GET_OPCODE(op), flags, dst, dstw, src1, src1w, src2, src2w);
2001
	}
2002

2003
	return SLJIT_SUCCESS;
2004
}
2005

2006
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2007
	sljit_s32 src1, sljit_sw src1w,
2008
	sljit_s32 src2, sljit_sw src2w)
2009
{
2010
	CHECK_ERROR();
2011
	CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2012

2013
	SLJIT_SKIP_CHECKS(compiler);
2014
	return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
2015
}
2016

2017
#undef TEST_ADD_FORM1
2018
#undef TEST_SUB_FORM2
2019
#undef TEST_SUB_FORM3
2020

2021
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2022
	sljit_s32 dst_reg,
2023
	sljit_s32 src1, sljit_sw src1w,
2024
	sljit_s32 src2, sljit_sw src2w)
2025
{
2026
	CHECK_ERROR();
2027
	CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2028

2029
	switch (GET_OPCODE(op)) {
2030
	case SLJIT_MULADD:
2031
		SLJIT_SKIP_CHECKS(compiler);
2032
		FAIL_IF(sljit_emit_op2(compiler, SLJIT_MUL | (op & SLJIT_32), TMP_REG2, 0, src1, src1w, src2, src2w));
2033
		return push_inst(compiler, ADD | D(dst_reg) | A(dst_reg) | B(TMP_REG2));
2034
	}
2035

2036
	return SLJIT_SUCCESS;
2037
}
2038

2039
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2040
	sljit_s32 dst_reg,
2041
	sljit_s32 src1_reg,
2042
	sljit_s32 src2_reg,
2043
	sljit_s32 src3, sljit_sw src3w)
2044
{
2045
	sljit_s32 is_right;
2046
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2047
	sljit_s32 inp_flags = ((op & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2048
	sljit_sw bit_length = (op & SLJIT_32) ? 32 : 64;
2049
#else /* !SLJIT_CONFIG_PPC_64 */
2050
	sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2051
	sljit_sw bit_length = 32;
2052
#endif /* SLJIT_CONFIG_PPC_64 */
2053

2054
	CHECK_ERROR();
2055
	CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2056

2057
	is_right = (GET_OPCODE(op) == SLJIT_LSHR || GET_OPCODE(op) == SLJIT_MLSHR);
2058

2059
	if (src1_reg == src2_reg) {
2060
		SLJIT_SKIP_CHECKS(compiler);
2061
		return sljit_emit_op2(compiler, (is_right ? SLJIT_ROTR : SLJIT_ROTL) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
2062
	}
2063

2064
	ADJUST_LOCAL_OFFSET(src3, src3w);
2065

2066
	if (src3 == SLJIT_IMM) {
2067
		src3w &= bit_length - 1;
2068

2069
		if (src3w == 0)
2070
			return SLJIT_SUCCESS;
2071

2072
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2073
		if (!(op & SLJIT_32)) {
2074
			if (is_right) {
2075
				FAIL_IF(push_inst(compiler, SRDI(src3w) | S(src1_reg) | A(dst_reg)));
2076
				return push_inst(compiler, RLDIMI | S(src2_reg) | A(dst_reg) | RLDI_SH(64 - src3w) | RLDI_MB(0));
2077
			}
2078

2079
			FAIL_IF(push_inst(compiler, SLDI(src3w) | S(src1_reg) | A(dst_reg)));
2080
			/* Computes SRDI(64 - src2w). */
2081
			FAIL_IF(push_inst(compiler, RLDICL | S(src2_reg) | A(TMP_REG1) | RLDI_SH(src3w) | RLDI_MB(64 - src3w)));
2082
			return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2083
		}
2084
#endif /* SLJIT_CONFIG_PPC_64 */
2085

2086
		if (is_right) {
2087
			FAIL_IF(push_inst(compiler, SRWI(src3w) | S(src1_reg) | A(dst_reg)));
2088
			return push_inst(compiler, RLWIMI | S(src2_reg) | A(dst_reg) | RLWI_SH(32 - src3w) | RLWI_MBE(0, src3w - 1));
2089
		}
2090

2091
		FAIL_IF(push_inst(compiler, SLWI(src3w) | S(src1_reg) | A(dst_reg)));
2092
		return push_inst(compiler, RLWIMI | S(src2_reg) | A(dst_reg) | RLWI_SH(src3w) | RLWI_MBE(32 - src3w, 31));
2093
	}
2094

2095
	if (src3 & SLJIT_MEM) {
2096
		FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG2, src3, src3w, TMP_REG2));
2097
		src3 = TMP_REG2;
2098
	}
2099

2100
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2101
	if (!(op & SLJIT_32)) {
2102
		if (GET_OPCODE(op) == SLJIT_MSHL || GET_OPCODE(op) == SLJIT_MLSHR || dst_reg == src3) {
2103
			FAIL_IF(push_inst(compiler, ANDI | S(src3) | A(TMP_REG2) | 0x3f));
2104
			src3 = TMP_REG2;
2105
		}
2106

2107
		FAIL_IF(push_inst(compiler, (is_right ? SRD : SLD) | S(src1_reg) | A(dst_reg) | B(src3)));
2108
		FAIL_IF(push_inst(compiler, (is_right ? SLDI(1) : SRDI(1)) | S(src2_reg) | A(TMP_REG1)));
2109
		FAIL_IF(push_inst(compiler, XORI | S(src3) | A(TMP_REG2) | 0x3f));
2110
		FAIL_IF(push_inst(compiler, (is_right ? SLD : SRD) | S(TMP_REG1) | A(TMP_REG1) | B(TMP_REG2)));
2111
		return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2112
	}
2113
#endif /* SLJIT_CONFIG_PPC_64 */
2114

2115
	if (GET_OPCODE(op) == SLJIT_MSHL || GET_OPCODE(op) == SLJIT_MLSHR || dst_reg == src3) {
2116
		FAIL_IF(push_inst(compiler, ANDI | S(src3) | A(TMP_REG2) | 0x1f));
2117
		src3 = TMP_REG2;
2118
	}
2119

2120
	FAIL_IF(push_inst(compiler, (is_right ? SRW : SLW) | S(src1_reg) | A(dst_reg) | B(src3)));
2121
	FAIL_IF(push_inst(compiler, (is_right ? SLWI(1) : SRWI(1)) | S(src2_reg) | A(TMP_REG1)));
2122
	FAIL_IF(push_inst(compiler, XORI | S(src3) | A(TMP_REG2) | 0x1f));
2123
	FAIL_IF(push_inst(compiler, (is_right ? SLW : SRW) | S(TMP_REG1) | A(TMP_REG1) | B(TMP_REG2)));
2124
	return push_inst(compiler, OR | S(dst_reg) | A(dst_reg) | B(TMP_REG1));
2125
}
2126

2127
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
2128
	sljit_s32 dst, sljit_sw dstw,
2129
	sljit_s32 src1, sljit_sw src1w,
2130
	sljit_s32 src2, sljit_sw src2w,
2131
	sljit_sw shift_arg)
2132
{
2133
	sljit_s32 dst_r, tmp_r;
2134

2135
	CHECK_ERROR();
2136
	CHECK(check_sljit_emit_op2_shift(compiler, op, dst, dstw, src1, src1w, src2, src2w, shift_arg));
2137
	ADJUST_LOCAL_OFFSET(dst, dstw);
2138
	ADJUST_LOCAL_OFFSET(src1, src1w);
2139
	ADJUST_LOCAL_OFFSET(src2, src2w);
2140

2141
	shift_arg &= (sljit_sw)((sizeof(sljit_sw) * 8) - 1);
2142

2143
	if (src2 == SLJIT_IMM) {
2144
		src2w = src2w << shift_arg;
2145
		shift_arg = 0;
2146
	}
2147

2148
	if (shift_arg == 0) {
2149
		SLJIT_SKIP_CHECKS(compiler);
2150
		return sljit_emit_op2(compiler, GET_OPCODE(op), dst, dstw, src1, src1w, src2, src2w);
2151
	}
2152

2153
	if (src1 == SLJIT_IMM) {
2154
		FAIL_IF(load_immediate(compiler, TMP_REG1, src1w));
2155
		src1 = TMP_REG1;
2156
	} else if (src1 & SLJIT_MEM) {
2157
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src1, src1w, TMP_REG1));
2158
		src1 = TMP_REG1;
2159
	}
2160

2161
	tmp_r = (src1 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
2162

2163
	if (src2 & SLJIT_MEM) {
2164
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, tmp_r, src2, src2w, tmp_r));
2165
		src2 = tmp_r;
2166
	}
2167

2168
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2169
	FAIL_IF(push_inst(compiler, SLWI_W(shift_arg) | S(src2) | A(tmp_r)));
2170
	FAIL_IF(push_inst(compiler, ADD | D(dst_r) | A(src1) | B(tmp_r)));
2171

2172
	if (dst & SLJIT_MEM)
2173
		return emit_op_mem(compiler, WORD_DATA, dst_r, dst, dstw, TMP_REG1);
2174
	return SLJIT_SUCCESS;
2175
}
2176

2177
static sljit_s32 emit_prefetch(struct sljit_compiler *compiler,
2178
        sljit_s32 src, sljit_sw srcw)
2179
{
2180
	if (!(src & OFFS_REG_MASK)) {
2181
		if (srcw == 0 && (src & REG_MASK))
2182
			return push_inst(compiler, DCBT | A(0) | B(src & REG_MASK));
2183

2184
		FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2185
		/* Works with SLJIT_MEM0() case as well. */
2186
		return push_inst(compiler, DCBT | A(src & REG_MASK) | B(TMP_REG1));
2187
	}
2188

2189
	srcw &= 0x3;
2190

2191
	if (srcw == 0)
2192
		return push_inst(compiler, DCBT | A(src & REG_MASK) | B(OFFS_REG(src)));
2193

2194
	FAIL_IF(push_inst(compiler, SLWI_W(srcw) | S(OFFS_REG(src)) | A(TMP_REG1)));
2195
	return push_inst(compiler, DCBT | A(src & REG_MASK) | B(TMP_REG1));
2196
}
2197

2198
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2199
	sljit_s32 src, sljit_sw srcw)
2200
{
2201
	CHECK_ERROR();
2202
	CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2203
	ADJUST_LOCAL_OFFSET(src, srcw);
2204

2205
	switch (op) {
2206
	case SLJIT_FAST_RETURN:
2207
		if (FAST_IS_REG(src))
2208
			FAIL_IF(push_inst(compiler, MTLR | S(src)));
2209
		else {
2210
			FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG2, src, srcw, TMP_REG2));
2211
			FAIL_IF(push_inst(compiler, MTLR | S(TMP_REG2)));
2212
		}
2213

2214
		return push_inst(compiler, BLR);
2215
	case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2216
		return SLJIT_SUCCESS;
2217
	case SLJIT_PREFETCH_L1:
2218
	case SLJIT_PREFETCH_L2:
2219
	case SLJIT_PREFETCH_L3:
2220
	case SLJIT_PREFETCH_ONCE:
2221
		return emit_prefetch(compiler, src, srcw);
2222
	}
2223

2224
	return SLJIT_SUCCESS;
2225
}
2226

2227
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2228
	sljit_s32 dst, sljit_sw dstw)
2229
{
2230
	sljit_s32 dst_r;
2231

2232
	CHECK_ERROR();
2233
	CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
2234
	ADJUST_LOCAL_OFFSET(dst, dstw);
2235

2236
	switch (op) {
2237
	case SLJIT_FAST_ENTER:
2238
		if (FAST_IS_REG(dst))
2239
			return push_inst(compiler, MFLR | D(dst));
2240

2241
		FAIL_IF(push_inst(compiler, MFLR | D(TMP_REG1)));
2242
		break;
2243
	case SLJIT_GET_RETURN_ADDRESS:
2244
		dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2245
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, dst_r, SLJIT_MEM1(SLJIT_SP), compiler->local_size + LR_SAVE_OFFSET, TMP_REG2));
2246
		break;
2247
	}
2248

2249
	if (dst & SLJIT_MEM)
2250
		return emit_op_mem(compiler, WORD_DATA, TMP_REG1, dst, dstw, TMP_REG2);
2251

2252
	return SLJIT_SUCCESS;
2253
}
2254

2255
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2256
{
2257
	CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
2258

2259
	if (type == SLJIT_GP_REGISTER)
2260
		return reg_map[reg];
2261

2262
	if (type != SLJIT_FLOAT_REGISTER)
2263
		return -1;
2264

2265
	return freg_map[reg];
2266
}
2267

2268
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2269
	void *instruction, sljit_u32 size)
2270
{
2271
	SLJIT_UNUSED_ARG(size);
2272

2273
	CHECK_ERROR();
2274
	CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2275

2276
	return push_inst(compiler, *(sljit_ins*)instruction);
2277
}
2278

2279
/* --------------------------------------------------------------------- */
2280
/*  Floating point operators                                             */
2281
/* --------------------------------------------------------------------- */
2282

2283
#define SELECT_FOP(op, single, double) ((sljit_ins)((op & SLJIT_32) ? single : double))
2284

2285
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2286
	sljit_s32 dst, sljit_sw dstw,
2287
	sljit_s32 src, sljit_sw srcw)
2288
{
2289
	if (src & SLJIT_MEM) {
2290
		/* We can ignore the temporary data store on the stack from caching point of view. */
2291
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src, srcw, TMP_REG1));
2292
		src = TMP_FREG1;
2293
	}
2294

2295
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2296
	op = GET_OPCODE(op);
2297
	FAIL_IF(push_inst(compiler, (op == SLJIT_CONV_S32_FROM_F64 ? FCTIWZ : FCTIDZ) | FD(TMP_FREG1) | FB(src)));
2298

2299
	if (op == SLJIT_CONV_SW_FROM_F64) {
2300
		if (FAST_IS_REG(dst)) {
2301
			FAIL_IF(push_inst(compiler, STFD | FS(TMP_FREG1) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2302
			return push_inst(compiler, LD | S(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2303
		}
2304
		return emit_op_mem(compiler, DOUBLE_DATA, TMP_FREG1, dst, dstw, TMP_REG1);
2305
	}
2306
#else /* !SLJIT_CONFIG_PPC_64 */
2307
	FAIL_IF(push_inst(compiler, FCTIWZ | FD(TMP_FREG1) | FB(src)));
2308
#endif /* SLJIT_CONFIG_PPC_64 */
2309

2310
	if (FAST_IS_REG(dst)) {
2311
		FAIL_IF(load_immediate(compiler, TMP_REG1, TMP_MEM_OFFSET));
2312
		FAIL_IF(push_inst(compiler, STFIWX | FS(TMP_FREG1) | A(SLJIT_SP) | B(TMP_REG1)));
2313
		return push_inst(compiler, LWZ | S(dst) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2314
	}
2315

2316
	SLJIT_ASSERT(dst & SLJIT_MEM);
2317

2318
	if (dst & OFFS_REG_MASK) {
2319
		dstw &= 0x3;
2320
		if (dstw) {
2321
			FAIL_IF(push_inst(compiler, SLWI_W(dstw) | S(OFFS_REG(dst)) | A(TMP_REG1)));
2322
			dstw = TMP_REG1;
2323
		} else
2324
			dstw = OFFS_REG(dst);
2325
	}
2326
	else {
2327
		if ((dst & REG_MASK) && !dstw) {
2328
			dstw = dst & REG_MASK;
2329
			dst = 0;
2330
		} else {
2331
			/* This works regardless we have SLJIT_MEM1 or SLJIT_MEM0. */
2332
			FAIL_IF(load_immediate(compiler, TMP_REG1, dstw));
2333
			dstw = TMP_REG1;
2334
		}
2335
	}
2336

2337
	return push_inst(compiler, STFIWX | FS(TMP_FREG1) | A(dst & REG_MASK) | B(dstw));
2338
}
2339

2340
static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2341
	sljit_s32 src1, sljit_sw src1w,
2342
	sljit_s32 src2, sljit_sw src2w)
2343
{
2344
	if (src1 & SLJIT_MEM) {
2345
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, TMP_REG1));
2346
		src1 = TMP_FREG1;
2347
	}
2348

2349
	if (src2 & SLJIT_MEM) {
2350
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, TMP_REG2));
2351
		src2 = TMP_FREG2;
2352
	}
2353

2354
	FAIL_IF(push_inst(compiler, FCMPU | CRD(4) | FA(src1) | FB(src2)));
2355

2356
	switch (GET_FLAG_TYPE(op)) {
2357
	case SLJIT_UNORDERED_OR_EQUAL:
2358
		return push_inst(compiler, CROR | ((4 + 2) << 21) | ((4 + 2) << 16) | ((4 + 3) << 11));
2359
	case SLJIT_UNORDERED_OR_LESS:
2360
		return push_inst(compiler, CROR | ((4 + 0) << 21) | ((4 + 0) << 16) | ((4 + 3) << 11));
2361
	case SLJIT_UNORDERED_OR_GREATER:
2362
		return push_inst(compiler, CROR | ((4 + 1) << 21) | ((4 + 1) << 16) | ((4 + 3) << 11));
2363
	}
2364

2365
	return SLJIT_SUCCESS;
2366
}
2367

2368
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2369
	sljit_s32 dst, sljit_sw dstw,
2370
	sljit_s32 src, sljit_sw srcw)
2371
{
2372
	sljit_s32 dst_r;
2373

2374
	CHECK_ERROR();
2375

2376
	SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100) && !(DOUBLE_DATA & 0x4), float_transfer_bit_error);
2377
	SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2378

2379
	if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2380
		op ^= SLJIT_32;
2381

2382
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2383

2384
	if (src & SLJIT_MEM) {
2385
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, dst_r, src, srcw, TMP_REG1));
2386
		src = dst_r;
2387
	}
2388

2389
	switch (GET_OPCODE(op)) {
2390
	case SLJIT_CONV_F64_FROM_F32:
2391
		op ^= SLJIT_32;
2392
		if (op & SLJIT_32) {
2393
			FAIL_IF(push_inst(compiler, FRSP | FD(dst_r) | FB(src)));
2394
			break;
2395
		}
2396
		SLJIT_FALLTHROUGH
2397
	case SLJIT_MOV_F64:
2398
		if (src != dst_r) {
2399
			if (!(dst & SLJIT_MEM))
2400
				FAIL_IF(push_inst(compiler, FMR | FD(dst_r) | FB(src)));
2401
			else
2402
				dst_r = src;
2403
		}
2404
		break;
2405
	case SLJIT_NEG_F64:
2406
		FAIL_IF(push_inst(compiler, FNEG | FD(dst_r) | FB(src)));
2407
		break;
2408
	case SLJIT_ABS_F64:
2409
		FAIL_IF(push_inst(compiler, FABS | FD(dst_r) | FB(src)));
2410
		break;
2411
	}
2412

2413
	if (dst & SLJIT_MEM)
2414
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op), dst_r, dst, dstw, TMP_REG1));
2415
	return SLJIT_SUCCESS;
2416
}
2417

2418
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2419
	sljit_s32 dst, sljit_sw dstw,
2420
	sljit_s32 src1, sljit_sw src1w,
2421
	sljit_s32 src2, sljit_sw src2w)
2422
{
2423
	sljit_s32 dst_r;
2424

2425
	CHECK_ERROR();
2426
	CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2427
	ADJUST_LOCAL_OFFSET(dst, dstw);
2428
	ADJUST_LOCAL_OFFSET(src1, src1w);
2429
	ADJUST_LOCAL_OFFSET(src2, src2w);
2430

2431
	dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG2;
2432

2433
	if (src1 & SLJIT_MEM) {
2434
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, TMP_REG1));
2435
		src1 = TMP_FREG1;
2436
	}
2437

2438
	if (src2 & SLJIT_MEM) {
2439
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, TMP_REG1));
2440
		src2 = TMP_FREG2;
2441
	}
2442

2443
	switch (GET_OPCODE(op)) {
2444
	case SLJIT_ADD_F64:
2445
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FADDS, FADD) | FD(dst_r) | FA(src1) | FB(src2)));
2446
		break;
2447
	case SLJIT_SUB_F64:
2448
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FSUBS, FSUB) | FD(dst_r) | FA(src1) | FB(src2)));
2449
		break;
2450
	case SLJIT_MUL_F64:
2451
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FMULS, FMUL) | FD(dst_r) | FA(src1) | FC(src2) /* FMUL use FC as src2 */));
2452
		break;
2453
	case SLJIT_DIV_F64:
2454
		FAIL_IF(push_inst(compiler, SELECT_FOP(op, FDIVS, FDIV) | FD(dst_r) | FA(src1) | FB(src2)));
2455
		break;
2456
	case SLJIT_COPYSIGN_F64:
2457
		FAIL_IF(push_inst(compiler, ((op & SLJIT_32) ? STFS : STFD) | FS(src2) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2458
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
2459
		FAIL_IF(push_inst(compiler, LWZ | S(TMP_REG1) | A(SLJIT_SP) | ((op & SLJIT_32) ? TMP_MEM_OFFSET : TMP_MEM_OFFSET_HI)));
2460
#else /* !SLJIT_CONFIG_PPC_32 */
2461
		FAIL_IF(push_inst(compiler, ((op & SLJIT_32) ? LWZ : LD) | S(TMP_REG1) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2462
#endif /* SLJIT_CONFIG_PPC_32 */
2463
		FAIL_IF(push_inst(compiler, FABS | FD(dst_r) | FB(src1)));
2464
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
2465
		FAIL_IF(push_inst(compiler, CMPI | CRD(0) | A(TMP_REG1) | 0));
2466
#else /* !SLJIT_CONFIG_PPC_32 */
2467
		FAIL_IF(push_inst(compiler, CMPI | CRD(0 | ((op & SLJIT_32) ? 0 : 1)) | A(TMP_REG1) | 0));
2468
#endif /* SLJIT_CONFIG_PPC_32 */
2469
		FAIL_IF(push_inst(compiler, BCx | (4 << 21) | (0 << 16) | 8));
2470
		return push_inst(compiler, FNEG | FD(dst_r) | FB(dst_r));
2471
	}
2472

2473
	if (dst & SLJIT_MEM)
2474
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(op), TMP_FREG2, dst, dstw, TMP_REG1));
2475

2476
	return SLJIT_SUCCESS;
2477
}
2478

2479
#undef SELECT_FOP
2480

2481
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2482
	sljit_s32 freg, sljit_f32 value)
2483
{
2484
	union {
2485
		sljit_s32 imm;
2486
		sljit_f32 value;
2487
	} u;
2488

2489
	CHECK_ERROR();
2490
	CHECK(check_sljit_emit_fset32(compiler, freg, value));
2491

2492
	u.value = value;
2493

2494
	if (u.imm != 0)
2495
		FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm));
2496

2497
	FAIL_IF(push_inst(compiler, STW | S(u.imm != 0 ? TMP_REG1 : TMP_ZERO) | A(SLJIT_SP) | TMP_MEM_OFFSET));
2498
	return push_inst(compiler, LFS | FS(freg) | A(SLJIT_SP) | TMP_MEM_OFFSET);
2499
}
2500

2501
/* --------------------------------------------------------------------- */
2502
/*  Conditional instructions                                             */
2503
/* --------------------------------------------------------------------- */
2504

2505
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2506
{
2507
	struct sljit_label *label;
2508

2509
	CHECK_ERROR_PTR();
2510
	CHECK_PTR(check_sljit_emit_label(compiler));
2511

2512
	if (compiler->last_label && compiler->last_label->size == compiler->size)
2513
		return compiler->last_label;
2514

2515
	label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2516
	PTR_FAIL_IF(!label);
2517
	set_label(label, compiler);
2518
	return label;
2519
}
2520

2521
SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler,
2522
	sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
2523
{
2524
	sljit_uw mask, i;
2525
	struct sljit_label *label;
2526
	struct sljit_label *next_label;
2527
	struct sljit_extended_label *ext_label;
2528

2529
	CHECK_ERROR_PTR();
2530
	CHECK_PTR(check_sljit_emit_aligned_label(compiler, alignment, buffers));
2531

2532
	sljit_reset_read_only_buffers(buffers);
2533

2534
	if (alignment <= SLJIT_LABEL_ALIGN_4) {
2535
		SLJIT_SKIP_CHECKS(compiler);
2536
		label = sljit_emit_label(compiler);
2537
		PTR_FAIL_IF(!label);
2538
	} else {
2539
		/* The used space is filled with NOPs. */
2540
		mask = ((sljit_uw)1 << alignment) - sizeof(sljit_ins);
2541

2542
		for (i = (mask >> 2); i != 0; i--)
2543
			PTR_FAIL_IF(push_inst(compiler, NOP));
2544

2545
		ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
2546
		PTR_FAIL_IF(!ext_label);
2547
		set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
2548
		label = &ext_label->label;
2549
	}
2550

2551
	if (buffers == NULL)
2552
		return label;
2553

2554
	next_label = label;
2555

2556
	while (1) {
2557
		buffers->u.label = next_label;
2558

2559
		for (i = (buffers->size + 3) >> 2; i > 0; i--)
2560
			PTR_FAIL_IF(push_inst(compiler, NOP));
2561

2562
		buffers = buffers->next;
2563

2564
		if (buffers == NULL)
2565
			break;
2566

2567
		SLJIT_SKIP_CHECKS(compiler);
2568
		next_label = sljit_emit_label(compiler);
2569
		PTR_FAIL_IF(!next_label);
2570
	}
2571

2572
	return label;
2573
}
2574

2575
static sljit_ins get_bo_bi_flags(struct sljit_compiler *compiler, sljit_s32 type)
2576
{
2577
	switch (type) {
2578
	case SLJIT_NOT_CARRY:
2579
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
2580
			return (4 << 21) | (2 << 16);
2581
		SLJIT_FALLTHROUGH
2582

2583
	case SLJIT_EQUAL:
2584
	case SLJIT_ATOMIC_STORED:
2585
		return (12 << 21) | (2 << 16);
2586

2587
	case SLJIT_CARRY:
2588
		if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB)
2589
			return (12 << 21) | (2 << 16);
2590
		SLJIT_FALLTHROUGH
2591

2592
	case SLJIT_NOT_EQUAL:
2593
	case SLJIT_ATOMIC_NOT_STORED:
2594
		return (4 << 21) | (2 << 16);
2595

2596
	case SLJIT_LESS:
2597
	case SLJIT_SIG_LESS:
2598
		return (12 << 21) | (0 << 16);
2599

2600
	case SLJIT_GREATER_EQUAL:
2601
	case SLJIT_SIG_GREATER_EQUAL:
2602
		return (4 << 21) | (0 << 16);
2603

2604
	case SLJIT_GREATER:
2605
	case SLJIT_SIG_GREATER:
2606
		return (12 << 21) | (1 << 16);
2607

2608
	case SLJIT_LESS_EQUAL:
2609
	case SLJIT_SIG_LESS_EQUAL:
2610
		return (4 << 21) | (1 << 16);
2611

2612
	case SLJIT_OVERFLOW:
2613
		return (12 << 21) | (3 << 16);
2614

2615
	case SLJIT_NOT_OVERFLOW:
2616
		return (4 << 21) | (3 << 16);
2617

2618
	case SLJIT_F_LESS:
2619
	case SLJIT_ORDERED_LESS:
2620
	case SLJIT_UNORDERED_OR_LESS:
2621
		return (12 << 21) | ((4 + 0) << 16);
2622

2623
	case SLJIT_F_GREATER_EQUAL:
2624
	case SLJIT_ORDERED_GREATER_EQUAL:
2625
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2626
		return (4 << 21) | ((4 + 0) << 16);
2627

2628
	case SLJIT_F_GREATER:
2629
	case SLJIT_ORDERED_GREATER:
2630
	case SLJIT_UNORDERED_OR_GREATER:
2631
		return (12 << 21) | ((4 + 1) << 16);
2632

2633
	case SLJIT_F_LESS_EQUAL:
2634
	case SLJIT_ORDERED_LESS_EQUAL:
2635
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
2636
		return (4 << 21) | ((4 + 1) << 16);
2637

2638
	case SLJIT_F_EQUAL:
2639
	case SLJIT_ORDERED_EQUAL:
2640
	case SLJIT_UNORDERED_OR_EQUAL:
2641
		return (12 << 21) | ((4 + 2) << 16);
2642

2643
	case SLJIT_F_NOT_EQUAL:
2644
	case SLJIT_ORDERED_NOT_EQUAL:
2645
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
2646
		return (4 << 21) | ((4 + 2) << 16);
2647

2648
	case SLJIT_UNORDERED:
2649
		return (12 << 21) | ((4 + 3) << 16);
2650

2651
	case SLJIT_ORDERED:
2652
		return (4 << 21) | ((4 + 3) << 16);
2653

2654
	default:
2655
		SLJIT_ASSERT(type >= SLJIT_JUMP && type <= SLJIT_CALL_REG_ARG);
2656
		return (20 << 21);
2657
	}
2658
}
2659

2660
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2661
{
2662
	struct sljit_jump *jump;
2663
	sljit_ins bo_bi_flags;
2664

2665
	CHECK_ERROR_PTR();
2666
	CHECK_PTR(check_sljit_emit_jump(compiler, type));
2667

2668
	bo_bi_flags = get_bo_bi_flags(compiler, type & 0xff);
2669
	if (!bo_bi_flags)
2670
		return NULL;
2671

2672
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2673
	PTR_FAIL_IF(!jump);
2674
	set_jump(jump, compiler, (sljit_u32)type & SLJIT_REWRITABLE_JUMP);
2675
	type &= 0xff;
2676

2677
	if ((type | 0x1) == SLJIT_NOT_CARRY)
2678
		PTR_FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG2) | A(TMP_ZERO) | B(TMP_ZERO)));
2679

2680
	/* In PPC, we don't need to touch the arguments. */
2681
	if (type < SLJIT_JUMP)
2682
		jump->flags |= IS_COND;
2683
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2684
	if (type >= SLJIT_CALL)
2685
		jump->flags |= IS_CALL;
2686
#endif
2687

2688
	jump->addr = compiler->size;
2689
	PTR_FAIL_IF(push_inst(compiler, BCCTR | bo_bi_flags | (type >= SLJIT_FAST_CALL ? 1 : 0)));
2690

2691
	/* Maximum number of instructions required for generating a constant. */
2692
	compiler->size += JUMP_MAX_SIZE - 1;
2693
	return jump;
2694
}
2695

2696
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2697
	sljit_s32 arg_types)
2698
{
2699
	SLJIT_UNUSED_ARG(arg_types);
2700

2701
	CHECK_ERROR_PTR();
2702
	CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2703

2704
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2705
	if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2706
		PTR_FAIL_IF(call_with_args(compiler, arg_types, NULL));
2707
#endif
2708

2709
	if (type & SLJIT_CALL_RETURN) {
2710
		PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
2711
		type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2712
	}
2713

2714
	SLJIT_SKIP_CHECKS(compiler);
2715
	return sljit_emit_jump(compiler, type);
2716
}
2717

2718
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2719
{
2720
	struct sljit_jump *jump = NULL;
2721
	sljit_s32 src_r;
2722

2723
	CHECK_ERROR();
2724
	CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2725

2726
	if (src == SLJIT_IMM) {
2727
		/* These jumps are converted to jump/call instructions when possible. */
2728
		jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2729
		FAIL_IF(!jump);
2730
		set_jump(jump, compiler, JUMP_ADDR);
2731
		jump->u.target = (sljit_uw)srcw;
2732

2733
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2734
		if (type >= SLJIT_CALL)
2735
			jump->flags |= IS_CALL;
2736
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2737

2738
		jump->addr = compiler->size;
2739
		FAIL_IF(push_inst(compiler, BCCTR | (20 << 21) | (type >= SLJIT_FAST_CALL ? 1 : 0)));
2740

2741
		/* Maximum number of instructions required for generating a constant. */
2742
		compiler->size += JUMP_MAX_SIZE - 1;
2743
		return SLJIT_SUCCESS;
2744
	}
2745

2746
	if (FAST_IS_REG(src)) {
2747
#if (defined SLJIT_PASS_ENTRY_ADDR_TO_CALL && SLJIT_PASS_ENTRY_ADDR_TO_CALL)
2748
		if (type >= SLJIT_CALL && src != TMP_CALL_REG) {
2749
			FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
2750
			src_r = TMP_CALL_REG;
2751
		} else
2752
			src_r = src;
2753
#else /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2754
		src_r = src;
2755
#endif /* SLJIT_PASS_ENTRY_ADDR_TO_CALL */
2756
	} else {
2757
		ADJUST_LOCAL_OFFSET(src, srcw);
2758
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
2759
		src_r = TMP_CALL_REG;
2760
	}
2761

2762
	FAIL_IF(push_inst(compiler, MTCTR | S(src_r)));
2763
	return push_inst(compiler, BCCTR | (20 << 21) | (type >= SLJIT_FAST_CALL ? 1 : 0));
2764
}
2765

2766
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2767
	sljit_s32 arg_types,
2768
	sljit_s32 src, sljit_sw srcw)
2769
{
2770
	SLJIT_UNUSED_ARG(arg_types);
2771

2772
	CHECK_ERROR();
2773
	CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2774

2775
	if (src & SLJIT_MEM) {
2776
		ADJUST_LOCAL_OFFSET(src, srcw);
2777
		FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_CALL_REG, src, srcw, TMP_CALL_REG));
2778
		src = TMP_CALL_REG;
2779
	}
2780

2781
	if (type & SLJIT_CALL_RETURN) {
2782
		if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
2783
			FAIL_IF(push_inst(compiler, OR | S(src) | A(TMP_CALL_REG) | B(src)));
2784
			src = TMP_CALL_REG;
2785
		}
2786

2787
		FAIL_IF(emit_stack_frame_release(compiler, 0));
2788
		type = SLJIT_JUMP;
2789
	}
2790

2791
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2792
	if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2793
		FAIL_IF(call_with_args(compiler, arg_types, &src));
2794
#endif
2795

2796
	SLJIT_SKIP_CHECKS(compiler);
2797
	return sljit_emit_ijump(compiler, type, src, srcw);
2798
}
2799

2800
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2801
	sljit_s32 dst, sljit_sw dstw,
2802
	sljit_s32 type)
2803
{
2804
	sljit_s32 reg, invert;
2805
	sljit_u32 bit, from_xer;
2806
	sljit_s32 saved_op = op;
2807
	sljit_sw saved_dstw = dstw;
2808
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2809
	sljit_s32 input_flags = ((op & SLJIT_32) || op == SLJIT_MOV32) ? INT_DATA : WORD_DATA;
2810
#else
2811
	sljit_s32 input_flags = WORD_DATA;
2812
#endif
2813

2814
	CHECK_ERROR();
2815
	CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2816
	ADJUST_LOCAL_OFFSET(dst, dstw);
2817

2818
	op = GET_OPCODE(op);
2819
	reg = (op < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG2;
2820

2821
	if (op >= SLJIT_ADD && (dst & SLJIT_MEM))
2822
		FAIL_IF(emit_op_mem(compiler, input_flags | LOAD_DATA, TMP_REG1, dst, dstw, TMP_REG1));
2823

2824
	invert = 0;
2825
	bit = 0;
2826
	from_xer = 0;
2827

2828
	switch (type) {
2829
	case SLJIT_LESS:
2830
	case SLJIT_SIG_LESS:
2831
		break;
2832

2833
	case SLJIT_GREATER_EQUAL:
2834
	case SLJIT_SIG_GREATER_EQUAL:
2835
		invert = 1;
2836
		break;
2837

2838
	case SLJIT_GREATER:
2839
	case SLJIT_SIG_GREATER:
2840
		bit = 1;
2841
		break;
2842

2843
	case SLJIT_LESS_EQUAL:
2844
	case SLJIT_SIG_LESS_EQUAL:
2845
		bit = 1;
2846
		invert = 1;
2847
		break;
2848

2849
	case SLJIT_EQUAL:
2850
	case SLJIT_ATOMIC_STORED:
2851
		bit = 2;
2852
		break;
2853

2854
	case SLJIT_NOT_EQUAL:
2855
	case SLJIT_ATOMIC_NOT_STORED:
2856
		bit = 2;
2857
		invert = 1;
2858
		break;
2859

2860
	case SLJIT_OVERFLOW:
2861
		from_xer = 1;
2862
		bit = 1;
2863
		break;
2864

2865
	case SLJIT_NOT_OVERFLOW:
2866
		from_xer = 1;
2867
		bit = 1;
2868
		invert = 1;
2869
		break;
2870

2871
	case SLJIT_CARRY:
2872
		from_xer = 1;
2873
		bit = 2;
2874
		invert = (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_SUB) != 0;
2875
		break;
2876

2877
	case SLJIT_NOT_CARRY:
2878
		from_xer = 1;
2879
		bit = 2;
2880
		invert = (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD) != 0;
2881
		break;
2882

2883
	case SLJIT_F_LESS:
2884
	case SLJIT_ORDERED_LESS:
2885
	case SLJIT_UNORDERED_OR_LESS:
2886
		bit = 4 + 0;
2887
		break;
2888

2889
	case SLJIT_F_GREATER_EQUAL:
2890
	case SLJIT_ORDERED_GREATER_EQUAL:
2891
	case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2892
		bit = 4 + 0;
2893
		invert = 1;
2894
		break;
2895

2896
	case SLJIT_F_GREATER:
2897
	case SLJIT_ORDERED_GREATER:
2898
	case SLJIT_UNORDERED_OR_GREATER:
2899
		bit = 4 + 1;
2900
		break;
2901

2902
	case SLJIT_F_LESS_EQUAL:
2903
	case SLJIT_ORDERED_LESS_EQUAL:
2904
	case SLJIT_UNORDERED_OR_LESS_EQUAL:
2905
		bit = 4 + 1;
2906
		invert = 1;
2907
		break;
2908

2909
	case SLJIT_F_EQUAL:
2910
	case SLJIT_ORDERED_EQUAL:
2911
	case SLJIT_UNORDERED_OR_EQUAL:
2912
		bit = 4 + 2;
2913
		break;
2914

2915
	case SLJIT_F_NOT_EQUAL:
2916
	case SLJIT_ORDERED_NOT_EQUAL:
2917
	case SLJIT_UNORDERED_OR_NOT_EQUAL:
2918
		bit = 4 + 2;
2919
		invert = 1;
2920
		break;
2921

2922
	case SLJIT_UNORDERED:
2923
		bit = 4 + 3;
2924
		break;
2925

2926
	case SLJIT_ORDERED:
2927
		bit = 4 + 3;
2928
		invert = 1;
2929
		break;
2930

2931
	default:
2932
		SLJIT_UNREACHABLE();
2933
		break;
2934
	}
2935

2936
	FAIL_IF(push_inst(compiler, (from_xer ? MFXER : MFCR) | D(reg)));
2937
	/* Simplified mnemonics: extrwi. */
2938
	FAIL_IF(push_inst(compiler, RLWINM | S(reg) | A(reg) | RLWI_SH(1 + bit) | RLWI_MBE(31, 31)));
2939

2940
	if (invert)
2941
		FAIL_IF(push_inst(compiler, XORI | S(reg) | A(reg) | 0x1));
2942

2943
	if (op < SLJIT_ADD) {
2944
		if (!(dst & SLJIT_MEM))
2945
			return SLJIT_SUCCESS;
2946
		return emit_op_mem(compiler, input_flags, reg, dst, dstw, TMP_REG1);
2947
	}
2948

2949
	SLJIT_SKIP_CHECKS(compiler);
2950

2951
	if (dst & SLJIT_MEM)
2952
		return sljit_emit_op2(compiler, saved_op, dst, saved_dstw, TMP_REG1, 0, TMP_REG2, 0);
2953
	return sljit_emit_op2(compiler, saved_op, dst, 0, dst, 0, TMP_REG2, 0);
2954
}
2955

2956
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2957
	sljit_s32 dst_reg,
2958
	sljit_s32 src1, sljit_sw src1w,
2959
	sljit_s32 src2_reg)
2960
{
2961
	sljit_ins *ptr;
2962
	sljit_uw size;
2963
	sljit_s32 is_compare = (type & SLJIT_COMPARE_SELECT);
2964
	sljit_ins ins;
2965
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2966
	sljit_s32 inp_flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2967
#else /* !SLJIT_CONFIG_PPC_64 */
2968
        sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2969
#endif /* SLJIT_CONFIG_PPC_64 */
2970

2971
	CHECK_ERROR();
2972
	CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
2973

2974
	ADJUST_LOCAL_OFFSET(src1, src1w);
2975

2976
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
2977
	if (src1 == SLJIT_IMM && (type & SLJIT_32))
2978
		src1w = (sljit_s32)src1w;
2979
#endif /* SLJIT_CONFIG_PPC_64 */
2980

2981
	type &= ~(SLJIT_32 | SLJIT_COMPARE_SELECT);
2982

2983
	if (is_compare) {
2984
		ins = 0;
2985

2986
		if (src1 & SLJIT_MEM) {
2987
			FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG1, src1, src1w, TMP_REG1));
2988
			src1 = TMP_REG1;
2989
			src1w = 0;
2990
		}
2991

2992
		if (src1 == SLJIT_IMM) {
2993
			if (type >= SLJIT_LESS && type <= SLJIT_LESS_EQUAL && src1w >= 0 && src1w <= UIMM_MAX)
2994
				ins = CMPLI | CRD(0) | IMM(src1w);
2995
			else if (type >= SLJIT_SIG_LESS && type <= SLJIT_SIG_LESS_EQUAL && src1w >= SIMM_MIN && src1w <= SIMM_MAX)
2996
				ins = CMPI | CRD(0) | IMM(src1w);
2997
			else {
2998
				FAIL_IF(load_immediate(compiler, TMP_REG1, src1w));
2999
				src1 = TMP_REG1;
3000
				src1w = 0;
3001
			}
3002
		}
3003

3004
		if (ins == 0)
3005
			ins = ((type >= SLJIT_LESS && type <= SLJIT_LESS_EQUAL) ? CMPL : CMP) | CRD(0) | B(src1);
3006

3007
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3008
		if (inp_flags == (WORD_DATA | LOAD_DATA))
3009
			ins |= CRD(1);
3010
#endif /* SLJIT_CONFIG_PPC_64 */
3011
		FAIL_IF(push_inst(compiler, ins | A(src2_reg)));
3012
		type ^= 0x1;
3013
	}
3014

3015
	if (dst_reg != src2_reg) {
3016
		if (dst_reg == src1) {
3017
				src1 = src2_reg;
3018
				src1w = 0;
3019
				type ^= 0x1;
3020
		} else {
3021
			if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
3022
				FAIL_IF(push_inst(compiler, OR | S(dst_reg) | A(TMP_REG1) | B(dst_reg)));
3023

3024
				if ((src1 & REG_MASK) == dst_reg)
3025
					src1 = (src1 & ~REG_MASK) | TMP_REG1;
3026

3027
				if (OFFS_REG(src1) == dst_reg)
3028
					src1 = (src1 & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG1);
3029
			}
3030

3031
			FAIL_IF(push_inst(compiler, OR | S(src2_reg) | A(dst_reg) | B(src2_reg)));
3032
		}
3033
	}
3034

3035
	if ((type | 0x1) == SLJIT_NOT_CARRY)
3036
		FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG1) | A(TMP_ZERO) | B(TMP_ZERO)));
3037

3038
	size = compiler->size;
3039

3040
	ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
3041
	FAIL_IF(!ptr);
3042
	compiler->size++;
3043

3044
	if (src1 & SLJIT_MEM) {
3045
		FAIL_IF(emit_op_mem(compiler, inp_flags, dst_reg, src1, src1w, TMP_REG1));
3046
	} else if (src1 == SLJIT_IMM) {
3047
		FAIL_IF(load_immediate(compiler, dst_reg, src1w));
3048
	} else
3049
		FAIL_IF(push_inst(compiler, OR | S(src1) | A(dst_reg) | B(src1)));
3050

3051
	*ptr = BCx | get_bo_bi_flags(compiler, type ^ 0x1) | (sljit_ins)((compiler->size - size) << 2);
3052
	return SLJIT_SUCCESS;
3053
}
3054

3055
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
3056
	sljit_s32 dst_freg,
3057
	sljit_s32 src1, sljit_sw src1w,
3058
	sljit_s32 src2_freg)
3059
{
3060
	sljit_ins *ptr;
3061
	sljit_uw size;
3062

3063
	CHECK_ERROR();
3064
	CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
3065

3066
	ADJUST_LOCAL_OFFSET(src1, src1w);
3067

3068
	if (dst_freg != src2_freg) {
3069
		if (dst_freg == src1) {
3070
			src1 = src2_freg;
3071
			src1w = 0;
3072
			type ^= 0x1;
3073
		} else
3074
			FAIL_IF(push_inst(compiler, FMR | FD(dst_freg) | FB(src2_freg)));
3075
	}
3076

3077
	if (((type & ~SLJIT_32) | 0x1) == SLJIT_NOT_CARRY)
3078
		FAIL_IF(push_inst(compiler, ADDE | RC(ALT_SET_FLAGS) | D(TMP_REG1) | A(TMP_ZERO) | B(TMP_ZERO)));
3079

3080
	size = compiler->size;
3081

3082
	ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
3083
	FAIL_IF(!ptr);
3084
	compiler->size++;
3085

3086
	if (src1 & SLJIT_MEM)
3087
		FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(type) | LOAD_DATA, dst_freg, src1, src1w, TMP_REG1));
3088
	else
3089
		FAIL_IF(push_inst(compiler, FMR | FD(dst_freg) | FB(src1)));
3090

3091
	*ptr = BCx | get_bo_bi_flags(compiler, (type ^ 0x1) & ~SLJIT_32) | (sljit_ins)((compiler->size - size) << 2);
3092
	return SLJIT_SUCCESS;
3093
}
3094

3095
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
3096

3097
#define EMIT_MEM_LOAD_IMM(inst, mem, memw) \
3098
	((sljit_s16)(memw) > SIMM_MAX - SSIZE_OF(sw))
3099

3100
#else /* !SLJIT_CONFIG_PPC_32 */
3101

3102
#define EMIT_MEM_LOAD_IMM(inst, mem, memw) \
3103
	((((inst) & INT_ALIGNED) && ((memw) & 0x3) != 0) \
3104
		|| ((sljit_s16)(memw) > SIMM_MAX - SSIZE_OF(sw)) \
3105
		|| ((memw) > 0x7fff7fffl || (memw) < -0x80000000l)) \
3106

3107
#endif /* SLJIT_CONFIG_PPC_32 */
3108

3109
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
3110
	sljit_s32 reg,
3111
	sljit_s32 mem, sljit_sw memw)
3112
{
3113
	sljit_ins inst;
3114

3115
	CHECK_ERROR();
3116
	CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
3117

3118
	if (!(reg & REG_PAIR_MASK))
3119
		return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
3120

3121
	ADJUST_LOCAL_OFFSET(mem, memw);
3122

3123
	inst = data_transfer_insts[WORD_DATA | ((type & SLJIT_MEM_STORE) ? 0 : LOAD_DATA)];
3124

3125
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3126
		memw &= 0x3;
3127

3128
		if (memw != 0) {
3129
			FAIL_IF(push_inst(compiler, SLWI_W(memw) | S(OFFS_REG(mem)) | A(TMP_REG1)));
3130
			FAIL_IF(push_inst(compiler, ADD | D(TMP_REG1) | A(TMP_REG1) | B(mem & REG_MASK)));
3131
		} else
3132
			FAIL_IF(push_inst(compiler, ADD | D(TMP_REG1) | A(mem & REG_MASK) | B(OFFS_REG(mem))));
3133

3134
		mem = TMP_REG1;
3135
		memw = 0;
3136
	} else {
3137
		if (EMIT_MEM_LOAD_IMM(inst, mem, memw)) {
3138
			if ((mem & REG_MASK) != 0) {
3139
				SLJIT_SKIP_CHECKS(compiler);
3140
				FAIL_IF(sljit_emit_op2(compiler, SLJIT_ADD, TMP_REG1, 0, mem & REG_MASK, 0, SLJIT_IMM, memw));
3141
			} else
3142
				FAIL_IF(load_immediate(compiler, TMP_REG1, memw));
3143

3144
			memw = 0;
3145
			mem = TMP_REG1;
3146
		} else if (memw > SIMM_MAX || memw < SIMM_MIN) {
3147
			FAIL_IF(push_inst(compiler, ADDIS | D(TMP_REG1) | A(mem & REG_MASK) | IMM((memw + 0x8000) >> 16)));
3148

3149
			memw &= 0xffff;
3150
			mem = TMP_REG1;
3151
		} else {
3152
			memw &= 0xffff;
3153
			mem &= REG_MASK;
3154
		}
3155
	}
3156

3157
	SLJIT_ASSERT((memw >= 0 && memw <= SIMM_MAX - SSIZE_OF(sw)) || (memw >= 0x8000 && memw <= 0xffff));
3158

3159
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3160
	inst &= (sljit_ins)~INT_ALIGNED;
3161
#endif /* SLJIT_CONFIG_PPC_64 */
3162

3163
	if (!(type & SLJIT_MEM_STORE) && mem == REG_PAIR_FIRST(reg)) {
3164
		FAIL_IF(push_inst(compiler, inst | D(REG_PAIR_SECOND(reg)) | A(mem) | IMM(memw + SSIZE_OF(sw))));
3165
		return push_inst(compiler, inst | D(REG_PAIR_FIRST(reg)) | A(mem) | IMM(memw));
3166
	}
3167

3168
	FAIL_IF(push_inst(compiler, inst | D(REG_PAIR_FIRST(reg)) | A(mem) | IMM(memw)));
3169
	return push_inst(compiler, inst | D(REG_PAIR_SECOND(reg)) | A(mem) | IMM(memw + SSIZE_OF(sw)));
3170
}
3171

3172
#undef EMIT_MEM_LOAD_IMM
3173

3174
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3175
	sljit_s32 reg,
3176
	sljit_s32 mem, sljit_sw memw)
3177
{
3178
	sljit_s32 mem_flags;
3179
	sljit_ins inst;
3180

3181
	CHECK_ERROR();
3182
	CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3183

3184
	if (type & SLJIT_MEM_POST)
3185
		return SLJIT_ERR_UNSUPPORTED;
3186

3187
	switch (type & 0xff) {
3188
	case SLJIT_MOV:
3189
	case SLJIT_MOV_P:
3190
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
3191
	case SLJIT_MOV_U32:
3192
	case SLJIT_MOV_S32:
3193
	case SLJIT_MOV32:
3194
#endif
3195
		mem_flags = WORD_DATA;
3196
		break;
3197

3198
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3199
	case SLJIT_MOV_U32:
3200
	case SLJIT_MOV32:
3201
		mem_flags = INT_DATA;
3202
		break;
3203

3204
	case SLJIT_MOV_S32:
3205
		mem_flags = INT_DATA;
3206

3207
		if (!(type & SLJIT_MEM_STORE) && !(type & SLJIT_32)) {
3208
			if (mem & OFFS_REG_MASK)
3209
				mem_flags |= SIGNED_DATA;
3210
			else
3211
				return SLJIT_ERR_UNSUPPORTED;
3212
		}
3213
		break;
3214
#endif
3215

3216
	case SLJIT_MOV_U8:
3217
	case SLJIT_MOV_S8:
3218
		mem_flags = BYTE_DATA;
3219
		break;
3220

3221
	case SLJIT_MOV_U16:
3222
		mem_flags = HALF_DATA;
3223
		break;
3224

3225
	case SLJIT_MOV_S16:
3226
		mem_flags = HALF_DATA | SIGNED_DATA;
3227
		break;
3228

3229
	default:
3230
		SLJIT_UNREACHABLE();
3231
		mem_flags = WORD_DATA;
3232
		break;
3233
	}
3234

3235
	if (!(type & SLJIT_MEM_STORE))
3236
		mem_flags |= LOAD_DATA;
3237

3238
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3239
		if (memw != 0)
3240
			return SLJIT_ERR_UNSUPPORTED;
3241

3242
		if (type & SLJIT_MEM_SUPP)
3243
			return SLJIT_SUCCESS;
3244

3245
		inst = updated_data_transfer_insts[mem_flags | INDEXED];
3246
		FAIL_IF(push_inst(compiler, INST_CODE_AND_DST(inst, 0, reg) | A(mem & REG_MASK) | B(OFFS_REG(mem))));
3247
	}
3248
	else {
3249
		if (memw > SIMM_MAX || memw < SIMM_MIN)
3250
			return SLJIT_ERR_UNSUPPORTED;
3251

3252
		inst = updated_data_transfer_insts[mem_flags];
3253

3254
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3255
		if ((inst & INT_ALIGNED) && (memw & 0x3) != 0)
3256
			return SLJIT_ERR_UNSUPPORTED;
3257
#endif
3258

3259
		if (type & SLJIT_MEM_SUPP)
3260
			return SLJIT_SUCCESS;
3261

3262
		FAIL_IF(push_inst(compiler, INST_CODE_AND_DST(inst, 0, reg) | A(mem & REG_MASK) | IMM(memw)));
3263
	}
3264

3265
	if ((mem_flags & LOAD_DATA) && (type & 0xff) == SLJIT_MOV_S8)
3266
		return push_inst(compiler, EXTSB | S(reg) | A(reg));
3267
	return SLJIT_SUCCESS;
3268
}
3269

3270
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3271
	sljit_s32 freg,
3272
	sljit_s32 mem, sljit_sw memw)
3273
{
3274
	sljit_s32 mem_flags;
3275
	sljit_ins inst;
3276

3277
	CHECK_ERROR();
3278
	CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3279

3280
	if (type & SLJIT_MEM_POST)
3281
		return SLJIT_ERR_UNSUPPORTED;
3282

3283
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3284
		if (memw != 0)
3285
			return SLJIT_ERR_UNSUPPORTED;
3286
	}
3287
	else {
3288
		if (memw > SIMM_MAX || memw < SIMM_MIN)
3289
			return SLJIT_ERR_UNSUPPORTED;
3290
	}
3291

3292
	if (type & SLJIT_MEM_SUPP)
3293
		return SLJIT_SUCCESS;
3294

3295
	mem_flags = FLOAT_DATA(type);
3296

3297
	if (!(type & SLJIT_MEM_STORE))
3298
		mem_flags |= LOAD_DATA;
3299

3300
	if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3301
		inst = updated_data_transfer_insts[mem_flags | INDEXED];
3302
		return push_inst(compiler, INST_CODE_AND_DST(inst, DOUBLE_DATA, freg) | A(mem & REG_MASK) | B(OFFS_REG(mem)));
3303
	}
3304

3305
	inst = updated_data_transfer_insts[mem_flags];
3306
	return push_inst(compiler, INST_CODE_AND_DST(inst, DOUBLE_DATA, freg) | A(mem & REG_MASK) | IMM(memw));
3307
}
3308

3309
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
3310
	sljit_s32 dst_reg,
3311
	sljit_s32 mem_reg)
3312
{
3313
	sljit_ins ins;
3314

3315
	CHECK_ERROR();
3316
	CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
3317

3318
	if (op & SLJIT_ATOMIC_USE_CAS)
3319
		return SLJIT_ERR_UNSUPPORTED;
3320

3321
	switch (GET_OPCODE(op)) {
3322
	case SLJIT_MOV:
3323
	case SLJIT_MOV_P:
3324
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3325
		ins = LDARX;
3326
		break;
3327
#endif /* SLJIT_CONFIG_PPC_64 */
3328
	case SLJIT_MOV_U32:
3329
	case SLJIT_MOV32:
3330
		ins = LWARX;
3331
		break;
3332

3333
	default:
3334
		return SLJIT_ERR_UNSUPPORTED;
3335
	}
3336

3337
	if (op & SLJIT_ATOMIC_TEST)
3338
		return SLJIT_SUCCESS;
3339

3340
	return push_inst(compiler, ins | D(dst_reg) | B(mem_reg));
3341
}
3342

3343
SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
3344
	sljit_s32 src_reg,
3345
	sljit_s32 mem_reg,
3346
	sljit_s32 temp_reg)
3347
{
3348
	sljit_ins ins;
3349

3350
	/* temp_reg == mem_reg is undefined so use another temp register */
3351
	SLJIT_UNUSED_ARG(temp_reg);
3352

3353
	CHECK_ERROR();
3354
	CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
3355

3356
	if (op & SLJIT_ATOMIC_USE_CAS)
3357
		return SLJIT_ERR_UNSUPPORTED;
3358

3359
	switch (GET_OPCODE(op)) {
3360
	case SLJIT_MOV:
3361
	case SLJIT_MOV_P:
3362
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3363
		ins = STDCX | 0x1;
3364
		break;
3365
#endif /* SLJIT_CONFIG_PPC_64 */
3366
	case SLJIT_MOV_U32:
3367
	case SLJIT_MOV32:
3368
		ins = STWCX | 0x1;
3369
		break;
3370

3371
	default:
3372
		return SLJIT_ERR_UNSUPPORTED;
3373
	}
3374

3375
	if (op & SLJIT_ATOMIC_TEST)
3376
		return SLJIT_SUCCESS;
3377

3378
	return push_inst(compiler, ins | D(src_reg) | B(mem_reg));
3379
}
3380

3381
SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
3382
	sljit_s32 dst, sljit_sw dstw,
3383
	sljit_sw init_value)
3384
{
3385
	struct sljit_const *const_;
3386
	sljit_s32 dst_r;
3387
	sljit_s32 mem_flags = WORD_DATA;
3388

3389
	CHECK_ERROR_PTR();
3390
	CHECK_PTR(check_sljit_emit_const(compiler, op, dst, dstw, init_value));
3391
	ADJUST_LOCAL_OFFSET(dst, dstw);
3392

3393
	const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3394
	PTR_FAIL_IF(!const_);
3395
	set_const(const_, compiler);
3396

3397
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3398

3399
	switch (GET_OPCODE(op)) {
3400
	case SLJIT_MOV_U8:
3401
		if (init_value & 0x100)
3402
			init_value |= 0xff00;
3403
		else
3404
			init_value &= 0xff;
3405

3406
		PTR_FAIL_IF(push_inst(compiler, ADDI | D(dst_r) | A(0) | IMM(init_value)));
3407
		mem_flags = BYTE_DATA;
3408
		break;
3409

3410
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3411
	case SLJIT_MOV32:
3412
		mem_flags = INT_DATA;
3413
		SLJIT_FALLTHROUGH
3414
	case SLJIT_MOV_S32:
3415
		PTR_FAIL_IF(push_inst(compiler, ADDIS | D(dst_r) | A(0) | IMM(init_value >> 16)));
3416
		PTR_FAIL_IF(push_inst(compiler, ORI | S(dst_r) | A(dst_r) | IMM(init_value)));
3417
		break;
3418
#endif /* SLJIT_CONFIG_PPC_64 */
3419

3420
	default:
3421
		PTR_FAIL_IF(emit_const(compiler, dst_r, init_value));
3422
		break;
3423
	}
3424

3425
	if (dst & SLJIT_MEM)
3426
		PTR_FAIL_IF(emit_op_mem(compiler, mem_flags, dst_r, dst, dstw, TMP_REG1));
3427

3428
	return const_;
3429
}
3430

3431
SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
3432
	sljit_s32 dst, sljit_sw dstw)
3433
{
3434
	struct sljit_jump *jump;
3435
	sljit_s32 dst_r, target_r;
3436
	SLJIT_UNUSED_ARG(op);
3437

3438
	CHECK_ERROR_PTR();
3439
	CHECK_PTR(check_sljit_emit_op_addr(compiler, op, dst, dstw));
3440
	ADJUST_LOCAL_OFFSET(dst, dstw);
3441

3442
	dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3443

3444
	if (op != SLJIT_ADD_ABS_ADDR)
3445
		target_r = dst_r;
3446
	else {
3447
		target_r = TMP_REG1;
3448

3449
		if (dst & SLJIT_MEM)
3450
			PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, dst_r, dst, dstw, TMP_REG1));
3451
	}
3452

3453
	jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3454
	PTR_FAIL_IF(!jump);
3455
	set_mov_addr(jump, compiler, 0);
3456

3457
	PTR_FAIL_IF(push_inst(compiler, (sljit_ins)target_r));
3458
#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32)
3459
	compiler->size++;
3460
#else
3461
	compiler->size += 4;
3462
#endif
3463

3464
	if (op == SLJIT_ADD_ABS_ADDR)
3465
		PTR_FAIL_IF(push_inst(compiler, ADD | D(dst_r) | A(dst_r) | B(TMP_REG1)));
3466

3467
	if (dst & SLJIT_MEM)
3468
		PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, dst_r, dst, dstw, TMP_REG1));
3469

3470
	return jump;
3471
}
3472

3473
SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_s32 op, sljit_sw new_constant, sljit_sw executable_offset)
3474
{
3475
	sljit_ins *inst;
3476

3477
	switch (GET_OPCODE(op)) {
3478
	case SLJIT_MOV_U8:
3479
		inst = (sljit_ins *)addr;
3480
		SLJIT_ASSERT((inst[0] & 0xfc1f0000) == ADDI);
3481

3482
		if (new_constant & 0x100)
3483
			new_constant |= 0xff00;
3484
		else
3485
			new_constant &= 0xff;
3486

3487
		SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0);
3488
		inst[0] = (inst[0] & 0xffff0000) | IMM(new_constant);
3489
		SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1);
3490
		inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3491
		SLJIT_CACHE_FLUSH(inst, inst + 1);
3492
		return;
3493

3494
#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
3495
	case SLJIT_MOV32:
3496
	case SLJIT_MOV_S32:
3497
		inst = (sljit_ins *)addr;
3498
		SLJIT_ASSERT((inst[0] & 0xfc1f0000) == ADDIS && (inst[1] & 0xfc000000) == ORI);
3499

3500
		SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0);
3501
		inst[0] = (inst[0] & 0xffff0000) | IMM(new_constant >> 16);
3502
		inst[1] = (inst[1] & 0xffff0000) | IMM(new_constant);
3503
		SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1);
3504
		inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3505
		SLJIT_CACHE_FLUSH(inst, inst + 2);
3506
		return;
3507
#endif /* SLJIT_CONFIG_PPC_64 */
3508

3509
	default:
3510
		sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3511
		return;
3512
	}
3513
}
3514

3515
Product

Resources

Company
