CoCalc -- visitor.rs

GitHub Repository: bytecodealliance/wasmtime
Path: blob/main/winch/codegen/src/visitor.rs
³⁰⁵⁰ views
1
//! This module is the central place for machine code emission.
2
//! It defines an implementation of wasmparser's Visitor trait
3
//! for `CodeGen`; which defines a visitor per op-code,
4
//! which validates and dispatches to the corresponding
5
//! machine code emitter.
6

7
use crate::abi::RetArea;
8
use crate::codegen::{
9
    Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, FnCall,
10
    UnconditionalBranch, control_index,
11
};
12
use crate::masm::{
13
    AtomicWaitKind, DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind,
14
    MacroAssembler, MulWideKind, OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp,
15
    RoundingMode, SPOffset, ShiftKind, Signed, SplatKind, SplatLoadKind, StoreKind, TruncKind,
16
    V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind,
17
    V128LoadExtendKind, V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind,
18
    V128SubKind, V128TruncKind, VectorCompareKind, VectorEqualityKind, Zero,
19
};
20
use crate::reg::{Reg, writable};
21
use crate::stack::{TypedReg, Val};
22
use crate::{Result, bail, ensure, format_err};
23
use regalloc2::RegClass;
24
use smallvec::{SmallVec, smallvec};
25
use wasmparser::{
26
    BlockType, BrTable, Ieee32, Ieee64, MemArg, V128, VisitOperator, VisitSimdOperator,
27
};
28
use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL;
29
use wasmtime_environ::{
30
    FUNCREF_INIT_BIT, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType,
31
    WasmValType,
32
};
33

34
/// A macro to define unsupported WebAssembly operators.
35
///
36
/// This macro calls itself recursively;
37
/// 1. It no-ops when matching a supported operator.
38
/// 2. Defines the visitor function and panics when
39
///    matching an unsupported operator.
40
macro_rules! def_unsupported {
41
    ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => {
42
        $(
43
            def_unsupported!(
44
                emit
45
                    $op
46

47
                fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output {
48
                    $($(let _ = $arg;)*)?
49

50
                    Err(format_err!(CodeGenError::unimplemented_wasm_instruction()))
51
                }
52
            );
53
        )*
54
    };
55

56
    (emit I32Const $($rest:tt)*) => {};
57
    (emit I64Const $($rest:tt)*) => {};
58
    (emit F32Const $($rest:tt)*) => {};
59
    (emit F64Const $($rest:tt)*) => {};
60
    (emit V128Const $($rest:tt)*) => {};
61
    (emit F32Add $($rest:tt)*) => {};
62
    (emit F64Add $($rest:tt)*) => {};
63
    (emit F32Sub $($rest:tt)*) => {};
64
    (emit F64Sub $($rest:tt)*) => {};
65
    (emit F32Mul $($rest:tt)*) => {};
66
    (emit F64Mul $($rest:tt)*) => {};
67
    (emit F32Div $($rest:tt)*) => {};
68
    (emit F64Div $($rest:tt)*) => {};
69
    (emit F32Min $($rest:tt)*) => {};
70
    (emit F64Min $($rest:tt)*) => {};
71
    (emit F32Max $($rest:tt)*) => {};
72
    (emit F64Max $($rest:tt)*) => {};
73
    (emit F32Copysign $($rest:tt)*) => {};
74
    (emit F64Copysign $($rest:tt)*) => {};
75
    (emit F32Abs $($rest:tt)*) => {};
76
    (emit F64Abs $($rest:tt)*) => {};
77
    (emit F32Neg $($rest:tt)*) => {};
78
    (emit F64Neg $($rest:tt)*) => {};
79
    (emit F32Floor $($rest:tt)*) => {};
80
    (emit F64Floor $($rest:tt)*) => {};
81
    (emit F32Ceil $($rest:tt)*) => {};
82
    (emit F64Ceil $($rest:tt)*) => {};
83
    (emit F32Nearest $($rest:tt)*) => {};
84
    (emit F64Nearest $($rest:tt)*) => {};
85
    (emit F32Trunc $($rest:tt)*) => {};
86
    (emit F64Trunc $($rest:tt)*) => {};
87
    (emit F32Sqrt $($rest:tt)*) => {};
88
    (emit F64Sqrt $($rest:tt)*) => {};
89
    (emit F32Eq $($rest:tt)*) => {};
90
    (emit F64Eq $($rest:tt)*) => {};
91
    (emit F32Ne $($rest:tt)*) => {};
92
    (emit F64Ne $($rest:tt)*) => {};
93
    (emit F32Lt $($rest:tt)*) => {};
94
    (emit F64Lt $($rest:tt)*) => {};
95
    (emit F32Gt $($rest:tt)*) => {};
96
    (emit F64Gt $($rest:tt)*) => {};
97
    (emit F32Le $($rest:tt)*) => {};
98
    (emit F64Le $($rest:tt)*) => {};
99
    (emit F32Ge $($rest:tt)*) => {};
100
    (emit F64Ge $($rest:tt)*) => {};
101
    (emit F32ConvertI32S $($rest:tt)*) => {};
102
    (emit F32ConvertI32U $($rest:tt)*) => {};
103
    (emit F32ConvertI64S $($rest:tt)*) => {};
104
    (emit F32ConvertI64U $($rest:tt)*) => {};
105
    (emit F64ConvertI32S $($rest:tt)*) => {};
106
    (emit F64ConvertI32U $($rest:tt)*) => {};
107
    (emit F64ConvertI64S $($rest:tt)*) => {};
108
    (emit F64ConvertI64U $($rest:tt)*) => {};
109
    (emit F32ReinterpretI32 $($rest:tt)*) => {};
110
    (emit F64ReinterpretI64 $($rest:tt)*) => {};
111
    (emit F32DemoteF64 $($rest:tt)*) => {};
112
    (emit F64PromoteF32 $($rest:tt)*) => {};
113
    (emit I32Add $($rest:tt)*) => {};
114
    (emit I64Add $($rest:tt)*) => {};
115
    (emit I32Sub $($rest:tt)*) => {};
116
    (emit I32Mul $($rest:tt)*) => {};
117
    (emit I32DivS $($rest:tt)*) => {};
118
    (emit I32DivU $($rest:tt)*) => {};
119
    (emit I64DivS $($rest:tt)*) => {};
120
    (emit I64DivU $($rest:tt)*) => {};
121
    (emit I64RemU $($rest:tt)*) => {};
122
    (emit I64RemS $($rest:tt)*) => {};
123
    (emit I32RemU $($rest:tt)*) => {};
124
    (emit I32RemS $($rest:tt)*) => {};
125
    (emit I64Mul $($rest:tt)*) => {};
126
    (emit I64Sub $($rest:tt)*) => {};
127
    (emit I32Eq $($rest:tt)*) => {};
128
    (emit I64Eq $($rest:tt)*) => {};
129
    (emit I32Ne $($rest:tt)*) => {};
130
    (emit I64Ne $($rest:tt)*) => {};
131
    (emit I32LtS $($rest:tt)*) => {};
132
    (emit I64LtS $($rest:tt)*) => {};
133
    (emit I32LtU $($rest:tt)*) => {};
134
    (emit I64LtU $($rest:tt)*) => {};
135
    (emit I32LeS $($rest:tt)*) => {};
136
    (emit I64LeS $($rest:tt)*) => {};
137
    (emit I32LeU $($rest:tt)*) => {};
138
    (emit I64LeU $($rest:tt)*) => {};
139
    (emit I32GtS $($rest:tt)*) => {};
140
    (emit I64GtS $($rest:tt)*) => {};
141
    (emit I32GtU $($rest:tt)*) => {};
142
    (emit I64GtU $($rest:tt)*) => {};
143
    (emit I32GeS $($rest:tt)*) => {};
144
    (emit I64GeS $($rest:tt)*) => {};
145
    (emit I32GeU $($rest:tt)*) => {};
146
    (emit I64GeU $($rest:tt)*) => {};
147
    (emit I32Eqz $($rest:tt)*) => {};
148
    (emit I64Eqz $($rest:tt)*) => {};
149
    (emit I32And $($rest:tt)*) => {};
150
    (emit I64And $($rest:tt)*) => {};
151
    (emit I32Or $($rest:tt)*) => {};
152
    (emit I64Or $($rest:tt)*) => {};
153
    (emit I32Xor $($rest:tt)*) => {};
154
    (emit I64Xor $($rest:tt)*) => {};
155
    (emit I32Shl $($rest:tt)*) => {};
156
    (emit I64Shl $($rest:tt)*) => {};
157
    (emit I32ShrS $($rest:tt)*) => {};
158
    (emit I64ShrS $($rest:tt)*) => {};
159
    (emit I32ShrU $($rest:tt)*) => {};
160
    (emit I64ShrU $($rest:tt)*) => {};
161
    (emit I32Rotl $($rest:tt)*) => {};
162
    (emit I64Rotl $($rest:tt)*) => {};
163
    (emit I32Rotr $($rest:tt)*) => {};
164
    (emit I64Rotr $($rest:tt)*) => {};
165
    (emit I32Clz $($rest:tt)*) => {};
166
    (emit I64Clz $($rest:tt)*) => {};
167
    (emit I32Ctz $($rest:tt)*) => {};
168
    (emit I64Ctz $($rest:tt)*) => {};
169
    (emit I32Popcnt $($rest:tt)*) => {};
170
    (emit I64Popcnt $($rest:tt)*) => {};
171
    (emit I32WrapI64 $($rest:tt)*) => {};
172
    (emit I64ExtendI32S $($rest:tt)*) => {};
173
    (emit I64ExtendI32U $($rest:tt)*) => {};
174
    (emit I32Extend8S $($rest:tt)*) => {};
175
    (emit I32Extend16S $($rest:tt)*) => {};
176
    (emit I64Extend8S $($rest:tt)*) => {};
177
    (emit I64Extend16S $($rest:tt)*) => {};
178
    (emit I64Extend32S $($rest:tt)*) => {};
179
    (emit I32TruncF32S $($rest:tt)*) => {};
180
    (emit I32TruncF32U $($rest:tt)*) => {};
181
    (emit I32TruncF64S $($rest:tt)*) => {};
182
    (emit I32TruncF64U $($rest:tt)*) => {};
183
    (emit I64TruncF32S $($rest:tt)*) => {};
184
    (emit I64TruncF32U $($rest:tt)*) => {};
185
    (emit I64TruncF64S $($rest:tt)*) => {};
186
    (emit I64TruncF64U $($rest:tt)*) => {};
187
    (emit I32ReinterpretF32 $($rest:tt)*) => {};
188
    (emit I64ReinterpretF64 $($rest:tt)*) => {};
189
    (emit LocalGet $($rest:tt)*) => {};
190
    (emit LocalSet $($rest:tt)*) => {};
191
    (emit Call $($rest:tt)*) => {};
192
    (emit End $($rest:tt)*) => {};
193
    (emit Nop $($rest:tt)*) => {};
194
    (emit If $($rest:tt)*) => {};
195
    (emit Else $($rest:tt)*) => {};
196
    (emit Block $($rest:tt)*) => {};
197
    (emit Loop $($rest:tt)*) => {};
198
    (emit Br $($rest:tt)*) => {};
199
    (emit BrIf $($rest:tt)*) => {};
200
    (emit Return $($rest:tt)*) => {};
201
    (emit Unreachable $($rest:tt)*) => {};
202
    (emit LocalTee $($rest:tt)*) => {};
203
    (emit GlobalGet $($rest:tt)*) => {};
204
    (emit GlobalSet $($rest:tt)*) => {};
205
    (emit Select $($rest:tt)*) => {};
206
    (emit Drop $($rest:tt)*) => {};
207
    (emit BrTable $($rest:tt)*) => {};
208
    (emit CallIndirect $($rest:tt)*) => {};
209
    (emit TableInit $($rest:tt)*) => {};
210
    (emit TableCopy $($rest:tt)*) => {};
211
    (emit TableGet $($rest:tt)*) => {};
212
    (emit TableSet $($rest:tt)*) => {};
213
    (emit TableGrow $($rest:tt)*) => {};
214
    (emit TableSize $($rest:tt)*) => {};
215
    (emit TableFill $($rest:tt)*) => {};
216
    (emit ElemDrop $($rest:tt)*) => {};
217
    (emit MemoryInit $($rest:tt)*) => {};
218
    (emit MemoryCopy $($rest:tt)*) => {};
219
    (emit DataDrop $($rest:tt)*) => {};
220
    (emit MemoryFill $($rest:tt)*) => {};
221
    (emit MemorySize $($rest:tt)*) => {};
222
    (emit MemoryGrow $($rest:tt)*) => {};
223
    (emit I32Load $($rest:tt)*) => {};
224
    (emit I32Load8S $($rest:tt)*) => {};
225
    (emit I32Load8U $($rest:tt)*) => {};
226
    (emit I32Load16S $($rest:tt)*) => {};
227
    (emit I32Load16U $($rest:tt)*) => {};
228
    (emit I64Load8S $($rest:tt)*) => {};
229
    (emit I64Load8U $($rest:tt)*) => {};
230
    (emit I64Load16S $($rest:tt)*) => {};
231
    (emit I64Load16U $($rest:tt)*) => {};
232
    (emit I64Load32S $($rest:tt)*) => {};
233
    (emit I64Load32U $($rest:tt)*) => {};
234
    (emit I64Load $($rest:tt)*) => {};
235
    (emit I32Store $($rest:tt)*) => {};
236
    (emit I32Store8 $($rest:tt)*) => {};
237
    (emit I32Store16 $($rest:tt)*) => {};
238
    (emit I64Store $($rest:tt)*) => {};
239
    (emit I64Store8 $($rest:tt)*) => {};
240
    (emit I64Store16 $($rest:tt)*) => {};
241
    (emit I64Store32 $($rest:tt)*) => {};
242
    (emit F32Load $($rest:tt)*) => {};
243
    (emit F32Store $($rest:tt)*) => {};
244
    (emit F64Load $($rest:tt)*) => {};
245
    (emit F64Store $($rest:tt)*) => {};
246
    (emit I32TruncSatF32S $($rest:tt)*) => {};
247
    (emit I32TruncSatF32U $($rest:tt)*) => {};
248
    (emit I32TruncSatF64S $($rest:tt)*) => {};
249
    (emit I32TruncSatF64U $($rest:tt)*) => {};
250
    (emit I64TruncSatF32S $($rest:tt)*) => {};
251
    (emit I64TruncSatF32U $($rest:tt)*) => {};
252
    (emit I64TruncSatF64S $($rest:tt)*) => {};
253
    (emit I64TruncSatF64U $($rest:tt)*) => {};
254
    (emit V128Load $($rest:tt)*) => {};
255
    (emit V128Store $($rest:tt)*) => {};
256
    (emit I64Add128 $($rest:tt)*) => {};
257
    (emit I64Sub128 $($rest:tt)*) => {};
258
    (emit I64MulWideS $($rest:tt)*) => {};
259
    (emit I64MulWideU $($rest:tt)*) => {};
260
    (emit I32AtomicLoad8U $($rest:tt)*) => {};
261
    (emit I32AtomicLoad16U $($rest:tt)*) => {};
262
    (emit I32AtomicLoad $($rest:tt)*) => {};
263
    (emit I64AtomicLoad8U $($rest:tt)*) => {};
264
    (emit I64AtomicLoad16U $($rest:tt)*) => {};
265
    (emit I64AtomicLoad32U $($rest:tt)*) => {};
266
    (emit I64AtomicLoad $($rest:tt)*) => {};
267
    (emit V128Load8x8S $($rest:tt)*) => {};
268
    (emit V128Load8x8U $($rest:tt)*) => {};
269
    (emit V128Load16x4S $($rest:tt)*) => {};
270
    (emit V128Load16x4U $($rest:tt)*) => {};
271
    (emit V128Load32x2S $($rest:tt)*) => {};
272
    (emit V128Load32x2U $($rest:tt)*) => {};
273
    (emit V128Load8Splat $($rest:tt)*) => {};
274
    (emit V128Load16Splat $($rest:tt)*) => {};
275
    (emit V128Load32Splat $($rest:tt)*) => {};
276
    (emit V128Load64Splat $($rest:tt)*) => {};
277
    (emit I8x16Splat $($rest:tt)*) => {};
278
    (emit I16x8Splat $($rest:tt)*) => {};
279
    (emit I32x4Splat $($rest:tt)*) => {};
280
    (emit I64x2Splat $($rest:tt)*) => {};
281
    (emit F32x4Splat $($rest:tt)*) => {};
282
    (emit F64x2Splat $($rest:tt)*) => {};
283
    (emit I32AtomicStore8 $($rest:tt)*) => {};
284
    (emit I32AtomicStore16 $($rest:tt)*) => {};
285
    (emit I32AtomicStore $($rest:tt)*) => {};
286
    (emit I64AtomicStore8 $($rest:tt)*) => {};
287
    (emit I64AtomicStore16 $($rest:tt)*) => {};
288
    (emit I64AtomicStore32 $($rest:tt)*) => {};
289
    (emit I64AtomicStore $($rest:tt)*) => {};
290
    (emit I32AtomicRmw8AddU $($rest:tt)*) => {};
291
    (emit I32AtomicRmw16AddU $($rest:tt)*) => {};
292
    (emit I32AtomicRmwAdd $($rest:tt)*) => {};
293
    (emit I64AtomicRmw8AddU $($rest:tt)*) => {};
294
    (emit I64AtomicRmw16AddU $($rest:tt)*) => {};
295
    (emit I64AtomicRmw32AddU $($rest:tt)*) => {};
296
    (emit I64AtomicRmwAdd $($rest:tt)*) => {};
297
    (emit I8x16Shuffle $($rest:tt)*) => {};
298
    (emit I8x16Swizzle $($rest:tt)*) => {};
299
    (emit I32AtomicRmw8SubU $($rest:tt)*) => {};
300
    (emit I32AtomicRmw16SubU $($rest:tt)*) => {};
301
    (emit I32AtomicRmwSub $($rest:tt)*) => {};
302
    (emit I64AtomicRmw8SubU $($rest:tt)*) => {};
303
    (emit I64AtomicRmw16SubU $($rest:tt)*) => {};
304
    (emit I64AtomicRmw32SubU $($rest:tt)*) => {};
305
    (emit I64AtomicRmwSub $($rest:tt)*) => {};
306
    (emit I32AtomicRmw8XchgU $($rest:tt)*) => {};
307
    (emit I32AtomicRmw16XchgU $($rest:tt)*) => {};
308
    (emit I32AtomicRmwXchg $($rest:tt)*) => {};
309
    (emit I64AtomicRmw8XchgU $($rest:tt)*) => {};
310
    (emit I64AtomicRmw16XchgU $($rest:tt)*) => {};
311
    (emit I64AtomicRmw32XchgU $($rest:tt)*) => {};
312
    (emit I64AtomicRmwXchg $($rest:tt)*) => {};
313
    (emit I8x16ExtractLaneS $($rest:tt)*) => {};
314
    (emit I8x16ExtractLaneU $($rest:tt)*) => {};
315
    (emit I16x8ExtractLaneS $($rest:tt)*) => {};
316
    (emit I16x8ExtractLaneU $($rest:tt)*) => {};
317
    (emit I32x4ExtractLane $($rest:tt)*) => {};
318
    (emit I64x2ExtractLane $($rest:tt)*) => {};
319
    (emit F32x4ExtractLane $($rest:tt)*) => {};
320
    (emit F64x2ExtractLane $($rest:tt)*) => {};
321
    (emit I32AtomicRmw8AndU $($rest:tt)*) => {};
322
    (emit I32AtomicRmw16AndU $($rest:tt)*) => {};
323
    (emit I32AtomicRmwAnd $($rest:tt)*) => {};
324
    (emit I64AtomicRmw8AndU $($rest:tt)*) => {};
325
    (emit I64AtomicRmw16AndU $($rest:tt)*) => {};
326
    (emit I64AtomicRmw32AndU $($rest:tt)*) => {};
327
    (emit I64AtomicRmwAnd $($rest:tt)*) => {};
328
    (emit I32AtomicRmw8OrU $($rest:tt)*) => {};
329
    (emit I32AtomicRmw16OrU $($rest:tt)*) => {};
330
    (emit I32AtomicRmwOr $($rest:tt)*) => {};
331
    (emit I64AtomicRmw8OrU $($rest:tt)*) => {};
332
    (emit I64AtomicRmw16OrU $($rest:tt)*) => {};
333
    (emit I64AtomicRmw32OrU $($rest:tt)*) => {};
334
    (emit I64AtomicRmwOr $($rest:tt)*) => {};
335
    (emit I32AtomicRmw8XorU $($rest:tt)*) => {};
336
    (emit I32AtomicRmw16XorU $($rest:tt)*) => {};
337
    (emit I32AtomicRmwXor $($rest:tt)*) => {};
338
    (emit I64AtomicRmw8XorU $($rest:tt)*) => {};
339
    (emit I64AtomicRmw16XorU $($rest:tt)*) => {};
340
    (emit I64AtomicRmw32XorU $($rest:tt)*) => {};
341
    (emit I64AtomicRmwXor $($rest:tt)*) => {};
342
    (emit I8x16ReplaceLane $($rest:tt)*) => {};
343
    (emit I16x8ReplaceLane $($rest:tt)*) => {};
344
    (emit I32x4ReplaceLane $($rest:tt)*) => {};
345
    (emit I64x2ReplaceLane $($rest:tt)*) => {};
346
    (emit F32x4ReplaceLane $($rest:tt)*) => {};
347
    (emit F64x2ReplaceLane $($rest:tt)*) => {};
348
    (emit I32AtomicRmw8CmpxchgU $($rest:tt)*) => {};
349
    (emit I32AtomicRmw16CmpxchgU $($rest:tt)*) => {};
350
    (emit I32AtomicRmwCmpxchg $($rest:tt)*) => {};
351
    (emit I64AtomicRmw8CmpxchgU $($rest:tt)*) => {};
352
    (emit I64AtomicRmw16CmpxchgU $($rest:tt)*) => {};
353
    (emit I64AtomicRmw32CmpxchgU $($rest:tt)*) => {};
354
    (emit I64AtomicRmwCmpxchg $($rest:tt)*) => {};
355
    (emit I8x16Eq $($rest:tt)*) => {};
356
    (emit I16x8Eq $($rest:tt)*) => {};
357
    (emit I32x4Eq $($rest:tt)*) => {};
358
    (emit I64x2Eq $($rest:tt)*) => {};
359
    (emit F32x4Eq $($rest:tt)*) => {};
360
    (emit F64x2Eq $($rest:tt)*) => {};
361
    (emit I8x16Ne $($rest:tt)*) => {};
362
    (emit I16x8Ne $($rest:tt)*) => {};
363
    (emit I32x4Ne $($rest:tt)*) => {};
364
    (emit I64x2Ne $($rest:tt)*) => {};
365
    (emit F32x4Ne $($rest:tt)*) => {};
366
    (emit F64x2Ne $($rest:tt)*) => {};
367
    (emit I8x16LtS $($rest:tt)*) => {};
368
    (emit I8x16LtU $($rest:tt)*) => {};
369
    (emit I16x8LtS $($rest:tt)*) => {};
370
    (emit I16x8LtU $($rest:tt)*) => {};
371
    (emit I32x4LtS $($rest:tt)*) => {};
372
    (emit I32x4LtU $($rest:tt)*) => {};
373
    (emit I64x2LtS $($rest:tt)*) => {};
374
    (emit F32x4Lt $($rest:tt)*) => {};
375
    (emit F64x2Lt $($rest:tt)*) => {};
376
    (emit I8x16LeS $($rest:tt)*) => {};
377
    (emit I8x16LeU $($rest:tt)*) => {};
378
    (emit I16x8LeS $($rest:tt)*) => {};
379
    (emit I16x8LeU $($rest:tt)*) => {};
380
    (emit I32x4LeS $($rest:tt)*) => {};
381
    (emit I32x4LeU $($rest:tt)*) => {};
382
    (emit I64x2LeS $($rest:tt)*) => {};
383
    (emit F32x4Le $($rest:tt)*) => {};
384
    (emit F64x2Le $($rest:tt)*) => {};
385
    (emit I8x16GtS $($rest:tt)*) => {};
386
    (emit I8x16GtU $($rest:tt)*) => {};
387
    (emit I16x8GtS $($rest:tt)*) => {};
388
    (emit I16x8GtU $($rest:tt)*) => {};
389
    (emit I32x4GtS $($rest:tt)*) => {};
390
    (emit I32x4GtU $($rest:tt)*) => {};
391
    (emit I64x2GtS $($rest:tt)*) => {};
392
    (emit F32x4Gt $($rest:tt)*) => {};
393
    (emit F64x2Gt $($rest:tt)*) => {};
394
    (emit I8x16GeS $($rest:tt)*) => {};
395
    (emit I8x16GeU $($rest:tt)*) => {};
396
    (emit I16x8GeS $($rest:tt)*) => {};
397
    (emit I16x8GeU $($rest:tt)*) => {};
398
    (emit I32x4GeS $($rest:tt)*) => {};
399
    (emit I32x4GeU $($rest:tt)*) => {};
400
    (emit I64x2GeS $($rest:tt)*) => {};
401
    (emit F32x4Ge $($rest:tt)*) => {};
402
    (emit F64x2Ge $($rest:tt)*) => {};
403
    (emit MemoryAtomicWait32 $($rest:tt)*) => {};
404
    (emit MemoryAtomicWait64 $($rest:tt)*) => {};
405
    (emit MemoryAtomicNotify $($rest:tt)*) => {};
406
    (emit AtomicFence $($rest:tt)*) => {};
407
    (emit V128Not $($rest:tt)*) => {};
408
    (emit V128And $($rest:tt)*) => {};
409
    (emit V128AndNot $($rest:tt)*) => {};
410
    (emit V128Or $($rest:tt)*) => {};
411
    (emit V128Xor $($rest:tt)*) => {};
412
    (emit V128Bitselect $($rest:tt)*) => {};
413
    (emit V128AnyTrue $($rest:tt)*) => {};
414
    (emit V128Load8Lane $($rest:tt)*) => {};
415
    (emit V128Load16Lane $($rest:tt)*) => {};
416
    (emit V128Load32Lane $($rest:tt)*) => {};
417
    (emit V128Load64Lane $($rest:tt)*) => {};
418
    (emit V128Store8Lane $($rest:tt)*) => {};
419
    (emit V128Store16Lane $($rest:tt)*) => {};
420
    (emit V128Store32Lane $($rest:tt)*) => {};
421
    (emit V128Store64Lane $($rest:tt)*) => {};
422
    (emit F32x4ConvertI32x4S $($rest:tt)*) => {};
423
    (emit F32x4ConvertI32x4U $($rest:tt)*) => {};
424
    (emit F64x2ConvertLowI32x4S $($rest:tt)*) => {};
425
    (emit F64x2ConvertLowI32x4U $($rest:tt)*) => {};
426
    (emit I8x16NarrowI16x8S $($rest:tt)*) => {};
427
    (emit I8x16NarrowI16x8U $($rest:tt)*) => {};
428
    (emit I16x8NarrowI32x4S $($rest:tt)*) => {};
429
    (emit I16x8NarrowI32x4U $($rest:tt)*) => {};
430
    (emit F32x4DemoteF64x2Zero $($rest:tt)*) => {};
431
    (emit F64x2PromoteLowF32x4 $($rest:tt)*) => {};
432
    (emit I16x8ExtendLowI8x16S $($rest:tt)*) => {};
433
    (emit I16x8ExtendHighI8x16S $($rest:tt)*) => {};
434
    (emit I16x8ExtendLowI8x16U $($rest:tt)*) => {};
435
    (emit I16x8ExtendHighI8x16U $($rest:tt)*) => {};
436
    (emit I32x4ExtendLowI16x8S $($rest:tt)*) => {};
437
    (emit I32x4ExtendHighI16x8S $($rest:tt)*) => {};
438
    (emit I32x4ExtendLowI16x8U $($rest:tt)*) => {};
439
    (emit I32x4ExtendHighI16x8U $($rest:tt)*) => {};
440
    (emit I64x2ExtendLowI32x4S $($rest:tt)*) => {};
441
    (emit I64x2ExtendHighI32x4S $($rest:tt)*) => {};
442
    (emit I64x2ExtendLowI32x4U $($rest:tt)*) => {};
443
    (emit I64x2ExtendHighI32x4U $($rest:tt)*) => {};
444
    (emit I8x16Add $($rest:tt)*) => {};
445
    (emit I16x8Add $($rest:tt)*) => {};
446
    (emit I32x4Add $($rest:tt)*) => {};
447
    (emit I64x2Add $($rest:tt)*) => {};
448
    (emit I8x16Sub $($rest:tt)*) => {};
449
    (emit I16x8Sub $($rest:tt)*) => {};
450
    (emit I32x4Sub $($rest:tt)*) => {};
451
    (emit I64x2Sub $($rest:tt)*) => {};
452
    (emit I16x8Mul $($rest:tt)*) => {};
453
    (emit I32x4Mul $($rest:tt)*) => {};
454
    (emit I64x2Mul $($rest:tt)*) => {};
455
    (emit I8x16AddSatS $($rest:tt)*) => {};
456
    (emit I16x8AddSatS $($rest:tt)*) => {};
457
    (emit I8x16AddSatU $($rest:tt)*) => {};
458
    (emit I16x8AddSatU $($rest:tt)*) => {};
459
    (emit I8x16SubSatS $($rest:tt)*) => {};
460
    (emit I16x8SubSatS $($rest:tt)*) => {};
461
    (emit I8x16SubSatU $($rest:tt)*) => {};
462
    (emit I16x8SubSatU $($rest:tt)*) => {};
463
    (emit I8x16Abs $($rest:tt)*) => {};
464
    (emit I16x8Abs $($rest:tt)*) => {};
465
    (emit I32x4Abs $($rest:tt)*) => {};
466
    (emit I64x2Abs $($rest:tt)*) => {};
467
    (emit F32x4Abs $($rest:tt)*) => {};
468
    (emit F64x2Abs $($rest:tt)*) => {};
469
    (emit I8x16Neg $($rest:tt)*) => {};
470
    (emit I16x8Neg $($rest:tt)*) => {};
471
    (emit I32x4Neg $($rest:tt)*) => {};
472
    (emit I64x2Neg $($rest:tt)*) => {};
473
    (emit I8x16Shl $($rest:tt)*) => {};
474
    (emit I16x8Shl $($rest:tt)*) => {};
475
    (emit I32x4Shl $($rest:tt)*) => {};
476
    (emit I64x2Shl $($rest:tt)*) => {};
477
    (emit I8x16ShrU $($rest:tt)*) => {};
478
    (emit I16x8ShrU $($rest:tt)*) => {};
479
    (emit I32x4ShrU $($rest:tt)*) => {};
480
    (emit I64x2ShrU $($rest:tt)*) => {};
481
    (emit I8x16ShrS $($rest:tt)*) => {};
482
    (emit I16x8ShrS $($rest:tt)*) => {};
483
    (emit I32x4ShrS $($rest:tt)*) => {};
484
    (emit I64x2ShrS $($rest:tt)*) => {};
485
    (emit I16x8Q15MulrSatS $($rest:tt)*) => {};
486
    (emit I8x16AllTrue $($rest:tt)*) => {};
487
    (emit I16x8AllTrue $($rest:tt)*) => {};
488
    (emit I32x4AllTrue $($rest:tt)*) => {};
489
    (emit I64x2AllTrue $($rest:tt)*) => {};
490
    (emit I8x16Bitmask $($rest:tt)*) => {};
491
    (emit I16x8Bitmask $($rest:tt)*) => {};
492
    (emit I32x4Bitmask $($rest:tt)*) => {};
493
    (emit I64x2Bitmask $($rest:tt)*) => {};
494
    (emit I32x4TruncSatF32x4S $($rest:tt)*) => {};
495
    (emit I32x4TruncSatF32x4U $($rest:tt)*) => {};
496
    (emit I32x4TruncSatF64x2SZero $($rest:tt)*) => {};
497
    (emit I32x4TruncSatF64x2UZero $($rest:tt)*) => {};
498
    (emit I8x16MinU $($rest:tt)*) => {};
499
    (emit I16x8MinU $($rest:tt)*) => {};
500
    (emit I32x4MinU $($rest:tt)*) => {};
501
    (emit I8x16MinS $($rest:tt)*) => {};
502
    (emit I16x8MinS $($rest:tt)*) => {};
503
    (emit I32x4MinS $($rest:tt)*) => {};
504
    (emit I8x16MaxU $($rest:tt)*) => {};
505
    (emit I16x8MaxU $($rest:tt)*) => {};
506
    (emit I32x4MaxU $($rest:tt)*) => {};
507
    (emit I8x16MaxS $($rest:tt)*) => {};
508
    (emit I16x8MaxS $($rest:tt)*) => {};
509
    (emit I32x4MaxS $($rest:tt)*) => {};
510
    (emit I16x8ExtMulLowI8x16S $($rest:tt)*) => {};
511
    (emit I32x4ExtMulLowI16x8S $($rest:tt)*) => {};
512
    (emit I64x2ExtMulLowI32x4S $($rest:tt)*) => {};
513
    (emit I16x8ExtMulHighI8x16S $($rest:tt)*) => {};
514
    (emit I32x4ExtMulHighI16x8S $($rest:tt)*) => {};
515
    (emit I64x2ExtMulHighI32x4S $($rest:tt)*) => {};
516
    (emit I16x8ExtMulLowI8x16U $($rest:tt)*) => {};
517
    (emit I32x4ExtMulLowI16x8U $($rest:tt)*) => {};
518
    (emit I64x2ExtMulLowI32x4U $($rest:tt)*) => {};
519
    (emit I16x8ExtMulHighI8x16U $($rest:tt)*) => {};
520
    (emit I32x4ExtMulHighI16x8U $($rest:tt)*) => {};
521
    (emit I64x2ExtMulHighI32x4U $($rest:tt)*) => {};
522
    (emit I16x8ExtAddPairwiseI8x16U $($rest:tt)*) => {};
523
    (emit I16x8ExtAddPairwiseI8x16S $($rest:tt)*) => {};
524
    (emit I32x4ExtAddPairwiseI16x8U $($rest:tt)*) => {};
525
    (emit I32x4ExtAddPairwiseI16x8S $($rest:tt)*) => {};
526
    (emit I32x4DotI16x8S $($rest:tt)*) => {};
527
    (emit I8x16Popcnt $($rest:tt)*) => {};
528
    (emit I8x16AvgrU $($rest:tt)*) => {};
529
    (emit I16x8AvgrU $($rest:tt)*) => {};
530
    (emit F32x4Add $($rest:tt)*) => {};
531
    (emit F64x2Add $($rest:tt)*) => {};
532
    (emit F32x4Sub $($rest:tt)*) => {};
533
    (emit F64x2Sub $($rest:tt)*) => {};
534
    (emit F32x4Mul $($rest:tt)*) => {};
535
    (emit F64x2Mul $($rest:tt)*) => {};
536
    (emit F32x4Div $($rest:tt)*) => {};
537
    (emit F64x2Div $($rest:tt)*) => {};
538
    (emit F32x4Neg $($rest:tt)*) => {};
539
    (emit F64x2Neg $($rest:tt)*) => {};
540
    (emit F32x4Sqrt $($rest:tt)*) => {};
541
    (emit F64x2Sqrt $($rest:tt)*) => {};
542
    (emit F32x4Ceil $($rest:tt)*) => {};
543
    (emit F64x2Ceil $($rest:tt)*) => {};
544
    (emit F32x4Floor $($rest:tt)*) => {};
545
    (emit F64x2Floor $($rest:tt)*) => {};
546
    (emit F32x4Nearest $($rest:tt)*) => {};
547
    (emit F64x2Nearest $($rest:tt)*) => {};
548
    (emit F32x4Trunc $($rest:tt)*) => {};
549
    (emit F64x2Trunc $($rest:tt)*) => {};
550
    (emit V128Load32Zero $($rest:tt)*) => {};
551
    (emit V128Load64Zero $($rest:tt)*) => {};
552
    (emit F32x4PMin $($rest:tt)*) => {};
553
    (emit F64x2PMin $($rest:tt)*) => {};
554
    (emit F32x4PMax $($rest:tt)*) => {};
555
    (emit F64x2PMax $($rest:tt)*) => {};
556
    (emit F32x4Min $($rest:tt)*) => {};
557
    (emit F64x2Min $($rest:tt)*) => {};
558
    (emit F32x4Max $($rest:tt)*) => {};
559
    (emit F64x2Max $($rest:tt)*) => {};
560

561
    (emit $unsupported:tt $($rest:tt)*) => {$($rest)*};
562
}
563

564
impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission>
565
where
566
    M: MacroAssembler,
567
{
568
    type Output = Result<()>;
569

570
    fn visit_i32_const(&mut self, val: i32) -> Self::Output {
571
        self.context.stack.push(Val::i32(val));
572

573
        Ok(())
574
    }
575

576
    fn visit_i64_const(&mut self, val: i64) -> Self::Output {
577
        self.context.stack.push(Val::i64(val));
578
        Ok(())
579
    }
580

581
    fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output {
582
        self.context.stack.push(Val::f32(val));
583
        Ok(())
584
    }
585

586
    fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output {
587
        self.context.stack.push(Val::f64(val));
588
        Ok(())
589
    }
590

591
    fn visit_f32_add(&mut self) -> Self::Output {
592
        self.context.binop(
593
            self.masm,
594
            OperandSize::S32,
595
            &mut |masm: &mut M, dst, src, size| {
596
                masm.float_add(writable!(dst), dst, src, size)?;
597
                Ok(TypedReg::f32(dst))
598
            },
599
        )
600
    }
601

602
    fn visit_f64_add(&mut self) -> Self::Output {
603
        self.context.binop(
604
            self.masm,
605
            OperandSize::S64,
606
            &mut |masm: &mut M, dst, src, size| {
607
                masm.float_add(writable!(dst), dst, src, size)?;
608
                Ok(TypedReg::f64(dst))
609
            },
610
        )
611
    }
612

613
    fn visit_f32_sub(&mut self) -> Self::Output {
614
        self.context.binop(
615
            self.masm,
616
            OperandSize::S32,
617
            &mut |masm: &mut M, dst, src, size| {
618
                masm.float_sub(writable!(dst), dst, src, size)?;
619
                Ok(TypedReg::f32(dst))
620
            },
621
        )
622
    }
623

624
    fn visit_f64_sub(&mut self) -> Self::Output {
625
        self.context.binop(
626
            self.masm,
627
            OperandSize::S64,
628
            &mut |masm: &mut M, dst, src, size| {
629
                masm.float_sub(writable!(dst), dst, src, size)?;
630
                Ok(TypedReg::f64(dst))
631
            },
632
        )
633
    }
634

635
    fn visit_f32_mul(&mut self) -> Self::Output {
636
        self.context.binop(
637
            self.masm,
638
            OperandSize::S32,
639
            &mut |masm: &mut M, dst, src, size| {
640
                masm.float_mul(writable!(dst), dst, src, size)?;
641
                Ok(TypedReg::f32(dst))
642
            },
643
        )
644
    }
645

646
    fn visit_f64_mul(&mut self) -> Self::Output {
647
        self.context.binop(
648
            self.masm,
649
            OperandSize::S64,
650
            &mut |masm: &mut M, dst, src, size| {
651
                masm.float_mul(writable!(dst), dst, src, size)?;
652
                Ok(TypedReg::f64(dst))
653
            },
654
        )
655
    }
656

657
    fn visit_f32_div(&mut self) -> Self::Output {
658
        self.context.binop(
659
            self.masm,
660
            OperandSize::S32,
661
            &mut |masm: &mut M, dst, src, size| {
662
                masm.float_div(writable!(dst), dst, src, size)?;
663
                Ok(TypedReg::f32(dst))
664
            },
665
        )
666
    }
667

668
    fn visit_f64_div(&mut self) -> Self::Output {
669
        self.context.binop(
670
            self.masm,
671
            OperandSize::S64,
672
            &mut |masm: &mut M, dst, src, size| {
673
                masm.float_div(writable!(dst), dst, src, size)?;
674
                Ok(TypedReg::f64(dst))
675
            },
676
        )
677
    }
678

679
    fn visit_f32_min(&mut self) -> Self::Output {
680
        self.context.binop(
681
            self.masm,
682
            OperandSize::S32,
683
            &mut |masm: &mut M, dst, src, size| {
684
                masm.float_min(writable!(dst), dst, src, size)?;
685
                Ok(TypedReg::f32(dst))
686
            },
687
        )
688
    }
689

690
    fn visit_f64_min(&mut self) -> Self::Output {
691
        self.context.binop(
692
            self.masm,
693
            OperandSize::S64,
694
            &mut |masm: &mut M, dst, src, size| {
695
                masm.float_min(writable!(dst), dst, src, size)?;
696
                Ok(TypedReg::f64(dst))
697
            },
698
        )
699
    }
700

701
    fn visit_f32_max(&mut self) -> Self::Output {
702
        self.context.binop(
703
            self.masm,
704
            OperandSize::S32,
705
            &mut |masm: &mut M, dst, src, size| {
706
                masm.float_max(writable!(dst), dst, src, size)?;
707
                Ok(TypedReg::f32(dst))
708
            },
709
        )
710
    }
711

712
    fn visit_f64_max(&mut self) -> Self::Output {
713
        self.context.binop(
714
            self.masm,
715
            OperandSize::S64,
716
            &mut |masm: &mut M, dst, src, size| {
717
                masm.float_max(writable!(dst), dst, src, size)?;
718
                Ok(TypedReg::f64(dst))
719
            },
720
        )
721
    }
722

723
    fn visit_f32_copysign(&mut self) -> Self::Output {
724
        self.context.binop(
725
            self.masm,
726
            OperandSize::S32,
727
            &mut |masm: &mut M, dst, src, size| {
728
                masm.float_copysign(writable!(dst), dst, src, size)?;
729
                Ok(TypedReg::f32(dst))
730
            },
731
        )
732
    }
733

734
    fn visit_f64_copysign(&mut self) -> Self::Output {
735
        self.context.binop(
736
            self.masm,
737
            OperandSize::S64,
738
            &mut |masm: &mut M, dst, src, size| {
739
                masm.float_copysign(writable!(dst), dst, src, size)?;
740
                Ok(TypedReg::f64(dst))
741
            },
742
        )
743
    }
744

745
    fn visit_f32_abs(&mut self) -> Self::Output {
746
        self.context.unop(self.masm, |masm, reg| {
747
            masm.float_abs(writable!(reg), OperandSize::S32)?;
748
            Ok(TypedReg::f32(reg))
749
        })
750
    }
751

752
    fn visit_f64_abs(&mut self) -> Self::Output {
753
        self.context.unop(self.masm, |masm, reg| {
754
            masm.float_abs(writable!(reg), OperandSize::S64)?;
755
            Ok(TypedReg::f64(reg))
756
        })
757
    }
758

759
    fn visit_f32_neg(&mut self) -> Self::Output {
760
        self.context.unop(self.masm, |masm, reg| {
761
            masm.float_neg(writable!(reg), OperandSize::S32)?;
762
            Ok(TypedReg::f32(reg))
763
        })
764
    }
765

766
    fn visit_f64_neg(&mut self) -> Self::Output {
767
        self.context.unop(self.masm, |masm, reg| {
768
            masm.float_neg(writable!(reg), OperandSize::S64)?;
769
            Ok(TypedReg::f64(reg))
770
        })
771
    }
772

773
    fn visit_f32_floor(&mut self) -> Self::Output {
774
        self.masm.float_round(
775
            RoundingMode::Down,
776
            &mut self.env,
777
            &mut self.context,
778
            OperandSize::S32,
779
            |env, cx, masm| {
780
                let builtin = env.builtins.floor_f32::<M::ABI>()?;
781
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
782
            },
783
        )
784
    }
785

786
    fn visit_f64_floor(&mut self) -> Self::Output {
787
        self.masm.float_round(
788
            RoundingMode::Down,
789
            &mut self.env,
790
            &mut self.context,
791
            OperandSize::S64,
792
            |env, cx, masm| {
793
                let builtin = env.builtins.floor_f64::<M::ABI>()?;
794
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
795
            },
796
        )
797
    }
798

799
    fn visit_f32_ceil(&mut self) -> Self::Output {
800
        self.masm.float_round(
801
            RoundingMode::Up,
802
            &mut self.env,
803
            &mut self.context,
804
            OperandSize::S32,
805
            |env, cx, masm| {
806
                let builtin = env.builtins.ceil_f32::<M::ABI>()?;
807
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
808
            },
809
        )
810
    }
811

812
    fn visit_f64_ceil(&mut self) -> Self::Output {
813
        self.masm.float_round(
814
            RoundingMode::Up,
815
            &mut self.env,
816
            &mut self.context,
817
            OperandSize::S64,
818
            |env, cx, masm| {
819
                let builtin = env.builtins.ceil_f64::<M::ABI>()?;
820
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
821
            },
822
        )
823
    }
824

825
    fn visit_f32_nearest(&mut self) -> Self::Output {
826
        self.masm.float_round(
827
            RoundingMode::Nearest,
828
            &mut self.env,
829
            &mut self.context,
830
            OperandSize::S32,
831
            |env, cx, masm| {
832
                let builtin = env.builtins.nearest_f32::<M::ABI>()?;
833
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
834
            },
835
        )
836
    }
837

838
    fn visit_f64_nearest(&mut self) -> Self::Output {
839
        self.masm.float_round(
840
            RoundingMode::Nearest,
841
            &mut self.env,
842
            &mut self.context,
843
            OperandSize::S64,
844
            |env, cx, masm| {
845
                let builtin = env.builtins.nearest_f64::<M::ABI>()?;
846
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
847
            },
848
        )
849
    }
850

851
    fn visit_f32_trunc(&mut self) -> Self::Output {
852
        self.masm.float_round(
853
            RoundingMode::Zero,
854
            &mut self.env,
855
            &mut self.context,
856
            OperandSize::S32,
857
            |env, cx, masm| {
858
                let builtin = env.builtins.trunc_f32::<M::ABI>()?;
859
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
860
            },
861
        )
862
    }
863

864
    fn visit_f64_trunc(&mut self) -> Self::Output {
865
        self.masm.float_round(
866
            RoundingMode::Zero,
867
            &mut self.env,
868
            &mut self.context,
869
            OperandSize::S64,
870
            |env, cx, masm| {
871
                let builtin = env.builtins.trunc_f64::<M::ABI>()?;
872
                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
873
            },
874
        )
875
    }
876

877
    fn visit_f32_sqrt(&mut self) -> Self::Output {
878
        self.context.unop(self.masm, |masm, reg| {
879
            masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?;
880
            Ok(TypedReg::f32(reg))
881
        })
882
    }
883

884
    fn visit_f64_sqrt(&mut self) -> Self::Output {
885
        self.context.unop(self.masm, |masm, reg| {
886
            masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?;
887
            Ok(TypedReg::f64(reg))
888
        })
889
    }
890

891
    fn visit_f32_eq(&mut self) -> Self::Output {
892
        self.context.float_cmp_op(
893
            self.masm,
894
            OperandSize::S32,
895
            &mut |masm: &mut M, dst, src1, src2, size| {
896
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size)
897
            },
898
        )
899
    }
900

901
    fn visit_f64_eq(&mut self) -> Self::Output {
902
        self.context.float_cmp_op(
903
            self.masm,
904
            OperandSize::S64,
905
            &mut |masm: &mut M, dst, src1, src2, size| {
906
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size)
907
            },
908
        )
909
    }
910

911
    fn visit_f32_ne(&mut self) -> Self::Output {
912
        self.context.float_cmp_op(
913
            self.masm,
914
            OperandSize::S32,
915
            &mut |masm: &mut M, dst, src1, src2, size| {
916
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size)
917
            },
918
        )
919
    }
920

921
    fn visit_f64_ne(&mut self) -> Self::Output {
922
        self.context.float_cmp_op(
923
            self.masm,
924
            OperandSize::S64,
925
            &mut |masm: &mut M, dst, src1, src2, size| {
926
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size)
927
            },
928
        )
929
    }
930

931
    fn visit_f32_lt(&mut self) -> Self::Output {
932
        self.context.float_cmp_op(
933
            self.masm,
934
            OperandSize::S32,
935
            &mut |masm: &mut M, dst, src1, src2, size| {
936
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size)
937
            },
938
        )
939
    }
940

941
    fn visit_f64_lt(&mut self) -> Self::Output {
942
        self.context.float_cmp_op(
943
            self.masm,
944
            OperandSize::S64,
945
            &mut |masm: &mut M, dst, src1, src2, size| {
946
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size)
947
            },
948
        )
949
    }
950

951
    fn visit_f32_gt(&mut self) -> Self::Output {
952
        self.context.float_cmp_op(
953
            self.masm,
954
            OperandSize::S32,
955
            &mut |masm: &mut M, dst, src1, src2, size| {
956
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size)
957
            },
958
        )
959
    }
960

961
    fn visit_f64_gt(&mut self) -> Self::Output {
962
        self.context.float_cmp_op(
963
            self.masm,
964
            OperandSize::S64,
965
            &mut |masm: &mut M, dst, src1, src2, size| {
966
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size)
967
            },
968
        )
969
    }
970

971
    fn visit_f32_le(&mut self) -> Self::Output {
972
        self.context.float_cmp_op(
973
            self.masm,
974
            OperandSize::S32,
975
            &mut |masm: &mut M, dst, src1, src2, size| {
976
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size)
977
            },
978
        )
979
    }
980

981
    fn visit_f64_le(&mut self) -> Self::Output {
982
        self.context.float_cmp_op(
983
            self.masm,
984
            OperandSize::S64,
985
            &mut |masm: &mut M, dst, src1, src2, size| {
986
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size)
987
            },
988
        )
989
    }
990

991
    fn visit_f32_ge(&mut self) -> Self::Output {
992
        self.context.float_cmp_op(
993
            self.masm,
994
            OperandSize::S32,
995
            &mut |masm: &mut M, dst, src1, src2, size| {
996
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size)
997
            },
998
        )
999
    }
1000

1001
    fn visit_f64_ge(&mut self) -> Self::Output {
1002
        self.context.float_cmp_op(
1003
            self.masm,
1004
            OperandSize::S64,
1005
            &mut |masm: &mut M, dst, src1, src2, size| {
1006
                masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size)
1007
            },
1008
        )
1009
    }
1010

1011
    fn visit_f32_convert_i32_s(&mut self) -> Self::Output {
1012
        self.context
1013
            .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| {
1014
                masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size)
1015
            })
1016
    }
1017

1018
    fn visit_f32_convert_i32_u(&mut self) -> Self::Output {
1019
        self.context.convert_op_with_tmp_reg(
1020
            self.masm,
1021
            WasmValType::F32,
1022
            RegClass::Int,
1023
            |masm, dst, src, tmp_gpr, dst_size| {
1024
                masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size)
1025
            },
1026
        )
1027
    }
1028

1029
    fn visit_f32_convert_i64_s(&mut self) -> Self::Output {
1030
        self.context
1031
            .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| {
1032
                masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size)
1033
            })
1034
    }
1035

1036
    fn visit_f32_convert_i64_u(&mut self) -> Self::Output {
1037
        self.context.convert_op_with_tmp_reg(
1038
            self.masm,
1039
            WasmValType::F32,
1040
            RegClass::Int,
1041
            |masm, dst, src, tmp_gpr, dst_size| {
1042
                masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size)
1043
            },
1044
        )
1045
    }
1046

1047
    fn visit_f64_convert_i32_s(&mut self) -> Self::Output {
1048
        self.context
1049
            .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| {
1050
                masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size)
1051
            })
1052
    }
1053

1054
    fn visit_f64_convert_i32_u(&mut self) -> Self::Output {
1055
        self.context.convert_op_with_tmp_reg(
1056
            self.masm,
1057
            WasmValType::F64,
1058
            RegClass::Int,
1059
            |masm, dst, src, tmp_gpr, dst_size| {
1060
                masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size)
1061
            },
1062
        )
1063
    }
1064

1065
    fn visit_f64_convert_i64_s(&mut self) -> Self::Output {
1066
        self.context
1067
            .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| {
1068
                masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size)
1069
            })
1070
    }
1071

1072
    fn visit_f64_convert_i64_u(&mut self) -> Self::Output {
1073
        self.context.convert_op_with_tmp_reg(
1074
            self.masm,
1075
            WasmValType::F64,
1076
            RegClass::Int,
1077
            |masm, dst, src, tmp_gpr, dst_size| {
1078
                masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size)
1079
            },
1080
        )
1081
    }
1082

1083
    fn visit_f32_reinterpret_i32(&mut self) -> Self::Output {
1084
        self.context
1085
            .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| {
1086
                masm.reinterpret_int_as_float(writable!(dst), src, size)
1087
            })
1088
    }
1089

1090
    fn visit_f64_reinterpret_i64(&mut self) -> Self::Output {
1091
        self.context
1092
            .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| {
1093
                masm.reinterpret_int_as_float(writable!(dst), src, size)
1094
            })
1095
    }
1096

1097
    fn visit_f32_demote_f64(&mut self) -> Self::Output {
1098
        self.context.unop(self.masm, |masm, reg| {
1099
            masm.demote(writable!(reg), reg)?;
1100
            Ok(TypedReg::f32(reg))
1101
        })
1102
    }
1103

1104
    fn visit_f64_promote_f32(&mut self) -> Self::Output {
1105
        self.context.unop(self.masm, |masm, reg| {
1106
            masm.promote(writable!(reg), reg)?;
1107
            Ok(TypedReg::f64(reg))
1108
        })
1109
    }
1110

1111
    fn visit_i32_add(&mut self) -> Self::Output {
1112
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1113
            masm.add(writable!(dst), dst, src, size)?;
1114
            Ok(TypedReg::i32(dst))
1115
        })
1116
    }
1117

1118
    fn visit_i64_add(&mut self) -> Self::Output {
1119
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1120
            masm.add(writable!(dst), dst, src, size)?;
1121
            Ok(TypedReg::i64(dst))
1122
        })
1123
    }
1124

1125
    fn visit_i32_sub(&mut self) -> Self::Output {
1126
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1127
            masm.sub(writable!(dst), dst, src, size)?;
1128
            Ok(TypedReg::i32(dst))
1129
        })
1130
    }
1131

1132
    fn visit_i64_sub(&mut self) -> Self::Output {
1133
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1134
            masm.sub(writable!(dst), dst, src, size)?;
1135
            Ok(TypedReg::i64(dst))
1136
        })
1137
    }
1138

1139
    fn visit_i32_mul(&mut self) -> Self::Output {
1140
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1141
            masm.mul(writable!(dst), dst, src, size)?;
1142
            Ok(TypedReg::i32(dst))
1143
        })
1144
    }
1145

1146
    fn visit_i64_mul(&mut self) -> Self::Output {
1147
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1148
            masm.mul(writable!(dst), dst, src, size)?;
1149
            Ok(TypedReg::i64(dst))
1150
        })
1151
    }
1152

1153
    fn visit_i32_div_s(&mut self) -> Self::Output {
1154
        use DivKind::*;
1155
        use OperandSize::*;
1156

1157
        self.masm.div(&mut self.context, Signed, S32)
1158
    }
1159

1160
    fn visit_i32_div_u(&mut self) -> Self::Output {
1161
        use DivKind::*;
1162
        use OperandSize::*;
1163

1164
        self.masm.div(&mut self.context, Unsigned, S32)
1165
    }
1166

1167
    fn visit_i64_div_s(&mut self) -> Self::Output {
1168
        use DivKind::*;
1169
        use OperandSize::*;
1170

1171
        self.masm.div(&mut self.context, Signed, S64)
1172
    }
1173

1174
    fn visit_i64_div_u(&mut self) -> Self::Output {
1175
        use DivKind::*;
1176
        use OperandSize::*;
1177

1178
        self.masm.div(&mut self.context, Unsigned, S64)
1179
    }
1180

1181
    fn visit_i32_rem_s(&mut self) -> Self::Output {
1182
        use OperandSize::*;
1183
        use RemKind::*;
1184

1185
        self.masm.rem(&mut self.context, Signed, S32)
1186
    }
1187

1188
    fn visit_i32_rem_u(&mut self) -> Self::Output {
1189
        use OperandSize::*;
1190
        use RemKind::*;
1191

1192
        self.masm.rem(&mut self.context, Unsigned, S32)
1193
    }
1194

1195
    fn visit_i64_rem_s(&mut self) -> Self::Output {
1196
        use OperandSize::*;
1197
        use RemKind::*;
1198

1199
        self.masm.rem(&mut self.context, Signed, S64)
1200
    }
1201

1202
    fn visit_i64_rem_u(&mut self) -> Self::Output {
1203
        use OperandSize::*;
1204
        use RemKind::*;
1205

1206
        self.masm.rem(&mut self.context, Unsigned, S64)
1207
    }
1208

1209
    fn visit_i32_eq(&mut self) -> Self::Output {
1210
        self.cmp_i32s(IntCmpKind::Eq)
1211
    }
1212

1213
    fn visit_i64_eq(&mut self) -> Self::Output {
1214
        self.cmp_i64s(IntCmpKind::Eq)
1215
    }
1216

1217
    fn visit_i32_ne(&mut self) -> Self::Output {
1218
        self.cmp_i32s(IntCmpKind::Ne)
1219
    }
1220

1221
    fn visit_i64_ne(&mut self) -> Self::Output {
1222
        self.cmp_i64s(IntCmpKind::Ne)
1223
    }
1224

1225
    fn visit_i32_lt_s(&mut self) -> Self::Output {
1226
        self.cmp_i32s(IntCmpKind::LtS)
1227
    }
1228

1229
    fn visit_i64_lt_s(&mut self) -> Self::Output {
1230
        self.cmp_i64s(IntCmpKind::LtS)
1231
    }
1232

1233
    fn visit_i32_lt_u(&mut self) -> Self::Output {
1234
        self.cmp_i32s(IntCmpKind::LtU)
1235
    }
1236

1237
    fn visit_i64_lt_u(&mut self) -> Self::Output {
1238
        self.cmp_i64s(IntCmpKind::LtU)
1239
    }
1240

1241
    fn visit_i32_le_s(&mut self) -> Self::Output {
1242
        self.cmp_i32s(IntCmpKind::LeS)
1243
    }
1244

1245
    fn visit_i64_le_s(&mut self) -> Self::Output {
1246
        self.cmp_i64s(IntCmpKind::LeS)
1247
    }
1248

1249
    fn visit_i32_le_u(&mut self) -> Self::Output {
1250
        self.cmp_i32s(IntCmpKind::LeU)
1251
    }
1252

1253
    fn visit_i64_le_u(&mut self) -> Self::Output {
1254
        self.cmp_i64s(IntCmpKind::LeU)
1255
    }
1256

1257
    fn visit_i32_gt_s(&mut self) -> Self::Output {
1258
        self.cmp_i32s(IntCmpKind::GtS)
1259
    }
1260

1261
    fn visit_i64_gt_s(&mut self) -> Self::Output {
1262
        self.cmp_i64s(IntCmpKind::GtS)
1263
    }
1264

1265
    fn visit_i32_gt_u(&mut self) -> Self::Output {
1266
        self.cmp_i32s(IntCmpKind::GtU)
1267
    }
1268

1269
    fn visit_i64_gt_u(&mut self) -> Self::Output {
1270
        self.cmp_i64s(IntCmpKind::GtU)
1271
    }
1272

1273
    fn visit_i32_ge_s(&mut self) -> Self::Output {
1274
        self.cmp_i32s(IntCmpKind::GeS)
1275
    }
1276

1277
    fn visit_i64_ge_s(&mut self) -> Self::Output {
1278
        self.cmp_i64s(IntCmpKind::GeS)
1279
    }
1280

1281
    fn visit_i32_ge_u(&mut self) -> Self::Output {
1282
        self.cmp_i32s(IntCmpKind::GeU)
1283
    }
1284

1285
    fn visit_i64_ge_u(&mut self) -> Self::Output {
1286
        self.cmp_i64s(IntCmpKind::GeU)
1287
    }
1288

1289
    fn visit_i32_eqz(&mut self) -> Self::Output {
1290
        use OperandSize::*;
1291

1292
        self.context.unop(self.masm, |masm, reg| {
1293
            masm.cmp_with_set(writable!(reg), RegImm::i32(0), IntCmpKind::Eq, S32)?;
1294
            Ok(TypedReg::i32(reg))
1295
        })
1296
    }
1297

1298
    fn visit_i64_eqz(&mut self) -> Self::Output {
1299
        use OperandSize::*;
1300

1301
        self.context.unop(self.masm, |masm, reg| {
1302
            masm.cmp_with_set(writable!(reg), RegImm::i64(0), IntCmpKind::Eq, S64)?;
1303
            Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`.
1304
        })
1305
    }
1306

1307
    fn visit_i32_clz(&mut self) -> Self::Output {
1308
        use OperandSize::*;
1309

1310
        self.context.unop(self.masm, |masm, reg| {
1311
            masm.clz(writable!(reg), reg, S32)?;
1312
            Ok(TypedReg::i32(reg))
1313
        })
1314
    }
1315

1316
    fn visit_i64_clz(&mut self) -> Self::Output {
1317
        use OperandSize::*;
1318

1319
        self.context.unop(self.masm, |masm, reg| {
1320
            masm.clz(writable!(reg), reg, S64)?;
1321
            Ok(TypedReg::i64(reg))
1322
        })
1323
    }
1324

1325
    fn visit_i32_ctz(&mut self) -> Self::Output {
1326
        use OperandSize::*;
1327

1328
        self.context.unop(self.masm, |masm, reg| {
1329
            masm.ctz(writable!(reg), reg, S32)?;
1330
            Ok(TypedReg::i32(reg))
1331
        })
1332
    }
1333

1334
    fn visit_i64_ctz(&mut self) -> Self::Output {
1335
        use OperandSize::*;
1336

1337
        self.context.unop(self.masm, |masm, reg| {
1338
            masm.ctz(writable!(reg), reg, S64)?;
1339
            Ok(TypedReg::i64(reg))
1340
        })
1341
    }
1342

1343
    fn visit_i32_and(&mut self) -> Self::Output {
1344
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1345
            masm.and(writable!(dst), dst, src, size)?;
1346
            Ok(TypedReg::i32(dst))
1347
        })
1348
    }
1349

1350
    fn visit_i64_and(&mut self) -> Self::Output {
1351
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1352
            masm.and(writable!(dst), dst, src, size)?;
1353
            Ok(TypedReg::i64(dst))
1354
        })
1355
    }
1356

1357
    fn visit_i32_or(&mut self) -> Self::Output {
1358
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1359
            masm.or(writable!(dst), dst, src, size)?;
1360
            Ok(TypedReg::i32(dst))
1361
        })
1362
    }
1363

1364
    fn visit_i64_or(&mut self) -> Self::Output {
1365
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1366
            masm.or(writable!(dst), dst, src, size)?;
1367
            Ok(TypedReg::i64(dst))
1368
        })
1369
    }
1370

1371
    fn visit_i32_xor(&mut self) -> Self::Output {
1372
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
1373
            masm.xor(writable!(dst), dst, src, size)?;
1374
            Ok(TypedReg::i32(dst))
1375
        })
1376
    }
1377

1378
    fn visit_i64_xor(&mut self) -> Self::Output {
1379
        self.context.i64_binop(self.masm, |masm, dst, src, size| {
1380
            masm.xor(writable!(dst), dst, src, size)?;
1381
            Ok(TypedReg::i64(dst))
1382
        })
1383
    }
1384

1385
    fn visit_i32_shl(&mut self) -> Self::Output {
1386
        use ShiftKind::*;
1387

1388
        self.context.i32_shift(self.masm, Shl)
1389
    }
1390

1391
    fn visit_i64_shl(&mut self) -> Self::Output {
1392
        use ShiftKind::*;
1393

1394
        self.context.i64_shift(self.masm, Shl)
1395
    }
1396

1397
    fn visit_i32_shr_s(&mut self) -> Self::Output {
1398
        use ShiftKind::*;
1399

1400
        self.context.i32_shift(self.masm, ShrS)
1401
    }
1402

1403
    fn visit_i64_shr_s(&mut self) -> Self::Output {
1404
        use ShiftKind::*;
1405

1406
        self.context.i64_shift(self.masm, ShrS)
1407
    }
1408

1409
    fn visit_i32_shr_u(&mut self) -> Self::Output {
1410
        use ShiftKind::*;
1411

1412
        self.context.i32_shift(self.masm, ShrU)
1413
    }
1414

1415
    fn visit_i64_shr_u(&mut self) -> Self::Output {
1416
        use ShiftKind::*;
1417

1418
        self.context.i64_shift(self.masm, ShrU)
1419
    }
1420

1421
    fn visit_i32_rotl(&mut self) -> Self::Output {
1422
        use ShiftKind::*;
1423

1424
        self.context.i32_shift(self.masm, Rotl)
1425
    }
1426

1427
    fn visit_i64_rotl(&mut self) -> Self::Output {
1428
        use ShiftKind::*;
1429

1430
        self.context.i64_shift(self.masm, Rotl)
1431
    }
1432

1433
    fn visit_i32_rotr(&mut self) -> Self::Output {
1434
        use ShiftKind::*;
1435

1436
        self.context.i32_shift(self.masm, Rotr)
1437
    }
1438

1439
    fn visit_i64_rotr(&mut self) -> Self::Output {
1440
        use ShiftKind::*;
1441

1442
        self.context.i64_shift(self.masm, Rotr)
1443
    }
1444

1445
    fn visit_end(&mut self) -> Self::Output {
1446
        if !self.context.reachable {
1447
            self.handle_unreachable_end()
1448
        } else {
1449
            let mut control = self.pop_control_frame()?;
1450
            control.emit_end(self.masm, &mut self.context)
1451
        }
1452
    }
1453

1454
    fn visit_i32_popcnt(&mut self) -> Self::Output {
1455
        use OperandSize::*;
1456
        self.masm.popcnt(&mut self.context, S32)
1457
    }
1458

1459
    fn visit_i64_popcnt(&mut self) -> Self::Output {
1460
        use OperandSize::*;
1461

1462
        self.masm.popcnt(&mut self.context, S64)
1463
    }
1464

1465
    fn visit_i32_wrap_i64(&mut self) -> Self::Output {
1466
        self.context.unop(self.masm, |masm, reg| {
1467
            masm.wrap(writable!(reg), reg)?;
1468
            Ok(TypedReg::i32(reg))
1469
        })
1470
    }
1471

1472
    fn visit_i64_extend_i32_s(&mut self) -> Self::Output {
1473
        self.context.unop(self.masm, |masm, reg| {
1474
            masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?;
1475
            Ok(TypedReg::i64(reg))
1476
        })
1477
    }
1478

1479
    fn visit_i64_extend_i32_u(&mut self) -> Self::Output {
1480
        self.context.unop(self.masm, |masm, reg| {
1481
            masm.extend(writable!(reg), reg, Extend::<Zero>::I64Extend32.into())?;
1482
            Ok(TypedReg::i64(reg))
1483
        })
1484
    }
1485

1486
    fn visit_i32_extend8_s(&mut self) -> Self::Output {
1487
        self.context.unop(self.masm, |masm, reg| {
1488
            masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend8.into())?;
1489
            Ok(TypedReg::i32(reg))
1490
        })
1491
    }
1492

1493
    fn visit_i32_extend16_s(&mut self) -> Self::Output {
1494
        self.context.unop(self.masm, |masm, reg| {
1495
            masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend16.into())?;
1496
            Ok(TypedReg::i32(reg))
1497
        })
1498
    }
1499

1500
    fn visit_i64_extend8_s(&mut self) -> Self::Output {
1501
        self.context.unop(self.masm, |masm, reg| {
1502
            masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend8.into())?;
1503
            Ok(TypedReg::i64(reg))
1504
        })
1505
    }
1506

1507
    fn visit_i64_extend16_s(&mut self) -> Self::Output {
1508
        self.context.unop(self.masm, |masm, reg| {
1509
            masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend16.into())?;
1510
            Ok(TypedReg::i64(reg))
1511
        })
1512
    }
1513

1514
    fn visit_i64_extend32_s(&mut self) -> Self::Output {
1515
        self.context.unop(self.masm, |masm, reg| {
1516
            masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?;
1517
            Ok(TypedReg::i64(reg))
1518
        })
1519
    }
1520

1521
    fn visit_i32_trunc_f32_s(&mut self) -> Self::Output {
1522
        use OperandSize::*;
1523

1524
        self.context
1525
            .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| {
1526
                masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked)
1527
            })
1528
    }
1529

1530
    fn visit_i32_trunc_f32_u(&mut self) -> Self::Output {
1531
        use OperandSize::*;
1532

1533
        self.masm
1534
            .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked)
1535
    }
1536

1537
    fn visit_i32_trunc_f64_s(&mut self) -> Self::Output {
1538
        use OperandSize::*;
1539

1540
        self.context
1541
            .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| {
1542
                masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked)
1543
            })
1544
    }
1545

1546
    fn visit_i32_trunc_f64_u(&mut self) -> Self::Output {
1547
        use OperandSize::*;
1548
        self.masm
1549
            .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked)
1550
    }
1551

1552
    fn visit_i64_trunc_f32_s(&mut self) -> Self::Output {
1553
        use OperandSize::*;
1554

1555
        self.context
1556
            .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| {
1557
                masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked)
1558
            })
1559
    }
1560

1561
    fn visit_i64_trunc_f32_u(&mut self) -> Self::Output {
1562
        use OperandSize::*;
1563

1564
        self.masm
1565
            .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked)
1566
    }
1567

1568
    fn visit_i64_trunc_f64_s(&mut self) -> Self::Output {
1569
        use OperandSize::*;
1570

1571
        self.context
1572
            .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| {
1573
                masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked)
1574
            })
1575
    }
1576

1577
    fn visit_i64_trunc_f64_u(&mut self) -> Self::Output {
1578
        use OperandSize::*;
1579

1580
        self.masm
1581
            .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked)
1582
    }
1583

1584
    fn visit_i32_reinterpret_f32(&mut self) -> Self::Output {
1585
        self.context
1586
            .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| {
1587
                masm.reinterpret_float_as_int(writable!(dst), src, size)
1588
            })
1589
    }
1590

1591
    fn visit_i64_reinterpret_f64(&mut self) -> Self::Output {
1592
        self.context
1593
            .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| {
1594
                masm.reinterpret_float_as_int(writable!(dst), src, size)
1595
            })
1596
    }
1597

1598
    fn visit_local_get(&mut self, index: u32) -> Self::Output {
1599
        use WasmValType::*;
1600
        let context = &mut self.context;
1601
        let slot = context.frame.get_wasm_local(index);
1602
        match slot.ty {
1603
            I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)),
1604
            Ref(rt) => match rt.heap_type {
1605
                WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)),
1606
                _ => bail!(CodeGenError::unsupported_wasm_type()),
1607
            },
1608
        }
1609

1610
        Ok(())
1611
    }
1612

1613
    fn visit_local_set(&mut self, index: u32) -> Self::Output {
1614
        let src = self.emit_set_local(index)?;
1615
        self.context.free_reg(src);
1616
        Ok(())
1617
    }
1618

1619
    fn visit_call(&mut self, index: u32) -> Self::Output {
1620
        let callee = self.env.callee_from_index(FuncIndex::from_u32(index));
1621
        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?;
1622
        Ok(())
1623
    }
1624

1625
    fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output {
1626
        // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit`
1627
        // invocations will both trigger spills since they both call functions.
1628
        // However, the machine instructions for the spill emitted by
1629
        // `emit_lazy_funcref` will be jumped over if the funcref was previously
1630
        // initialized which may result in the machine stack becoming
1631
        // unbalanced.
1632
        self.context.spill(self.masm)?;
1633

1634
        let type_index = TypeIndex::from_u32(type_index);
1635
        let table_index = TableIndex::from_u32(table_index);
1636

1637
        self.emit_lazy_init_funcref(table_index)?;
1638

1639
        // Perform the indirect call.
1640
        // This code assumes that [`Self::emit_lazy_init_funcref`] will
1641
        // push the funcref to the value stack.
1642
        let funcref_ptr = self
1643
            .context
1644
            .stack
1645
            .peek()
1646
            .map(|v| v.unwrap_reg())
1647
            .ok_or_else(|| CodeGenError::missing_values_in_stack())?;
1648
        self.masm
1649
            .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?;
1650
        self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?;
1651

1652
        let callee = self.env.funcref(type_index);
1653
        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?;
1654
        Ok(())
1655
    }
1656

1657
    fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output {
1658
        let at = self.context.stack.ensure_index_at(3)?;
1659

1660
        self.context
1661
            .stack
1662
            .insert_many(at, &[table.try_into()?, elem.try_into()?]);
1663

1664
        let builtin = self.env.builtins.table_init::<M::ABI>()?;
1665
        FnCall::emit::<M>(
1666
            &mut self.env,
1667
            self.masm,
1668
            &mut self.context,
1669
            Callee::Builtin(builtin.clone()),
1670
        )?;
1671
        self.context.pop_and_free(self.masm)
1672
    }
1673

1674
    fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output {
1675
        let at = self.context.stack.ensure_index_at(3)?;
1676
        self.context
1677
            .stack
1678
            .insert_many(at, &[dst.try_into()?, src.try_into()?]);
1679

1680
        let builtin = self.env.builtins.table_copy::<M::ABI>()?;
1681
        FnCall::emit::<M>(
1682
            &mut self.env,
1683
            self.masm,
1684
            &mut self.context,
1685
            Callee::Builtin(builtin),
1686
        )?;
1687
        self.context.pop_and_free(self.masm)
1688
    }
1689

1690
    fn visit_table_get(&mut self, table: u32) -> Self::Output {
1691
        let table_index = TableIndex::from_u32(table);
1692
        let table = self.env.table(table_index);
1693
        let heap_type = table.ref_type.heap_type;
1694

1695
        match heap_type {
1696
            WasmHeapType::Func => self.emit_lazy_init_funcref(table_index),
1697
            _ => Err(format_err!(CodeGenError::unsupported_wasm_type())),
1698
        }
1699
    }
1700

1701
    fn visit_table_grow(&mut self, table: u32) -> Self::Output {
1702
        let table_index = TableIndex::from_u32(table);
1703
        let table_ty = self.env.table(table_index);
1704
        let builtin = match table_ty.ref_type.heap_type {
1705
            WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI>()?,
1706
            _ => bail!(CodeGenError::unsupported_wasm_type()),
1707
        };
1708

1709
        let len = self.context.stack.len();
1710
        // table.grow` requires at least 2 elements on the value stack.
1711
        let at = self.context.stack.ensure_index_at(2)?;
1712

1713
        // The table_grow builtin expects the parameters in a different
1714
        // order.
1715
        // The value stack at this point should contain:
1716
        // [ init_value | delta ] (stack top)
1717
        // but the builtin function expects the init value as the last
1718
        // argument.
1719
        self.context.stack.inner_mut().swap(len - 1, len - 2);
1720

1721
        let builtin = self.prepare_builtin_defined_table_arg(table_index, at, builtin)?;
1722

1723
        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
1724

1725
        Ok(())
1726
    }
1727

1728
    fn visit_table_size(&mut self, table: u32) -> Self::Output {
1729
        let table_index = TableIndex::from_u32(table);
1730
        let table_data = self.env.resolve_table_data(table_index);
1731
        self.emit_compute_table_size(&table_data)
1732
    }
1733

1734
    fn visit_table_fill(&mut self, table: u32) -> Self::Output {
1735
        let table_index = TableIndex::from_u32(table);
1736
        let table_ty = self.env.table(table_index);
1737

1738
        ensure!(
1739
            table_ty.ref_type.heap_type == WasmHeapType::Func,
1740
            CodeGenError::unsupported_wasm_type()
1741
        );
1742

1743
        let builtin = self.env.builtins.table_fill_func_ref::<M::ABI>()?;
1744

1745
        let at = self.context.stack.ensure_index_at(3)?;
1746

1747
        self.context.stack.insert_many(at, &[table.try_into()?]);
1748
        FnCall::emit::<M>(
1749
            &mut self.env,
1750
            self.masm,
1751
            &mut self.context,
1752
            Callee::Builtin(builtin.clone()),
1753
        )?;
1754
        self.context.pop_and_free(self.masm)
1755
    }
1756

1757
    fn visit_table_set(&mut self, table: u32) -> Self::Output {
1758
        let ptr_type = self.env.ptr_type();
1759
        let table_index = TableIndex::from_u32(table);
1760
        let table_data = self.env.resolve_table_data(table_index);
1761
        let table = self.env.table(table_index);
1762
        match table.ref_type.heap_type {
1763
            WasmHeapType::Func => {
1764
                ensure!(
1765
                    self.tunables.table_lazy_init,
1766
                    CodeGenError::unsupported_table_eager_init()
1767
                );
1768
                let value = self.context.pop_to_reg(self.masm, None)?;
1769
                let index = self.context.pop_to_reg(self.masm, None)?;
1770
                let base = self.context.any_gpr(self.masm)?;
1771
                let elem_addr =
1772
                    self.emit_compute_table_elem_addr(index.into(), base, &table_data)?;
1773
                // Set the initialized bit.
1774
                self.masm.or(
1775
                    writable!(value.into()),
1776
                    value.into(),
1777
                    RegImm::i64(FUNCREF_INIT_BIT as i64),
1778
                    ptr_type.try_into()?,
1779
                )?;
1780

1781
                self.masm.store_ptr(value.into(), elem_addr)?;
1782

1783
                self.context.free_reg(value);
1784
                self.context.free_reg(index);
1785
                self.context.free_reg(base);
1786
                Ok(())
1787
            }
1788
            _ => Err(format_err!(CodeGenError::unsupported_wasm_type())),
1789
        }
1790
    }
1791

1792
    fn visit_elem_drop(&mut self, index: u32) -> Self::Output {
1793
        let elem_drop = self.env.builtins.elem_drop::<M::ABI>()?;
1794
        self.context.stack.extend([index.try_into()?]);
1795
        FnCall::emit::<M>(
1796
            &mut self.env,
1797
            self.masm,
1798
            &mut self.context,
1799
            Callee::Builtin(elem_drop),
1800
        )?;
1801
        self.context.pop_and_free(self.masm)
1802
    }
1803

1804
    fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output {
1805
        let at = self.context.stack.ensure_index_at(3)?;
1806
        self.context
1807
            .stack
1808
            .insert_many(at, &[mem.try_into()?, data_index.try_into()?]);
1809
        let builtin = self.env.builtins.memory_init::<M::ABI>()?;
1810
        FnCall::emit::<M>(
1811
            &mut self.env,
1812
            self.masm,
1813
            &mut self.context,
1814
            Callee::Builtin(builtin),
1815
        )?;
1816
        self.context.pop_and_free(self.masm)
1817
    }
1818

1819
    fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output {
1820
        // At this point, the stack is expected to contain:
1821
        //     [ dst_offset, src_offset, len ]
1822
        // The following code inserts the missing params, so that stack contains:
1823
        //     [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ]
1824
        // Which is the order expected by the builtin function.
1825
        let _ = self.context.stack.ensure_index_at(3)?;
1826
        let at = self.context.stack.ensure_index_at(2)?;
1827
        self.context.stack.insert_many(at, &[src_mem.try_into()?]);
1828

1829
        // One element was inserted above, so instead of 3, we use 4.
1830
        let at = self.context.stack.ensure_index_at(4)?;
1831
        self.context.stack.insert_many(at, &[dst_mem.try_into()?]);
1832

1833
        let builtin = self.env.builtins.memory_copy::<M::ABI>()?;
1834

1835
        FnCall::emit::<M>(
1836
            &mut self.env,
1837
            self.masm,
1838
            &mut self.context,
1839
            Callee::Builtin(builtin),
1840
        )?;
1841
        self.context.pop_and_free(self.masm)
1842
    }
1843

1844
    fn visit_memory_fill(&mut self, mem: u32) -> Self::Output {
1845
        let at = self.context.stack.ensure_index_at(3)?;
1846
        let mem = MemoryIndex::from_u32(mem);
1847

1848
        let builtin = self.env.builtins.memory_fill::<M::ABI>()?;
1849
        let builtin = self.prepare_builtin_defined_memory_arg(mem, at, builtin)?;
1850

1851
        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
1852
        self.context.pop_and_free(self.masm)
1853
    }
1854

1855
    fn visit_memory_size(&mut self, mem: u32) -> Self::Output {
1856
        let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem));
1857
        self.emit_compute_memory_size(&heap)
1858
    }
1859

1860
    fn visit_memory_grow(&mut self, mem: u32) -> Self::Output {
1861
        let at = self.context.stack.ensure_index_at(1)?;
1862
        let mem = MemoryIndex::from_u32(mem);
1863
        // The stack at this point contains: [ delta ]
1864
        // The desired state is
1865
        //   [ vmctx, delta, index ]
1866
        let builtin = self.env.builtins.memory_grow::<M::ABI>()?;
1867
        let builtin = self.prepare_builtin_defined_memory_arg(mem, at + 1, builtin)?;
1868

1869
        let heap = self.env.resolve_heap(mem);
1870
        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
1871

1872
        // The memory32_grow builtin returns a pointer type, therefore we must
1873
        // ensure that the return type is representative of the address space of
1874
        // the heap type.
1875
        match (self.env.ptr_type(), heap.index_type()) {
1876
            (WasmValType::I64, WasmValType::I64) => Ok(()),
1877
            // When the heap type is smaller than the pointer type, we adjust
1878
            // the result of the memory32_grow builtin.
1879
            (WasmValType::I64, WasmValType::I32) => {
1880
                let top: Reg = self.context.pop_to_reg(self.masm, None)?.into();
1881
                self.masm.wrap(writable!(top), top)?;
1882
                self.context.stack.push(TypedReg::i32(top).into());
1883
                Ok(())
1884
            }
1885
            _ => Err(format_err!(CodeGenError::unsupported_32_bit_platform())),
1886
        }
1887
    }
1888

1889
    fn visit_data_drop(&mut self, data_index: u32) -> Self::Output {
1890
        self.context.stack.extend([data_index.try_into()?]);
1891

1892
        let builtin = self.env.builtins.data_drop::<M::ABI>()?;
1893
        FnCall::emit::<M>(
1894
            &mut self.env,
1895
            self.masm,
1896
            &mut self.context,
1897
            Callee::Builtin(builtin),
1898
        )?;
1899
        self.context.pop_and_free(self.masm)
1900
    }
1901

1902
    fn visit_nop(&mut self) -> Self::Output {
1903
        Ok(())
1904
    }
1905

1906
    fn visit_if(&mut self, blockty: BlockType) -> Self::Output {
1907
        self.control_frames.push(ControlStackFrame::r#if(
1908
            self.env.resolve_block_sig(blockty)?,
1909
            self.masm,
1910
            &mut self.context,
1911
        )?);
1912

1913
        Ok(())
1914
    }
1915

1916
    fn visit_else(&mut self) -> Self::Output {
1917
        if !self.context.reachable {
1918
            self.handle_unreachable_else()
1919
        } else {
1920
            let control = self
1921
                .control_frames
1922
                .last_mut()
1923
                .ok_or_else(|| CodeGenError::control_frame_expected())?;
1924
            control.emit_else(self.masm, &mut self.context)
1925
        }
1926
    }
1927

1928
    fn visit_block(&mut self, blockty: BlockType) -> Self::Output {
1929
        self.control_frames.push(ControlStackFrame::block(
1930
            self.env.resolve_block_sig(blockty)?,
1931
            self.masm,
1932
            &mut self.context,
1933
        )?);
1934

1935
        Ok(())
1936
    }
1937

1938
    fn visit_loop(&mut self, blockty: BlockType) -> Self::Output {
1939
        self.control_frames.push(ControlStackFrame::r#loop(
1940
            self.env.resolve_block_sig(blockty)?,
1941
            self.masm,
1942
            &mut self.context,
1943
        )?);
1944

1945
        self.maybe_emit_epoch_check()?;
1946
        self.maybe_emit_fuel_check()
1947
    }
1948

1949
    fn visit_br(&mut self, depth: u32) -> Self::Output {
1950
        let index = control_index(depth, self.control_frames.len())?;
1951
        let frame = &mut self.control_frames[index];
1952
        self.context
1953
            .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, cx, frame| {
1954
                frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| {
1955
                    Ok(results.ret_area().copied())
1956
                })
1957
            })
1958
    }
1959

1960
    fn visit_br_if(&mut self, depth: u32) -> Self::Output {
1961
        let index = control_index(depth, self.control_frames.len())?;
1962
        let frame = &mut self.control_frames[index];
1963
        frame.set_as_target();
1964

1965
        let top = {
1966
            let top = self.context.without::<Result<TypedReg>, M, _>(
1967
                frame.results::<M>()?.regs(),
1968
                self.masm,
1969
                |ctx, masm| ctx.pop_to_reg(masm, None),
1970
            )??;
1971
            // Explicitly save any live registers and locals before setting up
1972
            // the branch state.
1973
            // In some cases, calculating the `top` value above, will result in
1974
            // a spill, thus the following one will result in a no-op.
1975
            self.context.spill(self.masm)?;
1976
            frame.top_abi_results::<M, _>(
1977
                &mut self.context,
1978
                self.masm,
1979
                |results, context, masm| {
1980
                    // In the case of `br_if` there's a possibility that we'll
1981
                    // exit early from the block or fallthrough, for
1982
                    // a fallthrough, we cannot rely on the pre-computed return area;
1983
                    // it must be recalculated so that any values that are
1984
                    // generated are correctly placed near the current stack
1985
                    // pointer.
1986
                    if results.on_stack() {
1987
                        let stack_consumed = context.stack.sizeof(results.stack_operands_len());
1988
                        let base = masm.sp_offset()?.as_u32() - stack_consumed;
1989
                        let offs = base + results.size();
1990
                        Ok(Some(RetArea::sp(SPOffset::from_u32(offs))))
1991
                    } else {
1992
                        Ok(None)
1993
                    }
1994
                },
1995
            )?;
1996
            top
1997
        };
1998

1999
        // Emit instructions to balance the machine stack.
2000
        let current_sp_offset = self.masm.sp_offset()?;
2001
        let unbalanced = frame.unbalanced(self.masm)?;
2002
        let (label, cmp) = if unbalanced {
2003
            (self.masm.get_label()?, IntCmpKind::Eq)
2004
        } else {
2005
            (*frame.label(), IntCmpKind::Ne)
2006
        };
2007

2008
        self.masm
2009
            .branch(cmp, top.reg, top.reg.into(), label, OperandSize::S32)?;
2010
        self.context.free_reg(top);
2011

2012
        if unbalanced {
2013
            self.context
2014
                .br::<_, _, ConditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?;
2015

2016
            // Restore sp_offset to what it was for falling through and emit
2017
            // fallthrough label.
2018
            self.masm.reset_stack_pointer(current_sp_offset)?;
2019
            self.masm.bind(label)?;
2020
        }
2021

2022
        Ok(())
2023
    }
2024

2025
    fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output {
2026
        // +1 to account for the default target.
2027
        let len = targets.len() + 1;
2028
        // SmallVec<[_; 5]> to match the binary emission layer (e.g
2029
        // see `JmpTableSeq'), but here we use 5 instead since we
2030
        // bundle the default target as the last element in the array.
2031
        let mut labels: SmallVec<[_; 5]> = smallvec![];
2032
        for _ in 0..len {
2033
            labels.push(self.masm.get_label()?);
2034
        }
2035

2036
        // Find the innermost target and use it as the relative frame
2037
        // for result handling below.
2038
        //
2039
        // This approach ensures that
2040
        // 1. The stack pointer offset is correctly positioned
2041
        //    according to the expectations of the innermost block end
2042
        //    sequence.
2043
        // 2. We meet the jump site invariants introduced by
2044
        //    `CodegenContext::br`, which take advantage of Wasm
2045
        //    semantics given that all jumps are "outward".
2046
        let mut innermost = targets.default();
2047
        for target in targets.targets() {
2048
            let target = target?;
2049
            if target < innermost {
2050
                innermost = target;
2051
            }
2052
        }
2053

2054
        let innermost_index = control_index(innermost, self.control_frames.len())?;
2055
        let innermost_frame = &mut self.control_frames[innermost_index];
2056
        let innermost_result = innermost_frame.results::<M>()?;
2057

2058
        let (index, tmp) = {
2059
            let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>(
2060
                innermost_result.regs(),
2061
                self.masm,
2062
                |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)),
2063
            )??;
2064

2065
            // Materialize any constants or locals into their result
2066
            // representation, so that when reachability is restored,
2067
            // they are correctly located.  NB: the results are popped
2068
            // in function of the innermost branch specified for
2069
            // `br_table`, which implies that the machine stack will
2070
            // be correctly balanced, by virtue of calling
2071
            // `pop_abi_results`.
2072

2073
            // It's possible that we need to balance the stack for the
2074
            // rest of the targets, which will be done before emitting
2075
            // the unconditional jump below.
2076
            innermost_frame.pop_abi_results::<M, _>(
2077
                &mut self.context,
2078
                self.masm,
2079
                |results, _, _| Ok(results.ret_area().copied()),
2080
            )?;
2081
            index_and_tmp
2082
        };
2083

2084
        self.masm.jmp_table(&labels, index.into(), tmp)?;
2085
        // Save the original stack pointer offset; we will reset the stack
2086
        // pointer to this offset after jumping to each of the targets. Each
2087
        // jump might adjust the stack according to the base offset of the
2088
        // target.
2089
        let current_sp = self.masm.sp_offset()?;
2090

2091
        for (t, l) in targets
2092
            .targets()
2093
            .chain(std::iter::once(Ok(targets.default())))
2094
            .zip(labels.iter())
2095
        {
2096
            let control_index = control_index(t?, self.control_frames.len())?;
2097
            let frame = &mut self.control_frames[control_index];
2098
            // Reset the stack pointer to its original offset. This is needed
2099
            // because each jump will potentially adjust the stack pointer
2100
            // according to the base offset of the target.
2101
            self.masm.reset_stack_pointer(current_sp)?;
2102

2103
            // NB: We don't perform any result handling as it was
2104
            // already taken care of above before jumping to the
2105
            // jump table.
2106
            self.masm.bind(*l)?;
2107
            // Ensure that the stack pointer is correctly positioned before
2108
            // jumping to the jump table code.
2109
            self.context
2110
                .br::<_, _, UnconditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?;
2111
        }
2112
        // Finally reset the stack pointer to the original location.
2113
        // The reachability analysis, will ensure it's correctly located
2114
        // once reachability is restored.
2115
        self.masm.reset_stack_pointer(current_sp)?;
2116
        self.context.reachable = false;
2117
        self.context.free_reg(index.reg);
2118
        self.context.free_reg(tmp);
2119

2120
        Ok(())
2121
    }
2122

2123
    fn visit_return(&mut self) -> Self::Output {
2124
        // Grab the outermost frame, which is the function's body
2125
        // frame. We don't rely on [`codegen::control_index`] since
2126
        // this frame is implicit and we know that it should exist at
2127
        // index 0.
2128
        let outermost = &mut self.control_frames[0];
2129
        self.context
2130
            .br::<_, _, UnconditionalBranch>(outermost, self.masm, |masm, cx, frame| {
2131
                frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| {
2132
                    Ok(results.ret_area().copied())
2133
                })
2134
            })
2135
    }
2136

2137
    fn visit_unreachable(&mut self) -> Self::Output {
2138
        self.masm.unreachable()?;
2139
        self.context.reachable = false;
2140
        // Set the implicit outermost frame as target to perform the necessary
2141
        // stack clean up.
2142
        let outermost = &mut self.control_frames[0];
2143
        outermost.set_as_target();
2144

2145
        Ok(())
2146
    }
2147

2148
    fn visit_local_tee(&mut self, index: u32) -> Self::Output {
2149
        let typed_reg = self.emit_set_local(index)?;
2150
        self.context.stack.push(typed_reg.into());
2151

2152
        Ok(())
2153
    }
2154

2155
    fn visit_global_get(&mut self, global_index: u32) -> Self::Output {
2156
        let index = GlobalIndex::from_u32(global_index);
2157
        let (ty, base, offset) = self.emit_get_global_addr(index)?;
2158
        let addr = self.masm.address_at_reg(base, offset)?;
2159
        let dst = self.context.reg_for_type(ty, self.masm)?;
2160
        self.masm.load(addr, writable!(dst), ty.try_into()?)?;
2161
        self.context.stack.push(Val::reg(dst, ty));
2162

2163
        self.context.free_reg(base);
2164

2165
        Ok(())
2166
    }
2167

2168
    fn visit_global_set(&mut self, global_index: u32) -> Self::Output {
2169
        let index = GlobalIndex::from_u32(global_index);
2170
        let (ty, base, offset) = self.emit_get_global_addr(index)?;
2171
        let addr = self.masm.address_at_reg(base, offset)?;
2172

2173
        let typed_reg = self.context.pop_to_reg(self.masm, None)?;
2174
        self.masm
2175
            .store(typed_reg.reg.into(), addr, ty.try_into()?)?;
2176
        self.context.free_reg(typed_reg.reg);
2177
        self.context.free_reg(base);
2178

2179
        Ok(())
2180
    }
2181

2182
    fn visit_drop(&mut self) -> Self::Output {
2183
        self.context.drop_last(1, |regalloc, val| match val {
2184
            Val::Reg(tr) => Ok(regalloc.free(tr.reg)),
2185
            Val::Memory(m) => self.masm.free_stack(m.slot.size),
2186
            _ => Ok(()),
2187
        })
2188
    }
2189

2190
    fn visit_select(&mut self) -> Self::Output {
2191
        let cond = self.context.pop_to_reg(self.masm, None)?;
2192
        let val2 = self.context.pop_to_reg(self.masm, None)?;
2193
        let val1 = self.context.pop_to_reg(self.masm, None)?;
2194
        self.masm.cmp(cond.reg, RegImm::i32(0), OperandSize::S32)?;
2195
        // Conditionally move val1 to val2 if the comparison is
2196
        // not zero.
2197
        self.masm.cmov(
2198
            writable!(val2.into()),
2199
            val1.into(),
2200
            IntCmpKind::Ne,
2201
            val1.ty.try_into()?,
2202
        )?;
2203
        self.context.stack.push(val2.into());
2204
        self.context.free_reg(val1.reg);
2205
        self.context.free_reg(cond);
2206

2207
        Ok(())
2208
    }
2209

2210
    fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output {
2211
        self.emit_wasm_load(
2212
            &memarg,
2213
            WasmValType::I32,
2214
            LoadKind::Operand(OperandSize::S32),
2215
        )
2216
    }
2217

2218
    fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output {
2219
        self.emit_wasm_load(
2220
            &memarg,
2221
            WasmValType::I32,
2222
            LoadKind::ScalarExtend(Extend::<Signed>::I32Extend8.into()),
2223
        )
2224
    }
2225

2226
    fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output {
2227
        self.emit_wasm_load(
2228
            &memarg,
2229
            WasmValType::I32,
2230
            LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()),
2231
        )
2232
    }
2233

2234
    fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output {
2235
        self.emit_wasm_load(
2236
            &memarg,
2237
            WasmValType::I32,
2238
            LoadKind::ScalarExtend(Extend::<Signed>::I32Extend16.into()),
2239
        )
2240
    }
2241

2242
    fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output {
2243
        self.emit_wasm_load(
2244
            &memarg,
2245
            WasmValType::I32,
2246
            LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()),
2247
        )
2248
    }
2249

2250
    fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output {
2251
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32))
2252
    }
2253

2254
    fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output {
2255
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8))
2256
    }
2257

2258
    fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output {
2259
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16))
2260
    }
2261

2262
    fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output {
2263
        self.emit_wasm_load(
2264
            &memarg,
2265
            WasmValType::I64,
2266
            LoadKind::ScalarExtend(Extend::<Signed>::I64Extend8.into()),
2267
        )
2268
    }
2269

2270
    fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output {
2271
        self.emit_wasm_load(
2272
            &memarg,
2273
            WasmValType::I64,
2274
            LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()),
2275
        )
2276
    }
2277

2278
    fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output {
2279
        self.emit_wasm_load(
2280
            &memarg,
2281
            WasmValType::I64,
2282
            LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()),
2283
        )
2284
    }
2285

2286
    fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output {
2287
        self.emit_wasm_load(
2288
            &memarg,
2289
            WasmValType::I64,
2290
            LoadKind::ScalarExtend(Extend::<Signed>::I64Extend16.into()),
2291
        )
2292
    }
2293

2294
    fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output {
2295
        self.emit_wasm_load(
2296
            &memarg,
2297
            WasmValType::I64,
2298
            LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()),
2299
        )
2300
    }
2301

2302
    fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output {
2303
        self.emit_wasm_load(
2304
            &memarg,
2305
            WasmValType::I64,
2306
            LoadKind::ScalarExtend(Extend::<Signed>::I64Extend32.into()),
2307
        )
2308
    }
2309

2310
    fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output {
2311
        self.emit_wasm_load(
2312
            &memarg,
2313
            WasmValType::I64,
2314
            LoadKind::Operand(OperandSize::S64),
2315
        )
2316
    }
2317

2318
    fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output {
2319
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64))
2320
    }
2321

2322
    fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output {
2323
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8))
2324
    }
2325

2326
    fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output {
2327
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16))
2328
    }
2329

2330
    fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output {
2331
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32))
2332
    }
2333

2334
    fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output {
2335
        self.emit_wasm_load(
2336
            &memarg,
2337
            WasmValType::F32,
2338
            LoadKind::Operand(OperandSize::S32),
2339
        )
2340
    }
2341

2342
    fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output {
2343
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32))
2344
    }
2345

2346
    fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output {
2347
        self.emit_wasm_load(
2348
            &memarg,
2349
            WasmValType::F64,
2350
            LoadKind::Operand(OperandSize::S64),
2351
        )
2352
    }
2353

2354
    fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output {
2355
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64))
2356
    }
2357

2358
    fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output {
2359
        use OperandSize::*;
2360

2361
        self.context
2362
            .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| {
2363
                masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked)
2364
            })
2365
    }
2366

2367
    fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output {
2368
        use OperandSize::*;
2369

2370
        self.masm
2371
            .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked)
2372
    }
2373

2374
    fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output {
2375
        use OperandSize::*;
2376

2377
        self.context
2378
            .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| {
2379
                masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked)
2380
            })
2381
    }
2382

2383
    fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output {
2384
        use OperandSize::*;
2385

2386
        self.masm
2387
            .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked)
2388
    }
2389

2390
    fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output {
2391
        use OperandSize::*;
2392

2393
        self.context
2394
            .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| {
2395
                masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked)
2396
            })
2397
    }
2398

2399
    fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output {
2400
        use OperandSize::*;
2401

2402
        self.masm
2403
            .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked)
2404
    }
2405

2406
    fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output {
2407
        use OperandSize::*;
2408

2409
        self.context
2410
            .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| {
2411
                masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked)
2412
            })
2413
    }
2414

2415
    fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output {
2416
        use OperandSize::*;
2417

2418
        self.masm
2419
            .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked)
2420
    }
2421

2422
    fn visit_i64_add128(&mut self) -> Self::Output {
2423
        self.context
2424
            .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| {
2425
                masm.add128(
2426
                    writable!(lhs_lo),
2427
                    writable!(lhs_hi),
2428
                    lhs_lo,
2429
                    lhs_hi,
2430
                    rhs_lo,
2431
                    rhs_hi,
2432
                )?;
2433
                Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)))
2434
            })
2435
    }
2436

2437
    fn visit_i64_sub128(&mut self) -> Self::Output {
2438
        self.context
2439
            .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| {
2440
                masm.sub128(
2441
                    writable!(lhs_lo),
2442
                    writable!(lhs_hi),
2443
                    lhs_lo,
2444
                    lhs_hi,
2445
                    rhs_lo,
2446
                    rhs_hi,
2447
                )?;
2448
                Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)))
2449
            })
2450
    }
2451

2452
    fn visit_i64_mul_wide_s(&mut self) -> Self::Output {
2453
        self.masm.mul_wide(&mut self.context, MulWideKind::Signed)
2454
    }
2455

2456
    fn visit_i64_mul_wide_u(&mut self) -> Self::Output {
2457
        self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned)
2458
    }
2459

2460
    fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output {
2461
        self.emit_wasm_load(
2462
            &memarg,
2463
            WasmValType::I32,
2464
            LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I32Extend8.into())),
2465
        )
2466
    }
2467

2468
    fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output {
2469
        self.emit_wasm_load(
2470
            &memarg,
2471
            WasmValType::I32,
2472
            LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I32Extend16.into())),
2473
        )
2474
    }
2475

2476
    fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output {
2477
        self.emit_wasm_load(
2478
            &memarg,
2479
            WasmValType::I32,
2480
            LoadKind::Atomic(OperandSize::S32, None),
2481
        )
2482
    }
2483

2484
    fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output {
2485
        self.emit_wasm_load(
2486
            &memarg,
2487
            WasmValType::I64,
2488
            LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I64Extend8.into())),
2489
        )
2490
    }
2491

2492
    fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output {
2493
        self.emit_wasm_load(
2494
            &memarg,
2495
            WasmValType::I64,
2496
            LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I64Extend16.into())),
2497
        )
2498
    }
2499

2500
    fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output {
2501
        self.emit_wasm_load(
2502
            &memarg,
2503
            WasmValType::I64,
2504
            LoadKind::Atomic(OperandSize::S32, Some(Extend::<Zero>::I64Extend32.into())),
2505
        )
2506
    }
2507

2508
    fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output {
2509
        self.emit_wasm_load(
2510
            &memarg,
2511
            WasmValType::I64,
2512
            LoadKind::Atomic(OperandSize::S64, None),
2513
        )
2514
    }
2515

2516
    fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output {
2517
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32))
2518
    }
2519

2520
    fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output {
2521
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S64))
2522
    }
2523

2524
    fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output {
2525
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8))
2526
    }
2527

2528
    fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output {
2529
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16))
2530
    }
2531

2532
    fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output {
2533
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8))
2534
    }
2535

2536
    fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output {
2537
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16))
2538
    }
2539

2540
    fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output {
2541
        self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32))
2542
    }
2543

2544
    fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output {
2545
        self.emit_atomic_rmw(
2546
            &arg,
2547
            RmwOp::Add,
2548
            OperandSize::S8,
2549
            Some(Extend::<Zero>::I32Extend8),
2550
        )
2551
    }
2552

2553
    fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output {
2554
        self.emit_atomic_rmw(
2555
            &arg,
2556
            RmwOp::Add,
2557
            OperandSize::S16,
2558
            Some(Extend::<Zero>::I32Extend16),
2559
        )
2560
    }
2561

2562
    fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output {
2563
        self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None)
2564
    }
2565

2566
    fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output {
2567
        self.emit_atomic_rmw(
2568
            &arg,
2569
            RmwOp::Add,
2570
            OperandSize::S8,
2571
            Some(Extend::<Zero>::I64Extend8),
2572
        )
2573
    }
2574

2575
    fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output {
2576
        self.emit_atomic_rmw(
2577
            &arg,
2578
            RmwOp::Add,
2579
            OperandSize::S16,
2580
            Some(Extend::<Zero>::I64Extend16),
2581
        )
2582
    }
2583

2584
    fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output {
2585
        self.emit_atomic_rmw(
2586
            &arg,
2587
            RmwOp::Add,
2588
            OperandSize::S32,
2589
            Some(Extend::<Zero>::I64Extend32),
2590
        )
2591
    }
2592

2593
    fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output {
2594
        self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None)
2595
    }
2596

2597
    fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output {
2598
        self.emit_atomic_rmw(
2599
            &arg,
2600
            RmwOp::Sub,
2601
            OperandSize::S8,
2602
            Some(Extend::<Zero>::I32Extend8),
2603
        )
2604
    }
2605
    fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output {
2606
        self.emit_atomic_rmw(
2607
            &arg,
2608
            RmwOp::Sub,
2609
            OperandSize::S16,
2610
            Some(Extend::<Zero>::I32Extend16),
2611
        )
2612
    }
2613

2614
    fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output {
2615
        self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None)
2616
    }
2617

2618
    fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output {
2619
        self.emit_atomic_rmw(
2620
            &arg,
2621
            RmwOp::Sub,
2622
            OperandSize::S8,
2623
            Some(Extend::<Zero>::I64Extend8),
2624
        )
2625
    }
2626

2627
    fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output {
2628
        self.emit_atomic_rmw(
2629
            &arg,
2630
            RmwOp::Sub,
2631
            OperandSize::S16,
2632
            Some(Extend::<Zero>::I64Extend16),
2633
        )
2634
    }
2635

2636
    fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output {
2637
        self.emit_atomic_rmw(
2638
            &arg,
2639
            RmwOp::Sub,
2640
            OperandSize::S32,
2641
            Some(Extend::<Zero>::I64Extend32),
2642
        )
2643
    }
2644

2645
    fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output {
2646
        self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None)
2647
    }
2648

2649
    fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output {
2650
        self.emit_atomic_rmw(
2651
            &arg,
2652
            RmwOp::Xchg,
2653
            OperandSize::S8,
2654
            Some(Extend::<Zero>::I32Extend8),
2655
        )
2656
    }
2657

2658
    fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output {
2659
        self.emit_atomic_rmw(
2660
            &arg,
2661
            RmwOp::Xchg,
2662
            OperandSize::S16,
2663
            Some(Extend::<Zero>::I32Extend16),
2664
        )
2665
    }
2666

2667
    fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output {
2668
        self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None)
2669
    }
2670

2671
    fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output {
2672
        self.emit_atomic_rmw(
2673
            &arg,
2674
            RmwOp::Xchg,
2675
            OperandSize::S8,
2676
            Some(Extend::<Zero>::I64Extend8),
2677
        )
2678
    }
2679

2680
    fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output {
2681
        self.emit_atomic_rmw(
2682
            &arg,
2683
            RmwOp::Xchg,
2684
            OperandSize::S16,
2685
            Some(Extend::<Zero>::I64Extend16),
2686
        )
2687
    }
2688

2689
    fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output {
2690
        self.emit_atomic_rmw(
2691
            &arg,
2692
            RmwOp::Xchg,
2693
            OperandSize::S32,
2694
            Some(Extend::<Zero>::I64Extend32),
2695
        )
2696
    }
2697

2698
    fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output {
2699
        self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None)
2700
    }
2701

2702
    fn visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output {
2703
        self.emit_atomic_rmw(
2704
            &arg,
2705
            RmwOp::And,
2706
            OperandSize::S8,
2707
            Some(Extend::<Zero>::I32Extend8),
2708
        )
2709
    }
2710

2711
    fn visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output {
2712
        self.emit_atomic_rmw(
2713
            &arg,
2714
            RmwOp::And,
2715
            OperandSize::S16,
2716
            Some(Extend::<Zero>::I32Extend16),
2717
        )
2718
    }
2719

2720
    fn visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output {
2721
        self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S32, None)
2722
    }
2723

2724
    fn visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output {
2725
        self.emit_atomic_rmw(
2726
            &arg,
2727
            RmwOp::And,
2728
            OperandSize::S8,
2729
            Some(Extend::<Zero>::I64Extend8),
2730
        )
2731
    }
2732

2733
    fn visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output {
2734
        self.emit_atomic_rmw(
2735
            &arg,
2736
            RmwOp::And,
2737
            OperandSize::S16,
2738
            Some(Extend::<Zero>::I64Extend16),
2739
        )
2740
    }
2741

2742
    fn visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output {
2743
        self.emit_atomic_rmw(
2744
            &arg,
2745
            RmwOp::And,
2746
            OperandSize::S32,
2747
            Some(Extend::<Zero>::I64Extend32),
2748
        )
2749
    }
2750

2751
    fn visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output {
2752
        self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S64, None)
2753
    }
2754

2755
    fn visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output {
2756
        self.emit_atomic_rmw(
2757
            &arg,
2758
            RmwOp::Or,
2759
            OperandSize::S8,
2760
            Some(Extend::<Zero>::I32Extend8),
2761
        )
2762
    }
2763

2764
    fn visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output {
2765
        self.emit_atomic_rmw(
2766
            &arg,
2767
            RmwOp::Or,
2768
            OperandSize::S16,
2769
            Some(Extend::<Zero>::I32Extend16),
2770
        )
2771
    }
2772

2773
    fn visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output {
2774
        self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S32, None)
2775
    }
2776

2777
    fn visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output {
2778
        self.emit_atomic_rmw(
2779
            &arg,
2780
            RmwOp::Or,
2781
            OperandSize::S8,
2782
            Some(Extend::<Zero>::I64Extend8),
2783
        )
2784
    }
2785

2786
    fn visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output {
2787
        self.emit_atomic_rmw(
2788
            &arg,
2789
            RmwOp::Or,
2790
            OperandSize::S16,
2791
            Some(Extend::<Zero>::I64Extend16),
2792
        )
2793
    }
2794

2795
    fn visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output {
2796
        self.emit_atomic_rmw(
2797
            &arg,
2798
            RmwOp::Or,
2799
            OperandSize::S32,
2800
            Some(Extend::<Zero>::I64Extend32),
2801
        )
2802
    }
2803

2804
    fn visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output {
2805
        self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S64, None)
2806
    }
2807

2808
    fn visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output {
2809
        self.emit_atomic_rmw(
2810
            &arg,
2811
            RmwOp::Xor,
2812
            OperandSize::S8,
2813
            Some(Extend::<Zero>::I32Extend8),
2814
        )
2815
    }
2816

2817
    fn visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output {
2818
        self.emit_atomic_rmw(
2819
            &arg,
2820
            RmwOp::Xor,
2821
            OperandSize::S16,
2822
            Some(Extend::<Zero>::I32Extend16),
2823
        )
2824
    }
2825

2826
    fn visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output {
2827
        self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S32, None)
2828
    }
2829

2830
    fn visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output {
2831
        self.emit_atomic_rmw(
2832
            &arg,
2833
            RmwOp::Xor,
2834
            OperandSize::S8,
2835
            Some(Extend::<Zero>::I64Extend8),
2836
        )
2837
    }
2838

2839
    fn visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output {
2840
        self.emit_atomic_rmw(
2841
            &arg,
2842
            RmwOp::Xor,
2843
            OperandSize::S16,
2844
            Some(Extend::<Zero>::I64Extend16),
2845
        )
2846
    }
2847

2848
    fn visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output {
2849
        self.emit_atomic_rmw(
2850
            &arg,
2851
            RmwOp::Xor,
2852
            OperandSize::S32,
2853
            Some(Extend::<Zero>::I64Extend32),
2854
        )
2855
    }
2856

2857
    fn visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output {
2858
        self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S64, None)
2859
    }
2860

2861
    fn visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output {
2862
        self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I32Extend8))
2863
    }
2864

2865
    fn visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output {
2866
        self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I32Extend16))
2867
    }
2868

2869
    fn visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output {
2870
        self.emit_atomic_cmpxchg(&arg, OperandSize::S32, None)
2871
    }
2872

2873
    fn visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output {
2874
        self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I64Extend8))
2875
    }
2876

2877
    fn visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output {
2878
        self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I64Extend16))
2879
    }
2880

2881
    fn visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output {
2882
        self.emit_atomic_cmpxchg(&arg, OperandSize::S32, Some(Extend::I64Extend32))
2883
    }
2884

2885
    fn visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output {
2886
        self.emit_atomic_cmpxchg(&arg, OperandSize::S64, None)
2887
    }
2888

2889
    fn visit_memory_atomic_wait32(&mut self, arg: MemArg) -> Self::Output {
2890
        self.emit_atomic_wait(&arg, AtomicWaitKind::Wait32)
2891
    }
2892

2893
    fn visit_memory_atomic_wait64(&mut self, arg: MemArg) -> Self::Output {
2894
        self.emit_atomic_wait(&arg, AtomicWaitKind::Wait64)
2895
    }
2896

2897
    fn visit_memory_atomic_notify(&mut self, arg: MemArg) -> Self::Output {
2898
        self.emit_atomic_notify(&arg)
2899
    }
2900

2901
    fn visit_atomic_fence(&mut self) -> Self::Output {
2902
        self.masm.fence()
2903
    }
2904

2905
    wasmparser::for_each_visit_operator!(def_unsupported);
2906
}
2907

2908
impl<'a, 'translation, 'data, M> VisitSimdOperator<'a>
2909
    for CodeGen<'a, 'translation, 'data, M, Emission>
2910
where
2911
    M: MacroAssembler,
2912
{
2913
    fn visit_v128_const(&mut self, val: V128) -> Self::Output {
2914
        self.context.stack.push(Val::v128(val.i128()));
2915
        Ok(())
2916
    }
2917

2918
    fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output {
2919
        self.emit_wasm_load(
2920
            &memarg,
2921
            WasmValType::V128,
2922
            LoadKind::Operand(OperandSize::S128),
2923
        )
2924
    }
2925

2926
    fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output {
2927
        self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S128))
2928
    }
2929

2930
    fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output {
2931
        self.emit_wasm_load(
2932
            &memarg,
2933
            WasmValType::V128,
2934
            LoadKind::VectorExtend(V128LoadExtendKind::E8x8S),
2935
        )
2936
    }
2937

2938
    fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output {
2939
        self.emit_wasm_load(
2940
            &memarg,
2941
            WasmValType::V128,
2942
            LoadKind::VectorExtend(V128LoadExtendKind::E8x8U),
2943
        )
2944
    }
2945

2946
    fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output {
2947
        self.emit_wasm_load(
2948
            &memarg,
2949
            WasmValType::V128,
2950
            LoadKind::VectorExtend(V128LoadExtendKind::E16x4S),
2951
        )
2952
    }
2953

2954
    fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output {
2955
        self.emit_wasm_load(
2956
            &memarg,
2957
            WasmValType::V128,
2958
            LoadKind::VectorExtend(V128LoadExtendKind::E16x4U),
2959
        )
2960
    }
2961

2962
    fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output {
2963
        self.emit_wasm_load(
2964
            &memarg,
2965
            WasmValType::V128,
2966
            LoadKind::VectorExtend(V128LoadExtendKind::E32x2S),
2967
        )
2968
    }
2969

2970
    fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output {
2971
        self.emit_wasm_load(
2972
            &memarg,
2973
            WasmValType::V128,
2974
            LoadKind::VectorExtend(V128LoadExtendKind::E32x2U),
2975
        )
2976
    }
2977

2978
    fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output {
2979
        self.emit_wasm_load(
2980
            &memarg,
2981
            WasmValType::V128,
2982
            LoadKind::Splat(SplatLoadKind::S8),
2983
        )
2984
    }
2985

2986
    fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output {
2987
        self.emit_wasm_load(
2988
            &memarg,
2989
            WasmValType::V128,
2990
            LoadKind::Splat(SplatLoadKind::S16),
2991
        )
2992
    }
2993

2994
    fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output {
2995
        self.emit_wasm_load(
2996
            &memarg,
2997
            WasmValType::V128,
2998
            LoadKind::Splat(SplatLoadKind::S32),
2999
        )
3000
    }
3001

3002
    fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output {
3003
        self.emit_wasm_load(
3004
            &memarg,
3005
            WasmValType::V128,
3006
            LoadKind::Splat(SplatLoadKind::S64),
3007
        )
3008
    }
3009

3010
    fn visit_i8x16_splat(&mut self) -> Self::Output {
3011
        self.masm.splat(&mut self.context, SplatKind::I8x16)
3012
    }
3013

3014
    fn visit_i16x8_splat(&mut self) -> Self::Output {
3015
        self.masm.splat(&mut self.context, SplatKind::I16x8)
3016
    }
3017

3018
    fn visit_i32x4_splat(&mut self) -> Self::Output {
3019
        self.masm.splat(&mut self.context, SplatKind::I32x4)
3020
    }
3021

3022
    fn visit_i64x2_splat(&mut self) -> Self::Output {
3023
        self.masm.splat(&mut self.context, SplatKind::I64x2)
3024
    }
3025

3026
    fn visit_f32x4_splat(&mut self) -> Self::Output {
3027
        self.masm.splat(&mut self.context, SplatKind::F32x4)
3028
    }
3029

3030
    fn visit_f64x2_splat(&mut self) -> Self::Output {
3031
        self.masm.splat(&mut self.context, SplatKind::F64x2)
3032
    }
3033

3034
    fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output {
3035
        let rhs = self.context.pop_to_reg(self.masm, None)?;
3036
        let lhs = self.context.pop_to_reg(self.masm, None)?;
3037
        self.masm
3038
            .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?;
3039
        self.context.stack.push(TypedReg::v128(lhs.into()).into());
3040
        self.context.free_reg(rhs);
3041
        Ok(())
3042
    }
3043

3044
    fn visit_i8x16_swizzle(&mut self) -> Self::Output {
3045
        let rhs = self.context.pop_to_reg(self.masm, None)?;
3046
        let lhs = self.context.pop_to_reg(self.masm, None)?;
3047
        self.masm
3048
            .swizzle(writable!(lhs.into()), lhs.into(), rhs.into())?;
3049
        self.context.stack.push(TypedReg::v128(lhs.into()).into());
3050
        self.context.free_reg(rhs);
3051
        Ok(())
3052
    }
3053

3054
    fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output {
3055
        self.context.extract_lane_op(
3056
            self.masm,
3057
            ExtractLaneKind::I8x16S,
3058
            |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind),
3059
        )
3060
    }
3061

3062
    fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output {
3063
        self.context.extract_lane_op(
3064
            self.masm,
3065
            ExtractLaneKind::I8x16U,
3066
            |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind),
3067
        )
3068
    }
3069

3070
    fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output {
3071
        self.context.extract_lane_op(
3072
            self.masm,
3073
            ExtractLaneKind::I16x8S,
3074
            |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind),
3075
        )
3076
    }
3077

3078
    fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output {
3079
        self.context.extract_lane_op(
3080
            self.masm,
3081
            ExtractLaneKind::I16x8U,
3082
            |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind),
3083
        )
3084
    }
3085

3086
    fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output {
3087
        self.context
3088
            .extract_lane_op(self.masm, ExtractLaneKind::I32x4, |masm, src, dst, kind| {
3089
                masm.extract_lane(src, dst, lane, kind)
3090
            })
3091
    }
3092

3093
    fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output {
3094
        self.context
3095
            .extract_lane_op(self.masm, ExtractLaneKind::I64x2, |masm, src, dst, kind| {
3096
                masm.extract_lane(src, dst, lane, kind)
3097
            })
3098
    }
3099

3100
    fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output {
3101
        self.context
3102
            .extract_lane_op(self.masm, ExtractLaneKind::F32x4, |masm, src, dst, kind| {
3103
                masm.extract_lane(src, dst, lane, kind)
3104
            })
3105
    }
3106

3107
    fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output {
3108
        self.context
3109
            .extract_lane_op(self.masm, ExtractLaneKind::F64x2, |masm, src, dst, kind| {
3110
                masm.extract_lane(src, dst, lane, kind)
3111
            })
3112
    }
3113

3114
    fn visit_i8x16_eq(&mut self) -> Self::Output {
3115
        self.context
3116
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3117
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I8x16)?;
3118
                Ok(TypedReg::v128(dst))
3119
            })
3120
    }
3121

3122
    fn visit_i16x8_eq(&mut self) -> Self::Output {
3123
        self.context
3124
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3125
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I16x8)?;
3126
                Ok(TypedReg::v128(dst))
3127
            })
3128
    }
3129

3130
    fn visit_i32x4_eq(&mut self) -> Self::Output {
3131
        self.context
3132
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3133
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I32x4)?;
3134
                Ok(TypedReg::v128(dst))
3135
            })
3136
    }
3137

3138
    fn visit_i64x2_eq(&mut self) -> Self::Output {
3139
        self.context
3140
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3141
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I64x2)?;
3142
                Ok(TypedReg::v128(dst))
3143
            })
3144
    }
3145

3146
    fn visit_f32x4_eq(&mut self) -> Self::Output {
3147
        self.context
3148
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3149
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F32x4)?;
3150
                Ok(TypedReg::v128(dst))
3151
            })
3152
    }
3153

3154
    fn visit_f64x2_eq(&mut self) -> Self::Output {
3155
        self.context
3156
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3157
                masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F64x2)?;
3158
                Ok(TypedReg::v128(dst))
3159
            })
3160
    }
3161

3162
    fn visit_i8x16_ne(&mut self) -> Self::Output {
3163
        self.context
3164
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3165
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I8x16)?;
3166
                Ok(TypedReg::v128(dst))
3167
            })
3168
    }
3169

3170
    fn visit_i16x8_ne(&mut self) -> Self::Output {
3171
        self.context
3172
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3173
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I16x8)?;
3174
                Ok(TypedReg::v128(dst))
3175
            })
3176
    }
3177

3178
    fn visit_i32x4_ne(&mut self) -> Self::Output {
3179
        self.context
3180
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3181
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I32x4)?;
3182
                Ok(TypedReg::v128(dst))
3183
            })
3184
    }
3185

3186
    fn visit_i64x2_ne(&mut self) -> Self::Output {
3187
        self.context
3188
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3189
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I64x2)?;
3190
                Ok(TypedReg::v128(dst))
3191
            })
3192
    }
3193

3194
    fn visit_f32x4_ne(&mut self) -> Self::Output {
3195
        self.context
3196
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3197
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F32x4)?;
3198
                Ok(TypedReg::v128(dst))
3199
            })
3200
    }
3201

3202
    fn visit_f64x2_ne(&mut self) -> Self::Output {
3203
        self.context
3204
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3205
                masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F64x2)?;
3206
                Ok(TypedReg::v128(dst))
3207
            })
3208
    }
3209

3210
    fn visit_i8x16_lt_s(&mut self) -> Self::Output {
3211
        self.context
3212
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3213
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?;
3214
                Ok(TypedReg::v128(dst))
3215
            })
3216
    }
3217

3218
    fn visit_i8x16_lt_u(&mut self) -> Self::Output {
3219
        self.context
3220
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3221
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?;
3222
                Ok(TypedReg::v128(dst))
3223
            })
3224
    }
3225

3226
    fn visit_i16x8_lt_s(&mut self) -> Self::Output {
3227
        self.context
3228
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3229
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?;
3230
                Ok(TypedReg::v128(dst))
3231
            })
3232
    }
3233

3234
    fn visit_i16x8_lt_u(&mut self) -> Self::Output {
3235
        self.context
3236
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3237
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?;
3238
                Ok(TypedReg::v128(dst))
3239
            })
3240
    }
3241

3242
    fn visit_i32x4_lt_s(&mut self) -> Self::Output {
3243
        self.context
3244
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3245
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?;
3246
                Ok(TypedReg::v128(dst))
3247
            })
3248
    }
3249

3250
    fn visit_i32x4_lt_u(&mut self) -> Self::Output {
3251
        self.context
3252
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3253
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?;
3254
                Ok(TypedReg::v128(dst))
3255
            })
3256
    }
3257

3258
    fn visit_i64x2_lt_s(&mut self) -> Self::Output {
3259
        self.context
3260
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3261
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?;
3262
                Ok(TypedReg::v128(dst))
3263
            })
3264
    }
3265

3266
    fn visit_f32x4_lt(&mut self) -> Self::Output {
3267
        self.context
3268
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3269
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F32x4)?;
3270
                Ok(TypedReg::v128(dst))
3271
            })
3272
    }
3273

3274
    fn visit_f64x2_lt(&mut self) -> Self::Output {
3275
        self.context
3276
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3277
                masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F64x2)?;
3278
                Ok(TypedReg::v128(dst))
3279
            })
3280
    }
3281

3282
    fn visit_i8x16_le_s(&mut self) -> Self::Output {
3283
        self.context
3284
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3285
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16S)?;
3286
                Ok(TypedReg::v128(dst))
3287
            })
3288
    }
3289

3290
    fn visit_i8x16_le_u(&mut self) -> Self::Output {
3291
        self.context
3292
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3293
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16U)?;
3294
                Ok(TypedReg::v128(dst))
3295
            })
3296
    }
3297

3298
    fn visit_i16x8_le_s(&mut self) -> Self::Output {
3299
        self.context
3300
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3301
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8S)?;
3302
                Ok(TypedReg::v128(dst))
3303
            })
3304
    }
3305

3306
    fn visit_i16x8_le_u(&mut self) -> Self::Output {
3307
        self.context
3308
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3309
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8U)?;
3310
                Ok(TypedReg::v128(dst))
3311
            })
3312
    }
3313

3314
    fn visit_i32x4_le_s(&mut self) -> Self::Output {
3315
        self.context
3316
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3317
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4S)?;
3318
                Ok(TypedReg::v128(dst))
3319
            })
3320
    }
3321

3322
    fn visit_i32x4_le_u(&mut self) -> Self::Output {
3323
        self.context
3324
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3325
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4U)?;
3326
                Ok(TypedReg::v128(dst))
3327
            })
3328
    }
3329

3330
    fn visit_i64x2_le_s(&mut self) -> Self::Output {
3331
        self.context
3332
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3333
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I64x2S)?;
3334
                Ok(TypedReg::v128(dst))
3335
            })
3336
    }
3337

3338
    fn visit_f32x4_le(&mut self) -> Self::Output {
3339
        self.context
3340
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3341
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F32x4)?;
3342
                Ok(TypedReg::v128(dst))
3343
            })
3344
    }
3345

3346
    fn visit_f64x2_le(&mut self) -> Self::Output {
3347
        self.context
3348
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3349
                masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F64x2)?;
3350
                Ok(TypedReg::v128(dst))
3351
            })
3352
    }
3353

3354
    fn visit_i8x16_gt_s(&mut self) -> Self::Output {
3355
        self.context
3356
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3357
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?;
3358
                Ok(TypedReg::v128(dst))
3359
            })
3360
    }
3361

3362
    fn visit_i8x16_gt_u(&mut self) -> Self::Output {
3363
        self.context
3364
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3365
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?;
3366
                Ok(TypedReg::v128(dst))
3367
            })
3368
    }
3369

3370
    fn visit_i16x8_gt_s(&mut self) -> Self::Output {
3371
        self.context
3372
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3373
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?;
3374
                Ok(TypedReg::v128(dst))
3375
            })
3376
    }
3377

3378
    fn visit_i16x8_gt_u(&mut self) -> Self::Output {
3379
        self.context
3380
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3381
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?;
3382
                Ok(TypedReg::v128(dst))
3383
            })
3384
    }
3385

3386
    fn visit_i32x4_gt_s(&mut self) -> Self::Output {
3387
        self.context
3388
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3389
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?;
3390
                Ok(TypedReg::v128(dst))
3391
            })
3392
    }
3393

3394
    fn visit_i32x4_gt_u(&mut self) -> Self::Output {
3395
        self.context
3396
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3397
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?;
3398
                Ok(TypedReg::v128(dst))
3399
            })
3400
    }
3401

3402
    fn visit_i64x2_gt_s(&mut self) -> Self::Output {
3403
        self.context
3404
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3405
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?;
3406
                Ok(TypedReg::v128(dst))
3407
            })
3408
    }
3409

3410
    fn visit_f32x4_gt(&mut self) -> Self::Output {
3411
        self.context
3412
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3413
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F32x4)?;
3414
                Ok(TypedReg::v128(dst))
3415
            })
3416
    }
3417

3418
    fn visit_f64x2_gt(&mut self) -> Self::Output {
3419
        self.context
3420
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3421
                masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F64x2)?;
3422
                Ok(TypedReg::v128(dst))
3423
            })
3424
    }
3425

3426
    fn visit_i8x16_ge_s(&mut self) -> Self::Output {
3427
        self.context
3428
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3429
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16S)?;
3430
                Ok(TypedReg::v128(dst))
3431
            })
3432
    }
3433

3434
    fn visit_i8x16_ge_u(&mut self) -> Self::Output {
3435
        self.context
3436
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3437
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16U)?;
3438
                Ok(TypedReg::v128(dst))
3439
            })
3440
    }
3441

3442
    fn visit_i16x8_ge_s(&mut self) -> Self::Output {
3443
        self.context
3444
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3445
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8S)?;
3446
                Ok(TypedReg::v128(dst))
3447
            })
3448
    }
3449

3450
    fn visit_i16x8_ge_u(&mut self) -> Self::Output {
3451
        self.context
3452
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3453
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8U)?;
3454
                Ok(TypedReg::v128(dst))
3455
            })
3456
    }
3457

3458
    fn visit_i32x4_ge_s(&mut self) -> Self::Output {
3459
        self.context
3460
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3461
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4S)?;
3462
                Ok(TypedReg::v128(dst))
3463
            })
3464
    }
3465

3466
    fn visit_i32x4_ge_u(&mut self) -> Self::Output {
3467
        self.context
3468
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3469
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4U)?;
3470
                Ok(TypedReg::v128(dst))
3471
            })
3472
    }
3473

3474
    fn visit_i64x2_ge_s(&mut self) -> Self::Output {
3475
        self.context
3476
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3477
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I64x2S)?;
3478
                Ok(TypedReg::v128(dst))
3479
            })
3480
    }
3481

3482
    fn visit_f32x4_ge(&mut self) -> Self::Output {
3483
        self.context
3484
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3485
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F32x4)?;
3486
                Ok(TypedReg::v128(dst))
3487
            })
3488
    }
3489

3490
    fn visit_f64x2_ge(&mut self) -> Self::Output {
3491
        self.context
3492
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3493
                masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F64x2)?;
3494
                Ok(TypedReg::v128(dst))
3495
            })
3496
    }
3497

3498
    fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output {
3499
        self.context
3500
            .replace_lane_op(self.masm, ReplaceLaneKind::I8x16, |masm, src, dst, kind| {
3501
                masm.replace_lane(src, dst, lane, kind)
3502
            })
3503
    }
3504

3505
    fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output {
3506
        self.context
3507
            .replace_lane_op(self.masm, ReplaceLaneKind::I16x8, |masm, src, dst, kind| {
3508
                masm.replace_lane(src, dst, lane, kind)
3509
            })
3510
    }
3511

3512
    fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output {
3513
        self.context
3514
            .replace_lane_op(self.masm, ReplaceLaneKind::I32x4, |masm, src, dst, kind| {
3515
                masm.replace_lane(src, dst, lane, kind)
3516
            })
3517
    }
3518

3519
    fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output {
3520
        self.context
3521
            .replace_lane_op(self.masm, ReplaceLaneKind::I64x2, |masm, src, dst, kind| {
3522
                masm.replace_lane(src, dst, lane, kind)
3523
            })
3524
    }
3525

3526
    fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output {
3527
        self.context
3528
            .replace_lane_op(self.masm, ReplaceLaneKind::F32x4, |masm, src, dst, kind| {
3529
                masm.replace_lane(src, dst, lane, kind)
3530
            })
3531
    }
3532

3533
    fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output {
3534
        self.context
3535
            .replace_lane_op(self.masm, ReplaceLaneKind::F64x2, |masm, src, dst, kind| {
3536
                masm.replace_lane(src, dst, lane, kind)
3537
            })
3538
    }
3539

3540
    fn visit_v128_not(&mut self) -> Self::Output {
3541
        self.context.unop(self.masm, |masm, reg| {
3542
            masm.v128_not(writable!(reg))?;
3543
            Ok(TypedReg::new(WasmValType::V128, reg))
3544
        })
3545
    }
3546

3547
    fn visit_v128_and(&mut self) -> Self::Output {
3548
        self.context
3549
            .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| {
3550
                masm.v128_and(dst, src, writable!(dst))?;
3551
                Ok(TypedReg::new(WasmValType::V128, dst))
3552
            })
3553
    }
3554

3555
    fn visit_v128_andnot(&mut self) -> Self::Output {
3556
        self.context
3557
            .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| {
3558
                // careful here: and_not is *not* commutative: dst = !src1 & src2
3559
                masm.v128_and_not(src, dst, writable!(dst))?;
3560
                Ok(TypedReg::new(WasmValType::V128, dst))
3561
            })
3562
    }
3563

3564
    fn visit_v128_or(&mut self) -> Self::Output {
3565
        self.context
3566
            .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| {
3567
                // careful here: and_not is *not* commutative: dst = !src1 & src2
3568
                masm.v128_or(src, dst, writable!(dst))?;
3569
                Ok(TypedReg::new(WasmValType::V128, dst))
3570
            })
3571
    }
3572

3573
    fn visit_v128_xor(&mut self) -> Self::Output {
3574
        self.context
3575
            .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| {
3576
                // careful here: and_not is *not* commutative: dst = !src1 & src2
3577
                masm.v128_xor(src, dst, writable!(dst))?;
3578
                Ok(TypedReg::new(WasmValType::V128, dst))
3579
            })
3580
    }
3581

3582
    fn visit_v128_bitselect(&mut self) -> Self::Output {
3583
        let mask = self.context.pop_to_reg(self.masm, None)?;
3584
        let op2 = self.context.pop_to_reg(self.masm, None)?;
3585
        let op1 = self.context.pop_to_reg(self.masm, None)?;
3586
        let dst = self.context.any_fpr(self.masm)?;
3587

3588
        // careful here: bitselect is *not* commutative.
3589
        self.masm
3590
            .v128_bitselect(op1.reg, op2.reg, mask.reg, writable!(dst))?;
3591

3592
        self.context
3593
            .stack
3594
            .push(TypedReg::new(WasmValType::V128, dst).into());
3595
        self.context.free_reg(op1);
3596
        self.context.free_reg(op2);
3597
        self.context.free_reg(mask);
3598

3599
        Ok(())
3600
    }
3601

3602
    fn visit_v128_any_true(&mut self) -> Self::Output {
3603
        let src = self.context.pop_to_reg(self.masm, None)?;
3604
        let dst = self.context.any_gpr(self.masm)?;
3605

3606
        self.masm.v128_any_true(src.reg, writable!(dst))?;
3607

3608
        self.context
3609
            .stack
3610
            .push(TypedReg::new(WasmValType::I32, dst).into());
3611
        self.context.free_reg(src);
3612

3613
        Ok(())
3614
    }
3615

3616
    fn visit_v128_load8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3617
        self.emit_wasm_load(
3618
            &arg,
3619
            WasmValType::V128,
3620
            LoadKind::vector_lane(lane, OperandSize::S8),
3621
        )
3622
    }
3623

3624
    fn visit_v128_load16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3625
        self.emit_wasm_load(
3626
            &arg,
3627
            WasmValType::V128,
3628
            LoadKind::vector_lane(lane, OperandSize::S16),
3629
        )
3630
    }
3631

3632
    fn visit_v128_load32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3633
        self.emit_wasm_load(
3634
            &arg,
3635
            WasmValType::V128,
3636
            LoadKind::vector_lane(lane, OperandSize::S32),
3637
        )
3638
    }
3639

3640
    fn visit_v128_load64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3641
        self.emit_wasm_load(
3642
            &arg,
3643
            WasmValType::V128,
3644
            LoadKind::vector_lane(lane, OperandSize::S64),
3645
        )
3646
    }
3647

3648
    fn visit_v128_store8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3649
        self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S8))
3650
    }
3651

3652
    fn visit_v128_store16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3653
        self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S16))
3654
    }
3655

3656
    fn visit_v128_store32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3657
        self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S32))
3658
    }
3659

3660
    fn visit_v128_store64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output {
3661
        self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S64))
3662
    }
3663

3664
    fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output {
3665
        self.context.unop(self.masm, |masm, reg| {
3666
            masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4S)?;
3667
            Ok(TypedReg::v128(reg))
3668
        })
3669
    }
3670

3671
    fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output {
3672
        self.context.unop(self.masm, |masm, reg| {
3673
            masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4U)?;
3674
            Ok(TypedReg::v128(reg))
3675
        })
3676
    }
3677

3678
    fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output {
3679
        self.context.unop(self.masm, |masm, reg| {
3680
            masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowS)?;
3681
            Ok(TypedReg::v128(reg))
3682
        })
3683
    }
3684

3685
    fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output {
3686
        self.context.unop(self.masm, |masm, reg| {
3687
            masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowU)?;
3688
            Ok(TypedReg::v128(reg))
3689
        })
3690
    }
3691

3692
    fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output {
3693
        self.context
3694
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3695
                masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8S)?;
3696
                Ok(TypedReg::v128(dst))
3697
            })
3698
    }
3699

3700
    fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output {
3701
        self.context
3702
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3703
                masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8U)?;
3704
                Ok(TypedReg::v128(dst))
3705
            })
3706
    }
3707

3708
    fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output {
3709
        self.context
3710
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3711
                masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4S)?;
3712
                Ok(TypedReg::v128(dst))
3713
            })
3714
    }
3715

3716
    fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output {
3717
        self.context
3718
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3719
                masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4U)?;
3720
                Ok(TypedReg::v128(dst))
3721
            })
3722
    }
3723

3724
    fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output {
3725
        self.context.unop(self.masm, |masm, reg| {
3726
            masm.v128_demote(reg, writable!(reg))?;
3727
            Ok(TypedReg::v128(reg))
3728
        })
3729
    }
3730

3731
    fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output {
3732
        self.context.unop(self.masm, |masm, reg| {
3733
            masm.v128_promote(reg, writable!(reg))?;
3734
            Ok(TypedReg::v128(reg))
3735
        })
3736
    }
3737

3738
    fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output {
3739
        self.context.unop(self.masm, |masm, reg| {
3740
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16S)?;
3741
            Ok(TypedReg::v128(reg))
3742
        })
3743
    }
3744

3745
    fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output {
3746
        self.context.unop(self.masm, |masm, reg| {
3747
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16S)?;
3748
            Ok(TypedReg::v128(reg))
3749
        })
3750
    }
3751

3752
    fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output {
3753
        self.context.unop(self.masm, |masm, reg| {
3754
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16U)?;
3755
            Ok(TypedReg::v128(reg))
3756
        })
3757
    }
3758

3759
    fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output {
3760
        self.context.unop(self.masm, |masm, reg| {
3761
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16U)?;
3762
            Ok(TypedReg::v128(reg))
3763
        })
3764
    }
3765

3766
    fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output {
3767
        self.context.unop(self.masm, |masm, reg| {
3768
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8S)?;
3769
            Ok(TypedReg::v128(reg))
3770
        })
3771
    }
3772

3773
    fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output {
3774
        self.context.unop(self.masm, |masm, reg| {
3775
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8S)?;
3776
            Ok(TypedReg::v128(reg))
3777
        })
3778
    }
3779

3780
    fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output {
3781
        self.context.unop(self.masm, |masm, reg| {
3782
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8U)?;
3783
            Ok(TypedReg::v128(reg))
3784
        })
3785
    }
3786

3787
    fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output {
3788
        self.context.unop(self.masm, |masm, reg| {
3789
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8U)?;
3790
            Ok(TypedReg::v128(reg))
3791
        })
3792
    }
3793

3794
    fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output {
3795
        self.context.unop(self.masm, |masm, reg| {
3796
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4S)?;
3797
            Ok(TypedReg::v128(reg))
3798
        })
3799
    }
3800

3801
    fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output {
3802
        self.context.unop(self.masm, |masm, reg| {
3803
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4S)?;
3804
            Ok(TypedReg::v128(reg))
3805
        })
3806
    }
3807

3808
    fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output {
3809
        self.context.unop(self.masm, |masm, reg| {
3810
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4U)?;
3811
            Ok(TypedReg::v128(reg))
3812
        })
3813
    }
3814

3815
    fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output {
3816
        self.context.unop(self.masm, |masm, reg| {
3817
            masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4U)?;
3818
            Ok(TypedReg::v128(reg))
3819
        })
3820
    }
3821

3822
    fn visit_i8x16_add(&mut self) -> Self::Output {
3823
        self.context
3824
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3825
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16)?;
3826
                Ok(TypedReg::new(WasmValType::V128, dst))
3827
            })
3828
    }
3829

3830
    fn visit_i16x8_add(&mut self) -> Self::Output {
3831
        self.context
3832
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3833
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8)?;
3834
                Ok(TypedReg::new(WasmValType::V128, dst))
3835
            })
3836
    }
3837

3838
    fn visit_i32x4_add(&mut self) -> Self::Output {
3839
        self.context
3840
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3841
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I32x4)?;
3842
                Ok(TypedReg::new(WasmValType::V128, dst))
3843
            })
3844
    }
3845

3846
    fn visit_i64x2_add(&mut self) -> Self::Output {
3847
        self.context
3848
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3849
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I64x2)?;
3850
                Ok(TypedReg::new(WasmValType::V128, dst))
3851
            })
3852
    }
3853

3854
    fn visit_i8x16_sub(&mut self) -> Self::Output {
3855
        self.context
3856
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3857
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16)?;
3858
                Ok(TypedReg::new(WasmValType::V128, dst))
3859
            })
3860
    }
3861

3862
    fn visit_i16x8_sub(&mut self) -> Self::Output {
3863
        self.context
3864
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3865
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8)?;
3866
                Ok(TypedReg::new(WasmValType::V128, dst))
3867
            })
3868
    }
3869

3870
    fn visit_i32x4_sub(&mut self) -> Self::Output {
3871
        self.context
3872
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
3873
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I32x4)?;
3874
                Ok(TypedReg::new(WasmValType::V128, dst))
3875
            })
3876
    }
3877

3878
    fn visit_i64x2_sub(&mut self) -> Self::Output {
3879
        self.context
3880
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
3881
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I64x2)?;
3882
                Ok(TypedReg::new(WasmValType::V128, dst))
3883
            })
3884
    }
3885

3886
    fn visit_i16x8_mul(&mut self) -> Self::Output {
3887
        self.masm.v128_mul(&mut self.context, V128MulKind::I16x8)
3888
    }
3889

3890
    fn visit_i32x4_mul(&mut self) -> Self::Output {
3891
        self.masm.v128_mul(&mut self.context, V128MulKind::I32x4)
3892
    }
3893

3894
    fn visit_i64x2_mul(&mut self) -> Self::Output {
3895
        self.masm.v128_mul(&mut self.context, V128MulKind::I64x2)
3896
    }
3897

3898
    fn visit_i8x16_add_sat_s(&mut self) -> Self::Output {
3899
        self.context
3900
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3901
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatS)?;
3902
                Ok(TypedReg::new(WasmValType::V128, dst))
3903
            })
3904
    }
3905

3906
    fn visit_i16x8_add_sat_s(&mut self) -> Self::Output {
3907
        self.context
3908
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3909
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatS)?;
3910
                Ok(TypedReg::new(WasmValType::V128, dst))
3911
            })
3912
    }
3913

3914
    fn visit_i8x16_add_sat_u(&mut self) -> Self::Output {
3915
        self.context
3916
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3917
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatU)?;
3918
                Ok(TypedReg::new(WasmValType::V128, dst))
3919
            })
3920
    }
3921

3922
    fn visit_i16x8_add_sat_u(&mut self) -> Self::Output {
3923
        self.context
3924
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3925
                masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatU)?;
3926
                Ok(TypedReg::new(WasmValType::V128, dst))
3927
            })
3928
    }
3929

3930
    fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output {
3931
        self.context
3932
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3933
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatS)?;
3934
                Ok(TypedReg::new(WasmValType::V128, dst))
3935
            })
3936
    }
3937

3938
    fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output {
3939
        self.context
3940
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3941
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatS)?;
3942
                Ok(TypedReg::new(WasmValType::V128, dst))
3943
            })
3944
    }
3945

3946
    fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output {
3947
        self.context
3948
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
3949
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatU)?;
3950
                Ok(TypedReg::new(WasmValType::V128, dst))
3951
            })
3952
    }
3953

3954
    fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output {
3955
        self.context
3956
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
3957
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatU)?;
3958
                Ok(TypedReg::new(WasmValType::V128, dst))
3959
            })
3960
    }
3961

3962
    fn visit_i8x16_abs(&mut self) -> Self::Output {
3963
        self.context.unop(self.masm, |masm, reg| {
3964
            masm.v128_abs(reg, writable!(reg), V128AbsKind::I8x16)?;
3965
            Ok(TypedReg::new(WasmValType::V128, reg))
3966
        })
3967
    }
3968

3969
    fn visit_i16x8_abs(&mut self) -> Self::Output {
3970
        self.context.unop(self.masm, |masm, reg| {
3971
            masm.v128_abs(reg, writable!(reg), V128AbsKind::I16x8)?;
3972
            Ok(TypedReg::new(WasmValType::V128, reg))
3973
        })
3974
    }
3975

3976
    fn visit_i32x4_abs(&mut self) -> Self::Output {
3977
        self.context.unop(self.masm, |masm, reg| {
3978
            masm.v128_abs(reg, writable!(reg), V128AbsKind::I32x4)?;
3979
            Ok(TypedReg::new(WasmValType::V128, reg))
3980
        })
3981
    }
3982

3983
    fn visit_i64x2_abs(&mut self) -> Self::Output {
3984
        self.context.unop(self.masm, |masm, reg| {
3985
            masm.v128_abs(reg, writable!(reg), V128AbsKind::I64x2)?;
3986
            Ok(TypedReg::new(WasmValType::V128, reg))
3987
        })
3988
    }
3989

3990
    fn visit_f32x4_abs(&mut self) -> Self::Output {
3991
        self.context.unop(self.masm, |masm, reg| {
3992
            masm.v128_abs(reg, writable!(reg), V128AbsKind::F32x4)?;
3993
            Ok(TypedReg::new(WasmValType::V128, reg))
3994
        })
3995
    }
3996

3997
    fn visit_f64x2_abs(&mut self) -> Self::Output {
3998
        self.context.unop(self.masm, |masm, reg| {
3999
            masm.v128_abs(reg, writable!(reg), V128AbsKind::F64x2)?;
4000
            Ok(TypedReg::new(WasmValType::V128, reg))
4001
        })
4002
    }
4003

4004
    fn visit_i8x16_neg(&mut self) -> Self::Output {
4005
        self.context.unop(self.masm, |masm, op| {
4006
            masm.v128_neg(writable!(op), V128NegKind::I8x16)?;
4007
            Ok(TypedReg::new(WasmValType::V128, op))
4008
        })
4009
    }
4010

4011
    fn visit_i16x8_neg(&mut self) -> Self::Output {
4012
        self.context.unop(self.masm, |masm, op| {
4013
            masm.v128_neg(writable!(op), V128NegKind::I16x8)?;
4014
            Ok(TypedReg::new(WasmValType::V128, op))
4015
        })
4016
    }
4017

4018
    fn visit_i32x4_neg(&mut self) -> Self::Output {
4019
        self.context.unop(self.masm, |masm, op| {
4020
            masm.v128_neg(writable!(op), V128NegKind::I32x4)?;
4021
            Ok(TypedReg::new(WasmValType::V128, op))
4022
        })
4023
    }
4024

4025
    fn visit_i64x2_neg(&mut self) -> Self::Output {
4026
        self.context.unop(self.masm, |masm, op| {
4027
            masm.v128_neg(writable!(op), V128NegKind::I64x2)?;
4028
            Ok(TypedReg::new(WasmValType::V128, op))
4029
        })
4030
    }
4031

4032
    fn visit_i8x16_shl(&mut self) -> Self::Output {
4033
        self.masm
4034
            .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::Shl)
4035
    }
4036

4037
    fn visit_i16x8_shl(&mut self) -> Self::Output {
4038
        self.masm
4039
            .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::Shl)
4040
    }
4041

4042
    fn visit_i32x4_shl(&mut self) -> Self::Output {
4043
        self.masm
4044
            .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::Shl)
4045
    }
4046

4047
    fn visit_i64x2_shl(&mut self) -> Self::Output {
4048
        self.masm
4049
            .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::Shl)
4050
    }
4051

4052
    fn visit_i8x16_shr_u(&mut self) -> Self::Output {
4053
        self.masm
4054
            .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrU)
4055
    }
4056

4057
    fn visit_i16x8_shr_u(&mut self) -> Self::Output {
4058
        self.masm
4059
            .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrU)
4060
    }
4061

4062
    fn visit_i32x4_shr_u(&mut self) -> Self::Output {
4063
        self.masm
4064
            .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrU)
4065
    }
4066

4067
    fn visit_i64x2_shr_u(&mut self) -> Self::Output {
4068
        self.masm
4069
            .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrU)
4070
    }
4071

4072
    fn visit_i8x16_shr_s(&mut self) -> Self::Output {
4073
        self.masm
4074
            .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrS)
4075
    }
4076

4077
    fn visit_i16x8_shr_s(&mut self) -> Self::Output {
4078
        self.masm
4079
            .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrS)
4080
    }
4081

4082
    fn visit_i32x4_shr_s(&mut self) -> Self::Output {
4083
        self.masm
4084
            .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrS)
4085
    }
4086

4087
    fn visit_i64x2_shr_s(&mut self) -> Self::Output {
4088
        self.masm
4089
            .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrS)
4090
    }
4091

4092
    fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output {
4093
        self.context
4094
            .binop(self.masm, OperandSize::S16, |masm, dst, src, size| {
4095
                masm.v128_q15mulr_sat_s(dst, src, writable!(dst), size)?;
4096
                Ok(TypedReg::v128(dst))
4097
            })
4098
    }
4099

4100
    fn visit_i8x16_min_s(&mut self) -> Self::Output {
4101
        self.context
4102
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
4103
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16S)?;
4104
                Ok(TypedReg::v128(dst))
4105
            })
4106
    }
4107

4108
    fn visit_i8x16_all_true(&mut self) -> Self::Output {
4109
        self.context.v128_all_true_op(self.masm, |masm, src, dst| {
4110
            masm.v128_all_true(src, writable!(dst), OperandSize::S8)
4111
        })
4112
    }
4113

4114
    fn visit_i16x8_all_true(&mut self) -> Self::Output {
4115
        self.context.v128_all_true_op(self.masm, |masm, src, dst| {
4116
            masm.v128_all_true(src, writable!(dst), OperandSize::S16)
4117
        })
4118
    }
4119

4120
    fn visit_i32x4_all_true(&mut self) -> Self::Output {
4121
        self.context.v128_all_true_op(self.masm, |masm, src, dst| {
4122
            masm.v128_all_true(src, writable!(dst), OperandSize::S32)
4123
        })
4124
    }
4125

4126
    fn visit_i64x2_all_true(&mut self) -> Self::Output {
4127
        self.context.v128_all_true_op(self.masm, |masm, src, dst| {
4128
            masm.v128_all_true(src, writable!(dst), OperandSize::S64)
4129
        })
4130
    }
4131

4132
    fn visit_i8x16_bitmask(&mut self) -> Self::Output {
4133
        self.context.v128_bitmask_op(self.masm, |masm, src, dst| {
4134
            masm.v128_bitmask(src, writable!(dst), OperandSize::S8)
4135
        })
4136
    }
4137

4138
    fn visit_i16x8_bitmask(&mut self) -> Self::Output {
4139
        self.context.v128_bitmask_op(self.masm, |masm, src, dst| {
4140
            masm.v128_bitmask(src, writable!(dst), OperandSize::S16)
4141
        })
4142
    }
4143

4144
    fn visit_i32x4_bitmask(&mut self) -> Self::Output {
4145
        self.context.v128_bitmask_op(self.masm, |masm, src, dst| {
4146
            masm.v128_bitmask(src, writable!(dst), OperandSize::S32)
4147
        })
4148
    }
4149

4150
    fn visit_i64x2_bitmask(&mut self) -> Self::Output {
4151
        self.context.v128_bitmask_op(self.masm, |masm, src, dst| {
4152
            masm.v128_bitmask(src, writable!(dst), OperandSize::S64)
4153
        })
4154
    }
4155

4156
    fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output {
4157
        self.masm
4158
            .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4S)
4159
    }
4160

4161
    fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output {
4162
        self.masm
4163
            .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4U)
4164
    }
4165

4166
    fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output {
4167
        self.masm
4168
            .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2SZero)
4169
    }
4170

4171
    fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output {
4172
        self.masm
4173
            .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2UZero)
4174
    }
4175

4176
    fn visit_i16x8_min_s(&mut self) -> Self::Output {
4177
        self.context
4178
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
4179
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8S)?;
4180
                Ok(TypedReg::v128(dst))
4181
            })
4182
    }
4183

4184
    fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output {
4185
        self.context
4186
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4187
                masm.v128_dot(dst, src, writable!(dst))?;
4188
                Ok(TypedReg::v128(dst))
4189
            })
4190
    }
4191

4192
    fn visit_i8x16_popcnt(&mut self) -> Self::Output {
4193
        self.masm.v128_popcnt(&mut self.context)
4194
    }
4195

4196
    fn visit_i8x16_avgr_u(&mut self) -> Self::Output {
4197
        self.context
4198
            .binop(self.masm, OperandSize::S8, |masm, dst, src, size| {
4199
                masm.v128_avgr(dst, src, writable!(dst), size)?;
4200
                Ok(TypedReg::v128(dst))
4201
            })
4202
    }
4203

4204
    fn visit_i32x4_min_s(&mut self) -> Self::Output {
4205
        self.context
4206
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4207
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4S)?;
4208
                Ok(TypedReg::v128(dst))
4209
            })
4210
    }
4211

4212
    fn visit_i8x16_min_u(&mut self) -> Self::Output {
4213
        self.context
4214
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
4215
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16U)?;
4216
                Ok(TypedReg::v128(dst))
4217
            })
4218
    }
4219

4220
    fn visit_i16x8_avgr_u(&mut self) -> Self::Output {
4221
        self.context
4222
            .binop(self.masm, OperandSize::S16, |masm, dst, src, size| {
4223
                masm.v128_avgr(dst, src, writable!(dst), size)?;
4224
                Ok(TypedReg::v128(dst))
4225
            })
4226
    }
4227

4228
    fn visit_i16x8_min_u(&mut self) -> Self::Output {
4229
        self.context
4230
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
4231
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8U)?;
4232
                Ok(TypedReg::v128(dst))
4233
            })
4234
    }
4235

4236
    fn visit_i32x4_min_u(&mut self) -> Self::Output {
4237
        self.context
4238
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4239
                masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4U)?;
4240
                Ok(TypedReg::v128(dst))
4241
            })
4242
    }
4243

4244
    fn visit_i8x16_max_s(&mut self) -> Self::Output {
4245
        self.context
4246
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
4247
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16S)?;
4248
                Ok(TypedReg::v128(dst))
4249
            })
4250
    }
4251

4252
    fn visit_i16x8_max_s(&mut self) -> Self::Output {
4253
        self.context
4254
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
4255
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8S)?;
4256
                Ok(TypedReg::v128(dst))
4257
            })
4258
    }
4259

4260
    fn visit_i32x4_max_s(&mut self) -> Self::Output {
4261
        self.context
4262
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4263
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4S)?;
4264
                Ok(TypedReg::v128(dst))
4265
            })
4266
    }
4267

4268
    fn visit_i8x16_max_u(&mut self) -> Self::Output {
4269
        self.context
4270
            .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| {
4271
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16U)?;
4272
                Ok(TypedReg::v128(dst))
4273
            })
4274
    }
4275

4276
    fn visit_i16x8_max_u(&mut self) -> Self::Output {
4277
        self.context
4278
            .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| {
4279
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8U)?;
4280
                Ok(TypedReg::v128(dst))
4281
            })
4282
    }
4283

4284
    fn visit_i32x4_max_u(&mut self) -> Self::Output {
4285
        self.context
4286
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4287
                masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4U)?;
4288
                Ok(TypedReg::v128(dst))
4289
            })
4290
    }
4291

4292
    fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output {
4293
        self.masm
4294
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16S)
4295
    }
4296

4297
    fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output {
4298
        self.masm
4299
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8S)
4300
    }
4301

4302
    fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output {
4303
        self.masm
4304
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4S)
4305
    }
4306

4307
    fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output {
4308
        self.masm
4309
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16U)
4310
    }
4311

4312
    fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output {
4313
        self.masm
4314
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8U)
4315
    }
4316

4317
    fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output {
4318
        self.masm
4319
            .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4U)
4320
    }
4321

4322
    fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output {
4323
        self.masm
4324
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16U)
4325
    }
4326

4327
    fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output {
4328
        self.masm
4329
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8U)
4330
    }
4331

4332
    fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output {
4333
        self.masm
4334
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4U)
4335
    }
4336

4337
    fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output {
4338
        self.masm
4339
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16S)
4340
    }
4341

4342
    fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output {
4343
        self.masm
4344
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8S)
4345
    }
4346

4347
    fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output {
4348
        self.masm
4349
            .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4S)
4350
    }
4351

4352
    fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output {
4353
        self.context.unop(self.masm, |masm, op| {
4354
            masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16S)?;
4355
            Ok(TypedReg::v128(op))
4356
        })
4357
    }
4358

4359
    fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output {
4360
        self.context.unop(self.masm, |masm, op| {
4361
            masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16U)?;
4362
            Ok(TypedReg::v128(op))
4363
        })
4364
    }
4365

4366
    fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output {
4367
        self.context.unop(self.masm, |masm, op| {
4368
            masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8S)?;
4369
            Ok(TypedReg::v128(op))
4370
        })
4371
    }
4372

4373
    fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output {
4374
        self.context.unop(self.masm, |masm, op| {
4375
            masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8U)?;
4376
            Ok(TypedReg::v128(op))
4377
        })
4378
    }
4379

4380
    fn visit_f32x4_add(&mut self) -> Self::Output {
4381
        self.context
4382
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4383
                masm.v128_add(dst, src, writable!(dst), V128AddKind::F32x4)?;
4384
                Ok(TypedReg::v128(dst))
4385
            })
4386
    }
4387

4388
    fn visit_f64x2_add(&mut self) -> Self::Output {
4389
        self.context
4390
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
4391
                masm.v128_add(dst, src, writable!(dst), V128AddKind::F64x2)?;
4392
                Ok(TypedReg::v128(dst))
4393
            })
4394
    }
4395

4396
    fn visit_f32x4_sub(&mut self) -> Self::Output {
4397
        self.context
4398
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4399
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::F32x4)?;
4400
                Ok(TypedReg::v128(dst))
4401
            })
4402
    }
4403

4404
    fn visit_f64x2_sub(&mut self) -> Self::Output {
4405
        self.context
4406
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
4407
                masm.v128_sub(dst, src, writable!(dst), V128SubKind::F64x2)?;
4408
                Ok(TypedReg::v128(dst))
4409
            })
4410
    }
4411

4412
    fn visit_f32x4_mul(&mut self) -> Self::Output {
4413
        self.masm.v128_mul(&mut self.context, V128MulKind::F32x4)
4414
    }
4415

4416
    fn visit_f64x2_mul(&mut self) -> Self::Output {
4417
        self.masm.v128_mul(&mut self.context, V128MulKind::F64x2)
4418
    }
4419

4420
    fn visit_f32x4_div(&mut self) -> Self::Output {
4421
        self.context
4422
            .binop(self.masm, OperandSize::S32, |masm, dst, src, size| {
4423
                masm.v128_div(dst, src, writable!(dst), size)?;
4424
                Ok(TypedReg::v128(dst))
4425
            })
4426
    }
4427

4428
    fn visit_f64x2_div(&mut self) -> Self::Output {
4429
        self.context
4430
            .binop(self.masm, OperandSize::S64, |masm, dst, src, size| {
4431
                masm.v128_div(dst, src, writable!(dst), size)?;
4432
                Ok(TypedReg::v128(dst))
4433
            })
4434
    }
4435

4436
    fn visit_f32x4_neg(&mut self) -> Self::Output {
4437
        self.context.unop(self.masm, |masm, reg| {
4438
            masm.v128_neg(writable!(reg), V128NegKind::F32x4)?;
4439
            Ok(TypedReg::v128(reg))
4440
        })
4441
    }
4442

4443
    fn visit_f32x4_ceil(&mut self) -> Self::Output {
4444
        self.context.unop(self.masm, |masm, reg| {
4445
            masm.v128_ceil(reg, writable!(reg), OperandSize::S32)?;
4446
            Ok(TypedReg::v128(reg))
4447
        })
4448
    }
4449

4450
    fn visit_f64x2_neg(&mut self) -> Self::Output {
4451
        self.context.unop(self.masm, |masm, reg| {
4452
            masm.v128_neg(writable!(reg), V128NegKind::F64x2)?;
4453
            Ok(TypedReg::v128(reg))
4454
        })
4455
    }
4456

4457
    fn visit_f64x2_ceil(&mut self) -> Self::Output {
4458
        self.context.unop(self.masm, |masm, reg| {
4459
            masm.v128_ceil(reg, writable!(reg), OperandSize::S64)?;
4460
            Ok(TypedReg::v128(reg))
4461
        })
4462
    }
4463

4464
    fn visit_f32x4_sqrt(&mut self) -> Self::Output {
4465
        self.context.unop(self.masm, |masm, reg| {
4466
            masm.v128_sqrt(reg, writable!(reg), OperandSize::S32)?;
4467
            Ok(TypedReg::v128(reg))
4468
        })
4469
    }
4470

4471
    fn visit_f32x4_floor(&mut self) -> Self::Output {
4472
        self.context.unop(self.masm, |masm, reg| {
4473
            masm.v128_floor(reg, writable!(reg), OperandSize::S32)?;
4474
            Ok(TypedReg::v128(reg))
4475
        })
4476
    }
4477

4478
    fn visit_f64x2_sqrt(&mut self) -> Self::Output {
4479
        self.context.unop(self.masm, |masm, reg| {
4480
            masm.v128_sqrt(reg, writable!(reg), OperandSize::S64)?;
4481
            Ok(TypedReg::v128(reg))
4482
        })
4483
    }
4484

4485
    fn visit_f64x2_floor(&mut self) -> Self::Output {
4486
        self.context.unop(self.masm, |masm, reg| {
4487
            masm.v128_floor(reg, writable!(reg), OperandSize::S64)?;
4488
            Ok(TypedReg::v128(reg))
4489
        })
4490
    }
4491

4492
    fn visit_f32x4_nearest(&mut self) -> Self::Output {
4493
        self.context.unop(self.masm, |masm, reg| {
4494
            masm.v128_nearest(reg, writable!(reg), OperandSize::S32)?;
4495
            Ok(TypedReg::v128(reg))
4496
        })
4497
    }
4498

4499
    fn visit_f64x2_nearest(&mut self) -> Self::Output {
4500
        self.context.unop(self.masm, |masm, reg| {
4501
            masm.v128_nearest(reg, writable!(reg), OperandSize::S64)?;
4502
            Ok(TypedReg::v128(reg))
4503
        })
4504
    }
4505

4506
    fn visit_f32x4_trunc(&mut self) -> Self::Output {
4507
        self.masm
4508
            .v128_trunc(&mut self.context, V128TruncKind::F32x4)
4509
    }
4510

4511
    fn visit_f64x2_trunc(&mut self) -> Self::Output {
4512
        self.masm
4513
            .v128_trunc(&mut self.context, V128TruncKind::F64x2)
4514
    }
4515

4516
    fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output {
4517
        self.emit_wasm_load(
4518
            &memarg,
4519
            WasmValType::V128,
4520
            LoadKind::VectorZero(OperandSize::S32),
4521
        )
4522
    }
4523

4524
    fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output {
4525
        self.emit_wasm_load(
4526
            &memarg,
4527
            WasmValType::V128,
4528
            LoadKind::VectorZero(OperandSize::S64),
4529
        )
4530
    }
4531

4532
    fn visit_f32x4_pmin(&mut self) -> Self::Output {
4533
        self.context
4534
            .binop(self.masm, OperandSize::S32, |masm, dst, src, size| {
4535
                masm.v128_pmin(dst, src, writable!(dst), size)?;
4536
                Ok(TypedReg::v128(dst))
4537
            })
4538
    }
4539

4540
    fn visit_f64x2_pmin(&mut self) -> Self::Output {
4541
        self.context
4542
            .binop(self.masm, OperandSize::S64, |masm, dst, src, size| {
4543
                masm.v128_pmin(dst, src, writable!(dst), size)?;
4544
                Ok(TypedReg::v128(dst))
4545
            })
4546
    }
4547

4548
    fn visit_f32x4_pmax(&mut self) -> Self::Output {
4549
        self.context
4550
            .binop(self.masm, OperandSize::S32, |masm, dst, src, size| {
4551
                masm.v128_pmax(dst, src, writable!(dst), size)?;
4552
                Ok(TypedReg::v128(dst))
4553
            })
4554
    }
4555

4556
    fn visit_f64x2_pmax(&mut self) -> Self::Output {
4557
        self.context
4558
            .binop(self.masm, OperandSize::S64, |masm, dst, src, size| {
4559
                masm.v128_pmax(dst, src, writable!(dst), size)?;
4560
                Ok(TypedReg::v128(dst))
4561
            })
4562
    }
4563

4564
    fn visit_f32x4_min(&mut self) -> Self::Output {
4565
        self.context
4566
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4567
                masm.v128_min(dst, src, writable!(dst), V128MinKind::F32x4)?;
4568
                Ok(TypedReg::v128(dst))
4569
            })
4570
    }
4571

4572
    fn visit_f64x2_min(&mut self) -> Self::Output {
4573
        self.context
4574
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
4575
                masm.v128_min(dst, src, writable!(dst), V128MinKind::F64x2)?;
4576
                Ok(TypedReg::v128(dst))
4577
            })
4578
    }
4579

4580
    fn visit_f32x4_max(&mut self) -> Self::Output {
4581
        self.context
4582
            .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| {
4583
                masm.v128_max(dst, src, writable!(dst), V128MaxKind::F32x4)?;
4584
                Ok(TypedReg::v128(dst))
4585
            })
4586
    }
4587

4588
    fn visit_f64x2_max(&mut self) -> Self::Output {
4589
        self.context
4590
            .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| {
4591
                masm.v128_max(dst, src, writable!(dst), V128MaxKind::F64x2)?;
4592
                Ok(TypedReg::v128(dst))
4593
            })
4594
    }
4595

4596
    wasmparser::for_each_visit_simd_operator!(def_unsupported);
4597
}
4598

4599
impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>
4600
where
4601
    M: MacroAssembler,
4602
{
4603
    fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> {
4604
        self.context.i32_binop(self.masm, |masm, dst, src, size| {
4605
            masm.cmp_with_set(writable!(dst), src, kind, size)?;
4606
            Ok(TypedReg::i32(dst))
4607
        })
4608
    }
4609

4610
    fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> {
4611
        self.context
4612
            .i64_binop(self.masm, move |masm, dst, src, size| {
4613
                masm.cmp_with_set(writable!(dst), src, kind, size)?;
4614
                Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`.
4615
            })
4616
    }
4617
}
4618

4619
impl TryFrom<WasmValType> for OperandSize {
4620
    type Error = crate::Error;
4621
    fn try_from(ty: WasmValType) -> Result<OperandSize> {
4622
        let ty = match ty {
4623
            WasmValType::I32 | WasmValType::F32 => OperandSize::S32,
4624
            WasmValType::I64 | WasmValType::F64 => OperandSize::S64,
4625
            WasmValType::V128 => OperandSize::S128,
4626
            WasmValType::Ref(rt) => {
4627
                match rt.heap_type {
4628
                    // TODO: Hardcoded size, assuming 64-bit support only. Once
4629
                    // Wasmtime supports 32-bit architectures, this will need
4630
                    // to be updated in such a way that the calculation of the
4631
                    // OperandSize will depend on the target's  pointer size.
4632
                    WasmHeapType::Func => OperandSize::S64,
4633
                    WasmHeapType::Extern => OperandSize::S64,
4634
                    _ => bail!(CodeGenError::unsupported_wasm_type()),
4635
                }
4636
            }
4637
        };
4638
        Ok(ty)
4639
    }
4640
}
4641

4642
Product

Resources

Company
