CoCalc -- assembler-sve-aarch64.cc

GitHub Repository: stenzek/duckstation
Path: blob/master/dep/vixl/src/aarch64/assembler-sve-aarch64.cc
⁴²⁶¹ views
1
// Copyright 2019, VIXL authors
2
// All rights reserved.
3
//
4
// Redistribution and use in source and binary forms, with or without
5
// modification, are permitted provided that the following conditions are met:
6
//
7
//   * Redistributions of source code must retain the above copyright notice,
8
//     this list of conditions and the following disclaimer.
9
//   * Redistributions in binary form must reproduce the above copyright notice,
10
//     this list of conditions and the following disclaimer in the documentation
11
//     and/or other materials provided with the distribution.
12
//   * Neither the name of ARM Limited nor the names of its contributors may be
13
//     used to endorse or promote products derived from this software without
14
//     specific prior written permission.
15
//
16
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26

27
#include "assembler-aarch64.h"
28

29
namespace vixl {
30
namespace aarch64 {
31

32
void Assembler::ResolveSVEImm8Shift(int* imm8, int* shift) {
33
  if (*shift < 0) {
34
    VIXL_ASSERT(*shift == -1);
35
    // Derive the shift amount from the immediate.
36
    if (IsInt8(*imm8)) {
37
      *shift = 0;
38
    } else if ((*imm8 % 256) == 0) {
39
      *imm8 /= 256;
40
      *shift = 8;
41
    }
42
  }
43

44
  VIXL_ASSERT(IsInt8(*imm8));
45
  VIXL_ASSERT((*shift == 0) || (*shift == 8));
46
}
47

48
// SVEAddressGeneration.
49

50
void Assembler::adr(const ZRegister& zd, const SVEMemOperand& addr) {
51
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
52
  VIXL_ASSERT(addr.IsVectorPlusVector());
53
  VIXL_ASSERT(
54
      AreSameLaneSize(zd, addr.GetVectorBase(), addr.GetVectorOffset()));
55

56
  int lane_size = zd.GetLaneSizeInBits();
57
  VIXL_ASSERT((lane_size == kSRegSize) || (lane_size == kDRegSize));
58

59
  int shift_amount = addr.GetShiftAmount();
60
  VIXL_ASSERT((shift_amount >= 0) && (shift_amount <= 3));
61

62
  Instr op = 0xffffffff;
63
  Instr msz = shift_amount << 10;
64
  SVEOffsetModifier mod = addr.GetOffsetModifier();
65
  switch (mod) {
66
    case SVE_UXTW:
67
      VIXL_ASSERT(lane_size == kDRegSize);
68
      op = ADR_z_az_d_u32_scaled;
69
      break;
70
    case SVE_SXTW:
71
      VIXL_ASSERT(lane_size == kDRegSize);
72
      op = ADR_z_az_d_s32_scaled;
73
      break;
74
    case SVE_LSL:
75
    case NO_SVE_OFFSET_MODIFIER:
76
      op = (lane_size == kSRegSize) ? ADR_z_az_s_same_scaled
77
                                    : ADR_z_az_d_same_scaled;
78
      break;
79
    default:
80
      VIXL_UNIMPLEMENTED();
81
  }
82
  Emit(op | msz | Rd(zd) | Rn(addr.GetVectorBase()) |
83
       Rm(addr.GetVectorOffset()));
84
}
85

86
void Assembler::SVELogicalImmediate(const ZRegister& zdn,
87
                                    uint64_t imm,
88
                                    Instr op) {
89
  unsigned bit_n, imm_s, imm_r;
90
  unsigned lane_size = zdn.GetLaneSizeInBits();
91
  // Check that the immediate can be encoded in the instruction.
92
  if (IsImmLogical(imm, lane_size, &bit_n, &imm_s, &imm_r)) {
93
    Emit(op | Rd(zdn) | SVEBitN(bit_n) | SVEImmRotate(imm_r, lane_size) |
94
         SVEImmSetBits(imm_s, lane_size));
95
  } else {
96
    VIXL_UNREACHABLE();
97
  }
98
}
99

100
void Assembler::and_(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
101
  USE(zn);
102
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
103
  VIXL_ASSERT(zd.Is(zn));
104
  SVELogicalImmediate(zd, imm, AND_z_zi);
105
}
106

107
void Assembler::dupm(const ZRegister& zd, uint64_t imm) {
108
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
109
  // DUPM_z_i is an SVEBroadcastBitmaskImmOp, but its encoding and constraints
110
  // are similar enough to SVEBitwiseLogicalWithImm_UnpredicatedOp, that we can
111
  // use the logical immediate encoder to get the correct behaviour.
112
  SVELogicalImmediate(zd, imm, DUPM_z_i);
113
}
114

115
void Assembler::eor(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
116
  USE(zn);
117
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
118
  VIXL_ASSERT(zd.Is(zn));
119
  SVELogicalImmediate(zd, imm, EOR_z_zi);
120
}
121

122
void Assembler::orr(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
123
  USE(zn);
124
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
125
  VIXL_ASSERT(zd.Is(zn));
126
  SVELogicalImmediate(zd, imm, ORR_z_zi);
127
}
128

129
// SVEBitwiseLogicalUnpredicated.
130
void Assembler::and_(const ZRegister& zd,
131
                     const ZRegister& zn,
132
                     const ZRegister& zm) {
133
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
134
  VIXL_ASSERT(zd.IsLaneSizeD());
135
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
136
  Emit(AND_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
137
}
138

139
void Assembler::bic(const ZRegister& zd,
140
                    const ZRegister& zn,
141
                    const ZRegister& zm) {
142
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
143
  VIXL_ASSERT(zd.IsLaneSizeD());
144
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
145
  Emit(BIC_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
146
}
147

148
void Assembler::eor(const ZRegister& zd,
149
                    const ZRegister& zn,
150
                    const ZRegister& zm) {
151
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
152
  VIXL_ASSERT(zd.IsLaneSizeD());
153
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
154
  Emit(EOR_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
155
}
156

157
void Assembler::orr(const ZRegister& zd,
158
                    const ZRegister& zn,
159
                    const ZRegister& zm) {
160
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
161
  VIXL_ASSERT(zd.IsLaneSizeD());
162
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
163
  Emit(ORR_z_zz | Rd(zd) | Rn(zn) | Rm(zm));
164
}
165

166
// SVEBitwiseShiftPredicated.
167

168
void Assembler::SVEBitwiseShiftImmediatePred(const ZRegister& zdn,
169
                                             const PRegisterM& pg,
170
                                             Instr encoded_imm_and_tsz,
171
                                             Instr op) {
172
  Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz)
173
                       << 5;
174
  Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22;
175
  Emit(op | tszh | tszl_and_imm | PgLow8(pg) | Rd(zdn));
176
}
177

178
void Assembler::asr(const ZRegister& zd,
179
                    const PRegisterM& pg,
180
                    const ZRegister& zn,
181
                    int shift) {
182
  // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
183
  //  0000 0100 ..00 0000 100. .... .... ....
184
  //  tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 0 | Pg<12:10> |
185
  //  tszl<9:8> | imm3<7:5> | Zdn<4:0>
186

187
  USE(zn);
188
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
189
  VIXL_ASSERT(zd.Is(zn));
190
  Instr encoded_imm =
191
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
192
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASR_z_p_zi);
193
}
194

195
void Assembler::asr(const ZRegister& zd,
196
                    const PRegisterM& pg,
197
                    const ZRegister& zn,
198
                    const ZRegister& zm) {
199
  // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
200
  //  0000 0100 ..01 1000 100. .... .... ....
201
  //  size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> |
202
  //  Zdn<4:0>
203

204
  USE(zn);
205
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
206
  VIXL_ASSERT(zd.Is(zn));
207
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
208
              ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
209
               (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
210
  Instr op = ASR_z_p_zw;
211
  if (AreSameLaneSize(zd, zn, zm)) {
212
    op = ASR_z_p_zz;
213
  }
214
  Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
215
}
216

217
void Assembler::asrd(const ZRegister& zd,
218
                     const PRegisterM& pg,
219
                     const ZRegister& zn,
220
                     int shift) {
221
  // ASRD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
222
  //  0000 0100 ..00 0100 100. .... .... ....
223
  //  tszh<23:22> | opc<19:18> = 01 | L<17> = 0 | U<16> = 0 | Pg<12:10> |
224
  //  tszl<9:8> | imm3<7:5> | Zdn<4:0>
225

226
  USE(zn);
227
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
228
  VIXL_ASSERT(zd.Is(zn));
229

230
  Instr encoded_imm =
231
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
232
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASRD_z_p_zi);
233
}
234

235
void Assembler::asrr(const ZRegister& zd,
236
                     const PRegisterM& pg,
237
                     const ZRegister& zn,
238
                     const ZRegister& zm) {
239
  // ASRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
240
  //  0000 0100 ..01 0100 100. .... .... ....
241
  //  size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> |
242
  //  Zdn<4:0>
243

244
  USE(zn);
245
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
246
  VIXL_ASSERT(zd.Is(zn));
247
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
248

249
  Emit(ASRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
250
}
251

252
void Assembler::lsl(const ZRegister& zd,
253
                    const PRegisterM& pg,
254
                    const ZRegister& zn,
255
                    int shift) {
256
  // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
257
  //  0000 0100 ..00 0011 100. .... .... ....
258
  //  tszh<23:22> | opc<19:18> = 00 | L<17> = 1 | U<16> = 1 | Pg<12:10> |
259
  //  tszl<9:8> | imm3<7:5> | Zdn<4:0>
260

261
  USE(zn);
262
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
263
  VIXL_ASSERT(zd.Is(zn));
264

265
  Instr encoded_imm =
266
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
267
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSL_z_p_zi);
268
}
269

270
void Assembler::lsl(const ZRegister& zd,
271
                    const PRegisterM& pg,
272
                    const ZRegister& zn,
273
                    const ZRegister& zm) {
274
  // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
275
  //  0000 0100 ..01 1011 100. .... .... ....
276
  //  size<23:22> | R<18> = 0 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
277
  //  Zdn<4:0>
278

279
  USE(zn);
280
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
281
  VIXL_ASSERT(zd.Is(zn));
282
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
283
              ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
284
               (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
285
  Instr op = LSL_z_p_zw;
286
  if (AreSameLaneSize(zd, zn, zm)) {
287
    op = LSL_z_p_zz;
288
  }
289
  Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
290
}
291

292
void Assembler::lslr(const ZRegister& zd,
293
                     const PRegisterM& pg,
294
                     const ZRegister& zn,
295
                     const ZRegister& zm) {
296
  // LSLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
297
  //  0000 0100 ..01 0111 100. .... .... ....
298
  //  size<23:22> | R<18> = 1 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
299
  //  Zdn<4:0>
300

301
  USE(zn);
302
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
303
  VIXL_ASSERT(zd.Is(zn));
304
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
305

306
  Emit(LSLR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
307
}
308

309
void Assembler::lsr(const ZRegister& zd,
310
                    const PRegisterM& pg,
311
                    const ZRegister& zn,
312
                    int shift) {
313
  // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
314
  //  0000 0100 ..00 0001 100. .... .... ....
315
  //  tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 1 | Pg<12:10> |
316
  //  tszl<9:8> | imm3<7:5> | Zdn<4:0>
317

318
  USE(zn);
319
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
320
  VIXL_ASSERT(zd.Is(zn));
321

322
  Instr encoded_imm =
323
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
324
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSR_z_p_zi);
325
}
326

327
void Assembler::lsr(const ZRegister& zd,
328
                    const PRegisterM& pg,
329
                    const ZRegister& zn,
330
                    const ZRegister& zm) {
331
  // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D
332
  //  0000 0100 ..01 1001 100. .... .... ....
333
  //  size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
334
  //  Zdn<4:0>
335

336
  USE(zn);
337
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
338
  VIXL_ASSERT(zd.Is(zn));
339
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) ||
340
              ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) &&
341
               (zd.GetLaneSizeInBytes() != kDRegSizeInBytes)));
342
  Instr op = LSR_z_p_zw;
343
  if (AreSameLaneSize(zd, zn, zm)) {
344
    op = LSR_z_p_zz;
345
  }
346
  Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
347
}
348

349
void Assembler::lsrr(const ZRegister& zd,
350
                     const PRegisterM& pg,
351
                     const ZRegister& zn,
352
                     const ZRegister& zm) {
353
  // LSRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
354
  //  0000 0100 ..01 0101 100. .... .... ....
355
  //  size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> |
356
  //  Zdn<4:0>
357

358
  USE(zn);
359
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
360
  VIXL_ASSERT(zd.Is(zn));
361
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
362

363
  Emit(LSRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
364
}
365

366
// SVEBitwiseShiftUnpredicated.
367

368
Instr Assembler::EncodeSVEShiftLeftImmediate(int shift, int lane_size_in_bits) {
369
  VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
370
  return lane_size_in_bits + shift;
371
}
372

373
Instr Assembler::EncodeSVEShiftRightImmediate(int shift,
374
                                              int lane_size_in_bits) {
375
  VIXL_ASSERT((shift > 0) && (shift <= lane_size_in_bits));
376
  return (2 * lane_size_in_bits) - shift;
377
}
378

379
void Assembler::SVEBitwiseShiftImmediate(const ZRegister& zd,
380
                                         const ZRegister& zn,
381
                                         Instr encoded_imm_and_tsz,
382
                                         Instr op) {
383
  Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz)
384
                       << 16;
385
  Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22;
386
  Emit(op | tszh | tszl_and_imm | Rd(zd) | Rn(zn));
387
}
388

389
void Assembler::asr(const ZRegister& zd, const ZRegister& zn, int shift) {
390
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
391
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
392
  Instr encoded_imm =
393
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
394
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, ASR_z_zi);
395
}
396

397
void Assembler::asr(const ZRegister& zd,
398
                    const ZRegister& zn,
399
                    const ZRegister& zm) {
400
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
401
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
402
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
403

404
  Emit(ASR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
405
}
406

407
void Assembler::lsl(const ZRegister& zd, const ZRegister& zn, int shift) {
408
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
409
  Instr encoded_imm =
410
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
411
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSL_z_zi);
412
}
413

414
void Assembler::lsl(const ZRegister& zd,
415
                    const ZRegister& zn,
416
                    const ZRegister& zm) {
417
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
418
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
419
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
420

421
  Emit(LSL_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
422
}
423

424
void Assembler::lsr(const ZRegister& zd, const ZRegister& zn, int shift) {
425
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
426
  Instr encoded_imm =
427
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
428
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSR_z_zi);
429
}
430

431
void Assembler::lsr(const ZRegister& zd,
432
                    const ZRegister& zn,
433
                    const ZRegister& zm) {
434
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
435
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
436
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes);
437

438
  Emit(LSR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
439
}
440

441
// SVEElementCount.
442

443
#define VIXL_SVE_INC_DEC_LIST(V) \
444
  V(cntb, CNTB_r_s)              \
445
  V(cnth, CNTH_r_s)              \
446
  V(cntw, CNTW_r_s)              \
447
  V(cntd, CNTD_r_s)              \
448
  V(decb, DECB_r_rs)             \
449
  V(dech, DECH_r_rs)             \
450
  V(decw, DECW_r_rs)             \
451
  V(decd, DECD_r_rs)             \
452
  V(incb, INCB_r_rs)             \
453
  V(inch, INCH_r_rs)             \
454
  V(incw, INCW_r_rs)             \
455
  V(incd, INCD_r_rs)             \
456
  V(sqdecb, SQDECB_r_rs_x)       \
457
  V(sqdech, SQDECH_r_rs_x)       \
458
  V(sqdecw, SQDECW_r_rs_x)       \
459
  V(sqdecd, SQDECD_r_rs_x)       \
460
  V(sqincb, SQINCB_r_rs_x)       \
461
  V(sqinch, SQINCH_r_rs_x)       \
462
  V(sqincw, SQINCW_r_rs_x)       \
463
  V(sqincd, SQINCD_r_rs_x)
464

465
#define VIXL_DEFINE_ASM_FUNC(FN, OP)                                     \
466
  void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \
467
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                              \
468
    VIXL_ASSERT(rdn.IsX());                                              \
469
    Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) |             \
470
         ImmUnsignedField<19, 16>(multiplier - 1));                      \
471
  }
472
VIXL_SVE_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC)
473
#undef VIXL_DEFINE_ASM_FUNC
474

475
#define VIXL_SVE_UQINC_UQDEC_LIST(V)                      \
476
  V(uqdecb, (rdn.IsX() ? UQDECB_r_rs_x : UQDECB_r_rs_uw)) \
477
  V(uqdech, (rdn.IsX() ? UQDECH_r_rs_x : UQDECH_r_rs_uw)) \
478
  V(uqdecw, (rdn.IsX() ? UQDECW_r_rs_x : UQDECW_r_rs_uw)) \
479
  V(uqdecd, (rdn.IsX() ? UQDECD_r_rs_x : UQDECD_r_rs_uw)) \
480
  V(uqincb, (rdn.IsX() ? UQINCB_r_rs_x : UQINCB_r_rs_uw)) \
481
  V(uqinch, (rdn.IsX() ? UQINCH_r_rs_x : UQINCH_r_rs_uw)) \
482
  V(uqincw, (rdn.IsX() ? UQINCW_r_rs_x : UQINCW_r_rs_uw)) \
483
  V(uqincd, (rdn.IsX() ? UQINCD_r_rs_x : UQINCD_r_rs_uw))
484

485
#define VIXL_DEFINE_ASM_FUNC(FN, OP)                                     \
486
  void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \
487
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                              \
488
    Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) |             \
489
         ImmUnsignedField<19, 16>(multiplier - 1));                      \
490
  }
491
VIXL_SVE_UQINC_UQDEC_LIST(VIXL_DEFINE_ASM_FUNC)
492
#undef VIXL_DEFINE_ASM_FUNC
493

494
#define VIXL_SVE_SQX_INC_DEC_LIST(V) \
495
  V(sqdecb, SQDECB)                  \
496
  V(sqdech, SQDECH)                  \
497
  V(sqdecw, SQDECW)                  \
498
  V(sqdecd, SQDECD)                  \
499
  V(sqincb, SQINCB)                  \
500
  V(sqinch, SQINCH)                  \
501
  V(sqincw, SQINCW)                  \
502
  V(sqincd, SQINCD)
503

504
#define VIXL_DEFINE_ASM_FUNC(FN, OP)                                  \
505
  void Assembler::FN(const Register& xd,                              \
506
                     const Register& wn,                              \
507
                     int pattern,                                     \
508
                     int multiplier) {                                \
509
    USE(wn);                                                          \
510
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                           \
511
    VIXL_ASSERT(wn.IsW() && xd.Is(wn.X()));                           \
512
    Emit(OP##_r_rs_sx | Rd(xd) | ImmSVEPredicateConstraint(pattern) | \
513
         ImmUnsignedField<19, 16>(multiplier - 1));                   \
514
  }
515
VIXL_SVE_SQX_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC)
516
#undef VIXL_DEFINE_ASM_FUNC
517

518
#define VIXL_SVE_INC_DEC_VEC_LIST(V) \
519
  V(dech, DEC, H)                    \
520
  V(decw, DEC, W)                    \
521
  V(decd, DEC, D)                    \
522
  V(inch, INC, H)                    \
523
  V(incw, INC, W)                    \
524
  V(incd, INC, D)                    \
525
  V(sqdech, SQDEC, H)                \
526
  V(sqdecw, SQDEC, W)                \
527
  V(sqdecd, SQDEC, D)                \
528
  V(sqinch, SQINC, H)                \
529
  V(sqincw, SQINC, W)                \
530
  V(sqincd, SQINC, D)                \
531
  V(uqdech, UQDEC, H)                \
532
  V(uqdecw, UQDEC, W)                \
533
  V(uqdecd, UQDEC, D)                \
534
  V(uqinch, UQINC, H)                \
535
  V(uqincw, UQINC, W)                \
536
  V(uqincd, UQINC, D)
537

538
#define VIXL_DEFINE_ASM_FUNC(FN, OP, T)                                   \
539
  void Assembler::FN(const ZRegister& zdn, int pattern, int multiplier) { \
540
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                               \
541
    VIXL_ASSERT(zdn.GetLaneSizeInBytes() == k##T##RegSizeInBytes);        \
542
    Emit(OP##T##_z_zs | Rd(zdn) | ImmSVEPredicateConstraint(pattern) |    \
543
         ImmUnsignedField<19, 16>(multiplier - 1));                       \
544
  }
545
VIXL_SVE_INC_DEC_VEC_LIST(VIXL_DEFINE_ASM_FUNC)
546
#undef VIXL_DEFINE_ASM_FUNC
547

548
// SVEFPAccumulatingReduction.
549

550
void Assembler::fadda(const VRegister& vd,
551
                      const PRegister& pg,
552
                      const VRegister& vn,
553
                      const ZRegister& zm) {
554
  // FADDA <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
555
  //  0110 0101 ..01 1000 001. .... .... ....
556
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
557

558
  USE(vn);
559
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
560
  VIXL_ASSERT(vd.Is(vn));
561
  VIXL_ASSERT(vd.IsScalar());
562
  VIXL_ASSERT(zm.GetLaneSizeInBytes() != kBRegSizeInBytes);
563
  VIXL_ASSERT(AreSameLaneSize(zm, vd));
564

565
  Emit(FADDA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
566
}
567

568
// SVEFPArithmetic_Predicated.
569

570
void Assembler::fabd(const ZRegister& zd,
571
                     const PRegisterM& pg,
572
                     const ZRegister& zn,
573
                     const ZRegister& zm) {
574
  // FABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
575
  //  0110 0101 ..00 1000 100. .... .... ....
576
  //  size<23:22> | opc<19:16> = 1000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
577

578
  USE(zn);
579
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
580
  VIXL_ASSERT(zd.Is(zn));
581
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
582
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
583

584
  Emit(FABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
585
}
586

587
void Assembler::fadd(const ZRegister& zd,
588
                     const PRegisterM& pg,
589
                     const ZRegister& zn,
590
                     double imm) {
591
  // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
592
  //  0110 0101 ..01 1000 100. ..00 00.. ....
593
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | i1<5> | Zdn<4:0>
594

595
  USE(zn);
596
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
597
  VIXL_ASSERT(zd.Is(zn));
598
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
599
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
600
  VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
601

602
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
603
  Emit(FADD_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
604
}
605

606
void Assembler::fadd(const ZRegister& zd,
607
                     const PRegisterM& pg,
608
                     const ZRegister& zn,
609
                     const ZRegister& zm) {
610
  // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
611
  //  0110 0101 ..00 0000 100. .... .... ....
612
  //  size<23:22> | opc<19:16> = 0000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
613

614
  USE(zn);
615
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
616
  VIXL_ASSERT(zd.Is(zn));
617
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
618
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
619

620
  Emit(FADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
621
}
622

623
void Assembler::fdiv(const ZRegister& zd,
624
                     const PRegisterM& pg,
625
                     const ZRegister& zn,
626
                     const ZRegister& zm) {
627
  // FDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
628
  //  0110 0101 ..00 1101 100. .... .... ....
629
  //  size<23:22> | opc<19:16> = 1101 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
630

631
  USE(zn);
632
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
633
  VIXL_ASSERT(zd.Is(zn));
634
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
635
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
636

637
  Emit(FDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
638
}
639

640
void Assembler::fdivr(const ZRegister& zd,
641
                      const PRegisterM& pg,
642
                      const ZRegister& zn,
643
                      const ZRegister& zm) {
644
  // FDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
645
  //  0110 0101 ..00 1100 100. .... .... ....
646
  //  size<23:22> | opc<19:16> = 1100 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
647

648
  USE(zn);
649
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
650
  VIXL_ASSERT(zd.Is(zn));
651
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
652
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
653

654
  Emit(FDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
655
}
656

657
void Assembler::fmax(const ZRegister& zd,
658
                     const PRegisterM& pg,
659
                     const ZRegister& zn,
660
                     double imm) {
661
  // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
662
  //  0110 0101 ..01 1110 100. ..00 00.. ....
663
  //  size<23:22> | opc<18:16> = 110 | Pg<12:10> | i1<5> | Zdn<4:0>
664

665
  USE(zn);
666
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
667
  VIXL_ASSERT(zd.Is(zn));
668
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
669
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
670
  VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
671

672
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
673
  Emit(FMAX_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
674
}
675

676
void Assembler::fmax(const ZRegister& zd,
677
                     const PRegisterM& pg,
678
                     const ZRegister& zn,
679
                     const ZRegister& zm) {
680
  // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
681
  //  0110 0101 ..00 0110 100. .... .... ....
682
  //  size<23:22> | opc<19:16> = 0110 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
683

684
  USE(zn);
685
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
686
  VIXL_ASSERT(zd.Is(zn));
687
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
688
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
689

690
  Emit(FMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
691
}
692

693
void Assembler::fmaxnm(const ZRegister& zd,
694
                       const PRegisterM& pg,
695
                       const ZRegister& zn,
696
                       double imm) {
697
  // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
698
  //  0110 0101 ..01 1100 100. ..00 00.. ....
699
  //  size<23:22> | opc<18:16> = 100 | Pg<12:10> | i1<5> | Zdn<4:0>
700

701
  USE(zn);
702
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
703
  VIXL_ASSERT(zd.Is(zn));
704
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
705
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
706
  VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
707

708
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
709
  Emit(FMAXNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
710
}
711

712
void Assembler::fmaxnm(const ZRegister& zd,
713
                       const PRegisterM& pg,
714
                       const ZRegister& zn,
715
                       const ZRegister& zm) {
716
  // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
717
  //  0110 0101 ..00 0100 100. .... .... ....
718
  //  size<23:22> | opc<19:16> = 0100 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
719

720
  USE(zn);
721
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
722
  VIXL_ASSERT(zd.Is(zn));
723
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
724
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
725

726
  Emit(FMAXNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
727
}
728

729
void Assembler::fmin(const ZRegister& zd,
730
                     const PRegisterM& pg,
731
                     const ZRegister& zn,
732
                     double imm) {
733
  // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
734
  //  0110 0101 ..01 1111 100. ..00 00.. ....
735
  //  size<23:22> | opc<18:16> = 111 | Pg<12:10> | i1<5> | Zdn<4:0>
736

737
  USE(zn);
738
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
739
  VIXL_ASSERT(zd.Is(zn));
740
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
741
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
742
  VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
743

744
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
745
  Emit(FMIN_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
746
}
747

748
void Assembler::fmin(const ZRegister& zd,
749
                     const PRegisterM& pg,
750
                     const ZRegister& zn,
751
                     const ZRegister& zm) {
752
  // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
753
  //  0110 0101 ..00 0111 100. .... .... ....
754
  //  size<23:22> | opc<19:16> = 0111 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
755

756
  USE(zn);
757
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
758
  VIXL_ASSERT(zd.Is(zn));
759
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
760
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
761

762
  Emit(FMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
763
}
764

765
void Assembler::fminnm(const ZRegister& zd,
766
                       const PRegisterM& pg,
767
                       const ZRegister& zn,
768
                       double imm) {
769
  // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
770
  //  0110 0101 ..01 1101 100. ..00 00.. ....
771
  //  size<23:22> | opc<18:16> = 101 | Pg<12:10> | i1<5> | Zdn<4:0>
772

773
  USE(zn);
774
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
775
  VIXL_ASSERT(zd.Is(zn));
776
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
777
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
778
  VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0));
779

780
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
781
  Emit(FMINNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
782
}
783

784
void Assembler::fminnm(const ZRegister& zd,
785
                       const PRegisterM& pg,
786
                       const ZRegister& zn,
787
                       const ZRegister& zm) {
788
  // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
789
  //  0110 0101 ..00 0101 100. .... .... ....
790
  //  size<23:22> | opc<19:16> = 0101 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
791

792
  USE(zn);
793
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
794
  VIXL_ASSERT(zd.Is(zn));
795
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
796
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
797

798
  Emit(FMINNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
799
}
800

801
void Assembler::fmul(const ZRegister& zd,
802
                     const PRegisterM& pg,
803
                     const ZRegister& zn,
804
                     double imm) {
805
  // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
806
  //  0110 0101 ..01 1010 100. ..00 00.. ....
807
  //  size<23:22> | opc<18:16> = 010 | Pg<12:10> | i1<5> | Zdn<4:0>
808

809
  USE(zn);
810
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
811
  VIXL_ASSERT(zd.Is(zn));
812
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
813
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
814
  VIXL_ASSERT((imm == 0.5) || (imm == 2.0));
815

816
  Instr i1 = (imm == 2.0) ? (1 << 5) : 0;
817
  Emit(FMUL_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
818
}
819

820
void Assembler::fmul(const ZRegister& zd,
821
                     const PRegisterM& pg,
822
                     const ZRegister& zn,
823
                     const ZRegister& zm) {
824
  // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
825
  //  0110 0101 ..00 0010 100. .... .... ....
826
  //  size<23:22> | opc<19:16> = 0010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
827

828
  USE(zn);
829
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
830
  VIXL_ASSERT(zd.Is(zn));
831
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
832
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
833

834
  Emit(FMUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
835
}
836

837
void Assembler::fmulx(const ZRegister& zd,
838
                      const PRegisterM& pg,
839
                      const ZRegister& zn,
840
                      const ZRegister& zm) {
841
  // FMULX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
842
  //  0110 0101 ..00 1010 100. .... .... ....
843
  //  size<23:22> | opc<19:16> = 1010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
844

845
  USE(zn);
846
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
847
  VIXL_ASSERT(zd.Is(zn));
848
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
849
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
850

851
  Emit(FMULX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
852
}
853

854
void Assembler::fscale(const ZRegister& zd,
855
                       const PRegisterM& pg,
856
                       const ZRegister& zn,
857
                       const ZRegister& zm) {
858
  // FSCALE <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
859
  //  0110 0101 ..00 1001 100. .... .... ....
860
  //  size<23:22> | opc<19:16> = 1001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
861

862
  USE(zn);
863
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
864
  VIXL_ASSERT(zd.Is(zn));
865
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
866
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
867

868
  Emit(FSCALE_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
869
}
870

871
void Assembler::fsub(const ZRegister& zd,
872
                     const PRegisterM& pg,
873
                     const ZRegister& zn,
874
                     double imm) {
875
  // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
876
  //  0110 0101 ..01 1001 100. ..00 00.. ....
877
  //  size<23:22> | opc<18:16> = 001 | Pg<12:10> | i1<5> | Zdn<4:0>
878

879
  USE(zn);
880
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
881
  VIXL_ASSERT(zd.Is(zn));
882
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
883
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
884
  VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
885

886
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
887
  Emit(FSUB_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
888
}
889

890
void Assembler::fsub(const ZRegister& zd,
891
                     const PRegisterM& pg,
892
                     const ZRegister& zn,
893
                     const ZRegister& zm) {
894
  // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
895
  //  0110 0101 ..00 0001 100. .... .... ....
896
  //  size<23:22> | opc<19:16> = 0001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
897

898
  USE(zn);
899
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
900
  VIXL_ASSERT(zd.Is(zn));
901
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
902
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
903

904
  Emit(FSUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
905
}
906

907
void Assembler::fsubr(const ZRegister& zd,
908
                      const PRegisterM& pg,
909
                      const ZRegister& zn,
910
                      double imm) {
911
  // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const>
912
  //  0110 0101 ..01 1011 100. ..00 00.. ....
913
  //  size<23:22> | opc<18:16> = 011 | Pg<12:10> | i1<5> | Zdn<4:0>
914

915
  USE(zn);
916
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
917
  VIXL_ASSERT(zd.Is(zn));
918
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
919
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
920
  VIXL_ASSERT((imm == 0.5) || (imm == 1.0));
921

922
  Instr i1 = (imm == 1.0) ? (1 << 5) : 0;
923
  Emit(FSUBR_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1);
924
}
925

926
void Assembler::fsubr(const ZRegister& zd,
927
                      const PRegisterM& pg,
928
                      const ZRegister& zn,
929
                      const ZRegister& zm) {
930
  // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
931
  //  0110 0101 ..00 0011 100. .... .... ....
932
  //  size<23:22> | opc<19:16> = 0011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
933

934
  USE(zn);
935
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
936
  VIXL_ASSERT(zd.Is(zn));
937
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
938
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
939

940
  Emit(FSUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
941
}
942

943
void Assembler::ftmad(const ZRegister& zd,
944
                      const ZRegister& zn,
945
                      const ZRegister& zm,
946
                      int imm3) {
947
  // FTMAD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<imm>
948
  //  0110 0101 ..01 0... 1000 00.. .... ....
949
  //  size<23:22> | imm3<18:16> | Zm<9:5> | Zdn<4:0>
950

951
  USE(zn);
952
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
953
  VIXL_ASSERT(zd.Is(zn));
954
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
955
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
956

957
  Emit(FTMAD_z_zzi | SVESize(zd) | Rd(zd) | Rn(zm) |
958
       ImmUnsignedField<18, 16>(imm3));
959
}
960

961
// SVEFPArithmeticUnpredicated.
962

963
void Assembler::fadd(const ZRegister& zd,
964
                     const ZRegister& zn,
965
                     const ZRegister& zm) {
966
  // FADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
967
  //  0110 0101 ..0. .... 0000 00.. .... ....
968
  //  size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
969

970
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
971
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
972
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
973

974
  Emit(FADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
975
}
976

977
void Assembler::fmul(const ZRegister& zd,
978
                     const ZRegister& zn,
979
                     const ZRegister& zm) {
980
  // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
981
  //  0110 0101 ..0. .... 0000 10.. .... ....
982
  //  size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0>
983

984
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
985
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
986
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
987

988
  Emit(FMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
989
}
990

991
void Assembler::frecps(const ZRegister& zd,
992
                       const ZRegister& zn,
993
                       const ZRegister& zm) {
994
  // FRECPS <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
995
  //  0110 0101 ..0. .... 0001 10.. .... ....
996
  //  size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0>
997

998
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
999
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1000
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1001

1002
  Emit(FRECPS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1003
}
1004

1005
void Assembler::frsqrts(const ZRegister& zd,
1006
                        const ZRegister& zn,
1007
                        const ZRegister& zm) {
1008
  // FRSQRTS <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1009
  //  0110 0101 ..0. .... 0001 11.. .... ....
1010
  //  size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0>
1011

1012
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1013
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1014
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1015

1016
  Emit(FRSQRTS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1017
}
1018

1019
void Assembler::fsub(const ZRegister& zd,
1020
                     const ZRegister& zn,
1021
                     const ZRegister& zm) {
1022
  // FSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1023
  //  0110 0101 ..0. .... 0000 01.. .... ....
1024
  //  size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
1025

1026
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1027
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1028
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1029

1030
  Emit(FSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1031
}
1032

1033
void Assembler::ftsmul(const ZRegister& zd,
1034
                       const ZRegister& zn,
1035
                       const ZRegister& zm) {
1036
  // FTSMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
1037
  //  0110 0101 ..0. .... 0000 11.. .... ....
1038
  //  size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0>
1039

1040
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1041
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1042
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1043

1044
  Emit(FTSMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
1045
}
1046

1047
// SVEFPCompareVectors.
1048

1049
void Assembler::facge(const PRegisterWithLaneSize& pd,
1050
                      const PRegisterZ& pg,
1051
                      const ZRegister& zn,
1052
                      const ZRegister& zm) {
1053
  // FACGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1054
  //  0110 0101 ..0. .... 110. .... ...1 ....
1055
  //  size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1056
  //  o3<4> = 1 | Pd<3:0>
1057

1058
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1059
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1060
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1061

1062
  Emit(FACGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1063
}
1064

1065
void Assembler::facgt(const PRegisterWithLaneSize& pd,
1066
                      const PRegisterZ& pg,
1067
                      const ZRegister& zn,
1068
                      const ZRegister& zm) {
1069
  // FACGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1070
  //  0110 0101 ..0. .... 111. .... ...1 ....
1071
  //  size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1072
  //  o3<4> = 1 | Pd<3:0>
1073

1074
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1075
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1076
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1077

1078
  Emit(FACGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1079
}
1080

1081
void Assembler::fcmeq(const PRegisterWithLaneSize& pd,
1082
                      const PRegisterZ& pg,
1083
                      const ZRegister& zn,
1084
                      const ZRegister& zm) {
1085
  // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1086
  //  0110 0101 ..0. .... 011. .... ...0 ....
1087
  //  size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1088
  //  o3<4> = 0 | Pd<3:0>
1089

1090
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1091
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1092
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1093

1094
  Emit(FCMEQ_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1095
}
1096

1097
void Assembler::fcmge(const PRegisterWithLaneSize& pd,
1098
                      const PRegisterZ& pg,
1099
                      const ZRegister& zn,
1100
                      const ZRegister& zm) {
1101
  // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1102
  //  0110 0101 ..0. .... 010. .... ...0 ....
1103
  //  size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1104
  //  o3<4> = 0 | Pd<3:0>
1105

1106
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1107
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1108
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1109

1110
  Emit(FCMGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1111
}
1112

1113
void Assembler::fcmgt(const PRegisterWithLaneSize& pd,
1114
                      const PRegisterZ& pg,
1115
                      const ZRegister& zn,
1116
                      const ZRegister& zm) {
1117
  // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1118
  //  0110 0101 ..0. .... 010. .... ...1 ....
1119
  //  size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1120
  //  o3<4> = 1 | Pd<3:0>
1121

1122
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1123
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1124
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1125

1126
  Emit(FCMGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1127
}
1128

1129
void Assembler::fcmne(const PRegisterWithLaneSize& pd,
1130
                      const PRegisterZ& pg,
1131
                      const ZRegister& zn,
1132
                      const ZRegister& zm) {
1133
  // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1134
  //  0110 0101 ..0. .... 011. .... ...1 ....
1135
  //  size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
1136
  //  o3<4> = 1 | Pd<3:0>
1137

1138
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1139
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1140
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1141

1142
  Emit(FCMNE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1143
}
1144

1145
void Assembler::fcmuo(const PRegisterWithLaneSize& pd,
1146
                      const PRegisterZ& pg,
1147
                      const ZRegister& zn,
1148
                      const ZRegister& zm) {
1149
  // FCMUO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
1150
  //  0110 0101 ..0. .... 110. .... ...0 ....
1151
  //  size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
1152
  //  o3<4> = 0 | Pd<3:0>
1153

1154
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1155
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
1156
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1157

1158
  Emit(FCMUO_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
1159
}
1160

1161
// SVEFPCompareWithZero.
1162

1163
void Assembler::fcmeq(const PRegisterWithLaneSize& pd,
1164
                      const PRegisterZ& pg,
1165
                      const ZRegister& zn,
1166
                      double zero) {
1167
  // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1168
  //  0110 0101 ..01 0010 001. .... ...0 ....
1169
  //  size<23:22> | eq<17> = 1 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1170
  //  Pd<3:0>
1171

1172
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1173
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1174
  VIXL_ASSERT(zero == 0.0);
1175
  USE(zero);
1176

1177
  Emit(FCMEQ_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1178
}
1179

1180
void Assembler::fcmge(const PRegisterWithLaneSize& pd,
1181
                      const PRegisterZ& pg,
1182
                      const ZRegister& zn,
1183
                      double zero) {
1184
  // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1185
  //  0110 0101 ..01 0000 001. .... ...0 ....
1186
  //  size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1187
  //  Pd<3:0>
1188

1189
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1190
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1191
  VIXL_ASSERT(zero == 0.0);
1192
  USE(zero);
1193

1194
  Emit(FCMGE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1195
}
1196

1197
void Assembler::fcmgt(const PRegisterWithLaneSize& pd,
1198
                      const PRegisterZ& pg,
1199
                      const ZRegister& zn,
1200
                      double zero) {
1201
  // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1202
  //  0110 0101 ..01 0000 001. .... ...1 ....
1203
  //  size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
1204
  //  Pd<3:0>
1205

1206
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1207
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1208
  VIXL_ASSERT(zero == 0.0);
1209
  USE(zero);
1210

1211
  Emit(FCMGT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1212
}
1213

1214
void Assembler::fcmle(const PRegisterWithLaneSize& pd,
1215
                      const PRegisterZ& pg,
1216
                      const ZRegister& zn,
1217
                      double zero) {
1218
  // FCMLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1219
  //  0110 0101 ..01 0001 001. .... ...1 ....
1220
  //  size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
1221
  //  Pd<3:0>
1222

1223
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1224
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1225
  VIXL_ASSERT(zero == 0.0);
1226
  USE(zero);
1227

1228
  Emit(FCMLE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1229
}
1230

1231
void Assembler::fcmlt(const PRegisterWithLaneSize& pd,
1232
                      const PRegisterZ& pg,
1233
                      const ZRegister& zn,
1234
                      double zero) {
1235
  // FCMLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1236
  //  0110 0101 ..01 0001 001. .... ...0 ....
1237
  //  size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1238
  //  Pd<3:0>
1239

1240
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1241
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1242
  VIXL_ASSERT(zero == 0.0);
1243
  USE(zero);
1244

1245
  Emit(FCMLT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1246
}
1247

1248
void Assembler::fcmne(const PRegisterWithLaneSize& pd,
1249
                      const PRegisterZ& pg,
1250
                      const ZRegister& zn,
1251
                      double zero) {
1252
  // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0
1253
  //  0110 0101 ..01 0011 001. .... ...0 ....
1254
  //  size<23:22> | eq<17> = 1 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
1255
  //  Pd<3:0>
1256

1257
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1258
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1259
  VIXL_ASSERT(zero == 0.0);
1260
  USE(zero);
1261

1262
  Emit(FCMNE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn));
1263
}
1264

1265
// SVEFPComplexAddition.
1266

1267
void Assembler::fcadd(const ZRegister& zd,
1268
                      const PRegisterM& pg,
1269
                      const ZRegister& zn,
1270
                      const ZRegister& zm,
1271
                      int rot) {
1272
  // FCADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>, <const>
1273
  //  0110 0100 ..00 000. 100. .... .... ....
1274
  //  size<23:22> | rot<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1275

1276
  USE(zn);
1277
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1278
  VIXL_ASSERT(zd.Is(zn));
1279
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1280
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1281
  VIXL_ASSERT((rot == 90) || (rot == 270));
1282

1283
  Instr rotate_bit = (rot == 90) ? 0 : (1 << 16);
1284
  Emit(FCADD_z_p_zz | rotate_bit | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
1285
}
1286

1287
// SVEFPComplexMulAdd.
1288

1289
void Assembler::fcmla(const ZRegister& zda,
1290
                      const PRegisterM& pg,
1291
                      const ZRegister& zn,
1292
                      const ZRegister& zm,
1293
                      int rot) {
1294
  // FCMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>, <const>
1295
  //  0110 0100 ..0. .... 0... .... .... ....
1296
  //  size<23:22> | Zm<20:16> | rot<14:13> | Pg<12:10> | Zn<9:5> | Zda<4:0>
1297

1298
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1299
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1300
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1301
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
1302

1303
  Instr rotate_bit = (rot / 90) << 13;
1304
  Emit(FCMLA_z_p_zzz | rotate_bit | SVESize(zda) | Rd(zda) | PgLow8(pg) |
1305
       Rn(zn) | Rm(zm));
1306
}
1307

1308
// SVEFPComplexMulAddIndex.
1309

1310
void Assembler::fcmla(const ZRegister& zda,
1311
                      const ZRegister& zn,
1312
                      const ZRegister& zm,
1313
                      int index,
1314
                      int rot) {
1315
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1316
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1317
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
1318

1319
  Instr rotate_bit = (rot / 90) << 10;
1320
  Emit(FCMLA_z_zzzi_h | SVEMulComplexIndexHelper(zm, index) | rotate_bit |
1321
       Rd(zda) | Rn(zn));
1322
}
1323

1324
// SVEFPFastReduction.
1325

1326
void Assembler::faddv(const VRegister& vd,
1327
                      const PRegister& pg,
1328
                      const ZRegister& zn) {
1329
  // FADDV <V><d>, <Pg>, <Zn>.<T>
1330
  //  0110 0101 ..00 0000 001. .... .... ....
1331
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1332

1333
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1334
  VIXL_ASSERT(vd.IsScalar());
1335
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1336
  VIXL_ASSERT(AreSameLaneSize(zn, vd));
1337

1338
  Emit(FADDV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1339
}
1340

1341
void Assembler::fmaxnmv(const VRegister& vd,
1342
                        const PRegister& pg,
1343
                        const ZRegister& zn) {
1344
  // FMAXNMV <V><d>, <Pg>, <Zn>.<T>
1345
  //  0110 0101 ..00 0100 001. .... .... ....
1346
  //  size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1347

1348
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1349
  VIXL_ASSERT(vd.IsScalar());
1350
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1351
  VIXL_ASSERT(AreSameLaneSize(zn, vd));
1352

1353
  Emit(FMAXNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1354
}
1355

1356
void Assembler::fmaxv(const VRegister& vd,
1357
                      const PRegister& pg,
1358
                      const ZRegister& zn) {
1359
  // FMAXV <V><d>, <Pg>, <Zn>.<T>
1360
  //  0110 0101 ..00 0110 001. .... .... ....
1361
  //  size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1362

1363
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1364
  VIXL_ASSERT(vd.IsScalar());
1365
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1366
  VIXL_ASSERT(AreSameLaneSize(zn, vd));
1367

1368
  Emit(FMAXV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1369
}
1370

1371
void Assembler::fminnmv(const VRegister& vd,
1372
                        const PRegister& pg,
1373
                        const ZRegister& zn) {
1374
  // FMINNMV <V><d>, <Pg>, <Zn>.<T>
1375
  //  0110 0101 ..00 0101 001. .... .... ....
1376
  //  size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1377

1378
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1379
  VIXL_ASSERT(vd.IsScalar());
1380
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1381
  VIXL_ASSERT(AreSameLaneSize(zn, vd));
1382

1383
  Emit(FMINNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1384
}
1385

1386
void Assembler::fminv(const VRegister& vd,
1387
                      const PRegister& pg,
1388
                      const ZRegister& zn) {
1389
  // FMINV <V><d>, <Pg>, <Zn>.<T>
1390
  //  0110 0101 ..00 0111 001. .... .... ....
1391
  //  size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Vd<4:0>
1392

1393
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1394
  VIXL_ASSERT(vd.IsScalar());
1395
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1396
  VIXL_ASSERT(AreSameLaneSize(zn, vd));
1397

1398
  Emit(FMINV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
1399
}
1400

1401
// SVEFPMulAdd.
1402

1403
void Assembler::fmad(const ZRegister& zdn,
1404
                     const PRegisterM& pg,
1405
                     const ZRegister& zm,
1406
                     const ZRegister& za) {
1407
  // FMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1408
  //  0110 0101 ..1. .... 100. .... .... ....
1409
  //  size<23:22> | Za<20:16> | opc<14:13> = 00 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1410

1411
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1412
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1413
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1414

1415
  Emit(FMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1416
}
1417

1418
void Assembler::fmla(const ZRegister& zda,
1419
                     const PRegisterM& pg,
1420
                     const ZRegister& zn,
1421
                     const ZRegister& zm) {
1422
  // FMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1423
  //  0110 0101 ..1. .... 000. .... .... ....
1424
  //  size<23:22> | Zm<20:16> | opc<14:13> = 00 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1425

1426
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1427
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1428
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1429

1430
  Emit(FMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1431
}
1432

1433
void Assembler::fmls(const ZRegister& zda,
1434
                     const PRegisterM& pg,
1435
                     const ZRegister& zn,
1436
                     const ZRegister& zm) {
1437
  // FMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1438
  //  0110 0101 ..1. .... 001. .... .... ....
1439
  //  size<23:22> | Zm<20:16> | opc<14:13> = 01 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1440

1441
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1442
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1443
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1444

1445
  Emit(FMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1446
}
1447

1448
void Assembler::fmsb(const ZRegister& zdn,
1449
                     const PRegisterM& pg,
1450
                     const ZRegister& zm,
1451
                     const ZRegister& za) {
1452
  // FMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1453
  //  0110 0101 ..1. .... 101. .... .... ....
1454
  //  size<23:22> | Za<20:16> | opc<14:13> = 01 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1455

1456
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1457
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1458
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1459

1460
  Emit(FMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1461
}
1462

1463
void Assembler::fnmad(const ZRegister& zdn,
1464
                      const PRegisterM& pg,
1465
                      const ZRegister& zm,
1466
                      const ZRegister& za) {
1467
  // FNMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1468
  //  0110 0101 ..1. .... 110. .... .... ....
1469
  //  size<23:22> | Za<20:16> | opc<14:13> = 10 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1470

1471
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1472
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1473
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1474

1475
  Emit(FNMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1476
}
1477

1478
void Assembler::fnmla(const ZRegister& zda,
1479
                      const PRegisterM& pg,
1480
                      const ZRegister& zn,
1481
                      const ZRegister& zm) {
1482
  // FNMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1483
  //  0110 0101 ..1. .... 010. .... .... ....
1484
  //  size<23:22> | Zm<20:16> | opc<14:13> = 10 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1485

1486
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1487
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1488
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1489

1490
  Emit(FNMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1491
}
1492

1493
void Assembler::fnmls(const ZRegister& zda,
1494
                      const PRegisterM& pg,
1495
                      const ZRegister& zn,
1496
                      const ZRegister& zm) {
1497
  // FNMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
1498
  //  0110 0101 ..1. .... 011. .... .... ....
1499
  //  size<23:22> | Zm<20:16> | opc<14:13> = 11 | Pg<12:10> | Zn<9:5> | Zda<4:0>
1500

1501
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1502
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1503
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
1504

1505
  Emit(FNMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
1506
}
1507

1508
void Assembler::fnmsb(const ZRegister& zdn,
1509
                      const PRegisterM& pg,
1510
                      const ZRegister& zm,
1511
                      const ZRegister& za) {
1512
  // FNMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
1513
  //  0110 0101 ..1. .... 111. .... .... ....
1514
  //  size<23:22> | Za<20:16> | opc<14:13> = 11 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
1515

1516
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1517
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
1518
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
1519

1520
  Emit(FNMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za));
1521
}
1522

1523
Instr Assembler::SVEMulIndexHelper(unsigned lane_size_in_bytes_log2,
1524
                                   const ZRegister& zm,
1525
                                   int index,
1526
                                   Instr op_h,
1527
                                   Instr op_s,
1528
                                   Instr op_d) {
1529
  Instr size = lane_size_in_bytes_log2 << SVESize_offset;
1530
  Instr zm_with_index = Rm(zm);
1531
  Instr op = 0xffffffff;
1532
  // Allowable register number and lane index depends on the lane size.
1533
  switch (lane_size_in_bytes_log2) {
1534
    case kHRegSizeInBytesLog2:
1535
      VIXL_ASSERT(zm.GetCode() <= 7);
1536
      VIXL_ASSERT(IsUint3(index));
1537
      // For H-sized lanes, size is encoded as 0b0x, where x is used as the top
1538
      // bit of the index. So, if index is less than four, the top bit of index
1539
      // is zero, and therefore size is 0b00. Otherwise, it's 0b01, the usual
1540
      // encoding for H-sized lanes.
1541
      if (index < 4) size = 0;
1542
      // Top two bits of "zm" encode the index.
1543
      zm_with_index |= (index & 3) << (Rm_offset + 3);
1544
      op = op_h;
1545
      break;
1546
    case kSRegSizeInBytesLog2:
1547
      VIXL_CHECK(zm.GetCode() <= 7);
1548
      VIXL_CHECK(IsUint2(index));
1549
      // Top two bits of "zm" encode the index.
1550
      zm_with_index |= (index & 3) << (Rm_offset + 3);
1551
      op = op_s;
1552
      break;
1553
    case kDRegSizeInBytesLog2:
1554
      VIXL_CHECK(zm.GetCode() <= 15);
1555
      VIXL_CHECK(IsUint1(index));
1556
      // Top bit of "zm" encodes the index.
1557
      zm_with_index |= (index & 1) << (Rm_offset + 4);
1558
      op = op_d;
1559
      break;
1560
    default:
1561
      VIXL_UNIMPLEMENTED();
1562
  }
1563
  return op | zm_with_index | size;
1564
}
1565

1566
Instr Assembler::SVEMulLongIndexHelper(const ZRegister& zm, int index) {
1567
  Instr imm_field;
1568
  Instr zm_id;
1569
  if (zm.IsLaneSizeH()) {
1570
    VIXL_CHECK(zm.GetCode() <= 7);
1571
    VIXL_CHECK(IsUint3(index));
1572
    imm_field = ExtractUnsignedBitfield32(2, 1, index) << 19;
1573
    zm_id = Rx<18, 16>(zm);
1574
  } else {
1575
    VIXL_ASSERT(zm.IsLaneSizeS());
1576
    VIXL_CHECK(zm.GetCode() <= 15);
1577
    VIXL_CHECK(IsUint2(index));
1578
    imm_field = ExtractBit(index, 1) << 20;
1579
    zm_id = Rx<19, 16>(zm);
1580
  }
1581

1582
  // Synthesize the low part of immediate encoding.
1583
  imm_field |= ExtractBit(index, 0) << 11;
1584

1585
  return zm_id | imm_field;
1586
}
1587

1588
Instr Assembler::SVEMulComplexIndexHelper(const ZRegister& zm, int index) {
1589
  Instr zm_idx_size;
1590
  if (zm.IsLaneSizeH()) {
1591
    // Zm<18:16> | i2<20:19>
1592
    VIXL_CHECK(zm.GetCode() <= 7);
1593
    VIXL_CHECK(IsUint2(index));
1594
    zm_idx_size = (index << 19) | Rx<18, 16>(zm) | 0;
1595
  } else {
1596
    VIXL_ASSERT(zm.IsLaneSizeS());
1597
    // Zm<19:16> | i1<20>
1598
    VIXL_CHECK(zm.GetCode() <= 15);
1599
    VIXL_CHECK(IsUint1(index));
1600
    zm_idx_size = (index << 20) | Rx<19, 16>(zm) | (1 << 22);
1601
  }
1602
  return zm_idx_size;
1603
}
1604

1605
// SVEFPMulAddIndex.
1606

1607
void Assembler::fmla(const ZRegister& zda,
1608
                     const ZRegister& zn,
1609
                     const ZRegister& zm,
1610
                     int index) {
1611
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1612
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1613

1614
  // The encoding of opcode, index, Zm, and size are synthesized in this
1615
  // variable.
1616
  Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
1617
                                           zm,
1618
                                           index,
1619
                                           FMLA_z_zzzi_h,
1620
                                           FMLA_z_zzzi_s,
1621
                                           FMLA_z_zzzi_d);
1622

1623
  Emit(synthesized_op | Rd(zda) | Rn(zn));
1624
}
1625

1626
void Assembler::fmls(const ZRegister& zda,
1627
                     const ZRegister& zn,
1628
                     const ZRegister& zm,
1629
                     int index) {
1630
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1631
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
1632

1633
  // The encoding of opcode, index, Zm, and size are synthesized in this
1634
  // variable.
1635
  Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
1636
                                           zm,
1637
                                           index,
1638
                                           FMLS_z_zzzi_h,
1639
                                           FMLS_z_zzzi_s,
1640
                                           FMLS_z_zzzi_d);
1641

1642
  Emit(synthesized_op | Rd(zda) | Rn(zn));
1643
}
1644

1645
// SVEFPMulIndex.
1646

1647
// This prototype maps to 3 instruction encodings:
1648
void Assembler::fmul(const ZRegister& zd,
1649
                     const ZRegister& zn,
1650
                     const ZRegister& zm,
1651
                     unsigned index) {
1652
  // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>[<imm>]
1653
  //  0110 0100 ..1. .... 0010 00.. .... ....
1654
  //  size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0>
1655

1656
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1657
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
1658

1659
  // The encoding of opcode, index, Zm, and size are synthesized in this
1660
  // variable.
1661
  Instr synthesized_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(),
1662
                                           zm,
1663
                                           index,
1664
                                           FMUL_z_zzi_h,
1665
                                           FMUL_z_zzi_s,
1666
                                           FMUL_z_zzi_d);
1667

1668
  Emit(synthesized_op | Rd(zd) | Rn(zn));
1669
}
1670

1671
// SVEFPUnaryOpPredicated.
1672

1673
void Assembler::fcvt(const ZRegister& zd,
1674
                     const PRegisterM& pg,
1675
                     const ZRegister& zn) {
1676
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1677

1678
  Instr op = 0xffffffff;
1679
  switch (zn.GetLaneSizeInBytes()) {
1680
    case kHRegSizeInBytes:
1681
      switch (zd.GetLaneSizeInBytes()) {
1682
        case kSRegSizeInBytes:
1683
          op = FCVT_z_p_z_h2s;
1684
          break;
1685
        case kDRegSizeInBytes:
1686
          op = FCVT_z_p_z_h2d;
1687
          break;
1688
      }
1689
      break;
1690
    case kSRegSizeInBytes:
1691
      switch (zd.GetLaneSizeInBytes()) {
1692
        case kHRegSizeInBytes:
1693
          op = FCVT_z_p_z_s2h;
1694
          break;
1695
        case kDRegSizeInBytes:
1696
          op = FCVT_z_p_z_s2d;
1697
          break;
1698
      }
1699
      break;
1700
    case kDRegSizeInBytes:
1701
      switch (zd.GetLaneSizeInBytes()) {
1702
        case kHRegSizeInBytes:
1703
          op = FCVT_z_p_z_d2h;
1704
          break;
1705
        case kSRegSizeInBytes:
1706
          op = FCVT_z_p_z_d2s;
1707
          break;
1708
      }
1709
      break;
1710
  }
1711
  VIXL_ASSERT(op != 0xffffffff);
1712

1713
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1714
}
1715

1716
void Assembler::fcvtzs(const ZRegister& zd,
1717
                       const PRegisterM& pg,
1718
                       const ZRegister& zn) {
1719
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1720
  Instr op = 0xffffffff;
1721
  switch (zn.GetLaneSizeInBytes()) {
1722
    case kHRegSizeInBytes:
1723
      switch (zd.GetLaneSizeInBytes()) {
1724
        case kHRegSizeInBytes:
1725
          op = FCVTZS_z_p_z_fp162h;
1726
          break;
1727
        case kSRegSizeInBytes:
1728
          op = FCVTZS_z_p_z_fp162w;
1729
          break;
1730
        case kDRegSizeInBytes:
1731
          op = FCVTZS_z_p_z_fp162x;
1732
          break;
1733
      }
1734
      break;
1735
    case kSRegSizeInBytes:
1736
      switch (zd.GetLaneSizeInBytes()) {
1737
        case kSRegSizeInBytes:
1738
          op = FCVTZS_z_p_z_s2w;
1739
          break;
1740
        case kDRegSizeInBytes:
1741
          op = FCVTZS_z_p_z_s2x;
1742
          break;
1743
      }
1744
      break;
1745
    case kDRegSizeInBytes:
1746
      switch (zd.GetLaneSizeInBytes()) {
1747
        case kSRegSizeInBytes:
1748
          op = FCVTZS_z_p_z_d2w;
1749
          break;
1750
        case kDRegSizeInBytes:
1751
          op = FCVTZS_z_p_z_d2x;
1752
          break;
1753
      }
1754
      break;
1755
  }
1756
  VIXL_ASSERT(op != 0xffffffff);
1757

1758
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1759
}
1760

1761
void Assembler::fcvtzu(const ZRegister& zd,
1762
                       const PRegisterM& pg,
1763
                       const ZRegister& zn) {
1764
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1765
  Instr op = 0xffffffff;
1766
  switch (zn.GetLaneSizeInBytes()) {
1767
    case kHRegSizeInBytes:
1768
      switch (zd.GetLaneSizeInBytes()) {
1769
        case kHRegSizeInBytes:
1770
          op = FCVTZU_z_p_z_fp162h;
1771
          break;
1772
        case kSRegSizeInBytes:
1773
          op = FCVTZU_z_p_z_fp162w;
1774
          break;
1775
        case kDRegSizeInBytes:
1776
          op = FCVTZU_z_p_z_fp162x;
1777
          break;
1778
      }
1779
      break;
1780
    case kSRegSizeInBytes:
1781
      switch (zd.GetLaneSizeInBytes()) {
1782
        case kSRegSizeInBytes:
1783
          op = FCVTZU_z_p_z_s2w;
1784
          break;
1785
        case kDRegSizeInBytes:
1786
          op = FCVTZU_z_p_z_s2x;
1787
          break;
1788
      }
1789
      break;
1790
    case kDRegSizeInBytes:
1791
      switch (zd.GetLaneSizeInBytes()) {
1792
        case kSRegSizeInBytes:
1793
          op = FCVTZU_z_p_z_d2w;
1794
          break;
1795
        case kDRegSizeInBytes:
1796
          op = FCVTZU_z_p_z_d2x;
1797
          break;
1798
      }
1799
      break;
1800
  }
1801
  VIXL_ASSERT(op != 0xffffffff);
1802

1803
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1804
}
1805

1806
void Assembler::frecpx(const ZRegister& zd,
1807
                       const PRegisterM& pg,
1808
                       const ZRegister& zn) {
1809
  // FRECPX <Zd>.<T>, <Pg>/M, <Zn>.<T>
1810
  //  0110 0101 ..00 1100 101. .... .... ....
1811
  //  size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0>
1812

1813
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1814
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1815
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1816

1817
  Emit(FRECPX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1818
}
1819

1820
void Assembler::frinta(const ZRegister& zd,
1821
                       const PRegisterM& pg,
1822
                       const ZRegister& zn) {
1823
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1824
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1825
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1826

1827
  Emit(FRINTA_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1828
}
1829

1830
void Assembler::frinti(const ZRegister& zd,
1831
                       const PRegisterM& pg,
1832
                       const ZRegister& zn) {
1833
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1834
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1835
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1836

1837
  Emit(FRINTI_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1838
}
1839

1840
void Assembler::frintm(const ZRegister& zd,
1841
                       const PRegisterM& pg,
1842
                       const ZRegister& zn) {
1843
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1844
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1845
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1846

1847
  Emit(FRINTM_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1848
}
1849

1850
void Assembler::frintn(const ZRegister& zd,
1851
                       const PRegisterM& pg,
1852
                       const ZRegister& zn) {
1853
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1854
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1855
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1856

1857
  Emit(FRINTN_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1858
}
1859

1860
void Assembler::frintp(const ZRegister& zd,
1861
                       const PRegisterM& pg,
1862
                       const ZRegister& zn) {
1863
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1864
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1865
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1866

1867
  Emit(FRINTP_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1868
}
1869

1870
void Assembler::frintx(const ZRegister& zd,
1871
                       const PRegisterM& pg,
1872
                       const ZRegister& zn) {
1873
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1874
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1875
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1876

1877
  Emit(FRINTX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1878
}
1879

1880
void Assembler::frintz(const ZRegister& zd,
1881
                       const PRegisterM& pg,
1882
                       const ZRegister& zn) {
1883
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1884
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1885
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1886

1887
  Emit(FRINTZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1888
}
1889

1890
void Assembler::fsqrt(const ZRegister& zd,
1891
                      const PRegisterM& pg,
1892
                      const ZRegister& zn) {
1893
  // FSQRT <Zd>.<T>, <Pg>/M, <Zn>.<T>
1894
  //  0110 0101 ..00 1101 101. .... .... ....
1895
  //  size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0>
1896

1897
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1898
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
1899
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
1900

1901
  Emit(FSQRT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
1902
}
1903

1904
void Assembler::scvtf(const ZRegister& zd,
1905
                      const PRegisterM& pg,
1906
                      const ZRegister& zn) {
1907
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1908
  Instr op = 0xffffffff;
1909
  switch (zn.GetLaneSizeInBytes()) {
1910
    case kHRegSizeInBytes:
1911
      switch (zd.GetLaneSizeInBytes()) {
1912
        case kHRegSizeInBytes:
1913
          op = SCVTF_z_p_z_h2fp16;
1914
          break;
1915
      }
1916
      break;
1917
    case kSRegSizeInBytes:
1918
      switch (zd.GetLaneSizeInBytes()) {
1919
        case kHRegSizeInBytes:
1920
          op = SCVTF_z_p_z_w2fp16;
1921
          break;
1922
        case kSRegSizeInBytes:
1923
          op = SCVTF_z_p_z_w2s;
1924
          break;
1925
        case kDRegSizeInBytes:
1926
          op = SCVTF_z_p_z_w2d;
1927
          break;
1928
      }
1929
      break;
1930
    case kDRegSizeInBytes:
1931
      switch (zd.GetLaneSizeInBytes()) {
1932
        case kHRegSizeInBytes:
1933
          op = SCVTF_z_p_z_x2fp16;
1934
          break;
1935
        case kSRegSizeInBytes:
1936
          op = SCVTF_z_p_z_x2s;
1937
          break;
1938
        case kDRegSizeInBytes:
1939
          op = SCVTF_z_p_z_x2d;
1940
          break;
1941
      }
1942
      break;
1943
  }
1944
  VIXL_ASSERT(op != 0xffffffff);
1945

1946
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1947
}
1948

1949
void Assembler::ucvtf(const ZRegister& zd,
1950
                      const PRegisterM& pg,
1951
                      const ZRegister& zn) {
1952
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
1953
  Instr op = 0xffffffff;
1954
  switch (zn.GetLaneSizeInBytes()) {
1955
    case kHRegSizeInBytes:
1956
      switch (zd.GetLaneSizeInBytes()) {
1957
        case kHRegSizeInBytes:
1958
          op = UCVTF_z_p_z_h2fp16;
1959
          break;
1960
      }
1961
      break;
1962
    case kSRegSizeInBytes:
1963
      switch (zd.GetLaneSizeInBytes()) {
1964
        case kHRegSizeInBytes:
1965
          op = UCVTF_z_p_z_w2fp16;
1966
          break;
1967
        case kSRegSizeInBytes:
1968
          op = UCVTF_z_p_z_w2s;
1969
          break;
1970
        case kDRegSizeInBytes:
1971
          op = UCVTF_z_p_z_w2d;
1972
          break;
1973
      }
1974
      break;
1975
    case kDRegSizeInBytes:
1976
      switch (zd.GetLaneSizeInBytes()) {
1977
        case kHRegSizeInBytes:
1978
          op = UCVTF_z_p_z_x2fp16;
1979
          break;
1980
        case kSRegSizeInBytes:
1981
          op = UCVTF_z_p_z_x2s;
1982
          break;
1983
        case kDRegSizeInBytes:
1984
          op = UCVTF_z_p_z_x2d;
1985
          break;
1986
      }
1987
      break;
1988
  }
1989
  VIXL_ASSERT(op != 0xffffffff);
1990

1991
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
1992
}
1993

1994
// SVEFPUnaryOpUnpredicated.
1995

1996
void Assembler::frecpe(const ZRegister& zd, const ZRegister& zn) {
1997
  // FRECPE <Zd>.<T>, <Zn>.<T>
1998
  //  0110 0101 ..00 1110 0011 00.. .... ....
1999
  //  size<23:22> | opc<18:16> = 110 | Zn<9:5> | Zd<4:0>
2000

2001
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2002
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
2003
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
2004

2005
  Emit(FRECPE_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
2006
}
2007

2008
void Assembler::frsqrte(const ZRegister& zd, const ZRegister& zn) {
2009
  // FRSQRTE <Zd>.<T>, <Zn>.<T>
2010
  //  0110 0101 ..00 1111 0011 00.. .... ....
2011
  //  size<23:22> | opc<18:16> = 111 | Zn<9:5> | Zd<4:0>
2012

2013
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2014
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
2015
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
2016

2017
  Emit(FRSQRTE_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
2018
}
2019

2020
// SVEIncDecByPredicateCount.
2021

2022
void Assembler::decp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2023
  // DECP <Xdn>, <Pg>.<T>
2024
  //  0010 0101 ..10 1101 1000 100. .... ....
2025
  //  size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> |
2026
  //  Rdn<4:0>
2027

2028
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2029
  VIXL_ASSERT(rdn.IsX());
2030

2031
  Emit(DECP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2032
}
2033

2034
void Assembler::decp(const ZRegister& zdn, const PRegister& pg) {
2035
  // DECP <Zdn>.<T>, <Pg>
2036
  //  0010 0101 ..10 1101 1000 000. .... ....
2037
  //  size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> |
2038
  //  Zdn<4:0>
2039

2040
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2041
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2042
  VIXL_ASSERT(pg.IsUnqualified());
2043

2044
  Emit(DECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2045
}
2046

2047
void Assembler::incp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2048
  // INCP <Xdn>, <Pg>.<T>
2049
  //  0010 0101 ..10 1100 1000 100. .... ....
2050
  //  size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> |
2051
  //  Rdn<4:0>
2052

2053
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2054
  VIXL_ASSERT(rdn.IsX());
2055

2056
  Emit(INCP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2057
}
2058

2059
void Assembler::incp(const ZRegister& zdn, const PRegister& pg) {
2060
  // INCP <Zdn>.<T>, <Pg>
2061
  //  0010 0101 ..10 1100 1000 000. .... ....
2062
  //  size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> |
2063
  //  Zdn<4:0>
2064

2065
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2066
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2067
  VIXL_ASSERT(pg.IsUnqualified());
2068

2069
  Emit(INCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2070
}
2071

2072
void Assembler::sqdecp(const Register& xd,
2073
                       const PRegisterWithLaneSize& pg,
2074
                       const Register& wn) {
2075
  // SQDECP <Xdn>, <Pg>.<T>, <Wdn>
2076
  //  0010 0101 ..10 1010 1000 100. .... ....
2077
  //  size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2078
  //  Rdn<4:0>
2079

2080
  USE(wn);
2081
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2082
  VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn));
2083

2084
  Emit(SQDECP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg));
2085
}
2086

2087
void Assembler::sqdecp(const Register& xdn, const PRegisterWithLaneSize& pg) {
2088
  // SQDECP <Xdn>, <Pg>.<T>
2089
  //  0010 0101 ..10 1010 1000 110. .... ....
2090
  //  size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> |
2091
  //  Rdn<4:0>
2092

2093
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2094
  VIXL_ASSERT(xdn.IsX());
2095

2096
  Emit(SQDECP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg));
2097
}
2098

2099
void Assembler::sqdecp(const ZRegister& zdn, const PRegister& pg) {
2100
  // SQDECP <Zdn>.<T>, <Pg>
2101
  //  0010 0101 ..10 1010 1000 000. .... ....
2102
  //  size<23:22> | D<17> = 1 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2103

2104
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2105
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2106
  VIXL_ASSERT(pg.IsUnqualified());
2107

2108
  Emit(SQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2109
}
2110

2111
void Assembler::sqincp(const Register& xd,
2112
                       const PRegisterWithLaneSize& pg,
2113
                       const Register& wn) {
2114
  // SQINCP <Xdn>, <Pg>.<T>, <Wdn>
2115
  //  0010 0101 ..10 1000 1000 100. .... ....
2116
  //  size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2117
  //  Rdn<4:0>
2118

2119
  USE(wn);
2120
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2121
  VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn));
2122

2123
  Emit(SQINCP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg));
2124
}
2125

2126
void Assembler::sqincp(const Register& xdn, const PRegisterWithLaneSize& pg) {
2127
  // SQINCP <Xdn>, <Pg>.<T>
2128
  //  0010 0101 ..10 1000 1000 110. .... ....
2129
  //  size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> |
2130
  //  Rdn<4:0>
2131

2132
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2133
  VIXL_ASSERT(xdn.IsX());
2134

2135
  Emit(SQINCP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg));
2136
}
2137

2138
void Assembler::sqincp(const ZRegister& zdn, const PRegister& pg) {
2139
  // SQINCP <Zdn>.<T>, <Pg>
2140
  //  0010 0101 ..10 1000 1000 000. .... ....
2141
  //  size<23:22> | D<17> = 0 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2142

2143
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2144
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2145
  VIXL_ASSERT(pg.IsUnqualified());
2146

2147
  Emit(SQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2148
}
2149

2150
void Assembler::uqdecp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2151
  // UQDECP <Wdn>, <Pg>.<T>
2152
  // UQDECP <Xdn>, <Pg>.<T>
2153
  //  0010 0101 ..10 1011 1000 10.. .... ....
2154
  //  size<23:22> | D<17> = 1 | U<16> = 1 | sf<10> | op<9> = 0 | Pg<8:5> |
2155
  //  Rdn<4:0>
2156

2157
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2158

2159
  Instr op = rdn.IsX() ? UQDECP_r_p_r_x : UQDECP_r_p_r_uw;
2160
  Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2161
}
2162

2163
void Assembler::uqdecp(const ZRegister& zdn, const PRegister& pg) {
2164
  // UQDECP <Zdn>.<T>, <Pg>
2165
  //  0010 0101 ..10 1011 1000 000. .... ....
2166
  //  size<23:22> | D<17> = 1 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2167

2168
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2169
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2170
  VIXL_ASSERT(pg.IsUnqualified());
2171

2172
  Emit(UQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2173
}
2174

2175
void Assembler::uqincp(const Register& rdn, const PRegisterWithLaneSize& pg) {
2176
  // UQINCP <Wdn>, <Pg>.<T>
2177
  //  0010 0101 ..10 1001 1000 100. .... ....
2178
  //  size<23:22> | D<17> = 0 | U<16> = 1 | sf<10> = 0 | op<9> = 0 | Pg<8:5> |
2179
  //  Rdn<4:0>
2180

2181
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2182

2183
  Instr op = rdn.IsX() ? UQINCP_r_p_r_x : UQINCP_r_p_r_uw;
2184
  Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg));
2185
}
2186

2187
void Assembler::uqincp(const ZRegister& zdn, const PRegister& pg) {
2188
  // UQINCP <Zdn>.<T>, <Pg>
2189
  //  0010 0101 ..10 1001 1000 000. .... ....
2190
  //  size<23:22> | D<17> = 0 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0>
2191

2192
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2193
  VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes);
2194
  VIXL_ASSERT(pg.IsUnqualified());
2195

2196
  Emit(UQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg));
2197
}
2198

2199
// SVEIndexGeneration.
2200

2201
void Assembler::index(const ZRegister& zd, int start, int step) {
2202
  // INDEX <Zd>.<T>, #<imm1>, #<imm2>
2203
  //  0000 0100 ..1. .... 0100 00.. .... ....
2204
  //  size<23:22> | step<20:16> | start<9:5> | Zd<4:0>
2205

2206
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2207

2208
  Emit(INDEX_z_ii | SVESize(zd) | ImmField<20, 16>(step) |
2209
       ImmField<9, 5>(start) | Rd(zd));
2210
}
2211

2212
void Assembler::index(const ZRegister& zd,
2213
                      const Register& rn,
2214
                      const Register& rm) {
2215
  // INDEX <Zd>.<T>, <R><n>, <R><m>
2216
  //  0000 0100 ..1. .... 0100 11.. .... ....
2217
  //  size<23:22> | Rm<20:16> | Rn<9:5> | Zd<4:0>
2218

2219
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2220
  VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
2221
              zd.GetLaneSizeInBits());
2222
  VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >=
2223
              zd.GetLaneSizeInBits());
2224

2225
  Emit(INDEX_z_rr | SVESize(zd) | Rd(zd) | Rn(rn) | Rm(rm));
2226
}
2227

2228
void Assembler::index(const ZRegister& zd, const Register& rn, int imm5) {
2229
  // INDEX <Zd>.<T>, <R><n>, #<imm>
2230
  //  0000 0100 ..1. .... 0100 01.. .... ....
2231
  //  size<23:22> | imm5<20:16> | Rn<9:5> | Zd<4:0>
2232

2233
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2234
  VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
2235
              zd.GetLaneSizeInBits());
2236

2237
  Emit(INDEX_z_ri | SVESize(zd) | Rd(zd) | Rn(rn) | ImmField<20, 16>(imm5));
2238
}
2239

2240
void Assembler::index(const ZRegister& zd, int imm5, const Register& rm) {
2241
  // INDEX <Zd>.<T>, #<imm>, <R><m>
2242
  //  0000 0100 ..1. .... 0100 10.. .... ....
2243
  //  size<23:22> | Rm<20:16> | imm5<9:5> | Zd<4:0>
2244

2245
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2246
  VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >=
2247
              zd.GetLaneSizeInBits());
2248

2249
  Emit(INDEX_z_ir | SVESize(zd) | Rd(zd) | ImmField<9, 5>(imm5) | Rm(rm));
2250
}
2251

2252
// SVEIntArithmeticUnpredicated.
2253

2254
void Assembler::add(const ZRegister& zd,
2255
                    const ZRegister& zn,
2256
                    const ZRegister& zm) {
2257
  // ADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2258
  //  0000 0100 ..1. .... 0000 00.. .... ....
2259
  //  size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
2260

2261
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2262
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2263

2264
  Emit(ADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2265
}
2266

2267
void Assembler::sqadd(const ZRegister& zd,
2268
                      const ZRegister& zn,
2269
                      const ZRegister& zm) {
2270
  // SQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2271
  //  0000 0100 ..1. .... 0001 00.. .... ....
2272
  //  size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0>
2273

2274
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2275
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2276

2277
  Emit(SQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2278
}
2279

2280
void Assembler::sqsub(const ZRegister& zd,
2281
                      const ZRegister& zn,
2282
                      const ZRegister& zm) {
2283
  // SQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2284
  //  0000 0100 ..1. .... 0001 10.. .... ....
2285
  //  size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0>
2286

2287
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2288
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2289

2290
  Emit(SQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2291
}
2292

2293
void Assembler::sub(const ZRegister& zd,
2294
                    const ZRegister& zn,
2295
                    const ZRegister& zm) {
2296
  // SUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2297
  //  0000 0100 ..1. .... 0000 01.. .... ....
2298
  //  size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
2299

2300
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2301
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2302

2303
  Emit(SUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2304
}
2305

2306
void Assembler::uqadd(const ZRegister& zd,
2307
                      const ZRegister& zn,
2308
                      const ZRegister& zm) {
2309
  // UQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2310
  //  0000 0100 ..1. .... 0001 01.. .... ....
2311
  //  size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0>
2312

2313
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2314
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2315

2316
  Emit(UQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2317
}
2318

2319
void Assembler::uqsub(const ZRegister& zd,
2320
                      const ZRegister& zn,
2321
                      const ZRegister& zm) {
2322
  // UQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
2323
  //  0000 0100 ..1. .... 0001 11.. .... ....
2324
  //  size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0>
2325

2326
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2327
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
2328

2329
  Emit(UQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
2330
}
2331

2332
// SVEIntBinaryArithmeticPredicated.
2333

2334
void Assembler::add(const ZRegister& zd,
2335
                    const PRegisterM& pg,
2336
                    const ZRegister& zn,
2337
                    const ZRegister& zm) {
2338
  // ADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2339
  //  0000 0100 ..00 0000 000. .... .... ....
2340
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2341

2342
  USE(zn);
2343
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2344
  VIXL_ASSERT(zd.Is(zn));
2345
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2346

2347
  Emit(ADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2348
}
2349

2350
void Assembler::and_(const ZRegister& zd,
2351
                     const PRegisterM& pg,
2352
                     const ZRegister& zn,
2353
                     const ZRegister& zm) {
2354
  // AND <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2355
  //  0000 0100 ..01 1010 000. .... .... ....
2356
  //  size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2357

2358
  USE(zn);
2359
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2360
  VIXL_ASSERT(zd.Is(zn));
2361
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2362

2363
  Emit(AND_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2364
}
2365

2366
void Assembler::bic(const ZRegister& zd,
2367
                    const PRegisterM& pg,
2368
                    const ZRegister& zn,
2369
                    const ZRegister& zm) {
2370
  // BIC <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2371
  //  0000 0100 ..01 1011 000. .... .... ....
2372
  //  size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2373

2374
  USE(zn);
2375
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2376
  VIXL_ASSERT(zd.Is(zn));
2377
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2378

2379
  Emit(BIC_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2380
}
2381

2382
void Assembler::eor(const ZRegister& zd,
2383
                    const PRegisterM& pg,
2384
                    const ZRegister& zn,
2385
                    const ZRegister& zm) {
2386
  // EOR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2387
  //  0000 0100 ..01 1001 000. .... .... ....
2388
  //  size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2389

2390
  USE(zn);
2391
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2392
  VIXL_ASSERT(zd.Is(zn));
2393
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2394

2395
  Emit(EOR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2396
}
2397

2398
void Assembler::mul(const ZRegister& zd,
2399
                    const PRegisterM& pg,
2400
                    const ZRegister& zn,
2401
                    const ZRegister& zm) {
2402
  // MUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2403
  //  0000 0100 ..01 0000 000. .... .... ....
2404
  //  size<23:22> | H<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2405

2406
  USE(zn);
2407
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2408
  VIXL_ASSERT(zd.Is(zn));
2409
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2410

2411
  Emit(MUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2412
}
2413

2414
void Assembler::orr(const ZRegister& zd,
2415
                    const PRegisterM& pg,
2416
                    const ZRegister& zn,
2417
                    const ZRegister& zm) {
2418
  // ORR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2419
  //  0000 0100 ..01 1000 000. .... .... ....
2420
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2421

2422
  USE(zn);
2423
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2424
  VIXL_ASSERT(zd.Is(zn));
2425
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2426

2427
  Emit(ORR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2428
}
2429

2430
void Assembler::sabd(const ZRegister& zd,
2431
                     const PRegisterM& pg,
2432
                     const ZRegister& zn,
2433
                     const ZRegister& zm) {
2434
  // SABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2435
  //  0000 0100 ..00 1100 000. .... .... ....
2436
  //  size<23:22> | opc<18:17> = 10 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2437

2438
  USE(zn);
2439
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2440
  VIXL_ASSERT(zd.Is(zn));
2441
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2442

2443
  Emit(SABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2444
}
2445

2446
void Assembler::sdiv(const ZRegister& zd,
2447
                     const PRegisterM& pg,
2448
                     const ZRegister& zn,
2449
                     const ZRegister& zm) {
2450
  // SDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2451
  //  0000 0100 ..01 0100 000. .... .... ....
2452
  //  size<23:22> | R<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2453

2454
  USE(zn);
2455
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2456
  VIXL_ASSERT(zd.Is(zn));
2457
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2458
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2459

2460
  Emit(SDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2461
}
2462

2463
void Assembler::sdivr(const ZRegister& zd,
2464
                      const PRegisterM& pg,
2465
                      const ZRegister& zn,
2466
                      const ZRegister& zm) {
2467
  // SDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2468
  //  0000 0100 ..01 0110 000. .... .... ....
2469
  //  size<23:22> | R<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2470

2471
  USE(zn);
2472
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2473
  VIXL_ASSERT(zd.Is(zn));
2474
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2475
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2476

2477
  Emit(SDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2478
}
2479

2480
void Assembler::smax(const ZRegister& zd,
2481
                     const PRegisterM& pg,
2482
                     const ZRegister& zn,
2483
                     const ZRegister& zm) {
2484
  // SMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2485
  //  0000 0100 ..00 1000 000. .... .... ....
2486
  //  size<23:22> | opc<18:17> = 00 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2487

2488
  USE(zn);
2489
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2490
  VIXL_ASSERT(zd.Is(zn));
2491
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2492

2493
  Emit(SMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2494
}
2495

2496
void Assembler::smin(const ZRegister& zd,
2497
                     const PRegisterM& pg,
2498
                     const ZRegister& zn,
2499
                     const ZRegister& zm) {
2500
  // SMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2501
  //  0000 0100 ..00 1010 000. .... .... ....
2502
  //  size<23:22> | opc<18:17> = 01 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2503

2504
  USE(zn);
2505
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2506
  VIXL_ASSERT(zd.Is(zn));
2507
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2508

2509
  Emit(SMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2510
}
2511

2512
void Assembler::smulh(const ZRegister& zd,
2513
                      const PRegisterM& pg,
2514
                      const ZRegister& zn,
2515
                      const ZRegister& zm) {
2516
  // SMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2517
  //  0000 0100 ..01 0010 000. .... .... ....
2518
  //  size<23:22> | H<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2519

2520
  USE(zn);
2521
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2522
  VIXL_ASSERT(zd.Is(zn));
2523
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2524

2525
  Emit(SMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2526
}
2527

2528
void Assembler::sub(const ZRegister& zd,
2529
                    const PRegisterM& pg,
2530
                    const ZRegister& zn,
2531
                    const ZRegister& zm) {
2532
  // SUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2533
  //  0000 0100 ..00 0001 000. .... .... ....
2534
  //  size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2535

2536
  USE(zn);
2537
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2538
  VIXL_ASSERT(zd.Is(zn));
2539
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2540

2541
  Emit(SUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2542
}
2543

2544
void Assembler::subr(const ZRegister& zd,
2545
                     const PRegisterM& pg,
2546
                     const ZRegister& zn,
2547
                     const ZRegister& zm) {
2548
  // SUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2549
  //  0000 0100 ..00 0011 000. .... .... ....
2550
  //  size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2551

2552
  USE(zn);
2553
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2554
  VIXL_ASSERT(zd.Is(zn));
2555
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2556

2557
  Emit(SUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2558
}
2559

2560
void Assembler::uabd(const ZRegister& zd,
2561
                     const PRegisterM& pg,
2562
                     const ZRegister& zn,
2563
                     const ZRegister& zm) {
2564
  // UABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2565
  //  0000 0100 ..00 1101 000. .... .... ....
2566
  //  size<23:22> | opc<18:17> = 10 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2567

2568
  USE(zn);
2569
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2570
  VIXL_ASSERT(zd.Is(zn));
2571
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2572

2573
  Emit(UABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2574
}
2575

2576
void Assembler::udiv(const ZRegister& zd,
2577
                     const PRegisterM& pg,
2578
                     const ZRegister& zn,
2579
                     const ZRegister& zm) {
2580
  // UDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2581
  //  0000 0100 ..01 0101 000. .... .... ....
2582
  //  size<23:22> | R<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2583

2584
  USE(zn);
2585
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2586
  VIXL_ASSERT(zd.Is(zn));
2587
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2588
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2589

2590
  Emit(UDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2591
}
2592

2593
void Assembler::udivr(const ZRegister& zd,
2594
                      const PRegisterM& pg,
2595
                      const ZRegister& zn,
2596
                      const ZRegister& zm) {
2597
  // UDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2598
  //  0000 0100 ..01 0111 000. .... .... ....
2599
  //  size<23:22> | R<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2600

2601
  USE(zn);
2602
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2603
  VIXL_ASSERT(zd.Is(zn));
2604
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2605
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
2606

2607
  Emit(UDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2608
}
2609

2610
void Assembler::umax(const ZRegister& zd,
2611
                     const PRegisterM& pg,
2612
                     const ZRegister& zn,
2613
                     const ZRegister& zm) {
2614
  // UMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2615
  //  0000 0100 ..00 1001 000. .... .... ....
2616
  //  size<23:22> | opc<18:17> = 00 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2617

2618
  USE(zn);
2619
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2620
  VIXL_ASSERT(zd.Is(zn));
2621
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2622

2623
  Emit(UMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2624
}
2625

2626
void Assembler::umin(const ZRegister& zd,
2627
                     const PRegisterM& pg,
2628
                     const ZRegister& zn,
2629
                     const ZRegister& zm) {
2630
  // UMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2631
  //  0000 0100 ..00 1011 000. .... .... ....
2632
  //  size<23:22> | opc<18:17> = 01 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2633

2634
  USE(zn);
2635
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2636
  VIXL_ASSERT(zd.Is(zn));
2637
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2638

2639
  Emit(UMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2640
}
2641

2642
void Assembler::umulh(const ZRegister& zd,
2643
                      const PRegisterM& pg,
2644
                      const ZRegister& zn,
2645
                      const ZRegister& zm) {
2646
  // UMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
2647
  //  0000 0100 ..01 0011 000. .... .... ....
2648
  //  size<23:22> | H<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
2649

2650
  USE(zn);
2651
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2652
  VIXL_ASSERT(zd.Is(zn));
2653
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
2654

2655
  Emit(UMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
2656
}
2657

2658
// SVEIntCompareScalars.
2659

2660
void Assembler::ctermeq(const Register& rn, const Register& rm) {
2661
  // CTERMEQ <R><n>, <R><m>
2662
  //  0010 0101 1.1. .... 0010 00.. ...0 0000
2663
  //  op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 0
2664

2665
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2666
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2667
  const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000;
2668

2669
  Emit(CTERMEQ_rr | sz | Rn(rn) | Rm(rm));
2670
}
2671

2672
void Assembler::ctermne(const Register& rn, const Register& rm) {
2673
  // CTERMNE <R><n>, <R><m>
2674
  //  0010 0101 1.1. .... 0010 00.. ...1 0000
2675
  //  op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 1
2676

2677
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2678
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2679
  const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000;
2680

2681
  Emit(CTERMNE_rr | sz | Rn(rn) | Rm(rm));
2682
}
2683

2684
void Assembler::whilele(const PRegisterWithLaneSize& pd,
2685
                        const Register& rn,
2686
                        const Register& rm) {
2687
  // WHILELE <Pd>.<T>, <R><n>, <R><m>
2688
  //  0010 0101 ..1. .... 000. 01.. ...1 ....
2689
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> |
2690
  //  eq<4> = 1 | Pd<3:0>
2691

2692
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2693
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2694
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2695

2696
  Emit(WHILELE_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2697
}
2698

2699
void Assembler::whilelo(const PRegisterWithLaneSize& pd,
2700
                        const Register& rn,
2701
                        const Register& rm) {
2702
  // WHILELO <Pd>.<T>, <R><n>, <R><m>
2703
  //  0010 0101 ..1. .... 000. 11.. ...0 ....
2704
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> |
2705
  //  eq<4> = 0 | Pd<3:0>
2706

2707
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2708
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2709
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2710

2711
  Emit(WHILELO_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2712
}
2713

2714
void Assembler::whilels(const PRegisterWithLaneSize& pd,
2715
                        const Register& rn,
2716
                        const Register& rm) {
2717
  // WHILELS <Pd>.<T>, <R><n>, <R><m>
2718
  //  0010 0101 ..1. .... 000. 11.. ...1 ....
2719
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> |
2720
  //  eq<4> = 1 | Pd<3:0>
2721

2722
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2723
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2724
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2725

2726
  Emit(WHILELS_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2727
}
2728

2729
void Assembler::whilelt(const PRegisterWithLaneSize& pd,
2730
                        const Register& rn,
2731
                        const Register& rm) {
2732
  // WHILELT <Pd>.<T>, <R><n>, <R><m>
2733
  //  0010 0101 ..1. .... 000. 01.. ...0 ....
2734
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> |
2735
  //  eq<4> = 0 | Pd<3:0>
2736

2737
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2738
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
2739
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
2740

2741
  Emit(WHILELT_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
2742
}
2743

2744
void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2745
                               const PRegisterZ& pg,
2746
                               const ZRegister& zn,
2747
                               const ZRegister& zm,
2748
                               SVEIntCompareVectorsOp op) {
2749
  Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
2750
}
2751

2752
void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2753
                               const PRegisterZ& pg,
2754
                               const ZRegister& zn,
2755
                               int imm,
2756
                               SVEIntCompareSignedImmOp op) {
2757
  Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm));
2758
}
2759

2760
void Assembler::CompareVectors(const PRegisterWithLaneSize& pd,
2761
                               const PRegisterZ& pg,
2762
                               const ZRegister& zn,
2763
                               unsigned imm,
2764
                               SVEIntCompareUnsignedImmOp op) {
2765
  Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) |
2766
       ImmUnsignedField<20, 14>(imm));
2767
}
2768

2769
void Assembler::cmp(Condition cond,
2770
                    const PRegisterWithLaneSize& pd,
2771
                    const PRegisterZ& pg,
2772
                    const ZRegister& zn,
2773
                    const ZRegister& zm) {
2774
  switch (cond) {
2775
    case eq:
2776
      cmpeq(pd, pg, zn, zm);
2777
      break;
2778
    case ge:
2779
      cmpge(pd, pg, zn, zm);
2780
      break;
2781
    case gt:
2782
      cmpgt(pd, pg, zn, zm);
2783
      break;
2784
    case le:
2785
      cmple(pd, pg, zn, zm);
2786
      break;
2787
    case lt:
2788
      cmplt(pd, pg, zn, zm);
2789
      break;
2790
    case ne:
2791
      cmpne(pd, pg, zn, zm);
2792
      break;
2793
    case hi:
2794
      cmphi(pd, pg, zn, zm);
2795
      break;
2796
    case hs:
2797
      cmphs(pd, pg, zn, zm);
2798
      break;
2799
    case lo:
2800
      cmplo(pd, pg, zn, zm);
2801
      break;
2802
    case ls:
2803
      cmpls(pd, pg, zn, zm);
2804
      break;
2805
    default:
2806
      VIXL_UNREACHABLE();
2807
  }
2808
}
2809

2810
// SVEIntCompareSignedImm.
2811

2812
void Assembler::cmpeq(const PRegisterWithLaneSize& pd,
2813
                      const PRegisterZ& pg,
2814
                      const ZRegister& zn,
2815
                      int imm5) {
2816
  // CMPEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2817
  //  0010 0101 ..0. .... 100. .... ...0 ....
2818
  //  size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2819
  //  | ne<4> = 0 | Pd<3:0>
2820

2821
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2822
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2823

2824
  CompareVectors(pd, pg, zn, imm5, CMPEQ_p_p_zi);
2825
}
2826

2827
void Assembler::cmpge(const PRegisterWithLaneSize& pd,
2828
                      const PRegisterZ& pg,
2829
                      const ZRegister& zn,
2830
                      int imm5) {
2831
  // CMPGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2832
  //  0010 0101 ..0. .... 000. .... ...0 ....
2833
  //  size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2834
  //  | ne<4> = 0 | Pd<3:0>
2835

2836
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2837
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2838

2839
  CompareVectors(pd, pg, zn, imm5, CMPGE_p_p_zi);
2840
}
2841

2842
void Assembler::cmpgt(const PRegisterWithLaneSize& pd,
2843
                      const PRegisterZ& pg,
2844
                      const ZRegister& zn,
2845
                      int imm5) {
2846
  // CMPGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2847
  //  0010 0101 ..0. .... 000. .... ...1 ....
2848
  //  size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2849
  //  | ne<4> = 1 | Pd<3:0>
2850

2851
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2852
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2853

2854
  CompareVectors(pd, pg, zn, imm5, CMPGT_p_p_zi);
2855
}
2856

2857
void Assembler::cmple(const PRegisterWithLaneSize& pd,
2858
                      const PRegisterZ& pg,
2859
                      const ZRegister& zn,
2860
                      int imm5) {
2861
  // CMPLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2862
  //  0010 0101 ..0. .... 001. .... ...1 ....
2863
  //  size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5>
2864
  //  | ne<4> = 1 | Pd<3:0>
2865

2866
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2867
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2868

2869
  CompareVectors(pd, pg, zn, imm5, CMPLE_p_p_zi);
2870
}
2871

2872
void Assembler::cmplt(const PRegisterWithLaneSize& pd,
2873
                      const PRegisterZ& pg,
2874
                      const ZRegister& zn,
2875
                      int imm5) {
2876
  // CMPLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2877
  //  0010 0101 ..0. .... 001. .... ...0 ....
2878
  //  size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5>
2879
  //  | ne<4> = 0 | Pd<3:0>
2880

2881
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2882
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2883

2884
  CompareVectors(pd, pg, zn, imm5, CMPLT_p_p_zi);
2885
}
2886

2887
void Assembler::cmpne(const PRegisterWithLaneSize& pd,
2888
                      const PRegisterZ& pg,
2889
                      const ZRegister& zn,
2890
                      int imm5) {
2891
  // CMPNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2892
  //  0010 0101 ..0. .... 100. .... ...1 ....
2893
  //  size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5>
2894
  //  | ne<4> = 1 | Pd<3:0>
2895

2896
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2897
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2898

2899
  CompareVectors(pd, pg, zn, imm5, CMPNE_p_p_zi);
2900
}
2901

2902
// SVEIntCompareUnsignedImm.
2903

2904
void Assembler::cmphi(const PRegisterWithLaneSize& pd,
2905
                      const PRegisterZ& pg,
2906
                      const ZRegister& zn,
2907
                      unsigned imm7) {
2908
  // CMPHI <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2909
  //  0010 0100 ..1. .... ..0. .... ...1 ....
2910
  //  size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
2911
  //  Pd<3:0>
2912

2913
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2914
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2915

2916
  CompareVectors(pd, pg, zn, imm7, CMPHI_p_p_zi);
2917
}
2918

2919
void Assembler::cmphs(const PRegisterWithLaneSize& pd,
2920
                      const PRegisterZ& pg,
2921
                      const ZRegister& zn,
2922
                      unsigned imm7) {
2923
  // CMPHS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2924
  //  0010 0100 ..1. .... ..0. .... ...0 ....
2925
  //  size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
2926
  //  Pd<3:0>
2927

2928
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2929
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2930

2931
  CompareVectors(pd, pg, zn, imm7, CMPHS_p_p_zi);
2932
}
2933

2934
void Assembler::cmplo(const PRegisterWithLaneSize& pd,
2935
                      const PRegisterZ& pg,
2936
                      const ZRegister& zn,
2937
                      unsigned imm7) {
2938
  // CMPLO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2939
  //  0010 0100 ..1. .... ..1. .... ...0 ....
2940
  //  size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 |
2941
  //  Pd<3:0>
2942

2943
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2944
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2945

2946
  CompareVectors(pd, pg, zn, imm7, CMPLO_p_p_zi);
2947
}
2948

2949
void Assembler::cmpls(const PRegisterWithLaneSize& pd,
2950
                      const PRegisterZ& pg,
2951
                      const ZRegister& zn,
2952
                      unsigned imm7) {
2953
  // CMPLS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm>
2954
  //  0010 0100 ..1. .... ..1. .... ...1 ....
2955
  //  size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 |
2956
  //  Pd<3:0>
2957

2958
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2959
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2960

2961
  CompareVectors(pd, pg, zn, imm7, CMPLS_p_p_zi);
2962
}
2963

2964
// SVEIntCompareVectors.
2965

2966
// This prototype maps to 2 instruction encodings:
2967
//  CMPEQ_p_p_zw
2968
//  CMPEQ_p_p_zz
2969
void Assembler::cmpeq(const PRegisterWithLaneSize& pd,
2970
                      const PRegisterZ& pg,
2971
                      const ZRegister& zn,
2972
                      const ZRegister& zm) {
2973
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2974
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2975
  SVEIntCompareVectorsOp op = CMPEQ_p_p_zz;
2976
  if (!AreSameLaneSize(zn, zm)) {
2977
    VIXL_ASSERT(zm.IsLaneSizeD());
2978
    op = CMPEQ_p_p_zw;
2979
  }
2980
  CompareVectors(pd, pg, zn, zm, op);
2981
}
2982

2983
// This prototype maps to 2 instruction encodings:
2984
//  CMPGE_p_p_zw
2985
//  CMPGE_p_p_zz
2986
void Assembler::cmpge(const PRegisterWithLaneSize& pd,
2987
                      const PRegisterZ& pg,
2988
                      const ZRegister& zn,
2989
                      const ZRegister& zm) {
2990
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
2991
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
2992
  SVEIntCompareVectorsOp op = CMPGE_p_p_zz;
2993
  if (!AreSameLaneSize(zn, zm)) {
2994
    VIXL_ASSERT(zm.IsLaneSizeD());
2995
    op = CMPGE_p_p_zw;
2996
  }
2997
  CompareVectors(pd, pg, zn, zm, op);
2998
}
2999

3000
// This prototype maps to 2 instruction encodings:
3001
//  CMPGT_p_p_zw
3002
//  CMPGT_p_p_zz
3003
void Assembler::cmpgt(const PRegisterWithLaneSize& pd,
3004
                      const PRegisterZ& pg,
3005
                      const ZRegister& zn,
3006
                      const ZRegister& zm) {
3007
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3008
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3009
  SVEIntCompareVectorsOp op = CMPGT_p_p_zz;
3010
  if (!AreSameLaneSize(zn, zm)) {
3011
    VIXL_ASSERT(zm.IsLaneSizeD());
3012
    op = CMPGT_p_p_zw;
3013
  }
3014
  CompareVectors(pd, pg, zn, zm, op);
3015
}
3016

3017
// This prototype maps to 2 instruction encodings:
3018
//  CMPHI_p_p_zw
3019
//  CMPHI_p_p_zz
3020
void Assembler::cmphi(const PRegisterWithLaneSize& pd,
3021
                      const PRegisterZ& pg,
3022
                      const ZRegister& zn,
3023
                      const ZRegister& zm) {
3024
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3025
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3026
  SVEIntCompareVectorsOp op = CMPHI_p_p_zz;
3027
  if (!AreSameLaneSize(zn, zm)) {
3028
    VIXL_ASSERT(zm.IsLaneSizeD());
3029
    op = CMPHI_p_p_zw;
3030
  }
3031
  CompareVectors(pd, pg, zn, zm, op);
3032
}
3033

3034
// This prototype maps to 2 instruction encodings:
3035
//  CMPHS_p_p_zw
3036
//  CMPHS_p_p_zz
3037
void Assembler::cmphs(const PRegisterWithLaneSize& pd,
3038
                      const PRegisterZ& pg,
3039
                      const ZRegister& zn,
3040
                      const ZRegister& zm) {
3041
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3042
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3043
  SVEIntCompareVectorsOp op = CMPHS_p_p_zz;
3044
  if (!AreSameLaneSize(zn, zm)) {
3045
    VIXL_ASSERT(zm.IsLaneSizeD());
3046
    op = CMPHS_p_p_zw;
3047
  }
3048
  CompareVectors(pd, pg, zn, zm, op);
3049
}
3050

3051
void Assembler::cmple(const PRegisterWithLaneSize& pd,
3052
                      const PRegisterZ& pg,
3053
                      const ZRegister& zn,
3054
                      const ZRegister& zm) {
3055
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3056
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3057
  if (AreSameLaneSize(zn, zm)) {
3058
    cmpge(pd, pg, zm, zn);
3059
    return;
3060
  }
3061
  VIXL_ASSERT(zm.IsLaneSizeD());
3062
  VIXL_ASSERT(!zn.IsLaneSizeD());
3063

3064
  CompareVectors(pd, pg, zn, zm, CMPLE_p_p_zw);
3065
}
3066

3067
void Assembler::cmplo(const PRegisterWithLaneSize& pd,
3068
                      const PRegisterZ& pg,
3069
                      const ZRegister& zn,
3070
                      const ZRegister& zm) {
3071
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3072
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3073
  if (AreSameLaneSize(zn, zm)) {
3074
    cmphi(pd, pg, zm, zn);
3075
    return;
3076
  }
3077
  VIXL_ASSERT(zm.IsLaneSizeD());
3078
  VIXL_ASSERT(!zn.IsLaneSizeD());
3079

3080
  CompareVectors(pd, pg, zn, zm, CMPLO_p_p_zw);
3081
}
3082

3083
void Assembler::cmpls(const PRegisterWithLaneSize& pd,
3084
                      const PRegisterZ& pg,
3085
                      const ZRegister& zn,
3086
                      const ZRegister& zm) {
3087
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3088
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3089
  if (AreSameLaneSize(zn, zm)) {
3090
    cmphs(pd, pg, zm, zn);
3091
    return;
3092
  }
3093
  VIXL_ASSERT(zm.IsLaneSizeD());
3094
  VIXL_ASSERT(!zn.IsLaneSizeD());
3095

3096
  CompareVectors(pd, pg, zn, zm, CMPLS_p_p_zw);
3097
}
3098

3099
void Assembler::cmplt(const PRegisterWithLaneSize& pd,
3100
                      const PRegisterZ& pg,
3101
                      const ZRegister& zn,
3102
                      const ZRegister& zm) {
3103
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3104
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3105
  if (AreSameLaneSize(zn, zm)) {
3106
    cmpgt(pd, pg, zm, zn);
3107
    return;
3108
  }
3109
  VIXL_ASSERT(zm.IsLaneSizeD());
3110
  VIXL_ASSERT(!zn.IsLaneSizeD());
3111

3112
  CompareVectors(pd, pg, zn, zm, CMPLT_p_p_zw);
3113
}
3114

3115
// This prototype maps to 2 instruction encodings:
3116
//  CMPNE_p_p_zw
3117
//  CMPNE_p_p_zz
3118
void Assembler::cmpne(const PRegisterWithLaneSize& pd,
3119
                      const PRegisterZ& pg,
3120
                      const ZRegister& zn,
3121
                      const ZRegister& zm) {
3122
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3123
  VIXL_ASSERT(AreSameLaneSize(pd, zn));
3124
  SVEIntCompareVectorsOp op = CMPNE_p_p_zz;
3125
  if (!AreSameLaneSize(zn, zm)) {
3126
    VIXL_ASSERT(zm.IsLaneSizeD());
3127
    op = CMPNE_p_p_zw;
3128
  }
3129
  CompareVectors(pd, pg, zn, zm, op);
3130
}
3131

3132
// SVEIntMiscUnpredicated.
3133

3134
void Assembler::fexpa(const ZRegister& zd, const ZRegister& zn) {
3135
  // FEXPA <Zd>.<T>, <Zn>.<T>
3136
  //  0000 0100 ..10 0000 1011 10.. .... ....
3137
  //  size<23:22> | opc<20:16> = 00000 | Zn<9:5> | Zd<4:0>
3138

3139
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3140
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3141
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3142

3143
  Emit(FEXPA_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
3144
}
3145

3146
void Assembler::ftssel(const ZRegister& zd,
3147
                       const ZRegister& zn,
3148
                       const ZRegister& zm) {
3149
  // FTSSEL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
3150
  //  0000 0100 ..1. .... 1011 00.. .... ....
3151
  //  size<23:22> | Zm<20:16> | op<10> = 0 | Zn<9:5> | Zd<4:0>
3152

3153
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3154
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
3155
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3156

3157
  Emit(FTSSEL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
3158
}
3159

3160
void Assembler::movprfx(const ZRegister& zd, const ZRegister& zn) {
3161
  // MOVPRFX <Zd>, <Zn>
3162
  //  0000 0100 0010 0000 1011 11.. .... ....
3163
  //  opc<23:22> = 00 | opc2<20:16> = 00000 | Zn<9:5> | Zd<4:0>
3164

3165
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3166
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3167

3168
  Emit(MOVPRFX_z_z | Rd(zd) | Rn(zn));
3169
}
3170

3171
// SVEIntMulAddPredicated.
3172

3173
void Assembler::mad(const ZRegister& zdn,
3174
                    const PRegisterM& pg,
3175
                    const ZRegister& zm,
3176
                    const ZRegister& za) {
3177
  // MAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
3178
  //  0000 0100 ..0. .... 110. .... .... ....
3179
  //  size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Za<9:5> | Zdn<4:0>
3180

3181
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3182
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
3183

3184
  Emit(MAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za));
3185
}
3186

3187
void Assembler::mla(const ZRegister& zda,
3188
                    const PRegisterM& pg,
3189
                    const ZRegister& zn,
3190
                    const ZRegister& zm) {
3191
  // MLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
3192
  //  0000 0100 ..0. .... 010. .... .... ....
3193
  //  size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Zn<9:5> | Zda<4:0>
3194

3195
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3196
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
3197

3198
  Emit(MLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
3199
}
3200

3201
void Assembler::mls(const ZRegister& zda,
3202
                    const PRegisterM& pg,
3203
                    const ZRegister& zn,
3204
                    const ZRegister& zm) {
3205
  // MLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>
3206
  //  0000 0100 ..0. .... 011. .... .... ....
3207
  //  size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Zn<9:5> | Zda<4:0>
3208

3209
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3210
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
3211

3212
  Emit(MLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm));
3213
}
3214

3215
void Assembler::msb(const ZRegister& zdn,
3216
                    const PRegisterM& pg,
3217
                    const ZRegister& zm,
3218
                    const ZRegister& za) {
3219
  // MSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T>
3220
  //  0000 0100 ..0. .... 111. .... .... ....
3221
  //  size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Za<9:5> | Zdn<4:0>
3222

3223
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3224
  VIXL_ASSERT(AreSameLaneSize(zdn, zm, za));
3225

3226
  Emit(MSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za));
3227
}
3228

3229
// SVEIntMulAddUnpredicated.
3230

3231
void Assembler::sdot(const ZRegister& zda,
3232
                     const ZRegister& zn,
3233
                     const ZRegister& zm) {
3234
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3235
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
3236
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
3237
  VIXL_ASSERT(AreSameLaneSize(zm, zn));
3238

3239
  Emit(SDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
3240
}
3241

3242
void Assembler::udot(const ZRegister& zda,
3243
                     const ZRegister& zn,
3244
                     const ZRegister& zm) {
3245
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3246
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
3247
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
3248
  VIXL_ASSERT(AreSameLaneSize(zm, zn));
3249

3250
  Emit(UDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
3251
}
3252

3253
// SVEIntReduction.
3254

3255
void Assembler::andv(const VRegister& vd,
3256
                     const PRegister& pg,
3257
                     const ZRegister& zn) {
3258
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3259
  VIXL_ASSERT(vd.IsScalar());
3260

3261
  Emit(ANDV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3262
}
3263

3264
void Assembler::eorv(const VRegister& vd,
3265
                     const PRegister& pg,
3266
                     const ZRegister& zn) {
3267
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3268
  VIXL_ASSERT(vd.IsScalar());
3269

3270
  Emit(EORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3271
}
3272

3273
void Assembler::movprfx(const ZRegister& zd,
3274
                        const PRegister& pg,
3275
                        const ZRegister& zn) {
3276
  // MOVPRFX <Zd>.<T>, <Pg>/<ZM>, <Zn>.<T>
3277
  //  0000 0100 ..01 000. 001. .... .... ....
3278
  //  size<23:22> | opc<18:17> = 00 | M<16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
3279

3280
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3281
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3282
  VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
3283
  VIXL_ASSERT(!pg.HasLaneSize());
3284

3285
  Instr m = pg.IsMerging() ? 0x00010000 : 0x00000000;
3286
  Emit(MOVPRFX_z_p_z | SVESize(zd) | m | Rd(zd) | PgLow8(pg) | Rn(zn));
3287
}
3288

3289
void Assembler::orv(const VRegister& vd,
3290
                    const PRegister& pg,
3291
                    const ZRegister& zn) {
3292
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3293
  VIXL_ASSERT(vd.IsScalar());
3294

3295
  Emit(ORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3296
}
3297

3298
void Assembler::saddv(const VRegister& dd,
3299
                      const PRegister& pg,
3300
                      const ZRegister& zn) {
3301
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3302
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kDRegSizeInBytes);
3303

3304
  Emit(SADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn));
3305
}
3306

3307
void Assembler::smaxv(const VRegister& vd,
3308
                      const PRegister& pg,
3309
                      const ZRegister& zn) {
3310
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3311
  VIXL_ASSERT(vd.IsScalar());
3312

3313
  Emit(SMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3314
}
3315

3316
void Assembler::sminv(const VRegister& vd,
3317
                      const PRegister& pg,
3318
                      const ZRegister& zn) {
3319
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3320
  VIXL_ASSERT(vd.IsScalar());
3321

3322
  Emit(SMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3323
}
3324

3325
void Assembler::uaddv(const VRegister& dd,
3326
                      const PRegister& pg,
3327
                      const ZRegister& zn) {
3328
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3329

3330
  Emit(UADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn));
3331
}
3332

3333
void Assembler::umaxv(const VRegister& vd,
3334
                      const PRegister& pg,
3335
                      const ZRegister& zn) {
3336
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3337
  VIXL_ASSERT(vd.IsScalar());
3338

3339
  Emit(UMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3340
}
3341

3342
void Assembler::uminv(const VRegister& vd,
3343
                      const PRegister& pg,
3344
                      const ZRegister& zn) {
3345
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3346
  VIXL_ASSERT(vd.IsScalar());
3347

3348
  Emit(UMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
3349
}
3350

3351
// SVEIntUnaryArithmeticPredicated.
3352

3353
void Assembler::abs(const ZRegister& zd,
3354
                    const PRegisterM& pg,
3355
                    const ZRegister& zn) {
3356
  // ABS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3357
  //  0000 0100 ..01 0110 101. .... .... ....
3358
  //  size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3359

3360
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3361
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3362

3363
  Emit(ABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3364
}
3365

3366
void Assembler::cls(const ZRegister& zd,
3367
                    const PRegisterM& pg,
3368
                    const ZRegister& zn) {
3369
  // CLS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3370
  //  0000 0100 ..01 1000 101. .... .... ....
3371
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3372

3373
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3374
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3375

3376
  Emit(CLS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3377
}
3378

3379
void Assembler::clz(const ZRegister& zd,
3380
                    const PRegisterM& pg,
3381
                    const ZRegister& zn) {
3382
  // CLZ <Zd>.<T>, <Pg>/M, <Zn>.<T>
3383
  //  0000 0100 ..01 1001 101. .... .... ....
3384
  //  size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3385

3386
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3387
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3388

3389
  Emit(CLZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3390
}
3391

3392
void Assembler::cnot(const ZRegister& zd,
3393
                     const PRegisterM& pg,
3394
                     const ZRegister& zn) {
3395
  // CNOT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3396
  //  0000 0100 ..01 1011 101. .... .... ....
3397
  //  size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3398

3399
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3400
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3401

3402
  Emit(CNOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3403
}
3404

3405
void Assembler::cnt(const ZRegister& zd,
3406
                    const PRegisterM& pg,
3407
                    const ZRegister& zn) {
3408
  // CNT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3409
  //  0000 0100 ..01 1010 101. .... .... ....
3410
  //  size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3411

3412
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3413
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3414

3415
  Emit(CNT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3416
}
3417

3418
void Assembler::fabs(const ZRegister& zd,
3419
                     const PRegisterM& pg,
3420
                     const ZRegister& zn) {
3421
  // FABS <Zd>.<T>, <Pg>/M, <Zn>.<T>
3422
  //  0000 0100 ..01 1100 101. .... .... ....
3423
  //  size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3424

3425
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3426
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3427
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3428

3429
  Emit(FABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3430
}
3431

3432
void Assembler::fneg(const ZRegister& zd,
3433
                     const PRegisterM& pg,
3434
                     const ZRegister& zn) {
3435
  // FNEG <Zd>.<T>, <Pg>/M, <Zn>.<T>
3436
  //  0000 0100 ..01 1101 101. .... .... ....
3437
  //  size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3438

3439
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3440
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3441
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3442

3443
  Emit(FNEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3444
}
3445

3446
void Assembler::neg(const ZRegister& zd,
3447
                    const PRegisterM& pg,
3448
                    const ZRegister& zn) {
3449
  // NEG <Zd>.<T>, <Pg>/M, <Zn>.<T>
3450
  //  0000 0100 ..01 0111 101. .... .... ....
3451
  //  size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3452

3453
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3454
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3455

3456
  Emit(NEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3457
}
3458

3459
void Assembler::not_(const ZRegister& zd,
3460
                     const PRegisterM& pg,
3461
                     const ZRegister& zn) {
3462
  // NOT <Zd>.<T>, <Pg>/M, <Zn>.<T>
3463
  //  0000 0100 ..01 1110 101. .... .... ....
3464
  //  size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3465

3466
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3467
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3468

3469
  Emit(NOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3470
}
3471

3472
void Assembler::sxtb(const ZRegister& zd,
3473
                     const PRegisterM& pg,
3474
                     const ZRegister& zn) {
3475
  // SXTB <Zd>.<T>, <Pg>/M, <Zn>.<T>
3476
  //  0000 0100 ..01 0000 101. .... .... ....
3477
  //  size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3478

3479
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3480
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3481
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes);
3482

3483
  Emit(SXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3484
}
3485

3486
void Assembler::sxth(const ZRegister& zd,
3487
                     const PRegisterM& pg,
3488
                     const ZRegister& zn) {
3489
  // SXTH <Zd>.<T>, <Pg>/M, <Zn>.<T>
3490
  //  0000 0100 ..01 0010 101. .... .... ....
3491
  //  size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3492

3493
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3494
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3495
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes);
3496

3497
  Emit(SXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3498
}
3499

3500
void Assembler::sxtw(const ZRegister& zd,
3501
                     const PRegisterM& pg,
3502
                     const ZRegister& zn) {
3503
  // SXTW <Zd>.D, <Pg>/M, <Zn>.D
3504
  //  0000 0100 ..01 0100 101. .... .... ....
3505
  //  size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3506

3507
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3508
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3509
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes);
3510

3511
  Emit(SXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3512
}
3513

3514
void Assembler::uxtb(const ZRegister& zd,
3515
                     const PRegisterM& pg,
3516
                     const ZRegister& zn) {
3517
  // UXTB <Zd>.<T>, <Pg>/M, <Zn>.<T>
3518
  //  0000 0100 ..01 0001 101. .... .... ....
3519
  //  size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3520

3521
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3522
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3523
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes);
3524

3525
  Emit(UXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3526
}
3527

3528
void Assembler::uxth(const ZRegister& zd,
3529
                     const PRegisterM& pg,
3530
                     const ZRegister& zn) {
3531
  // UXTH <Zd>.<T>, <Pg>/M, <Zn>.<T>
3532
  //  0000 0100 ..01 0011 101. .... .... ....
3533
  //  size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3534

3535
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3536
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3537
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes);
3538

3539
  Emit(UXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3540
}
3541

3542
void Assembler::uxtw(const ZRegister& zd,
3543
                     const PRegisterM& pg,
3544
                     const ZRegister& zn) {
3545
  // UXTW <Zd>.D, <Pg>/M, <Zn>.D
3546
  //  0000 0100 ..01 0101 101. .... .... ....
3547
  //  size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0>
3548

3549
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3550
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3551
  VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes);
3552

3553
  Emit(UXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
3554
}
3555

3556
// SVEIntWideImmPredicated.
3557

3558
void Assembler::cpy(const ZRegister& zd,
3559
                    const PRegister& pg,
3560
                    int imm8,
3561
                    int shift) {
3562
  // CPY <Zd>.<T>, <Pg>/<ZM>, #<imm>{, <shift>}
3563
  //  0000 0101 ..01 .... 0... .... .... ....
3564
  //  size<23:22> | Pg<19:16> | M<14> | sh<13> | imm8<12:5> | Zd<4:0>
3565

3566
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3567
  VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
3568

3569
  ResolveSVEImm8Shift(&imm8, &shift);
3570

3571
  Instr sh = (shift > 0) ? (1 << 13) : 0;
3572
  Instr m = pg.IsMerging() ? (1 << 14) : 0;
3573
  Emit(CPY_z_p_i | m | sh | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) |
3574
       ImmField<12, 5>(imm8));
3575
}
3576

3577
void Assembler::fcpy(const ZRegister& zd, const PRegisterM& pg, double imm) {
3578
  // FCPY <Zd>.<T>, <Pg>/M, #<const>
3579
  //  0000 0101 ..01 .... 110. .... .... ....
3580
  //  size<23:22> | Pg<19:16> | imm8<12:5> | Zd<4:0>
3581

3582
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3583
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3584

3585
  Instr imm_field = ImmUnsignedField<12, 5>(FP64ToImm8(imm));
3586
  Emit(FCPY_z_p_i | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | imm_field);
3587
}
3588

3589
// SVEIntAddSubtractImmUnpredicated.
3590

3591
void Assembler::SVEIntAddSubtractImmUnpredicatedHelper(
3592
    SVEIntAddSubtractImm_UnpredicatedOp op,
3593
    const ZRegister& zd,
3594
    int imm8,
3595
    int shift) {
3596
  if (shift < 0) {
3597
    VIXL_ASSERT(shift == -1);
3598
    // Derive the shift amount from the immediate.
3599
    if (IsUint8(imm8)) {
3600
      shift = 0;
3601
    } else if (IsUint16(imm8) && ((imm8 % 256) == 0)) {
3602
      imm8 /= 256;
3603
      shift = 8;
3604
    }
3605
  }
3606

3607
  VIXL_ASSERT(IsUint8(imm8));
3608
  VIXL_ASSERT((shift == 0) || (shift == 8));
3609

3610
  Instr shift_bit = (shift > 0) ? (1 << 13) : 0;
3611
  Emit(op | SVESize(zd) | Rd(zd) | shift_bit | ImmUnsignedField<12, 5>(imm8));
3612
}
3613

3614
void Assembler::add(const ZRegister& zd,
3615
                    const ZRegister& zn,
3616
                    int imm8,
3617
                    int shift) {
3618
  // ADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3619
  //  0010 0101 ..10 0000 11.. .... .... ....
3620
  //  size<23:22> | opc<18:16> = 000 | sh<13> | imm8<12:5> | Zdn<4:0>
3621

3622
  USE(zn);
3623
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3624
  VIXL_ASSERT(zd.Is(zn));
3625
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3626

3627
  SVEIntAddSubtractImmUnpredicatedHelper(ADD_z_zi, zd, imm8, shift);
3628
}
3629

3630
void Assembler::dup(const ZRegister& zd, int imm8, int shift) {
3631
  // DUP <Zd>.<T>, #<imm>{, <shift>}
3632
  //  0010 0101 ..11 1000 11.. .... .... ....
3633
  //  size<23:22> | opc<18:17> = 00 | sh<13> | imm8<12:5> | Zd<4:0>
3634

3635
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3636

3637
  ResolveSVEImm8Shift(&imm8, &shift);
3638
  VIXL_ASSERT((shift < 8) || !zd.IsLaneSizeB());
3639

3640
  Instr shift_bit = (shift > 0) ? (1 << 13) : 0;
3641
  Emit(DUP_z_i | SVESize(zd) | Rd(zd) | shift_bit | ImmField<12, 5>(imm8));
3642
}
3643

3644
void Assembler::fdup(const ZRegister& zd, double imm) {
3645
  // FDUP <Zd>.<T>, #<const>
3646
  //  0010 0101 ..11 1001 110. .... .... ....
3647
  //  size<23:22> | opc<18:17> = 00 | o2<13> = 0 | imm8<12:5> | Zd<4:0>
3648

3649
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3650
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
3651

3652
  Instr encoded_imm = FP64ToImm8(imm) << 5;
3653
  Emit(FDUP_z_i | SVESize(zd) | encoded_imm | Rd(zd));
3654
}
3655

3656
void Assembler::mul(const ZRegister& zd, const ZRegister& zn, int imm8) {
3657
  // MUL <Zdn>.<T>, <Zdn>.<T>, #<imm>
3658
  //  0010 0101 ..11 0000 110. .... .... ....
3659
  //  size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3660

3661
  USE(zn);
3662
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3663
  VIXL_ASSERT(zd.Is(zn));
3664
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3665

3666
  Emit(MUL_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3667
}
3668

3669
void Assembler::smax(const ZRegister& zd, const ZRegister& zn, int imm8) {
3670
  // SMAX <Zdn>.<T>, <Zdn>.<T>, #<imm>
3671
  //  0010 0101 ..10 1000 110. .... .... ....
3672
  //  size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3673

3674
  USE(zn);
3675
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3676
  VIXL_ASSERT(zd.Is(zn));
3677
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3678

3679
  Emit(SMAX_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3680
}
3681

3682
void Assembler::smin(const ZRegister& zd, const ZRegister& zn, int imm8) {
3683
  // SMIN <Zdn>.<T>, <Zdn>.<T>, #<imm>
3684
  //  0010 0101 ..10 1010 110. .... .... ....
3685
  //  size<23:22> | opc<18:16> = 010 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3686

3687
  USE(zn);
3688
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3689
  VIXL_ASSERT(zd.Is(zn));
3690
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3691

3692
  Emit(SMIN_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8));
3693
}
3694

3695
void Assembler::sqadd(const ZRegister& zd,
3696
                      const ZRegister& zn,
3697
                      int imm8,
3698
                      int shift) {
3699
  // SQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3700
  //  0010 0101 ..10 0100 11.. .... .... ....
3701
  //  size<23:22> | opc<18:16> = 100 | sh<13> | imm8<12:5> | Zdn<4:0>
3702

3703
  USE(zn);
3704
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3705
  VIXL_ASSERT(zd.Is(zn));
3706
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3707

3708
  SVEIntAddSubtractImmUnpredicatedHelper(SQADD_z_zi, zd, imm8, shift);
3709
}
3710

3711
void Assembler::sqsub(const ZRegister& zd,
3712
                      const ZRegister& zn,
3713
                      int imm8,
3714
                      int shift) {
3715
  // SQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3716
  //  0010 0101 ..10 0110 11.. .... .... ....
3717
  //  size<23:22> | opc<18:16> = 110 | sh<13> | imm8<12:5> | Zdn<4:0>
3718

3719
  USE(zn);
3720
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3721
  VIXL_ASSERT(zd.Is(zn));
3722
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3723

3724
  SVEIntAddSubtractImmUnpredicatedHelper(SQSUB_z_zi, zd, imm8, shift);
3725
}
3726

3727
void Assembler::sub(const ZRegister& zd,
3728
                    const ZRegister& zn,
3729
                    int imm8,
3730
                    int shift) {
3731
  // SUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3732
  //  0010 0101 ..10 0001 11.. .... .... ....
3733
  //  size<23:22> | opc<18:16> = 001 | sh<13> | imm8<12:5> | Zdn<4:0>
3734

3735
  USE(zn);
3736
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3737
  VIXL_ASSERT(zd.Is(zn));
3738
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3739

3740
  SVEIntAddSubtractImmUnpredicatedHelper(SUB_z_zi, zd, imm8, shift);
3741
}
3742

3743
void Assembler::subr(const ZRegister& zd,
3744
                     const ZRegister& zn,
3745
                     int imm8,
3746
                     int shift) {
3747
  // SUBR <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3748
  //  0010 0101 ..10 0011 11.. .... .... ....
3749
  //  size<23:22> | opc<18:16> = 011 | sh<13> | imm8<12:5> | Zdn<4:0>
3750

3751
  USE(zn);
3752
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3753
  VIXL_ASSERT(zd.Is(zn));
3754
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3755

3756
  SVEIntAddSubtractImmUnpredicatedHelper(SUBR_z_zi, zd, imm8, shift);
3757
}
3758

3759
void Assembler::umax(const ZRegister& zd, const ZRegister& zn, int imm8) {
3760
  // UMAX <Zdn>.<T>, <Zdn>.<T>, #<imm>
3761
  //  0010 0101 ..10 1001 110. .... .... ....
3762
  //  size<23:22> | opc<18:16> = 001 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3763

3764
  USE(zn);
3765
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3766
  VIXL_ASSERT(zd.Is(zn));
3767
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3768

3769
  Emit(UMAX_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8));
3770
}
3771

3772
void Assembler::umin(const ZRegister& zd, const ZRegister& zn, int imm8) {
3773
  // UMIN <Zdn>.<T>, <Zdn>.<T>, #<imm>
3774
  //  0010 0101 ..10 1011 110. .... .... ....
3775
  //  size<23:22> | opc<18:16> = 011 | o2<13> = 0 | imm8<12:5> | Zdn<4:0>
3776

3777
  USE(zn);
3778
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3779
  VIXL_ASSERT(zd.Is(zn));
3780
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3781

3782
  Emit(UMIN_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8));
3783
}
3784

3785
void Assembler::uqadd(const ZRegister& zd,
3786
                      const ZRegister& zn,
3787
                      int imm8,
3788
                      int shift) {
3789
  // UQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3790
  //  0010 0101 ..10 0101 11.. .... .... ....
3791
  //  size<23:22> | opc<18:16> = 101 | sh<13> | imm8<12:5> | Zdn<4:0>
3792

3793
  USE(zn);
3794
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3795
  VIXL_ASSERT(zd.Is(zn));
3796
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3797

3798
  SVEIntAddSubtractImmUnpredicatedHelper(UQADD_z_zi, zd, imm8, shift);
3799
}
3800

3801
void Assembler::uqsub(const ZRegister& zd,
3802
                      const ZRegister& zn,
3803
                      int imm8,
3804
                      int shift) {
3805
  // UQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>}
3806
  //  0010 0101 ..10 0111 11.. .... .... ....
3807
  //  size<23:22> | opc<18:16> = 111 | sh<13> | imm8<12:5> | Zdn<4:0>
3808

3809
  USE(zn);
3810
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
3811
  VIXL_ASSERT(zd.Is(zn));
3812
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
3813

3814
  SVEIntAddSubtractImmUnpredicatedHelper(UQSUB_z_zi, zd, imm8, shift);
3815
}
3816

3817
// SVEMemLoad.
3818

3819
void Assembler::SVELdSt1Helper(unsigned msize_in_bytes_log2,
3820
                               const ZRegister& zt,
3821
                               const PRegister& pg,
3822
                               const SVEMemOperand& addr,
3823
                               bool is_signed,
3824
                               Instr op) {
3825
  VIXL_ASSERT(addr.IsContiguous());
3826

3827
  Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr);
3828
  Instr dtype =
3829
      SVEDtype(msize_in_bytes_log2, zt.GetLaneSizeInBytesLog2(), is_signed);
3830
  Emit(op | mem_op | dtype | Rt(zt) | PgLow8(pg));
3831
}
3832

3833
void Assembler::SVELdSt234Helper(int num_regs,
3834
                                 const ZRegister& zt1,
3835
                                 const PRegister& pg,
3836
                                 const SVEMemOperand& addr,
3837
                                 Instr op) {
3838
  VIXL_ASSERT((num_regs >= 2) && (num_regs <= 4));
3839

3840
  unsigned msize_in_bytes_log2 = zt1.GetLaneSizeInBytesLog2();
3841
  Instr num = (num_regs - 1) << 21;
3842
  Instr msz = msize_in_bytes_log2 << 23;
3843
  Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, num_regs, addr);
3844
  Emit(op | mem_op | msz | num | Rt(zt1) | PgLow8(pg));
3845
}
3846

3847
void Assembler::SVELd1Helper(unsigned msize_in_bytes_log2,
3848
                             const ZRegister& zt,
3849
                             const PRegisterZ& pg,
3850
                             const SVEMemOperand& addr,
3851
                             bool is_signed) {
3852
  VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
3853
  if (is_signed) {
3854
    // Sign-extension is only possible when the vector elements are larger than
3855
    // the elements in memory.
3856
    VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
3857
  }
3858

3859
  if (addr.IsScatterGather()) {
3860
    bool is_load = true;
3861
    bool is_ff = false;
3862
    SVEScatterGatherHelper(msize_in_bytes_log2,
3863
                           zt,
3864
                           pg,
3865
                           addr,
3866
                           is_load,
3867
                           is_signed,
3868
                           is_ff);
3869
    return;
3870
  }
3871

3872
  Instr op = 0xffffffff;
3873
  if (addr.IsScalarPlusImmediate()) {
3874
    op = SVEContiguousLoad_ScalarPlusImmFixed;
3875
  } else if (addr.IsScalarPlusScalar()) {
3876
    // Rm must not be xzr.
3877
    VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
3878
    op = SVEContiguousLoad_ScalarPlusScalarFixed;
3879
  } else {
3880
    VIXL_UNIMPLEMENTED();
3881
  }
3882
  SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op);
3883
}
3884

3885
void Assembler::SVELdff1Helper(unsigned msize_in_bytes_log2,
3886
                               const ZRegister& zt,
3887
                               const PRegisterZ& pg,
3888
                               const SVEMemOperand& addr,
3889
                               bool is_signed) {
3890
  VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
3891
  if (is_signed) {
3892
    // Sign-extension is only possible when the vector elements are larger than
3893
    // the elements in memory.
3894
    VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
3895
  }
3896

3897
  if (addr.IsScatterGather()) {
3898
    bool is_load = true;
3899
    bool is_ff = true;
3900
    SVEScatterGatherHelper(msize_in_bytes_log2,
3901
                           zt,
3902
                           pg,
3903
                           addr,
3904
                           is_load,
3905
                           is_signed,
3906
                           is_ff);
3907
    return;
3908
  }
3909

3910
  if (addr.IsPlainScalar()) {
3911
    // SVEMemOperand(x0) is treated as a scalar-plus-immediate form ([x0, #0]).
3912
    // In these instructions, we want to treat it as [x0, xzr].
3913
    SVEMemOperand addr_scalar_plus_scalar(addr.GetScalarBase(), xzr);
3914
    // Guard against infinite recursion.
3915
    VIXL_ASSERT(!addr_scalar_plus_scalar.IsPlainScalar());
3916
    SVELdff1Helper(msize_in_bytes_log2,
3917
                   zt,
3918
                   pg,
3919
                   addr_scalar_plus_scalar,
3920
                   is_signed);
3921
    return;
3922
  }
3923

3924
  Instr op = 0xffffffff;
3925
  if (addr.IsScalarPlusScalar()) {
3926
    op = SVEContiguousFirstFaultLoad_ScalarPlusScalarFixed;
3927
  } else {
3928
    VIXL_UNIMPLEMENTED();
3929
  }
3930
  SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op);
3931
}
3932

3933
void Assembler::SVEScatterGatherHelper(unsigned msize_in_bytes_log2,
3934
                                       const ZRegister& zt,
3935
                                       const PRegister& pg,
3936
                                       const SVEMemOperand& addr,
3937
                                       bool is_load,
3938
                                       bool is_signed,
3939
                                       bool is_first_fault) {
3940
  VIXL_ASSERT(addr.IsScatterGather());
3941
  VIXL_ASSERT(zt.IsLaneSizeS() || zt.IsLaneSizeD());
3942
  VIXL_ASSERT(is_load || !is_first_fault);
3943
  VIXL_ASSERT(is_load || !is_signed);
3944

3945
  Instr op = 0xffffffff;
3946
  if (addr.IsVectorPlusImmediate()) {
3947
    VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorBase()));
3948
    if (is_load) {
3949
      if (zt.IsLaneSizeS()) {
3950
        op = SVE32BitGatherLoad_VectorPlusImmFixed;
3951
      } else {
3952
        op = SVE64BitGatherLoad_VectorPlusImmFixed;
3953
      }
3954
    } else {
3955
      if (zt.IsLaneSizeS()) {
3956
        op = SVE32BitScatterStore_VectorPlusImmFixed;
3957
      } else {
3958
        op = SVE64BitScatterStore_VectorPlusImmFixed;
3959
      }
3960
    }
3961
  } else {
3962
    VIXL_ASSERT(addr.IsScalarPlusVector());
3963
    VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorOffset()));
3964
    SVEOffsetModifier mod = addr.GetOffsetModifier();
3965
    if (zt.IsLaneSizeS()) {
3966
      VIXL_ASSERT((mod == SVE_UXTW) || (mod == SVE_SXTW));
3967
      unsigned shift_amount = addr.GetShiftAmount();
3968
      if (shift_amount == 0) {
3969
        if (is_load) {
3970
          op = SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsetsFixed;
3971
        } else {
3972
          op = SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsetsFixed;
3973
        }
3974
      } else if (shift_amount == 1) {
3975
        VIXL_ASSERT(msize_in_bytes_log2 == kHRegSizeInBytesLog2);
3976
        if (is_load) {
3977
          op = SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsetsFixed;
3978
        } else {
3979
          op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed;
3980
        }
3981
      } else {
3982
        VIXL_ASSERT(shift_amount == 2);
3983
        VIXL_ASSERT(msize_in_bytes_log2 == kSRegSizeInBytesLog2);
3984
        if (is_load) {
3985
          op = SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsetsFixed;
3986
        } else {
3987
          op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed;
3988
        }
3989
      }
3990
    } else if (zt.IsLaneSizeD()) {
3991
      switch (mod) {
3992
        case NO_SVE_OFFSET_MODIFIER:
3993
          if (is_load) {
3994
            op = SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsetsFixed;
3995
          } else {
3996
            op = SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsetsFixed;
3997
          }
3998
          break;
3999
        case SVE_LSL:
4000
          if (is_load) {
4001
            op = SVE64BitGatherLoad_ScalarPlus64BitScaledOffsetsFixed;
4002
          } else {
4003
            op = SVE64BitScatterStore_ScalarPlus64BitScaledOffsetsFixed;
4004
          }
4005
          break;
4006
        case SVE_UXTW:
4007
        case SVE_SXTW: {
4008
          unsigned shift_amount = addr.GetShiftAmount();
4009
          if (shift_amount == 0) {
4010
            if (is_load) {
4011
              op =
4012
                  SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsetsFixed;
4013
            } else {
4014
              op =
4015
                  SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsetsFixed;
4016
            }
4017
          } else {
4018
            VIXL_ASSERT(shift_amount == msize_in_bytes_log2);
4019
            if (is_load) {
4020
              op = SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsetsFixed;
4021
            } else {
4022
              op =
4023
                  SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsetsFixed;
4024
            }
4025
          }
4026
          break;
4027
        }
4028
        default:
4029
          VIXL_UNIMPLEMENTED();
4030
      }
4031
    }
4032
  }
4033

4034
  Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr, is_load);
4035
  Instr msz = ImmUnsignedField<24, 23>(msize_in_bytes_log2);
4036
  Instr u = (!is_load || is_signed) ? 0 : (1 << 14);
4037
  Instr ff = is_first_fault ? (1 << 13) : 0;
4038
  Emit(op | mem_op | msz | u | ff | Rt(zt) | PgLow8(pg));
4039
}
4040

4041
void Assembler::SVELd234Helper(int num_regs,
4042
                               const ZRegister& zt1,
4043
                               const PRegisterZ& pg,
4044
                               const SVEMemOperand& addr) {
4045
  if (addr.IsScalarPlusScalar()) {
4046
    // Rm must not be xzr.
4047
    VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
4048
  }
4049

4050
  Instr op;
4051
  if (addr.IsScalarPlusImmediate()) {
4052
    op = SVELoadMultipleStructures_ScalarPlusImmFixed;
4053
  } else if (addr.IsScalarPlusScalar()) {
4054
    op = SVELoadMultipleStructures_ScalarPlusScalarFixed;
4055
  } else {
4056
    // These instructions don't support any other addressing modes.
4057
    VIXL_ABORT();
4058
  }
4059
  SVELdSt234Helper(num_regs, zt1, pg, addr, op);
4060
}
4061

4062
// SVEMemContiguousLoad.
4063

4064
#define VIXL_DEFINE_LD1(MSZ, LANE_SIZE)                                  \
4065
  void Assembler::ld1##MSZ(const ZRegister& zt,                          \
4066
                           const PRegisterZ& pg,                         \
4067
                           const SVEMemOperand& addr) {                  \
4068
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                              \
4069
    SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \
4070
  }
4071
#define VIXL_DEFINE_LD2(MSZ, LANE_SIZE)                 \
4072
  void Assembler::ld2##MSZ(const ZRegister& zt1,        \
4073
                           const ZRegister& zt2,        \
4074
                           const PRegisterZ& pg,        \
4075
                           const SVEMemOperand& addr) { \
4076
    USE(zt2);                                           \
4077
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
4078
    VIXL_ASSERT(AreConsecutive(zt1, zt2));              \
4079
    VIXL_ASSERT(AreSameFormat(zt1, zt2));               \
4080
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
4081
    SVELd234Helper(2, zt1, pg, addr);                   \
4082
  }
4083
#define VIXL_DEFINE_LD3(MSZ, LANE_SIZE)                 \
4084
  void Assembler::ld3##MSZ(const ZRegister& zt1,        \
4085
                           const ZRegister& zt2,        \
4086
                           const ZRegister& zt3,        \
4087
                           const PRegisterZ& pg,        \
4088
                           const SVEMemOperand& addr) { \
4089
    USE(zt2, zt3);                                      \
4090
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
4091
    VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3));         \
4092
    VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3));          \
4093
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
4094
    SVELd234Helper(3, zt1, pg, addr);                   \
4095
  }
4096
#define VIXL_DEFINE_LD4(MSZ, LANE_SIZE)                 \
4097
  void Assembler::ld4##MSZ(const ZRegister& zt1,        \
4098
                           const ZRegister& zt2,        \
4099
                           const ZRegister& zt3,        \
4100
                           const ZRegister& zt4,        \
4101
                           const PRegisterZ& pg,        \
4102
                           const SVEMemOperand& addr) { \
4103
    USE(zt2, zt3, zt4);                                 \
4104
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
4105
    VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4));    \
4106
    VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4));     \
4107
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
4108
    SVELd234Helper(4, zt1, pg, addr);                   \
4109
  }
4110

4111
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD1)
4112
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD2)
4113
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD3)
4114
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD4)
4115

4116
#define VIXL_DEFINE_LD1S(MSZ, LANE_SIZE)                                \
4117
  void Assembler::ld1s##MSZ(const ZRegister& zt,                        \
4118
                            const PRegisterZ& pg,                       \
4119
                            const SVEMemOperand& addr) {                \
4120
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                             \
4121
    SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \
4122
  }
4123
VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LD1S)
4124

4125
// SVEMem32BitGatherAndUnsizedContiguous.
4126

4127
void Assembler::SVELd1BroadcastHelper(unsigned msize_in_bytes_log2,
4128
                                      const ZRegister& zt,
4129
                                      const PRegisterZ& pg,
4130
                                      const SVEMemOperand& addr,
4131
                                      bool is_signed) {
4132
  VIXL_ASSERT(addr.IsScalarPlusImmediate());
4133
  VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2);
4134
  if (is_signed) {
4135
    // Sign-extension is only possible when the vector elements are larger than
4136
    // the elements in memory.
4137
    VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2);
4138
  }
4139

4140
  int64_t imm = addr.GetImmediateOffset();
4141
  int divisor = 1 << msize_in_bytes_log2;
4142
  VIXL_ASSERT(imm % divisor == 0);
4143
  Instr dtype = SVEDtypeSplit(msize_in_bytes_log2,
4144
                              zt.GetLaneSizeInBytesLog2(),
4145
                              is_signed);
4146

4147
  Emit(SVELoadAndBroadcastElementFixed | dtype | RnSP(addr.GetScalarBase()) |
4148
       ImmUnsignedField<21, 16>(imm / divisor) | Rt(zt) | PgLow8(pg));
4149
}
4150

4151
// This prototype maps to 4 instruction encodings:
4152
//  LD1RB_z_p_bi_u16
4153
//  LD1RB_z_p_bi_u32
4154
//  LD1RB_z_p_bi_u64
4155
//  LD1RB_z_p_bi_u8
4156
void Assembler::ld1rb(const ZRegister& zt,
4157
                      const PRegisterZ& pg,
4158
                      const SVEMemOperand& addr) {
4159
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4160

4161
  SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, false);
4162
}
4163

4164
// This prototype maps to 3 instruction encodings:
4165
//  LD1RH_z_p_bi_u16
4166
//  LD1RH_z_p_bi_u32
4167
//  LD1RH_z_p_bi_u64
4168
void Assembler::ld1rh(const ZRegister& zt,
4169
                      const PRegisterZ& pg,
4170
                      const SVEMemOperand& addr) {
4171
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4172

4173
  SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, false);
4174
}
4175

4176
// This prototype maps to 2 instruction encodings:
4177
//  LD1RW_z_p_bi_u32
4178
//  LD1RW_z_p_bi_u64
4179
void Assembler::ld1rw(const ZRegister& zt,
4180
                      const PRegisterZ& pg,
4181
                      const SVEMemOperand& addr) {
4182
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4183

4184
  SVELd1BroadcastHelper(kSRegSizeInBytesLog2, zt, pg, addr, false);
4185
}
4186

4187
void Assembler::ld1rd(const ZRegister& zt,
4188
                      const PRegisterZ& pg,
4189
                      const SVEMemOperand& addr) {
4190
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4191

4192
  SVELd1BroadcastHelper(kDRegSizeInBytesLog2, zt, pg, addr, false);
4193
}
4194

4195
// This prototype maps to 3 instruction encodings:
4196
//  LD1RSB_z_p_bi_s16
4197
//  LD1RSB_z_p_bi_s32
4198
//  LD1RSB_z_p_bi_s64
4199
void Assembler::ld1rsb(const ZRegister& zt,
4200
                       const PRegisterZ& pg,
4201
                       const SVEMemOperand& addr) {
4202
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4203

4204
  SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, true);
4205
}
4206

4207
// This prototype maps to 2 instruction encodings:
4208
//  LD1RSH_z_p_bi_s32
4209
//  LD1RSH_z_p_bi_s64
4210
void Assembler::ld1rsh(const ZRegister& zt,
4211
                       const PRegisterZ& pg,
4212
                       const SVEMemOperand& addr) {
4213
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4214

4215
  SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, true);
4216
}
4217

4218
void Assembler::ld1rsw(const ZRegister& zt,
4219
                       const PRegisterZ& pg,
4220
                       const SVEMemOperand& addr) {
4221
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4222

4223
  SVELd1BroadcastHelper(kWRegSizeInBytesLog2, zt, pg, addr, true);
4224
}
4225

4226
void Assembler::ldr(const CPURegister& rt, const SVEMemOperand& addr) {
4227
  // LDR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}]
4228

4229
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4230
  VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister());
4231
  VIXL_ASSERT(addr.IsPlainScalar() ||
4232
              (addr.IsScalarPlusImmediate() &&
4233
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4234
  int64_t imm9 = addr.GetImmediateOffset();
4235
  VIXL_ASSERT(IsInt9(imm9));
4236
  Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10;
4237
  Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16;
4238

4239
  Instr op = LDR_z_bi;
4240
  if (rt.IsPRegister()) {
4241
    op = LDR_p_bi;
4242
  }
4243
  Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l);
4244
}
4245

4246
// SVEMem64BitGather.
4247

4248
// This prototype maps to 3 instruction encodings:
4249
//  LDFF1B_z_p_bz_d_64_unscaled
4250
//  LDFF1B_z_p_bz_d_x32_unscaled
4251
void Assembler::ldff1b(const ZRegister& zt,
4252
                       const PRegisterZ& pg,
4253
                       const Register& xn,
4254
                       const ZRegister& zm) {
4255
  // LDFF1B { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D]
4256
  //  1100 0100 010. .... 111. .... .... ....
4257
  //  msz<24:23> = 00 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4258
  //  | Zt<4:0>
4259

4260
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4261

4262
  Emit(LDFF1B_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4263
}
4264

4265
// This prototype maps to 2 instruction encodings:
4266
//  LDFF1B_z_p_ai_d
4267
//  LDFF1B_z_p_ai_s
4268
void Assembler::ldff1b(const ZRegister& zt,
4269
                       const PRegisterZ& pg,
4270
                       const ZRegister& zn,
4271
                       int imm5) {
4272
  // LDFF1B { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4273
  //  1100 0100 001. .... 111. .... .... ....
4274
  //  msz<24:23> = 00 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4275
  //  Zn<9:5> | Zt<4:0>
4276

4277
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4278

4279
  Emit(LDFF1B_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4280
}
4281

4282
// This prototype maps to 4 instruction encodings:
4283
//  LDFF1D_z_p_bz_d_64_scaled
4284
//  LDFF1D_z_p_bz_d_64_unscaled
4285
//  LDFF1D_z_p_bz_d_x32_scaled
4286
//  LDFF1D_z_p_bz_d_x32_unscaled
4287
void Assembler::ldff1d(const ZRegister& zt,
4288
                       const PRegisterZ& pg,
4289
                       const Register& xn,
4290
                       const ZRegister& zm) {
4291
  // LDFF1D { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #3]
4292
  //  1100 0101 111. .... 111. .... .... ....
4293
  //  msz<24:23> = 11 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4294
  //  | Zt<4:0>
4295

4296
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4297

4298
  Emit(LDFF1D_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4299
}
4300

4301
void Assembler::ldff1d(const ZRegister& zt,
4302
                       const PRegisterZ& pg,
4303
                       const ZRegister& zn,
4304
                       int imm5) {
4305
  // LDFF1D { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4306
  //  1100 0101 101. .... 111. .... .... ....
4307
  //  msz<24:23> = 11 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4308
  //  Zn<9:5> | Zt<4:0>
4309

4310
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4311

4312
  Emit(LDFF1D_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4313
}
4314

4315
// This prototype maps to 6 instruction encodings:
4316
//  LDFF1H_z_p_bz_d_64_scaled
4317
//  LDFF1H_z_p_bz_d_64_unscaled
4318
//  LDFF1H_z_p_bz_d_x32_scaled
4319
//  LDFF1H_z_p_bz_d_x32_unscaled
4320
void Assembler::ldff1h(const ZRegister& zt,
4321
                       const PRegisterZ& pg,
4322
                       const Register& xn,
4323
                       const ZRegister& zm) {
4324
  // LDFF1H { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1]
4325
  //  1100 0100 111. .... 111. .... .... ....
4326
  //  msz<24:23> = 01 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4327
  //  | Zt<4:0>
4328

4329
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4330

4331
  Emit(LDFF1H_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4332
}
4333

4334
// This prototype maps to 2 instruction encodings:
4335
//  LDFF1H_z_p_ai_d
4336
//  LDFF1H_z_p_ai_s
4337
void Assembler::ldff1h(const ZRegister& zt,
4338
                       const PRegisterZ& pg,
4339
                       const ZRegister& zn,
4340
                       int imm5) {
4341
  // LDFF1H { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4342
  //  1100 0100 101. .... 111. .... .... ....
4343
  //  msz<24:23> = 01 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4344
  //  Zn<9:5> | Zt<4:0>
4345

4346
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4347

4348
  Emit(LDFF1H_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4349
}
4350

4351
// This prototype maps to 3 instruction encodings:
4352
//  LDFF1SB_z_p_bz_d_64_unscaled
4353
//  LDFF1SB_z_p_bz_d_x32_unscaled
4354
void Assembler::ldff1sb(const ZRegister& zt,
4355
                        const PRegisterZ& pg,
4356
                        const Register& xn,
4357
                        const ZRegister& zm) {
4358
  // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D]
4359
  //  1100 0100 010. .... 101. .... .... ....
4360
  //  msz<24:23> = 00 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4361
  //  | Zt<4:0>
4362

4363
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4364

4365
  Emit(LDFF1SB_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4366
}
4367

4368
// This prototype maps to 2 instruction encodings:
4369
//  LDFF1SB_z_p_ai_d
4370
//  LDFF1SB_z_p_ai_s
4371
void Assembler::ldff1sb(const ZRegister& zt,
4372
                        const PRegisterZ& pg,
4373
                        const ZRegister& zn,
4374
                        int imm5) {
4375
  // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4376
  //  1100 0100 001. .... 101. .... .... ....
4377
  //  msz<24:23> = 00 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4378
  //  Zn<9:5> | Zt<4:0>
4379

4380
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4381

4382
  Emit(LDFF1SB_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4383
       ImmField<20, 16>(imm5));
4384
}
4385

4386
// This prototype maps to 6 instruction encodings:
4387
//  LDFF1SH_z_p_bz_d_64_scaled
4388
//  LDFF1SH_z_p_bz_d_64_unscaled
4389
//  LDFF1SH_z_p_bz_d_x32_scaled
4390
//  LDFF1SH_z_p_bz_d_x32_unscaled
4391
void Assembler::ldff1sh(const ZRegister& zt,
4392
                        const PRegisterZ& pg,
4393
                        const Register& xn,
4394
                        const ZRegister& zm) {
4395
  // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1]
4396
  //  1100 0100 111. .... 101. .... .... ....
4397
  //  msz<24:23> = 01 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4398
  //  | Zt<4:0>
4399

4400
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4401

4402
  Emit(LDFF1SH_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4403
}
4404

4405
// This prototype maps to 2 instruction encodings:
4406
//  LDFF1SH_z_p_ai_d
4407
//  LDFF1SH_z_p_ai_s
4408
void Assembler::ldff1sh(const ZRegister& zt,
4409
                        const PRegisterZ& pg,
4410
                        const ZRegister& zn,
4411
                        int imm5) {
4412
  // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4413
  //  1100 0100 101. .... 101. .... .... ....
4414
  //  msz<24:23> = 01 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4415
  //  Zn<9:5> | Zt<4:0>
4416

4417
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4418

4419
  Emit(LDFF1SH_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4420
       ImmField<20, 16>(imm5));
4421
}
4422

4423
// This prototype maps to 4 instruction encodings:
4424
//  LDFF1SW_z_p_bz_d_64_scaled
4425
//  LDFF1SW_z_p_bz_d_64_unscaled
4426
//  LDFF1SW_z_p_bz_d_x32_scaled
4427
//  LDFF1SW_z_p_bz_d_x32_unscaled
4428
void Assembler::ldff1sw(const ZRegister& zt,
4429
                        const PRegisterZ& pg,
4430
                        const Register& xn,
4431
                        const ZRegister& zm) {
4432
  // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2]
4433
  //  1100 0101 011. .... 101. .... .... ....
4434
  //  msz<24:23> = 10 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4435
  //  | Zt<4:0>
4436

4437
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4438

4439
  Emit(LDFF1SW_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4440
}
4441

4442
void Assembler::ldff1sw(const ZRegister& zt,
4443
                        const PRegisterZ& pg,
4444
                        const ZRegister& zn,
4445
                        int imm5) {
4446
  // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4447
  //  1100 0101 001. .... 101. .... .... ....
4448
  //  msz<24:23> = 10 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> |
4449
  //  Zn<9:5> | Zt<4:0>
4450

4451
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4452

4453
  Emit(LDFF1SW_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) |
4454
       ImmField<20, 16>(imm5));
4455
}
4456

4457
// This prototype maps to 6 instruction encodings:
4458
//  LDFF1W_z_p_bz_d_64_scaled
4459
//  LDFF1W_z_p_bz_d_64_unscaled
4460
//  LDFF1W_z_p_bz_d_x32_scaled
4461
//  LDFF1W_z_p_bz_d_x32_unscaled
4462
void Assembler::ldff1w(const ZRegister& zt,
4463
                       const PRegisterZ& pg,
4464
                       const Register& xn,
4465
                       const ZRegister& zm) {
4466
  // LDFF1W { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2]
4467
  //  1100 0101 011. .... 111. .... .... ....
4468
  //  msz<24:23> = 10 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5>
4469
  //  | Zt<4:0>
4470

4471
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4472

4473
  Emit(LDFF1W_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm));
4474
}
4475

4476
// This prototype maps to 2 instruction encodings:
4477
//  LDFF1W_z_p_ai_d
4478
//  LDFF1W_z_p_ai_s
4479
void Assembler::ldff1w(const ZRegister& zt,
4480
                       const PRegisterZ& pg,
4481
                       const ZRegister& zn,
4482
                       int imm5) {
4483
  // LDFF1W { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}]
4484
  //  1100 0101 001. .... 111. .... .... ....
4485
  //  msz<24:23> = 10 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> |
4486
  //  Zn<9:5> | Zt<4:0>
4487

4488
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4489

4490
  Emit(LDFF1W_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5));
4491
}
4492

4493
void Assembler::SVEGatherPrefetchVectorPlusImmediateHelper(
4494
    PrefetchOperation prfop,
4495
    const PRegister& pg,
4496
    const SVEMemOperand& addr,
4497
    int prefetch_size) {
4498
  VIXL_ASSERT(addr.IsVectorPlusImmediate());
4499
  ZRegister zn = addr.GetVectorBase();
4500
  VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
4501

4502
  Instr op = 0xffffffff;
4503
  switch (prefetch_size) {
4504
    case kBRegSize:
4505
      op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_ai_s)
4506
                            : static_cast<Instr>(PRFB_i_p_ai_d);
4507
      break;
4508
    case kHRegSize:
4509
      op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_ai_s)
4510
                            : static_cast<Instr>(PRFH_i_p_ai_d);
4511
      break;
4512
    case kSRegSize:
4513
      op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_ai_s)
4514
                            : static_cast<Instr>(PRFW_i_p_ai_d);
4515
      break;
4516
    case kDRegSize:
4517
      op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_ai_s)
4518
                            : static_cast<Instr>(PRFD_i_p_ai_d);
4519
      break;
4520
    default:
4521
      VIXL_UNIMPLEMENTED();
4522
      break;
4523
  }
4524

4525
  int64_t imm5 = addr.GetImmediateOffset();
4526
  Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | Rn(zn) |
4527
       ImmUnsignedField<20, 16>(imm5));
4528
}
4529

4530
void Assembler::SVEGatherPrefetchScalarPlusImmediateHelper(
4531
    PrefetchOperation prfop,
4532
    const PRegister& pg,
4533
    const SVEMemOperand& addr,
4534
    int prefetch_size) {
4535
  VIXL_ASSERT(addr.IsScalarPlusImmediate());
4536
  int64_t imm6 = addr.GetImmediateOffset();
4537

4538
  Instr op = 0xffffffff;
4539
  switch (prefetch_size) {
4540
    case kBRegSize:
4541
      op = PRFB_i_p_bi_s;
4542
      break;
4543
    case kHRegSize:
4544
      op = PRFH_i_p_bi_s;
4545
      break;
4546
    case kSRegSize:
4547
      op = PRFW_i_p_bi_s;
4548
      break;
4549
    case kDRegSize:
4550
      op = PRFD_i_p_bi_s;
4551
      break;
4552
    default:
4553
      VIXL_UNIMPLEMENTED();
4554
      break;
4555
  }
4556

4557
  Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) |
4558
       RnSP(addr.GetScalarBase()) | ImmField<21, 16>(imm6));
4559
}
4560

4561
void Assembler::SVEContiguousPrefetchScalarPlusScalarHelper(
4562
    PrefetchOperation prfop,
4563
    const PRegister& pg,
4564
    const SVEMemOperand& addr,
4565
    int prefetch_size) {
4566
  VIXL_ASSERT(addr.IsScalarPlusScalar());
4567
  Instr op = 0xffffffff;
4568

4569
  switch (prefetch_size) {
4570
    case kBRegSize:
4571
      VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER);
4572
      op = PRFB_i_p_br_s;
4573
      break;
4574
    case kHRegSize:
4575
      VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4576
      VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2);
4577
      op = PRFH_i_p_br_s;
4578
      break;
4579
    case kSRegSize:
4580
      VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4581
      VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2);
4582
      op = PRFW_i_p_br_s;
4583
      break;
4584
    case kDRegSize:
4585
      VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL);
4586
      VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2);
4587
      op = PRFD_i_p_br_s;
4588
      break;
4589
    default:
4590
      VIXL_UNIMPLEMENTED();
4591
      break;
4592
  }
4593

4594
  VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
4595
  Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) |
4596
       RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset()));
4597
}
4598

4599
void Assembler::SVEContiguousPrefetchScalarPlusVectorHelper(
4600
    PrefetchOperation prfop,
4601
    const PRegister& pg,
4602
    const SVEMemOperand& addr,
4603
    int prefetch_size) {
4604
  VIXL_ASSERT(addr.IsScalarPlusVector());
4605
  ZRegister zm = addr.GetVectorOffset();
4606
  SVEOffsetModifier mod = addr.GetOffsetModifier();
4607

4608
  // All prefetch scalar-plus-vector addressing modes use a shift corresponding
4609
  // to the element size.
4610
  switch (prefetch_size) {
4611
    case kBRegSize:
4612
      VIXL_ASSERT(addr.GetShiftAmount() == kBRegSizeInBytesLog2);
4613
      break;
4614
    case kHRegSize:
4615
      VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2);
4616
      break;
4617
    case kSRegSize:
4618
      VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2);
4619
      break;
4620
    case kDRegSize:
4621
      VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2);
4622
      break;
4623
    default:
4624
      VIXL_UNIMPLEMENTED();
4625
      break;
4626
  }
4627

4628
  Instr sx = 0;
4629
  Instr op = 0xffffffff;
4630
  if ((mod == NO_SVE_OFFSET_MODIFIER) || (mod == SVE_LSL)) {
4631
    VIXL_ASSERT(zm.IsLaneSizeD());
4632

4633
    switch (prefetch_size) {
4634
      case kBRegSize:
4635
        VIXL_ASSERT(mod == NO_SVE_OFFSET_MODIFIER);
4636
        op = PRFB_i_p_bz_d_64_scaled;
4637
        break;
4638
      case kHRegSize:
4639
        VIXL_ASSERT(mod == SVE_LSL);
4640
        op = PRFH_i_p_bz_d_64_scaled;
4641
        break;
4642
      case kSRegSize:
4643
        VIXL_ASSERT(mod == SVE_LSL);
4644
        op = PRFW_i_p_bz_d_64_scaled;
4645
        break;
4646
      case kDRegSize:
4647
        VIXL_ASSERT(mod == SVE_LSL);
4648
        op = PRFD_i_p_bz_d_64_scaled;
4649
        break;
4650
      default:
4651
        VIXL_UNIMPLEMENTED();
4652
        break;
4653
    }
4654
  } else {
4655
    VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW));
4656
    VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD());
4657

4658
    switch (prefetch_size) {
4659
      case kBRegSize:
4660
        op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_bz_s_x32_scaled)
4661
                              : static_cast<Instr>(PRFB_i_p_bz_d_x32_scaled);
4662
        break;
4663
      case kHRegSize:
4664
        op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_bz_s_x32_scaled)
4665
                              : static_cast<Instr>(PRFH_i_p_bz_d_x32_scaled);
4666
        break;
4667
      case kSRegSize:
4668
        op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_bz_s_x32_scaled)
4669
                              : static_cast<Instr>(PRFW_i_p_bz_d_x32_scaled);
4670
        break;
4671
      case kDRegSize:
4672
        op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_bz_s_x32_scaled)
4673
                              : static_cast<Instr>(PRFD_i_p_bz_d_x32_scaled);
4674
        break;
4675
      default:
4676
        VIXL_UNIMPLEMENTED();
4677
        break;
4678
    }
4679

4680
    if (mod == SVE_SXTW) {
4681
      sx = 1 << 22;
4682
    }
4683
  }
4684

4685
  Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | sx |
4686
       RnSP(addr.GetScalarBase()) | Rm(zm));
4687
}
4688

4689
void Assembler::SVEPrefetchHelper(PrefetchOperation prfop,
4690
                                  const PRegister& pg,
4691
                                  const SVEMemOperand& addr,
4692
                                  int prefetch_size) {
4693
  if (addr.IsVectorPlusImmediate()) {
4694
    // For example:
4695
    //   [z0.s, #0]
4696
    SVEGatherPrefetchVectorPlusImmediateHelper(prfop, pg, addr, prefetch_size);
4697

4698
  } else if (addr.IsScalarPlusImmediate()) {
4699
    // For example:
4700
    //   [x0, #42, mul vl]
4701
    SVEGatherPrefetchScalarPlusImmediateHelper(prfop, pg, addr, prefetch_size);
4702

4703
  } else if (addr.IsScalarPlusVector()) {
4704
    // For example:
4705
    //   [x0, z0.s, sxtw]
4706
    SVEContiguousPrefetchScalarPlusVectorHelper(prfop, pg, addr, prefetch_size);
4707

4708
  } else if (addr.IsScalarPlusScalar()) {
4709
    // For example:
4710
    //   [x0, x1]
4711
    SVEContiguousPrefetchScalarPlusScalarHelper(prfop, pg, addr, prefetch_size);
4712

4713
  } else {
4714
    VIXL_UNIMPLEMENTED();
4715
  }
4716
}
4717

4718
void Assembler::prfb(PrefetchOperation prfop,
4719
                     const PRegister& pg,
4720
                     const SVEMemOperand& addr) {
4721
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4722
  SVEPrefetchHelper(prfop, pg, addr, kBRegSize);
4723
}
4724

4725
void Assembler::prfd(PrefetchOperation prfop,
4726
                     const PRegister& pg,
4727
                     const SVEMemOperand& addr) {
4728
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4729
  SVEPrefetchHelper(prfop, pg, addr, kDRegSize);
4730
}
4731

4732
void Assembler::prfh(PrefetchOperation prfop,
4733
                     const PRegister& pg,
4734
                     const SVEMemOperand& addr) {
4735
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4736
  SVEPrefetchHelper(prfop, pg, addr, kHRegSize);
4737
}
4738

4739
void Assembler::prfw(PrefetchOperation prfop,
4740
                     const PRegister& pg,
4741
                     const SVEMemOperand& addr) {
4742
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4743
  SVEPrefetchHelper(prfop, pg, addr, kSRegSize);
4744
}
4745

4746
void Assembler::SVELd1St1ScaImmHelper(const ZRegister& zt,
4747
                                      const PRegister& pg,
4748
                                      const SVEMemOperand& addr,
4749
                                      Instr regoffset_op,
4750
                                      Instr immoffset_op,
4751
                                      int imm_divisor) {
4752
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4753
  VIXL_ASSERT(addr.IsScalarPlusScalar() || addr.IsScalarPlusImmediate());
4754

4755
  Instr op;
4756
  if (addr.IsScalarPlusScalar()) {
4757
    op = regoffset_op | Rm(addr.GetScalarOffset());
4758
  } else {
4759
    int64_t imm = addr.GetImmediateOffset();
4760
    VIXL_ASSERT(((imm % imm_divisor) == 0) && IsInt4(imm / imm_divisor));
4761
    op = immoffset_op | ImmField<19, 16>(imm / imm_divisor);
4762
  }
4763
  Emit(op | Rt(zt) | PgLow8(pg) | RnSP(addr.GetScalarBase()));
4764
}
4765

4766
void Assembler::SVELd1VecScaHelper(const ZRegister& zt,
4767
                                   const PRegister& pg,
4768
                                   const SVEMemOperand& addr,
4769
                                   uint32_t msize_bytes_log2,
4770
                                   bool is_signed) {
4771
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
4772
  VIXL_ASSERT(addr.IsVectorPlusScalar());
4773
  ZRegister zn = addr.GetVectorBase();
4774
  VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
4775
  VIXL_ASSERT(AreSameLaneSize(zn, zt));
4776

4777
  uint32_t esize = zn.GetLaneSizeInBytesLog2();
4778
  uint32_t b14_13 = 0;
4779
  if (!is_signed) b14_13 = zn.IsLaneSizeS() ? 0x1 : 0x2;
4780

4781
  Instr op = 0x04008000;  // LDNT1 with vector plus scalar addressing mode.
4782
  op |= (esize << 30) | (msize_bytes_log2 << 23) | (b14_13 << 13);
4783
  Emit(op | Rt(zt) | PgLow8(pg) |
4784
       SVEMemOperandHelper(msize_bytes_log2, 1, addr, true));
4785
}
4786

4787
void Assembler::SVESt1VecScaHelper(const ZRegister& zt,
4788
                                   const PRegister& pg,
4789
                                   const SVEMemOperand& addr,
4790
                                   uint32_t msize_bytes_log2) {
4791
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
4792
  VIXL_ASSERT(addr.IsVectorPlusScalar());
4793
  ZRegister zn = addr.GetVectorBase();
4794
  VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
4795
  VIXL_ASSERT(AreSameLaneSize(zn, zt));
4796

4797
  uint32_t bit22 = zn.IsLaneSizeS() ? (1 << 22) : 0;
4798
  Instr op = 0xe4002000;  // STNT1 with vector plus scalar addressing mode.
4799
  op |= bit22 | (msize_bytes_log2 << 23);
4800
  Emit(op | Rt(zt) | PgLow8(pg) |
4801
       SVEMemOperandHelper(msize_bytes_log2, 1, addr, true));
4802
}
4803

4804
#define VIXL_SVE_LD1R_LIST(V)                                  \
4805
  V(qb, 0, B, LD1RQB_z_p_br_contiguous, LD1RQB_z_p_bi_u8, 16)  \
4806
  V(qh, 1, H, LD1RQH_z_p_br_contiguous, LD1RQH_z_p_bi_u16, 16) \
4807
  V(qw, 2, S, LD1RQW_z_p_br_contiguous, LD1RQW_z_p_bi_u32, 16) \
4808
  V(qd, 3, D, LD1RQD_z_p_br_contiguous, LD1RQD_z_p_bi_u64, 16) \
4809
  V(ob, 0, B, 0xa4200000, 0xa4202000, 32)                      \
4810
  V(oh, 1, H, 0xa4a00000, 0xa4a02000, 32)                      \
4811
  V(ow, 2, S, 0xa5200000, 0xa5202000, 32)                      \
4812
  V(od, 3, D, 0xa5a00000, 0xa5a02000, 32)
4813

4814
#define VIXL_DEFINE_ASM_FUNC(FN, SH, SZ, SCA, IMM, BYTES)                    \
4815
  void Assembler::ld1r##FN(const ZRegister& zt,                              \
4816
                           const PRegisterZ& pg,                             \
4817
                           const SVEMemOperand& addr) {                      \
4818
    VIXL_ASSERT((BYTES == 16) ||                                             \
4819
                ((BYTES == 32) && (CPUHas(CPUFeatures::kSVEF64MM))));        \
4820
    VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(SH)); \
4821
    VIXL_ASSERT(zt.IsLaneSize##SZ());                                        \
4822
    SVELd1St1ScaImmHelper(zt, pg, addr, SCA, IMM, BYTES);                    \
4823
  }
4824
VIXL_SVE_LD1R_LIST(VIXL_DEFINE_ASM_FUNC)
4825
#undef VIXL_DEFINE_ASM_FUNC
4826
#undef VIXL_SVE_LD1R_LIST
4827

4828
#define VIXL_DEFINE_LDFF1(MSZ, LANE_SIZE)                                  \
4829
  void Assembler::ldff1##MSZ(const ZRegister& zt,                          \
4830
                             const PRegisterZ& pg,                         \
4831
                             const SVEMemOperand& addr) {                  \
4832
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                                \
4833
    SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \
4834
  }
4835
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LDFF1)
4836

4837
#define VIXL_DEFINE_LDFF1S(MSZ, LANE_SIZE)                                \
4838
  void Assembler::ldff1s##MSZ(const ZRegister& zt,                        \
4839
                              const PRegisterZ& pg,                       \
4840
                              const SVEMemOperand& addr) {                \
4841
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                               \
4842
    SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \
4843
  }
4844
VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LDFF1S)
4845

4846
void Assembler::ldnf1b(const ZRegister& zt,
4847
                       const PRegisterZ& pg,
4848
                       const SVEMemOperand& addr) {
4849
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4850
  VIXL_ASSERT(addr.IsPlainRegister() ||
4851
              (addr.IsScalarPlusImmediate() &&
4852
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4853

4854
  SVELdSt1Helper(0,
4855
                 zt,
4856
                 pg,
4857
                 addr,
4858
                 /* is_signed = */ false,
4859
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4860
}
4861

4862
void Assembler::ldnf1d(const ZRegister& zt,
4863
                       const PRegisterZ& pg,
4864
                       const SVEMemOperand& addr) {
4865
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4866
  VIXL_ASSERT(addr.IsPlainRegister() ||
4867
              (addr.IsScalarPlusImmediate() &&
4868
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4869

4870
  SVELdSt1Helper(3,
4871
                 zt,
4872
                 pg,
4873
                 addr,
4874
                 /* is_signed = */ false,
4875
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4876
}
4877

4878
void Assembler::ldnf1h(const ZRegister& zt,
4879
                       const PRegisterZ& pg,
4880
                       const SVEMemOperand& addr) {
4881
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4882
  VIXL_ASSERT(addr.IsPlainRegister() ||
4883
              (addr.IsScalarPlusImmediate() &&
4884
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4885

4886
  SVELdSt1Helper(1,
4887
                 zt,
4888
                 pg,
4889
                 addr,
4890
                 /* is_signed = */ false,
4891
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4892
}
4893

4894
void Assembler::ldnf1sb(const ZRegister& zt,
4895
                        const PRegisterZ& pg,
4896
                        const SVEMemOperand& addr) {
4897
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4898
  VIXL_ASSERT(addr.IsPlainRegister() ||
4899
              (addr.IsScalarPlusImmediate() &&
4900
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4901

4902
  SVELdSt1Helper(0,
4903
                 zt,
4904
                 pg,
4905
                 addr,
4906
                 /* is_signed = */ true,
4907
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4908
}
4909

4910
void Assembler::ldnf1sh(const ZRegister& zt,
4911
                        const PRegisterZ& pg,
4912
                        const SVEMemOperand& addr) {
4913
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4914
  VIXL_ASSERT(addr.IsPlainRegister() ||
4915
              (addr.IsScalarPlusImmediate() &&
4916
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4917

4918
  SVELdSt1Helper(1,
4919
                 zt,
4920
                 pg,
4921
                 addr,
4922
                 /* is_signed = */ true,
4923
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4924
}
4925

4926
void Assembler::ldnf1sw(const ZRegister& zt,
4927
                        const PRegisterZ& pg,
4928
                        const SVEMemOperand& addr) {
4929
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4930
  VIXL_ASSERT(addr.IsPlainRegister() ||
4931
              (addr.IsScalarPlusImmediate() &&
4932
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4933

4934
  SVELdSt1Helper(2,
4935
                 zt,
4936
                 pg,
4937
                 addr,
4938
                 /* is_signed = */ true,
4939
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4940
}
4941

4942
void Assembler::ldnf1w(const ZRegister& zt,
4943
                       const PRegisterZ& pg,
4944
                       const SVEMemOperand& addr) {
4945
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
4946
  VIXL_ASSERT(addr.IsPlainRegister() ||
4947
              (addr.IsScalarPlusImmediate() &&
4948
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
4949

4950
  SVELdSt1Helper(2,
4951
                 zt,
4952
                 pg,
4953
                 addr,
4954
                 /* is_signed = */ false,
4955
                 SVEContiguousNonFaultLoad_ScalarPlusImmFixed);
4956
}
4957

4958
void Assembler::ldnt1b(const ZRegister& zt,
4959
                       const PRegisterZ& pg,
4960
                       const SVEMemOperand& addr) {
4961
  VIXL_ASSERT(addr.IsPlainScalar() ||
4962
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4963
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)) ||
4964
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
4965
  if (addr.IsVectorPlusScalar()) {
4966
    SVELd1VecScaHelper(zt, pg, addr, 0, /* is_signed = */ false);
4967
  } else {
4968
    SVELd1St1ScaImmHelper(zt,
4969
                          pg,
4970
                          addr,
4971
                          LDNT1B_z_p_br_contiguous,
4972
                          LDNT1B_z_p_bi_contiguous);
4973
  }
4974
}
4975

4976
void Assembler::ldnt1d(const ZRegister& zt,
4977
                       const PRegisterZ& pg,
4978
                       const SVEMemOperand& addr) {
4979
  VIXL_ASSERT(addr.IsPlainScalar() ||
4980
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4981
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)) ||
4982
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
4983
  if (addr.IsVectorPlusScalar()) {
4984
    SVELd1VecScaHelper(zt, pg, addr, 3, /* is_signed = */ false);
4985
  } else {
4986
    SVELd1St1ScaImmHelper(zt,
4987
                          pg,
4988
                          addr,
4989
                          LDNT1D_z_p_br_contiguous,
4990
                          LDNT1D_z_p_bi_contiguous);
4991
  }
4992
}
4993

4994
void Assembler::ldnt1h(const ZRegister& zt,
4995
                       const PRegisterZ& pg,
4996
                       const SVEMemOperand& addr) {
4997
  VIXL_ASSERT(addr.IsPlainScalar() ||
4998
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
4999
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)) ||
5000
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5001
  if (addr.IsVectorPlusScalar()) {
5002
    SVELd1VecScaHelper(zt, pg, addr, 1, /* is_signed = */ false);
5003
  } else {
5004
    SVELd1St1ScaImmHelper(zt,
5005
                          pg,
5006
                          addr,
5007
                          LDNT1H_z_p_br_contiguous,
5008
                          LDNT1H_z_p_bi_contiguous);
5009
  }
5010
}
5011

5012
void Assembler::ldnt1w(const ZRegister& zt,
5013
                       const PRegisterZ& pg,
5014
                       const SVEMemOperand& addr) {
5015
  VIXL_ASSERT(addr.IsPlainScalar() ||
5016
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5017
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)) ||
5018
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5019
  if (addr.IsVectorPlusScalar()) {
5020
    SVELd1VecScaHelper(zt, pg, addr, 2, /* is_signed = */ false);
5021
  } else {
5022
    SVELd1St1ScaImmHelper(zt,
5023
                          pg,
5024
                          addr,
5025
                          LDNT1W_z_p_br_contiguous,
5026
                          LDNT1W_z_p_bi_contiguous);
5027
  }
5028
}
5029

5030
void Assembler::ldnt1sb(const ZRegister& zt,
5031
                        const PRegisterZ& pg,
5032
                        const SVEMemOperand& addr) {
5033
  VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2));
5034
  SVELd1VecScaHelper(zt, pg, addr, 0, /* is_signed = */ true);
5035
}
5036

5037
void Assembler::ldnt1sh(const ZRegister& zt,
5038
                        const PRegisterZ& pg,
5039
                        const SVEMemOperand& addr) {
5040
  VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2));
5041
  SVELd1VecScaHelper(zt, pg, addr, 1, /* is_signed = */ true);
5042
}
5043

5044
void Assembler::ldnt1sw(const ZRegister& zt,
5045
                        const PRegisterZ& pg,
5046
                        const SVEMemOperand& addr) {
5047
  VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2));
5048
  SVELd1VecScaHelper(zt, pg, addr, 2, /* is_signed = */ true);
5049
}
5050

5051
Instr Assembler::SVEMemOperandHelper(unsigned msize_in_bytes_log2,
5052
                                     int num_regs,
5053
                                     const SVEMemOperand& addr,
5054
                                     bool is_load) {
5055
  VIXL_ASSERT((num_regs >= 1) && (num_regs <= 4));
5056

5057
  Instr op = 0xfffffff;
5058
  if (addr.IsScalarPlusImmediate()) {
5059
    VIXL_ASSERT((addr.GetImmediateOffset() == 0) || addr.IsMulVl());
5060
    int64_t imm = addr.GetImmediateOffset();
5061
    VIXL_ASSERT((imm % num_regs) == 0);
5062
    op = RnSP(addr.GetScalarBase()) | ImmField<19, 16>(imm / num_regs);
5063

5064
  } else if (addr.IsScalarPlusScalar()) {
5065
    VIXL_ASSERT(addr.GetScalarOffset().IsZero() ||
5066
                addr.IsEquivalentToLSL(msize_in_bytes_log2));
5067
    op = RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset());
5068

5069
  } else if (addr.IsVectorPlusImmediate()) {
5070
    ZRegister zn = addr.GetVectorBase();
5071
    uint64_t imm = addr.GetImmediateOffset();
5072
    VIXL_ASSERT(num_regs == 1);
5073
    VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
5074
    VIXL_ASSERT(IsMultiple(imm, (1 << msize_in_bytes_log2)));
5075
    op = Rn(zn) | ImmUnsignedField<20, 16>(imm >> msize_in_bytes_log2);
5076
  } else if (addr.IsVectorPlusScalar()) {
5077
    VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER);
5078
    VIXL_ASSERT(addr.GetShiftAmount() == 0);
5079
    ZRegister zn = addr.GetVectorBase();
5080
    VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD());
5081
    Register xm = addr.GetScalarOffset();
5082
    op = Rn(zn) | Rm(xm);
5083
  } else if (addr.IsScalarPlusVector()) {
5084
    // We have to support several different addressing modes. Some instructions
5085
    // support a subset of these, but the SVEMemOperand encoding is consistent.
5086
    Register xn = addr.GetScalarBase();
5087
    ZRegister zm = addr.GetVectorOffset();
5088
    SVEOffsetModifier mod = addr.GetOffsetModifier();
5089
    Instr modifier_bit = 1 << (is_load ? 22 : 14);
5090
    Instr xs = (mod == SVE_SXTW) ? modifier_bit : 0;
5091
    VIXL_ASSERT(num_regs == 1);
5092

5093
    if (mod == SVE_LSL) {
5094
      // 64-bit scaled offset:            [<Xn|SP>, <Zm>.D, LSL #<shift>]
5095
      VIXL_ASSERT(zm.IsLaneSizeD());
5096
      VIXL_ASSERT(addr.GetShiftAmount() == msize_in_bytes_log2);
5097
    } else if (mod == NO_SVE_OFFSET_MODIFIER) {
5098
      // 64-bit unscaled offset:          [<Xn|SP>, <Zm>.D]
5099
      VIXL_ASSERT(zm.IsLaneSizeD());
5100
      VIXL_ASSERT(addr.GetShiftAmount() == 0);
5101
    } else {
5102
      // 32-bit scaled offset:            [<Xn|SP>, <Zm>.S, <mod> #<shift>]
5103
      // 32-bit unscaled offset:          [<Xn|SP>, <Zm>.S, <mod>]
5104
      // 32-bit unpacked scaled offset:   [<Xn|SP>, <Zm>.D, <mod> #<shift>]
5105
      // 32-bit unpacked unscaled offset: [<Xn|SP>, <Zm>.D, <mod>]
5106
      VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD());
5107
      VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW));
5108
      VIXL_ASSERT((addr.GetShiftAmount() == 0) ||
5109
                  (addr.GetShiftAmount() == msize_in_bytes_log2));
5110
    }
5111

5112
    // The form itself is encoded in the instruction opcode.
5113
    op = RnSP(xn) | Rm(zm) | xs;
5114
  } else {
5115
    VIXL_UNIMPLEMENTED();
5116
  }
5117

5118
  return op;
5119
}
5120

5121
// SVEMemStore.
5122

5123
void Assembler::SVESt1Helper(unsigned msize_in_bytes_log2,
5124
                             const ZRegister& zt,
5125
                             const PRegister& pg,
5126
                             const SVEMemOperand& addr) {
5127
  if (addr.IsScalarPlusScalar()) {
5128
    // Rm must not be xzr.
5129
    VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
5130
  }
5131

5132
  if (addr.IsScatterGather()) {
5133
    bool is_load = false;
5134
    bool is_signed = false;
5135
    bool is_ff = false;
5136
    SVEScatterGatherHelper(msize_in_bytes_log2,
5137
                           zt,
5138
                           pg,
5139
                           addr,
5140
                           is_load,
5141
                           is_signed,
5142
                           is_ff);
5143
    return;
5144
  }
5145

5146
  Instr op;
5147
  if (addr.IsScalarPlusImmediate()) {
5148
    op = SVEContiguousStore_ScalarPlusImmFixed;
5149
  } else if (addr.IsScalarPlusScalar()) {
5150
    op = SVEContiguousStore_ScalarPlusScalarFixed;
5151
  } else {
5152
    VIXL_UNIMPLEMENTED();
5153
    op = 0xffffffff;
5154
  }
5155
  SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, false, op);
5156
}
5157

5158
void Assembler::SVESt234Helper(int num_regs,
5159
                               const ZRegister& zt1,
5160
                               const PRegister& pg,
5161
                               const SVEMemOperand& addr) {
5162
  if (addr.IsScalarPlusScalar()) {
5163
    // Rm must not be xzr.
5164
    VIXL_ASSERT(!addr.GetScalarOffset().IsZero());
5165
  }
5166

5167
  Instr op;
5168
  if (addr.IsScalarPlusImmediate()) {
5169
    op = SVEStoreMultipleStructures_ScalarPlusImmFixed;
5170
  } else if (addr.IsScalarPlusScalar()) {
5171
    op = SVEStoreMultipleStructures_ScalarPlusScalarFixed;
5172
  } else {
5173
    // These instructions don't support any other addressing modes.
5174
    VIXL_ABORT();
5175
  }
5176
  SVELdSt234Helper(num_regs, zt1, pg, addr, op);
5177
}
5178

5179
#define VIXL_DEFINE_ST1(MSZ, LANE_SIZE)                           \
5180
  void Assembler::st1##MSZ(const ZRegister& zt,                   \
5181
                           const PRegister& pg,                   \
5182
                           const SVEMemOperand& addr) {           \
5183
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));                       \
5184
    SVESt1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr); \
5185
  }
5186
#define VIXL_DEFINE_ST2(MSZ, LANE_SIZE)                 \
5187
  void Assembler::st2##MSZ(const ZRegister& zt1,        \
5188
                           const ZRegister& zt2,        \
5189
                           const PRegister& pg,         \
5190
                           const SVEMemOperand& addr) { \
5191
    USE(zt2);                                           \
5192
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
5193
    VIXL_ASSERT(AreConsecutive(zt1, zt2));              \
5194
    VIXL_ASSERT(AreSameFormat(zt1, zt2));               \
5195
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
5196
    SVESt234Helper(2, zt1, pg, addr);                   \
5197
  }
5198
#define VIXL_DEFINE_ST3(MSZ, LANE_SIZE)                 \
5199
  void Assembler::st3##MSZ(const ZRegister& zt1,        \
5200
                           const ZRegister& zt2,        \
5201
                           const ZRegister& zt3,        \
5202
                           const PRegister& pg,         \
5203
                           const SVEMemOperand& addr) { \
5204
    USE(zt2, zt3);                                      \
5205
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
5206
    VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3));         \
5207
    VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3));          \
5208
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
5209
    SVESt234Helper(3, zt1, pg, addr);                   \
5210
  }
5211
#define VIXL_DEFINE_ST4(MSZ, LANE_SIZE)                 \
5212
  void Assembler::st4##MSZ(const ZRegister& zt1,        \
5213
                           const ZRegister& zt2,        \
5214
                           const ZRegister& zt3,        \
5215
                           const ZRegister& zt4,        \
5216
                           const PRegister& pg,         \
5217
                           const SVEMemOperand& addr) { \
5218
    USE(zt2, zt3, zt4);                                 \
5219
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));             \
5220
    VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4));    \
5221
    VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4));     \
5222
    VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE());           \
5223
    SVESt234Helper(4, zt1, pg, addr);                   \
5224
  }
5225

5226
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST1)
5227
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST2)
5228
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST3)
5229
VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST4)
5230

5231
void Assembler::stnt1b(const ZRegister& zt,
5232
                       const PRegister& pg,
5233
                       const SVEMemOperand& addr) {
5234
  VIXL_ASSERT(addr.IsPlainScalar() ||
5235
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5236
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)) ||
5237
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5238
  if (addr.IsVectorPlusScalar()) {
5239
    SVESt1VecScaHelper(zt, pg, addr, 0);
5240
  } else {
5241
    SVELd1St1ScaImmHelper(zt,
5242
                          pg,
5243
                          addr,
5244
                          STNT1B_z_p_br_contiguous,
5245
                          STNT1B_z_p_bi_contiguous);
5246
  }
5247
}
5248

5249
void Assembler::stnt1d(const ZRegister& zt,
5250
                       const PRegister& pg,
5251
                       const SVEMemOperand& addr) {
5252
  VIXL_ASSERT(addr.IsPlainScalar() ||
5253
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5254
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)) ||
5255
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5256
  if (addr.IsVectorPlusScalar()) {
5257
    SVESt1VecScaHelper(zt, pg, addr, 3);
5258
  } else {
5259
    SVELd1St1ScaImmHelper(zt,
5260
                          pg,
5261
                          addr,
5262
                          STNT1D_z_p_br_contiguous,
5263
                          STNT1D_z_p_bi_contiguous);
5264
  }
5265
}
5266

5267
void Assembler::stnt1h(const ZRegister& zt,
5268
                       const PRegister& pg,
5269
                       const SVEMemOperand& addr) {
5270
  VIXL_ASSERT(addr.IsPlainScalar() ||
5271
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5272
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)) ||
5273
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5274
  if (addr.IsVectorPlusScalar()) {
5275
    SVESt1VecScaHelper(zt, pg, addr, 1);
5276
  } else {
5277
    SVELd1St1ScaImmHelper(zt,
5278
                          pg,
5279
                          addr,
5280
                          STNT1H_z_p_br_contiguous,
5281
                          STNT1H_z_p_bi_contiguous);
5282
  }
5283
}
5284

5285
void Assembler::stnt1w(const ZRegister& zt,
5286
                       const PRegister& pg,
5287
                       const SVEMemOperand& addr) {
5288
  VIXL_ASSERT(addr.IsPlainScalar() ||
5289
              (addr.IsScalarPlusImmediate() && addr.IsMulVl()) ||
5290
              (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)) ||
5291
              (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)));
5292
  if (addr.IsVectorPlusScalar()) {
5293
    SVESt1VecScaHelper(zt, pg, addr, 2);
5294
  } else {
5295
    SVELd1St1ScaImmHelper(zt,
5296
                          pg,
5297
                          addr,
5298
                          STNT1W_z_p_br_contiguous,
5299
                          STNT1W_z_p_bi_contiguous);
5300
  }
5301
}
5302

5303
void Assembler::str(const CPURegister& rt, const SVEMemOperand& addr) {
5304
  // STR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}]
5305

5306
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5307
  VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister());
5308
  VIXL_ASSERT(addr.IsPlainScalar() ||
5309
              (addr.IsScalarPlusImmediate() &&
5310
               (addr.GetOffsetModifier() == SVE_MUL_VL)));
5311
  int64_t imm9 = addr.GetImmediateOffset();
5312
  VIXL_ASSERT(IsInt9(imm9));
5313
  Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10;
5314
  Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16;
5315

5316
  Instr op = STR_z_bi;
5317
  if (rt.IsPRegister()) {
5318
    op = STR_p_bi;
5319
  }
5320
  Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l);
5321
}
5322

5323
// SVEMulIndex.
5324

5325
void Assembler::sdot(const ZRegister& zda,
5326
                     const ZRegister& zn,
5327
                     const ZRegister& zm,
5328
                     int index) {
5329
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5330
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
5331
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
5332

5333
  Instr op = 0xffffffff;
5334
  switch (zda.GetLaneSizeInBits()) {
5335
    case kSRegSize:
5336
      VIXL_ASSERT(IsUint2(index));
5337
      op = SDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn);
5338
      break;
5339
    case kDRegSize:
5340
      VIXL_ASSERT(IsUint1(index));
5341
      op = SDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn);
5342
      break;
5343
    default:
5344
      VIXL_UNIMPLEMENTED();
5345
      break;
5346
  }
5347

5348
  Emit(op);
5349
}
5350

5351
void Assembler::udot(const ZRegister& zda,
5352
                     const ZRegister& zn,
5353
                     const ZRegister& zm,
5354
                     int index) {
5355
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5356
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
5357
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
5358

5359
  Instr op = 0xffffffff;
5360
  switch (zda.GetLaneSizeInBits()) {
5361
    case kSRegSize:
5362
      VIXL_ASSERT(IsUint2(index));
5363
      op = UDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn);
5364
      break;
5365
    case kDRegSize:
5366
      VIXL_ASSERT(IsUint1(index));
5367
      op = UDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn);
5368
      break;
5369
    default:
5370
      VIXL_UNIMPLEMENTED();
5371
      break;
5372
  }
5373

5374
  Emit(op);
5375
}
5376

5377
// SVEPartitionBreak.
5378

5379
void Assembler::brka(const PRegisterWithLaneSize& pd,
5380
                     const PRegister& pg,
5381
                     const PRegisterWithLaneSize& pn) {
5382
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5383
  VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
5384
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5385

5386
  Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000;
5387
  Emit(BRKA_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn));
5388
}
5389

5390
void Assembler::brkas(const PRegisterWithLaneSize& pd,
5391
                      const PRegisterZ& pg,
5392
                      const PRegisterWithLaneSize& pn) {
5393
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5394
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5395

5396
  Emit(BRKAS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5397
}
5398

5399
void Assembler::brkb(const PRegisterWithLaneSize& pd,
5400
                     const PRegister& pg,
5401
                     const PRegisterWithLaneSize& pn) {
5402
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5403
  VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
5404
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5405

5406
  Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000;
5407
  Emit(BRKB_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn));
5408
}
5409

5410
void Assembler::brkbs(const PRegisterWithLaneSize& pd,
5411
                      const PRegisterZ& pg,
5412
                      const PRegisterWithLaneSize& pn) {
5413
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5414
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5415

5416
  Emit(BRKBS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5417
}
5418

5419
void Assembler::brkn(const PRegisterWithLaneSize& pd,
5420
                     const PRegisterZ& pg,
5421
                     const PRegisterWithLaneSize& pn,
5422
                     const PRegisterWithLaneSize& pm) {
5423
  USE(pm);
5424
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5425
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5426
  VIXL_ASSERT(pd.Is(pm));
5427

5428
  Emit(BRKN_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5429
}
5430

5431
void Assembler::brkns(const PRegisterWithLaneSize& pd,
5432
                      const PRegisterZ& pg,
5433
                      const PRegisterWithLaneSize& pn,
5434
                      const PRegisterWithLaneSize& pm) {
5435
  USE(pm);
5436
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5437
  VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB());
5438
  VIXL_ASSERT(pd.Is(pm));
5439

5440
  Emit(BRKNS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn));
5441
}
5442

5443
// SVEPermutePredicate.
5444

5445
void Assembler::punpkhi(const PRegisterWithLaneSize& pd,
5446
                        const PRegisterWithLaneSize& pn) {
5447
  // PUNPKHI <Pd>.H, <Pn>.B
5448
  //  0000 0101 0011 0001 0100 000. ...0 ....
5449
  //  H<16> = 1 | Pn<8:5> | Pd<3:0>
5450

5451
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5452
  VIXL_ASSERT(pd.IsLaneSizeH());
5453
  VIXL_ASSERT(pn.IsLaneSizeB());
5454

5455
  Emit(PUNPKHI_p_p | Pd(pd) | Pn(pn));
5456
}
5457

5458
void Assembler::punpklo(const PRegisterWithLaneSize& pd,
5459
                        const PRegisterWithLaneSize& pn) {
5460
  // PUNPKLO <Pd>.H, <Pn>.B
5461
  //  0000 0101 0011 0000 0100 000. ...0 ....
5462
  //  H<16> = 0 | Pn<8:5> | Pd<3:0>
5463

5464
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5465
  VIXL_ASSERT(pd.IsLaneSizeH());
5466
  VIXL_ASSERT(pn.IsLaneSizeB());
5467

5468
  Emit(PUNPKLO_p_p | Pd(pd) | Pn(pn));
5469
}
5470

5471
void Assembler::rev(const PRegisterWithLaneSize& pd,
5472
                    const PRegisterWithLaneSize& pn) {
5473
  // REV <Pd>.<T>, <Pn>.<T>
5474
  //  0000 0101 ..11 0100 0100 000. ...0 ....
5475
  //  size<23:22> | Pn<8:5> | Pd<3:0>
5476

5477
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5478
  VIXL_ASSERT(AreSameLaneSize(pd, pn));
5479

5480
  Emit(REV_p_p | SVESize(pd) | Pd(pd) | Rx<8, 5>(pn));
5481
}
5482

5483
void Assembler::trn1(const PRegisterWithLaneSize& pd,
5484
                     const PRegisterWithLaneSize& pn,
5485
                     const PRegisterWithLaneSize& pm) {
5486
  // TRN1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5487
  //  0000 0101 ..10 .... 0101 000. ...0 ....
5488
  //  size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5489

5490
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5491
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5492

5493
  Emit(TRN1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5494
}
5495

5496
void Assembler::trn2(const PRegisterWithLaneSize& pd,
5497
                     const PRegisterWithLaneSize& pn,
5498
                     const PRegisterWithLaneSize& pm) {
5499
  // TRN2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5500
  //  0000 0101 ..10 .... 0101 010. ...0 ....
5501
  //  size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5502

5503
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5504
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5505

5506
  Emit(TRN2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5507
}
5508

5509
void Assembler::uzp1(const PRegisterWithLaneSize& pd,
5510
                     const PRegisterWithLaneSize& pn,
5511
                     const PRegisterWithLaneSize& pm) {
5512
  // UZP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5513
  //  0000 0101 ..10 .... 0100 100. ...0 ....
5514
  //  size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5515

5516
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5517
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5518

5519
  Emit(UZP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5520
}
5521

5522
void Assembler::uzp2(const PRegisterWithLaneSize& pd,
5523
                     const PRegisterWithLaneSize& pn,
5524
                     const PRegisterWithLaneSize& pm) {
5525
  // UZP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5526
  //  0000 0101 ..10 .... 0100 110. ...0 ....
5527
  //  size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5528

5529
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5530
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5531

5532
  Emit(UZP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5533
}
5534

5535
void Assembler::zip1(const PRegisterWithLaneSize& pd,
5536
                     const PRegisterWithLaneSize& pn,
5537
                     const PRegisterWithLaneSize& pm) {
5538
  // ZIP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5539
  //  0000 0101 ..10 .... 0100 000. ...0 ....
5540
  //  size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 0 | Pn<8:5> | Pd<3:0>
5541

5542
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5543
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5544

5545
  Emit(ZIP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5546
}
5547

5548
void Assembler::zip2(const PRegisterWithLaneSize& pd,
5549
                     const PRegisterWithLaneSize& pn,
5550
                     const PRegisterWithLaneSize& pm) {
5551
  // ZIP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
5552
  //  0000 0101 ..10 .... 0100 010. ...0 ....
5553
  //  size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 1 | Pn<8:5> | Pd<3:0>
5554

5555
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5556
  VIXL_ASSERT(AreSameLaneSize(pd, pn, pm));
5557

5558
  Emit(ZIP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm));
5559
}
5560

5561
// SVEPermuteVectorExtract.
5562

5563
void Assembler::ext(const ZRegister& zd,
5564
                    const ZRegister& zn,
5565
                    const ZRegister& zm,
5566
                    unsigned offset) {
5567
  // EXT <Zdn>.B, <Zdn>.B, <Zm>.B, #<imm>
5568
  //  0000 0101 001. .... 000. .... .... ....
5569
  //  imm8h<20:16> | imm8l<12:10> | Zm<9:5> | Zdn<4:0>
5570

5571
  // EXT <Zd>.B, { <Zn1>.B, <Zn2>.B }, #<imm>
5572
  //  0000 0101 011. .... 000. .... .... ....
5573
  //  imm8h<20:16> | imm8l<12:10> | Zn<9:5> | Zd<4:0>
5574

5575
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5576
  VIXL_ASSERT(IsUint8(offset));
5577

5578
  int imm8h = ExtractUnsignedBitfield32(7, 3, offset);
5579
  int imm8l = ExtractUnsignedBitfield32(2, 0, offset);
5580

5581
  Instr op;
5582
  if (zd.Is(zn)) {
5583
    // Destructive form.
5584
    op = EXT_z_zi_des | Rn(zm);
5585
  } else {
5586
    // Constructive form (requires SVE2).
5587
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2) && AreConsecutive(zn, zm));
5588
    op = 0x05600000 | Rn(zn);
5589
  }
5590

5591
  Emit(op | Rd(zd) | ImmUnsignedField<20, 16>(imm8h) |
5592
       ImmUnsignedField<12, 10>(imm8l));
5593
}
5594

5595
// SVEPermuteVectorInterleaving.
5596

5597
void Assembler::trn1(const ZRegister& zd,
5598
                     const ZRegister& zn,
5599
                     const ZRegister& zm) {
5600
  // TRN1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5601
  //  0000 0101 ..1. .... 0111 00.. .... ....
5602
  //  size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0>
5603

5604
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5605
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5606

5607
  Emit(TRN1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5608
}
5609

5610
void Assembler::trn2(const ZRegister& zd,
5611
                     const ZRegister& zn,
5612
                     const ZRegister& zm) {
5613
  // TRN2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5614
  //  0000 0101 ..1. .... 0111 01.. .... ....
5615
  //  size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0>
5616

5617
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5618
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5619

5620
  Emit(TRN2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5621
}
5622

5623
void Assembler::uzp1(const ZRegister& zd,
5624
                     const ZRegister& zn,
5625
                     const ZRegister& zm) {
5626
  // UZP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5627
  //  0000 0101 ..1. .... 0110 10.. .... ....
5628
  //  size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0>
5629

5630
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5631
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5632

5633
  Emit(UZP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5634
}
5635

5636
void Assembler::uzp2(const ZRegister& zd,
5637
                     const ZRegister& zn,
5638
                     const ZRegister& zm) {
5639
  // UZP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5640
  //  0000 0101 ..1. .... 0110 11.. .... ....
5641
  //  size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0>
5642

5643
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5644
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5645

5646
  Emit(UZP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5647
}
5648

5649
void Assembler::zip1(const ZRegister& zd,
5650
                     const ZRegister& zn,
5651
                     const ZRegister& zm) {
5652
  // ZIP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5653
  //  0000 0101 ..1. .... 0110 00.. .... ....
5654
  //  size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0>
5655

5656
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5657
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5658

5659
  Emit(ZIP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5660
}
5661

5662
void Assembler::zip2(const ZRegister& zd,
5663
                     const ZRegister& zn,
5664
                     const ZRegister& zm) {
5665
  // ZIP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
5666
  //  0000 0101 ..1. .... 0110 01.. .... ....
5667
  //  size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0>
5668

5669
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5670
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5671

5672
  Emit(ZIP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
5673
}
5674

5675
// SVEPermuteVectorPredicated.
5676

5677
void Assembler::clasta(const Register& rd,
5678
                       const PRegister& pg,
5679
                       const Register& rn,
5680
                       const ZRegister& zm) {
5681
  // CLASTA <R><dn>, <Pg>, <R><dn>, <Zm>.<T>
5682
  //  0000 0101 ..11 0000 101. .... .... ....
5683
  //  size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Rdn<4:0>
5684

5685
  USE(rn);
5686
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5687
  VIXL_ASSERT(rd.Is(rn));
5688

5689
  Emit(CLASTA_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm));
5690
}
5691

5692
void Assembler::clasta(const VRegister& vd,
5693
                       const PRegister& pg,
5694
                       const VRegister& vn,
5695
                       const ZRegister& zm) {
5696
  // CLASTA <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
5697
  //  0000 0101 ..10 1010 100. .... .... ....
5698
  //  size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
5699

5700
  USE(vn);
5701
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5702
  VIXL_ASSERT(vd.Is(vn));
5703
  VIXL_ASSERT(vd.IsScalar());
5704
  VIXL_ASSERT(AreSameLaneSize(vd, zm));
5705

5706
  Emit(CLASTA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
5707
}
5708

5709
void Assembler::clasta(const ZRegister& zd,
5710
                       const PRegister& pg,
5711
                       const ZRegister& zn,
5712
                       const ZRegister& zm) {
5713
  // CLASTA <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5714
  //  0000 0101 ..10 1000 100. .... .... ....
5715
  //  size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5716

5717
  USE(zn);
5718
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5719
  VIXL_ASSERT(zd.Is(zn));
5720
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5721

5722
  Emit(CLASTA_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5723
}
5724

5725
void Assembler::clastb(const Register& rd,
5726
                       const PRegister& pg,
5727
                       const Register& rn,
5728
                       const ZRegister& zm) {
5729
  // CLASTB <R><dn>, <Pg>, <R><dn>, <Zm>.<T>
5730
  //  0000 0101 ..11 0001 101. .... .... ....
5731
  //  size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Rdn<4:0>
5732

5733
  USE(rn);
5734
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5735
  VIXL_ASSERT(rd.Is(rn));
5736

5737
  Emit(CLASTB_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm));
5738
}
5739

5740
void Assembler::clastb(const VRegister& vd,
5741
                       const PRegister& pg,
5742
                       const VRegister& vn,
5743
                       const ZRegister& zm) {
5744
  // CLASTB <V><dn>, <Pg>, <V><dn>, <Zm>.<T>
5745
  //  0000 0101 ..10 1011 100. .... .... ....
5746
  //  size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Vdn<4:0>
5747

5748
  USE(vn);
5749
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5750
  VIXL_ASSERT(vd.Is(vn));
5751
  VIXL_ASSERT(vd.IsScalar());
5752
  VIXL_ASSERT(AreSameLaneSize(vd, zm));
5753

5754
  Emit(CLASTB_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm));
5755
}
5756

5757
void Assembler::clastb(const ZRegister& zd,
5758
                       const PRegister& pg,
5759
                       const ZRegister& zn,
5760
                       const ZRegister& zm) {
5761
  // CLASTB <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5762
  //  0000 0101 ..10 1001 100. .... .... ....
5763
  //  size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5764

5765
  USE(zn);
5766
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5767
  VIXL_ASSERT(zd.Is(zn));
5768
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5769

5770
  Emit(CLASTB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5771
}
5772

5773
void Assembler::compact(const ZRegister& zd,
5774
                        const PRegister& pg,
5775
                        const ZRegister& zn) {
5776
  // COMPACT <Zd>.<T>, <Pg>, <Zn>.<T>
5777
  //  0000 0101 1.10 0001 100. .... .... ....
5778
  //  sz<22> | Pg<12:10> | Zn<9:5> | Zd<4:0>
5779

5780
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5781
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
5782
  VIXL_ASSERT((zd.GetLaneSizeInBits() == kSRegSize) ||
5783
              (zd.GetLaneSizeInBits() == kDRegSize));
5784

5785
  Instr sz = (zd.GetLaneSizeInBits() == kDRegSize) ? (1 << 22) : 0;
5786
  Emit(COMPACT_z_p_z | sz | Rd(zd) | PgLow8(pg) | Rn(zn));
5787
}
5788

5789
void Assembler::cpy(const ZRegister& zd,
5790
                    const PRegisterM& pg,
5791
                    const Register& rn) {
5792
  // CPY <Zd>.<T>, <Pg>/M, <R><n|SP>
5793
  //  0000 0101 ..10 1000 101. .... .... ....
5794
  //  size<23:22> | Pg<12:10> | Rn<9:5> | Zd<4:0>
5795

5796
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5797
  VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >=
5798
              zd.GetLaneSizeInBits());
5799

5800
  Emit(CPY_z_p_r | SVESize(zd) | Rd(zd) | PgLow8(pg) | RnSP(rn));
5801
}
5802

5803
void Assembler::cpy(const ZRegister& zd,
5804
                    const PRegisterM& pg,
5805
                    const VRegister& vn) {
5806
  // CPY <Zd>.<T>, <Pg>/M, <V><n>
5807
  //  0000 0101 ..10 0000 100. .... .... ....
5808
  //  size<23:22> | Pg<12:10> | Vn<9:5> | Zd<4:0>
5809

5810
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5811
  VIXL_ASSERT(vn.IsScalar());
5812
  VIXL_ASSERT(static_cast<unsigned>(vn.GetSizeInBits()) ==
5813
              zd.GetLaneSizeInBits());
5814

5815
  Emit(CPY_z_p_v | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(vn));
5816
}
5817

5818
void Assembler::lasta(const Register& rd,
5819
                      const PRegister& pg,
5820
                      const ZRegister& zn) {
5821
  // LASTA <R><d>, <Pg>, <Zn>.<T>
5822
  //  0000 0101 ..10 0000 101. .... .... ....
5823
  //  size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Rd<4:0>
5824

5825
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5826

5827
  Emit(LASTA_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn));
5828
}
5829

5830
void Assembler::lasta(const VRegister& vd,
5831
                      const PRegister& pg,
5832
                      const ZRegister& zn) {
5833
  // LASTA <V><d>, <Pg>, <Zn>.<T>
5834
  //  0000 0101 ..10 0010 100. .... .... ....
5835
  //  size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Vd<4:0>
5836

5837
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5838
  VIXL_ASSERT(vd.IsScalar());
5839

5840
  Emit(LASTA_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
5841
}
5842

5843
void Assembler::lastb(const Register& rd,
5844
                      const PRegister& pg,
5845
                      const ZRegister& zn) {
5846
  // LASTB <R><d>, <Pg>, <Zn>.<T>
5847
  //  0000 0101 ..10 0001 101. .... .... ....
5848
  //  size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Rd<4:0>
5849

5850
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5851

5852
  Emit(LASTB_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn));
5853
}
5854

5855
void Assembler::lastb(const VRegister& vd,
5856
                      const PRegister& pg,
5857
                      const ZRegister& zn) {
5858
  // LASTB <V><d>, <Pg>, <Zn>.<T>
5859
  //  0000 0101 ..10 0011 100. .... .... ....
5860
  //  size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Vd<4:0>
5861

5862
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5863
  VIXL_ASSERT(vd.IsScalar());
5864

5865
  Emit(LASTB_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn));
5866
}
5867

5868
void Assembler::rbit(const ZRegister& zd,
5869
                     const PRegisterM& pg,
5870
                     const ZRegister& zn) {
5871
  // RBIT <Zd>.<T>, <Pg>/M, <Zn>.<T>
5872
  //  0000 0101 ..10 0111 100. .... .... ....
5873
  //  size<23:22> | opc<17:16> = 11 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5874

5875
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5876
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
5877

5878
  Emit(RBIT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5879
}
5880

5881
void Assembler::revb(const ZRegister& zd,
5882
                     const PRegisterM& pg,
5883
                     const ZRegister& zn) {
5884
  // REVB <Zd>.<T>, <Pg>/M, <Zn>.<T>
5885
  //  0000 0101 ..10 0100 100. .... .... ....
5886
  //  size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5887

5888
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5889
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
5890
  VIXL_ASSERT(zd.IsLaneSizeH() || zd.IsLaneSizeS() || zd.IsLaneSizeD());
5891

5892
  Emit(REVB_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5893
}
5894

5895
void Assembler::revh(const ZRegister& zd,
5896
                     const PRegisterM& pg,
5897
                     const ZRegister& zn) {
5898
  // REVH <Zd>.<T>, <Pg>/M, <Zn>.<T>
5899
  //  0000 0101 ..10 0101 100. .... .... ....
5900
  //  size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5901

5902
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5903
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
5904
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
5905

5906
  Emit(REVH_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5907
}
5908

5909
void Assembler::revw(const ZRegister& zd,
5910
                     const PRegisterM& pg,
5911
                     const ZRegister& zn) {
5912
  // REVW <Zd>.D, <Pg>/M, <Zn>.D
5913
  //  0000 0101 ..10 0110 100. .... .... ....
5914
  //  size<23:22> | opc<17:16> = 10 | Pg<12:10> | Zn<9:5> | Zd<4:0>
5915

5916
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5917
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
5918
  VIXL_ASSERT(zd.IsLaneSizeD());
5919

5920
  Emit(REVW_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
5921
}
5922

5923
void Assembler::splice(const ZRegister& zd,
5924
                       const PRegister& pg,
5925
                       const ZRegister& zn,
5926
                       const ZRegister& zm) {
5927
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
5928

5929
  if (zd.Aliases(zn)) {
5930
    // SPLICE <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T>
5931
    //  0000 0101 ..10 1100 100. .... .... ....
5932
    //  size<23:22> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
5933

5934
    USE(zn);
5935
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5936
    VIXL_ASSERT(zd.Is(zn));
5937

5938
    Emit(SPLICE_z_p_zz_des | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
5939
  } else {
5940
    splice_con(zd, pg, zn, zm);
5941
  }
5942
}
5943

5944
void Assembler::splice_con(const ZRegister& zd,
5945
                           const PRegister& pg,
5946
                           const ZRegister& zn1,
5947
                           const ZRegister& zn2) {
5948
  // SPLICE <Zd>.<T>, <Pg>, { <Zn1>.<T>, <Zn2>.<T> }
5949
  //  0000 0101 ..10 1101 100. .... .... ....
5950
  //  size<23:22> | Pg<12:10> | Zn<9:5> | Zd<4:0>
5951

5952
  USE(zn2);
5953
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
5954
  VIXL_ASSERT(AreConsecutive(zn1, zn2));
5955
  VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2));
5956

5957
  Emit(0x052d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn1));
5958
}
5959

5960
// SVEPermuteVectorUnpredicated.
5961

5962
void Assembler::dup(const ZRegister& zd, const Register& xn) {
5963
  // DUP <Zd>.<T>, <R><n|SP>
5964
  //  0000 0101 ..10 0000 0011 10.. .... ....
5965
  //  size<23:22> | Rn<9:5> | Zd<4:0>
5966

5967
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5968

5969
  Emit(DUP_z_r | SVESize(zd) | Rd(zd) | RnSP(xn));
5970
}
5971

5972
void Assembler::dup(const ZRegister& zd, const ZRegister& zn, unsigned index) {
5973
  // DUP <Zd>.<T>, <Zn>.<T>[<imm>]
5974
  //  0000 0101 ..1. .... 0010 00.. .... ....
5975
  //  imm2<23:22> | tsz<20:16> | Zn<9:5> | Zd<4:0>
5976

5977
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5978
  VIXL_ASSERT(AreSameFormat(zd, zn));
5979
  VIXL_ASSERT((index * zd.GetLaneSizeInBits()) < 512);
5980
  int n = zd.GetLaneSizeInBytesLog2();
5981
  unsigned imm_7 = (index << (n + 1)) | (1 << n);
5982
  VIXL_ASSERT(IsUint7(imm_7));
5983
  unsigned imm_2 = ExtractUnsignedBitfield32(6, 5, imm_7);
5984
  unsigned tsz_5 = ExtractUnsignedBitfield32(4, 0, imm_7);
5985

5986
  Emit(DUP_z_zi | ImmUnsignedField<23, 22>(imm_2) |
5987
       ImmUnsignedField<20, 16>(tsz_5) | Rd(zd) | Rn(zn));
5988
}
5989

5990
void Assembler::insr(const ZRegister& zdn, const Register& rm) {
5991
  // INSR <Zdn>.<T>, <R><m>
5992
  //  0000 0101 ..10 0100 0011 10.. .... ....
5993
  //  size<23:22> | Rm<9:5> | Zdn<4:0>
5994

5995
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
5996

5997
  Emit(INSR_z_r | SVESize(zdn) | Rd(zdn) | Rn(rm));
5998
}
5999

6000
void Assembler::insr(const ZRegister& zdn, const VRegister& vm) {
6001
  // INSR <Zdn>.<T>, <V><m>
6002
  //  0000 0101 ..11 0100 0011 10.. .... ....
6003
  //  size<23:22> | Vm<9:5> | Zdn<4:0>
6004

6005
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6006
  VIXL_ASSERT(vm.IsScalar());
6007

6008
  Emit(INSR_z_v | SVESize(zdn) | Rd(zdn) | Rn(vm));
6009
}
6010

6011
void Assembler::rev(const ZRegister& zd, const ZRegister& zn) {
6012
  // REV <Zd>.<T>, <Zn>.<T>
6013
  //  0000 0101 ..11 1000 0011 10.. .... ....
6014
  //  size<23:22> | Zn<9:5> | Zd<4:0>
6015

6016
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6017
  VIXL_ASSERT(AreSameFormat(zd, zn));
6018

6019
  Emit(REV_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
6020
}
6021

6022
void Assembler::sunpkhi(const ZRegister& zd, const ZRegister& zn) {
6023
  // SUNPKHI <Zd>.<T>, <Zn>.<Tb>
6024
  //  0000 0101 ..11 0001 0011 10.. .... ....
6025
  //  size<23:22> | U<17> = 0 | H<16> = 1 | Zn<9:5> | Zd<4:0>
6026

6027
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6028
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
6029
  VIXL_ASSERT(!zd.IsLaneSizeB());
6030

6031
  Emit(SUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
6032
}
6033

6034
void Assembler::sunpklo(const ZRegister& zd, const ZRegister& zn) {
6035
  // SUNPKLO <Zd>.<T>, <Zn>.<Tb>
6036
  //  0000 0101 ..11 0000 0011 10.. .... ....
6037
  //  size<23:22> | U<17> = 0 | H<16> = 0 | Zn<9:5> | Zd<4:0>
6038

6039
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6040
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
6041
  VIXL_ASSERT(!zd.IsLaneSizeB());
6042

6043
  Emit(SUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
6044
}
6045

6046
void Assembler::tbl(const ZRegister& zd,
6047
                    const ZRegister& zn,
6048
                    const ZRegister& zm) {
6049
  // TBL <Zd>.<T>, { <Zn>.<T> }, <Zm>.<T>
6050
  //  0000 0101 ..1. .... 0011 00.. .... ....
6051
  //  size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0>
6052

6053
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6054
  VIXL_ASSERT(AreSameFormat(zd, zn, zm));
6055

6056
  Emit(TBL_z_zz_1 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6057
}
6058

6059
void Assembler::uunpkhi(const ZRegister& zd, const ZRegister& zn) {
6060
  // UUNPKHI <Zd>.<T>, <Zn>.<Tb>
6061
  //  0000 0101 ..11 0011 0011 10.. .... ....
6062
  //  size<23:22> | U<17> = 1 | H<16> = 1 | Zn<9:5> | Zd<4:0>
6063

6064
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6065
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
6066
  VIXL_ASSERT(!zd.IsLaneSizeB());
6067

6068
  Emit(UUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
6069
}
6070

6071
void Assembler::uunpklo(const ZRegister& zd, const ZRegister& zn) {
6072
  // UUNPKLO <Zd>.<T>, <Zn>.<Tb>
6073
  //  0000 0101 ..11 0010 0011 10.. .... ....
6074
  //  size<23:22> | U<17> = 1 | H<16> = 0 | Zn<9:5> | Zd<4:0>
6075

6076
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6077
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
6078
  VIXL_ASSERT(!zd.IsLaneSizeB());
6079

6080
  Emit(UUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn));
6081
}
6082

6083
// SVEPredicateCount.
6084

6085
void Assembler::cntp(const Register& xd,
6086
                     const PRegister& pg,
6087
                     const PRegisterWithLaneSize& pn) {
6088
  // CNTP <Xd>, <Pg>, <Pn>.<T>
6089
  //  0010 0101 ..10 0000 10.. ..0. .... ....
6090
  //  size<23:22> | opc<18:16> = 000 | Pg<13:10> | o2<9> = 0 | Pn<8:5> | Rd<4:0>
6091

6092
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6093
  VIXL_ASSERT(xd.IsX());
6094
  VIXL_ASSERT(pg.IsUnqualified());
6095
  if (pg.HasLaneSize()) VIXL_ASSERT(AreSameFormat(pg, pn));
6096

6097
  Emit(CNTP_r_p_p | SVESize(pn) | Rd(xd) | Pg<13, 10>(pg) | Pn(pn));
6098
}
6099

6100
// SVEPredicateLogicalOp.
6101
void Assembler::and_(const PRegisterWithLaneSize& pd,
6102
                     const PRegisterZ& pg,
6103
                     const PRegisterWithLaneSize& pn,
6104
                     const PRegisterWithLaneSize& pm) {
6105
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6106
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6107
  VIXL_ASSERT(pd.IsLaneSizeB());
6108
  Emit(AND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6109
}
6110

6111
void Assembler::ands(const PRegisterWithLaneSize& pd,
6112
                     const PRegisterZ& pg,
6113
                     const PRegisterWithLaneSize& pn,
6114
                     const PRegisterWithLaneSize& pm) {
6115
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6116
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6117
  VIXL_ASSERT(pd.IsLaneSizeB());
6118
  Emit(ANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6119
}
6120

6121
void Assembler::bic(const PRegisterWithLaneSize& pd,
6122
                    const PRegisterZ& pg,
6123
                    const PRegisterWithLaneSize& pn,
6124
                    const PRegisterWithLaneSize& pm) {
6125
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6126
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6127
  VIXL_ASSERT(pd.IsLaneSizeB());
6128
  Emit(BIC_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6129
}
6130

6131
void Assembler::bics(const PRegisterWithLaneSize& pd,
6132
                     const PRegisterZ& pg,
6133
                     const PRegisterWithLaneSize& pn,
6134
                     const PRegisterWithLaneSize& pm) {
6135
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6136
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6137
  VIXL_ASSERT(pd.IsLaneSizeB());
6138
  Emit(BICS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6139
}
6140

6141
void Assembler::eor(const PRegisterWithLaneSize& pd,
6142
                    const PRegisterZ& pg,
6143
                    const PRegisterWithLaneSize& pn,
6144
                    const PRegisterWithLaneSize& pm) {
6145
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6146
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6147
  VIXL_ASSERT(pd.IsLaneSizeB());
6148
  Emit(EOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6149
}
6150

6151
void Assembler::eors(const PRegisterWithLaneSize& pd,
6152
                     const PRegisterZ& pg,
6153
                     const PRegisterWithLaneSize& pn,
6154
                     const PRegisterWithLaneSize& pm) {
6155
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6156
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6157
  VIXL_ASSERT(pd.IsLaneSizeB());
6158
  Emit(EORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6159
}
6160

6161
void Assembler::nand(const PRegisterWithLaneSize& pd,
6162
                     const PRegisterZ& pg,
6163
                     const PRegisterWithLaneSize& pn,
6164
                     const PRegisterWithLaneSize& pm) {
6165
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6166
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6167
  VIXL_ASSERT(pd.IsLaneSizeB());
6168
  Emit(NAND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6169
}
6170

6171
void Assembler::nands(const PRegisterWithLaneSize& pd,
6172
                      const PRegisterZ& pg,
6173
                      const PRegisterWithLaneSize& pn,
6174
                      const PRegisterWithLaneSize& pm) {
6175
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6176
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6177
  VIXL_ASSERT(pd.IsLaneSizeB());
6178
  Emit(NANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6179
}
6180

6181
void Assembler::nor(const PRegisterWithLaneSize& pd,
6182
                    const PRegisterZ& pg,
6183
                    const PRegisterWithLaneSize& pn,
6184
                    const PRegisterWithLaneSize& pm) {
6185
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6186
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6187
  VIXL_ASSERT(pd.IsLaneSizeB());
6188
  Emit(NOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6189
}
6190

6191
void Assembler::nors(const PRegisterWithLaneSize& pd,
6192
                     const PRegisterZ& pg,
6193
                     const PRegisterWithLaneSize& pn,
6194
                     const PRegisterWithLaneSize& pm) {
6195
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6196
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6197
  VIXL_ASSERT(pd.IsLaneSizeB());
6198
  Emit(NORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6199
}
6200

6201
void Assembler::orn(const PRegisterWithLaneSize& pd,
6202
                    const PRegisterZ& pg,
6203
                    const PRegisterWithLaneSize& pn,
6204
                    const PRegisterWithLaneSize& pm) {
6205
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6206
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6207
  VIXL_ASSERT(pd.IsLaneSizeB());
6208
  Emit(ORN_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6209
}
6210

6211
void Assembler::orns(const PRegisterWithLaneSize& pd,
6212
                     const PRegisterZ& pg,
6213
                     const PRegisterWithLaneSize& pn,
6214
                     const PRegisterWithLaneSize& pm) {
6215
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6216
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6217
  VIXL_ASSERT(pd.IsLaneSizeB());
6218
  Emit(ORNS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6219
}
6220

6221
void Assembler::orr(const PRegisterWithLaneSize& pd,
6222
                    const PRegisterZ& pg,
6223
                    const PRegisterWithLaneSize& pn,
6224
                    const PRegisterWithLaneSize& pm) {
6225
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6226
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6227
  VIXL_ASSERT(pd.IsLaneSizeB());
6228
  Emit(ORR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6229
}
6230

6231
void Assembler::orrs(const PRegisterWithLaneSize& pd,
6232
                     const PRegisterZ& pg,
6233
                     const PRegisterWithLaneSize& pn,
6234
                     const PRegisterWithLaneSize& pm) {
6235
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6236
  VIXL_ASSERT(AreSameFormat(pd, pn, pm));
6237
  VIXL_ASSERT(pd.IsLaneSizeB());
6238
  Emit(ORRS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6239
}
6240

6241
void Assembler::sel(const PRegisterWithLaneSize& pd,
6242
                    const PRegister& pg,
6243
                    const PRegisterWithLaneSize& pn,
6244
                    const PRegisterWithLaneSize& pm) {
6245
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6246
  Emit(SEL_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6247
}
6248

6249
// SVEPredicateMisc.
6250

6251
void Assembler::pfalse(const PRegisterWithLaneSize& pd) {
6252
  // PFALSE <Pd>.B
6253
  //  0010 0101 0001 1000 1110 0100 0000 ....
6254
  //  op<23> = 0 | S<22> = 0 | Pd<3:0>
6255

6256
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6257
  // Ignore the lane size, since it makes no difference to the operation.
6258

6259
  Emit(PFALSE_p | Pd(pd));
6260
}
6261

6262
void Assembler::pfirst(const PRegisterWithLaneSize& pd,
6263
                       const PRegister& pg,
6264
                       const PRegisterWithLaneSize& pn) {
6265
  // PFIRST <Pdn>.B, <Pg>, <Pdn>.B
6266
  //  0010 0101 0101 1000 1100 000. ...0 ....
6267
  //  op<23> = 0 | S<22> = 1 | Pg<8:5> | Pdn<3:0>
6268

6269
  USE(pn);
6270
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6271
  VIXL_ASSERT(pd.Is(pn));
6272
  VIXL_ASSERT(pd.IsLaneSizeB());
6273

6274
  Emit(PFIRST_p_p_p | Pd(pd) | Pg<8, 5>(pg));
6275
}
6276

6277
void Assembler::pnext(const PRegisterWithLaneSize& pd,
6278
                      const PRegister& pg,
6279
                      const PRegisterWithLaneSize& pn) {
6280
  // PNEXT <Pdn>.<T>, <Pg>, <Pdn>.<T>
6281
  //  0010 0101 ..01 1001 1100 010. ...0 ....
6282
  //  size<23:22> | Pg<8:5> | Pdn<3:0>
6283

6284
  USE(pn);
6285
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6286
  VIXL_ASSERT(pd.Is(pn));
6287

6288
  Emit(PNEXT_p_p_p | SVESize(pd) | Pd(pd) | Pg<8, 5>(pg));
6289
}
6290

6291
void Assembler::ptest(const PRegister& pg, const PRegisterWithLaneSize& pn) {
6292
  // PTEST <Pg>, <Pn>.B
6293
  //  0010 0101 0101 0000 11.. ..0. ...0 0000
6294
  //  op<23> = 0 | S<22> = 1 | Pg<13:10> | Pn<8:5> | opc2<3:0> = 0000
6295

6296
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6297
  VIXL_ASSERT(pn.IsLaneSizeB());
6298

6299
  Emit(PTEST_p_p | Pg<13, 10>(pg) | Rx<8, 5>(pn));
6300
}
6301

6302
void Assembler::ptrue(const PRegisterWithLaneSize& pd, int pattern) {
6303
  // PTRUE <Pd>.<T>{, <pattern>}
6304
  //  0010 0101 ..01 1000 1110 00.. ...0 ....
6305
  //  size<23:22> | S<16> = 0 | pattern<9:5> | Pd<3:0>
6306

6307
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6308

6309
  Emit(PTRUE_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern));
6310
}
6311

6312
void Assembler::ptrues(const PRegisterWithLaneSize& pd, int pattern) {
6313
  // PTRUES <Pd>.<T>{, <pattern>}
6314
  //  0010 0101 ..01 1001 1110 00.. ...0 ....
6315
  //  size<23:22> | S<16> = 1 | pattern<9:5> | Pd<3:0>
6316

6317
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6318

6319
  Emit(PTRUES_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern));
6320
}
6321

6322
void Assembler::rdffr(const PRegisterWithLaneSize& pd) {
6323
  // RDFFR <Pd>.B
6324
  //  0010 0101 0001 1001 1111 0000 0000 ....
6325
  //  op<23> = 0 | S<22> = 0 | Pd<3:0>
6326

6327
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6328

6329
  Emit(RDFFR_p_f | Pd(pd));
6330
}
6331

6332
void Assembler::rdffr(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) {
6333
  // RDFFR <Pd>.B, <Pg>/Z
6334
  //  0010 0101 0001 1000 1111 000. ...0 ....
6335
  //  op<23> = 0 | S<22> = 0 | Pg<8:5> | Pd<3:0>
6336

6337
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6338

6339
  Emit(RDFFR_p_p_f | Pd(pd) | Pg<8, 5>(pg));
6340
}
6341

6342
void Assembler::rdffrs(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) {
6343
  // RDFFRS <Pd>.B, <Pg>/Z
6344
  //  0010 0101 0101 1000 1111 000. ...0 ....
6345
  //  op<23> = 0 | S<22> = 1 | Pg<8:5> | Pd<3:0>
6346

6347
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6348

6349
  Emit(RDFFRS_p_p_f | Pd(pd) | Pg<8, 5>(pg));
6350
}
6351

6352
// SVEPropagateBreak.
6353

6354
void Assembler::brkpa(const PRegisterWithLaneSize& pd,
6355
                      const PRegisterZ& pg,
6356
                      const PRegisterWithLaneSize& pn,
6357
                      const PRegisterWithLaneSize& pm) {
6358
  // BRKPA <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6359
  //  0010 0101 0000 .... 11.. ..0. ...0 ....
6360
  //  op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 |
6361
  //  Pd<3:0>
6362

6363
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6364

6365
  Emit(BRKPA_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6366
}
6367

6368
void Assembler::brkpas(const PRegisterWithLaneSize& pd,
6369
                       const PRegisterZ& pg,
6370
                       const PRegisterWithLaneSize& pn,
6371
                       const PRegisterWithLaneSize& pm) {
6372
  // BRKPAS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6373
  //  0010 0101 0100 .... 11.. ..0. ...0 ....
6374
  //  op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 |
6375
  //  Pd<3:0>
6376

6377
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6378

6379
  Emit(BRKPAS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6380
}
6381

6382
void Assembler::brkpb(const PRegisterWithLaneSize& pd,
6383
                      const PRegisterZ& pg,
6384
                      const PRegisterWithLaneSize& pn,
6385
                      const PRegisterWithLaneSize& pm) {
6386
  // BRKPB <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6387
  //  0010 0101 0000 .... 11.. ..0. ...1 ....
6388
  //  op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 |
6389
  //  Pd<3:0>
6390

6391
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6392

6393
  Emit(BRKPB_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6394
}
6395

6396
void Assembler::brkpbs(const PRegisterWithLaneSize& pd,
6397
                       const PRegisterZ& pg,
6398
                       const PRegisterWithLaneSize& pn,
6399
                       const PRegisterWithLaneSize& pm) {
6400
  // BRKPBS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B
6401
  //  0010 0101 0100 .... 11.. ..0. ...1 ....
6402
  //  op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 |
6403
  //  Pd<3:0>
6404

6405
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6406

6407
  Emit(BRKPBS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm));
6408
}
6409

6410
// SVEStackFrameAdjustment.
6411

6412
void Assembler::addpl(const Register& xd, const Register& xn, int imm6) {
6413
  // ADDPL <Xd|SP>, <Xn|SP>, #<imm>
6414
  //  0000 0100 011. .... 0101 0... .... ....
6415
  //  op<22> = 1 | Rn<20:16> | imm6<10:5> | Rd<4:0>
6416

6417
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6418
  VIXL_ASSERT(xd.IsX());
6419
  VIXL_ASSERT(xn.IsX());
6420

6421
  Emit(ADDPL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6));
6422
}
6423

6424
void Assembler::addvl(const Register& xd, const Register& xn, int imm6) {
6425
  // ADDVL <Xd|SP>, <Xn|SP>, #<imm>
6426
  //  0000 0100 001. .... 0101 0... .... ....
6427
  //  op<22> = 0 | Rn<20:16> | imm6<10:5> | Rd<4:0>
6428

6429
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6430
  VIXL_ASSERT(xd.IsX());
6431
  VIXL_ASSERT(xn.IsX());
6432

6433
  Emit(ADDVL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6));
6434
}
6435

6436
// SVEStackFrameSize.
6437

6438
void Assembler::rdvl(const Register& xd, int imm6) {
6439
  // RDVL <Xd>, #<imm>
6440
  //  0000 0100 1011 1111 0101 0... .... ....
6441
  //  op<22> = 0 | opc2<20:16> = 11111 | imm6<10:5> | Rd<4:0>
6442

6443
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6444
  VIXL_ASSERT(xd.IsX());
6445

6446
  Emit(RDVL_r_i | Rd(xd) | ImmField<10, 5>(imm6));
6447
}
6448

6449
// SVEVectorSelect.
6450

6451
void Assembler::sel(const ZRegister& zd,
6452
                    const PRegister& pg,
6453
                    const ZRegister& zn,
6454
                    const ZRegister& zm) {
6455
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6456
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6457

6458
  Emit(SEL_z_p_zz | SVESize(zd) | Rd(zd) | Pg<13, 10>(pg) | Rn(zn) | Rm(zm));
6459
}
6460

6461
// SVEWriteFFR.
6462

6463
void Assembler::setffr() {
6464
  // SETFFR
6465
  //  0010 0101 0010 1100 1001 0000 0000 0000
6466
  //  opc<23:22> = 00
6467

6468
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6469

6470
  Emit(SETFFR_f);
6471
}
6472

6473
void Assembler::wrffr(const PRegisterWithLaneSize& pn) {
6474
  // WRFFR <Pn>.B
6475
  //  0010 0101 0010 1000 1001 000. ...0 0000
6476
  //  opc<23:22> = 00 | Pn<8:5>
6477

6478
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6479

6480
  Emit(WRFFR_f_p | Rx<8, 5>(pn));
6481
}
6482

6483
// Aliases.
6484

6485
void Assembler::bic(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6486
  and_(zd, zn, ~imm);
6487
}
6488

6489
void Assembler::eon(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6490
  eor(zd, zn, ~imm);
6491
}
6492

6493
void Assembler::orn(const ZRegister& zd, const ZRegister& zn, uint64_t imm) {
6494
  orr(zd, zn, ~imm);
6495
}
6496

6497

6498
void Assembler::fmov(const ZRegister& zd, const PRegisterM& pg, double imm) {
6499
  if (IsPositiveZero(imm)) {
6500
    cpy(zd, pg, 0);
6501
  } else {
6502
    fcpy(zd, pg, imm);
6503
  }
6504
}
6505

6506
void Assembler::fmov(const ZRegister& zd, double imm) {
6507
  if (IsPositiveZero(imm)) {
6508
    dup(zd, 0);
6509
  } else {
6510
    fdup(zd, imm);
6511
  }
6512
}
6513

6514
void Assembler::mov(const PRegister& pd, const PRegister& pn) {
6515
  // If the inputs carry a lane size, they must match.
6516
  VIXL_ASSERT((!pd.HasLaneSize() && !pn.HasLaneSize()) ||
6517
              AreSameLaneSize(pd, pn));
6518
  orr(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB());
6519
}
6520

6521
void Assembler::mov(const PRegisterWithLaneSize& pd,
6522
                    const PRegisterM& pg,
6523
                    const PRegisterWithLaneSize& pn) {
6524
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6525
  sel(pd, pg, pn, pd);
6526
}
6527

6528
void Assembler::mov(const PRegisterWithLaneSize& pd,
6529
                    const PRegisterZ& pg,
6530
                    const PRegisterWithLaneSize& pn) {
6531
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6532
  and_(pd, pg, pn, pn);
6533
}
6534

6535
void Assembler::mov(const ZRegister& zd,
6536
                    const PRegister& pg,
6537
                    int imm8,
6538
                    int shift) {
6539
  VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing());
6540
  cpy(zd, pg, imm8, shift);
6541
}
6542

6543
void Assembler::mov(const ZRegister& zd, const Register& xn) { dup(zd, xn); }
6544

6545
void Assembler::mov(const ZRegister& zd, const VRegister& vn) {
6546
  VIXL_ASSERT(vn.IsScalar());
6547
  VIXL_ASSERT(AreSameLaneSize(zd, vn));
6548
  dup(zd, vn.Z().WithSameLaneSizeAs(vn), 0);
6549
}
6550

6551
void Assembler::mov(const ZRegister& zd, const ZRegister& zn) {
6552
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
6553
  orr(zd.VnD(), zn.VnD(), zn.VnD());
6554
}
6555

6556
void Assembler::mov(const ZRegister& zd, const ZRegister& zn, unsigned index) {
6557
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
6558
  dup(zd, zn, index);
6559
}
6560

6561
void Assembler::mov(const ZRegister& zd,
6562
                    const PRegisterM& pg,
6563
                    const Register& rn) {
6564
  cpy(zd, pg, rn);
6565
}
6566

6567
void Assembler::mov(const ZRegister& zd,
6568
                    const PRegisterM& pg,
6569
                    const VRegister& vn) {
6570
  VIXL_ASSERT(vn.IsScalar());
6571
  VIXL_ASSERT(AreSameLaneSize(zd, vn));
6572
  cpy(zd, pg, vn);
6573
}
6574

6575
void Assembler::mov(const ZRegister& zd,
6576
                    const PRegisterM& pg,
6577
                    const ZRegister& zn) {
6578
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
6579
  sel(zd, pg, zn, zd);
6580
}
6581

6582
void Assembler::mov(const ZRegister& zd, uint64_t imm) {
6583
  // Mov is an alias of dupm for certain values of imm. Whilst this matters in
6584
  // the disassembler, for the assembler, we don't distinguish between the
6585
  // two mnemonics, and simply call dupm.
6586
  dupm(zd, imm);
6587
}
6588

6589
void Assembler::mov(const ZRegister& zd, int imm8, int shift) {
6590
  dup(zd, imm8, shift);
6591
}
6592

6593
void Assembler::movs(const PRegister& pd, const PRegister& pn) {
6594
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6595
  orrs(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB());
6596
}
6597

6598
void Assembler::movs(const PRegisterWithLaneSize& pd,
6599
                     const PRegisterZ& pg,
6600
                     const PRegisterWithLaneSize& pn) {
6601
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6602
  ands(pd, pg, pn, pn);
6603
}
6604

6605
void Assembler::not_(const PRegisterWithLaneSize& pd,
6606
                     const PRegisterZ& pg,
6607
                     const PRegisterWithLaneSize& pn) {
6608
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6609
  eor(pd, pg, pn, pg.VnB());
6610
}
6611

6612
void Assembler::nots(const PRegisterWithLaneSize& pd,
6613
                     const PRegisterZ& pg,
6614
                     const PRegisterWithLaneSize& pn) {
6615
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
6616
  eors(pd, pg, pn, pg.VnB());
6617
}
6618

6619
// SVE2
6620

6621
void Assembler::adclb(const ZRegister& zda,
6622
                      const ZRegister& zn,
6623
                      const ZRegister& zm) {
6624
  // ADCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
6625
  //  0100 0101 0.0. .... 1101 00.. .... ....
6626
  //  size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0>
6627

6628
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6629
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
6630
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
6631

6632
  Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0;
6633
  Emit(0x4500d000 | sz | Rd(zda) | Rn(zn) | Rm(zm));
6634
}
6635

6636
void Assembler::adclt(const ZRegister& zda,
6637
                      const ZRegister& zn,
6638
                      const ZRegister& zm) {
6639
  // ADCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
6640
  //  0100 0101 0.0. .... 1101 01.. .... ....
6641
  //  size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0>
6642

6643
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6644
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
6645
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
6646

6647
  Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0;
6648
  Emit(0x4500d400 | sz | Rd(zda) | Rn(zn) | Rm(zm));
6649
}
6650

6651
void Assembler::addhnb(const ZRegister& zd,
6652
                       const ZRegister& zn,
6653
                       const ZRegister& zm) {
6654
  // ADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
6655
  //  0100 0101 ..1. .... 0110 00.. .... ....
6656
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
6657

6658
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6659
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
6660
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
6661
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
6662

6663
  Emit(0x45206000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
6664
}
6665

6666
void Assembler::addhnt(const ZRegister& zd,
6667
                       const ZRegister& zn,
6668
                       const ZRegister& zm) {
6669
  // ADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
6670
  //  0100 0101 ..1. .... 0110 01.. .... ....
6671
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
6672

6673
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6674
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
6675
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
6676
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
6677

6678
  Emit(0x45206400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
6679
}
6680

6681
void Assembler::addp(const ZRegister& zd,
6682
                     const PRegisterM& pg,
6683
                     const ZRegister& zn,
6684
                     const ZRegister& zm) {
6685
  // ADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
6686
  //  0100 0100 ..01 0001 101. .... .... ....
6687
  //  size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
6688

6689
  USE(zn);
6690
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6691
  VIXL_ASSERT(zd.Is(zn));
6692
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6693

6694
  Emit(0x4411a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
6695
}
6696

6697
void Assembler::bcax(const ZRegister& zd,
6698
                     const ZRegister& zn,
6699
                     const ZRegister& zm,
6700
                     const ZRegister& zk) {
6701
  // BCAX <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
6702
  //  0000 0100 011. .... 0011 10.. .... ....
6703
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
6704

6705
  USE(zn);
6706
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6707
  VIXL_ASSERT(zd.Is(zn));
6708
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
6709
  VIXL_ASSERT(zd.IsLaneSizeD());
6710

6711
  Emit(0x04603800 | Rd(zd) | Rm(zm) | Rn(zk));
6712
}
6713

6714
void Assembler::bdep(const ZRegister& zd,
6715
                     const ZRegister& zn,
6716
                     const ZRegister& zm) {
6717
  // BDEP <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
6718
  //  0100 0101 ..0. .... 1011 01.. .... ....
6719
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
6720

6721
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6722
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm));
6723
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6724

6725
  Emit(0x4500b400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6726
}
6727

6728
void Assembler::bext(const ZRegister& zd,
6729
                     const ZRegister& zn,
6730
                     const ZRegister& zm) {
6731
  // BEXT <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
6732
  //  0100 0101 ..0. .... 1011 00.. .... ....
6733
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
6734

6735
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6736
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm));
6737
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6738

6739
  Emit(0x4500b000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6740
}
6741

6742
void Assembler::bgrp(const ZRegister& zd,
6743
                     const ZRegister& zn,
6744
                     const ZRegister& zm) {
6745
  // BGRP <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
6746
  //  0100 0101 ..0. .... 1011 10.. .... ....
6747
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
6748

6749
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6750
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm));
6751
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6752

6753
  Emit(0x4500b800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6754
}
6755

6756
void Assembler::bsl(const ZRegister& zd,
6757
                    const ZRegister& zn,
6758
                    const ZRegister& zm,
6759
                    const ZRegister& zk) {
6760
  // BSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
6761
  //  0000 0100 001. .... 0011 11.. .... ....
6762
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
6763

6764
  USE(zn);
6765
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6766
  VIXL_ASSERT(zd.Is(zn));
6767
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
6768
  VIXL_ASSERT(zd.IsLaneSizeD());
6769

6770
  Emit(0x04203c00 | Rd(zd) | Rm(zm) | Rn(zk));
6771
}
6772

6773
void Assembler::bsl1n(const ZRegister& zd,
6774
                      const ZRegister& zn,
6775
                      const ZRegister& zm,
6776
                      const ZRegister& zk) {
6777
  // BSL1N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
6778
  //  0000 0100 011. .... 0011 11.. .... ....
6779
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
6780

6781
  USE(zn);
6782
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6783
  VIXL_ASSERT(zd.Is(zn));
6784
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
6785
  VIXL_ASSERT(zd.IsLaneSizeD());
6786

6787
  Emit(0x04603c00 | Rd(zd) | Rm(zm) | Rn(zk));
6788
}
6789

6790
void Assembler::bsl2n(const ZRegister& zd,
6791
                      const ZRegister& zn,
6792
                      const ZRegister& zm,
6793
                      const ZRegister& zk) {
6794
  // BSL2N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
6795
  //  0000 0100 101. .... 0011 11.. .... ....
6796
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
6797

6798
  USE(zn);
6799
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6800
  VIXL_ASSERT(zd.Is(zn));
6801
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
6802
  VIXL_ASSERT(zd.IsLaneSizeD());
6803

6804
  Emit(0x04a03c00 | Rd(zd) | Rm(zm) | Rn(zk));
6805
}
6806

6807
void Assembler::cadd(const ZRegister& zd,
6808
                     const ZRegister& zn,
6809
                     const ZRegister& zm,
6810
                     int rot) {
6811
  // CADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const>
6812
  //  0100 0101 ..00 0000 1101 1... .... ....
6813
  //  size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0>
6814

6815
  USE(zn);
6816
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6817
  VIXL_ASSERT(zd.Is(zn));
6818
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6819
  VIXL_ASSERT((rot == 90) || (rot == 270));
6820

6821
  Instr rotate_bit = (rot == 90) ? 0 : (1 << 10);
6822
  Emit(0x4500d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm));
6823
}
6824

6825
void Assembler::cdot(const ZRegister& zda,
6826
                     const ZRegister& zn,
6827
                     const ZRegister& zm,
6828
                     int index,
6829
                     int rot) {
6830
  // CDOT <Zda>.D, <Zn>.H, <Zm>.H[<imm>], <const>
6831
  //  0100 0100 111. .... 0100 .... .... ....
6832
  //  size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0>
6833

6834
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6835
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
6836
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
6837
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
6838
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
6839
  VIXL_ASSERT(index >= 0);
6840

6841
  Instr zm_and_idx = 0;
6842
  if (zm.IsLaneSizeB()) {
6843
    // Zm<18:16> | i2<20:19>
6844
    VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 3));
6845
    zm_and_idx = (index << 19) | Rx<18, 16>(zm);
6846
  } else {
6847
    // Zm<19:16> | i1<20>
6848
    VIXL_ASSERT(zm.IsLaneSizeH());
6849
    VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 1));
6850
    zm_and_idx = (index << 20) | Rx<19, 16>(zm);
6851
  }
6852

6853
  Instr rotate_bits = (rot / 90) << 10;
6854
  Emit(0x44a04000 | zm_and_idx | rotate_bits | SVESize(zda) | Rd(zda) | Rn(zn));
6855
}
6856

6857
void Assembler::cdot(const ZRegister& zda,
6858
                     const ZRegister& zn,
6859
                     const ZRegister& zm,
6860
                     int rot) {
6861
  // CDOT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>, <const>
6862
  //  0100 0100 ..0. .... 0001 .... .... ....
6863
  //  size<23:22> | Zm<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0>
6864

6865
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6866
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
6867
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
6868
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
6869
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4));
6870

6871
  Instr rotate_bits = (rot / 90) << 10;
6872
  Emit(0x44001000 | rotate_bits | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
6873
}
6874

6875
void Assembler::cmla(const ZRegister& zda,
6876
                     const ZRegister& zn,
6877
                     const ZRegister& zm,
6878
                     int index,
6879
                     int rot) {
6880
  // CMLA <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const>
6881
  //  0100 0100 101. .... 0110 .... .... ....
6882
  //  size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0>
6883

6884
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6885
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
6886
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
6887

6888
  Instr rotate_bit = (rot / 90) << 10;
6889
  Emit(0x44a06000 | SVEMulComplexIndexHelper(zm, index) | rotate_bit | Rd(zda) |
6890
       Rn(zn));
6891
}
6892

6893
void Assembler::cmla(const ZRegister& zda,
6894
                     const ZRegister& zn,
6895
                     const ZRegister& zm,
6896
                     int rot) {
6897
  // CMLA <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const>
6898
  //  0100 0100 ..0. .... 0010 .... .... ....
6899
  //  size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0>
6900

6901
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6902
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
6903
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
6904

6905
  Instr rotate_bit = (rot / 90) << 10;
6906
  Emit(0x44002000 | rotate_bit | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
6907
}
6908

6909
void Assembler::eor3(const ZRegister& zd,
6910
                     const ZRegister& zn,
6911
                     const ZRegister& zm,
6912
                     const ZRegister& zk) {
6913
  // EOR3 <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
6914
  //  0000 0100 001. .... 0011 10.. .... ....
6915
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
6916

6917
  USE(zn);
6918
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6919
  VIXL_ASSERT(zd.Is(zn));
6920
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
6921
  VIXL_ASSERT(zd.IsLaneSizeD());
6922

6923
  Emit(0x04203800 | Rd(zd) | Rm(zm) | Rn(zk));
6924
}
6925

6926
void Assembler::eorbt(const ZRegister& zd,
6927
                      const ZRegister& zn,
6928
                      const ZRegister& zm) {
6929
  // EORBT <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
6930
  //  0100 0101 ..0. .... 1001 00.. .... ....
6931
  //  size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0>
6932

6933
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6934
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6935

6936
  Emit(0x45009000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6937
}
6938

6939
void Assembler::eortb(const ZRegister& zd,
6940
                      const ZRegister& zn,
6941
                      const ZRegister& zm) {
6942
  // EORTB <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
6943
  //  0100 0101 ..0. .... 1001 01.. .... ....
6944
  //  size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0>
6945

6946
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6947
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6948

6949
  Emit(0x45009400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
6950
}
6951

6952
void Assembler::faddp(const ZRegister& zd,
6953
                      const PRegisterM& pg,
6954
                      const ZRegister& zn,
6955
                      const ZRegister& zm) {
6956
  // FADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
6957
  //  0110 0100 ..01 0000 100. .... .... ....
6958
  //  size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
6959

6960
  USE(zn);
6961
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6962
  VIXL_ASSERT(zd.Is(zn));
6963
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
6964
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
6965

6966
  Emit(0x64108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
6967
}
6968

6969
void Assembler::fcvtlt(const ZRegister& zd,
6970
                       const PRegisterM& pg,
6971
                       const ZRegister& zn) {
6972
  // FCVTLT <Zd>.S, <Pg>/M, <Zn>.H
6973
  //  0110 0100 1000 1001 101. .... .... ....
6974
  //  opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
6975

6976
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6977

6978
  Instr op;
6979
  if (zd.IsLaneSizeD() && zn.IsLaneSizeS()) {
6980
    op = 0x64cba000;
6981
  } else {
6982
    VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeH());
6983
    op = 0x6489a000;
6984
  }
6985

6986
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
6987
}
6988

6989
void Assembler::fcvtnt(const ZRegister& zd,
6990
                       const PRegisterM& pg,
6991
                       const ZRegister& zn) {
6992
  // FCVTNT <Zd>.S, <Pg>/M, <Zn>.D
6993
  //  0110 0100 1100 1010 101. .... .... ....
6994
  //  opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
6995

6996
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
6997

6998
  Instr op;
6999
  if (zd.IsLaneSizeS() && zn.IsLaneSizeD()) {
7000
    op = 0x64caa000;
7001
  } else {
7002
    VIXL_ASSERT(zd.IsLaneSizeH() && zn.IsLaneSizeS());
7003
    op = 0x6488a000;
7004
  }
7005
  Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn));
7006
}
7007

7008
void Assembler::fcvtx(const ZRegister& zd,
7009
                      const PRegisterM& pg,
7010
                      const ZRegister& zn) {
7011
  // FCVTX <Zd>.S, <Pg>/M, <Zn>.D
7012
  //  0110 0101 0000 1010 101. .... .... ....
7013
  //  opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
7014

7015
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7016
  VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeD());
7017

7018
  Emit(0x650aa000 | Rd(zd) | PgLow8(pg) | Rn(zn));
7019
}
7020

7021
void Assembler::fcvtxnt(const ZRegister& zd,
7022
                        const PRegisterM& pg,
7023
                        const ZRegister& zn) {
7024
  // FCVTXNT <Zd>.S, <Pg>/M, <Zn>.D
7025
  //  0110 0100 0000 1010 101. .... .... ....
7026
  //  opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
7027

7028
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7029

7030
  Emit(0x640aa000 | Rd(zd) | PgLow8(pg) | Rn(zn));
7031
}
7032

7033
void Assembler::flogb(const ZRegister& zd,
7034
                      const PRegisterM& pg,
7035
                      const ZRegister& zn) {
7036
  // FLOGB <Zd>.<T>, <Pg>/M, <Zn>.<T>
7037
  //  0110 0101 0001 1..0 101. .... .... ....
7038
  //  opc<23:22> | opc2<18:17> | U<16> | Pg<12:10> | Zn<9:5> | Zd<4:0> | size<>
7039

7040
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7041
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
7042
  VIXL_ASSERT(!zd.IsLaneSizeB());
7043

7044
  // Size field is encoded in bits <18:17> rather than <23:22>.
7045
  Instr size = SVESize(zd) >> 5;
7046
  Emit(0x6518a000 | size | Rd(zd) | PgLow8(pg) | Rn(zn));
7047
}
7048

7049
void Assembler::fmaxnmp(const ZRegister& zd,
7050
                        const PRegisterM& pg,
7051
                        const ZRegister& zn,
7052
                        const ZRegister& zm) {
7053
  // FMAXNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7054
  //  0110 0100 ..01 0100 100. .... .... ....
7055
  //  size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7056

7057
  USE(zn);
7058
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7059
  VIXL_ASSERT(zd.Is(zn));
7060
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7061
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7062

7063
  Emit(0x64148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7064
}
7065

7066
void Assembler::fmaxp(const ZRegister& zd,
7067
                      const PRegisterM& pg,
7068
                      const ZRegister& zn,
7069
                      const ZRegister& zm) {
7070
  // FMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7071
  //  0110 0100 ..01 0110 100. .... .... ....
7072
  //  size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7073

7074
  USE(zn);
7075
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7076
  VIXL_ASSERT(zd.Is(zn));
7077
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7078
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7079

7080
  Emit(0x64168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7081
}
7082

7083
void Assembler::fminnmp(const ZRegister& zd,
7084
                        const PRegisterM& pg,
7085
                        const ZRegister& zn,
7086
                        const ZRegister& zm) {
7087
  // FMINNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7088
  //  0110 0100 ..01 0101 100. .... .... ....
7089
  //  size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7090

7091
  USE(zn);
7092
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7093
  VIXL_ASSERT(zd.Is(zn));
7094
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7095
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7096

7097
  Emit(0x64158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7098
}
7099

7100
void Assembler::fminp(const ZRegister& zd,
7101
                      const PRegisterM& pg,
7102
                      const ZRegister& zn,
7103
                      const ZRegister& zm) {
7104
  // FMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7105
  //  0110 0100 ..01 0111 100. .... .... ....
7106
  //  size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7107

7108
  USE(zn);
7109
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7110
  VIXL_ASSERT(zd.Is(zn));
7111
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7112
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7113

7114
  Emit(0x64178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7115
}
7116

7117
void Assembler::fmlalb(const ZRegister& zda,
7118
                       const ZRegister& zn,
7119
                       const ZRegister& zm) {
7120
  // FMLALB <Zda>.S, <Zn>.H, <Zm>.H
7121
  //  0110 0100 101. .... 1000 00.. .... ....
7122
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7123

7124
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7125
  VIXL_ASSERT(zda.IsLaneSizeS());
7126
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7127

7128
  Emit(0x64a08000 | Rd(zda) | Rn(zn) | Rm(zm));
7129
}
7130

7131
void Assembler::fmlalb(const ZRegister& zda,
7132
                       const ZRegister& zn,
7133
                       const ZRegister& zm,
7134
                       int index) {
7135
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7136
  VIXL_ASSERT(zda.IsLaneSizeS());
7137
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7138
  VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
7139
  Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
7140
                     (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
7141

7142
  Emit(0x64a04000 | Rd(zda) | Rn(zn) | zm_and_idx);
7143
}
7144

7145
void Assembler::fmlalt(const ZRegister& zda,
7146
                       const ZRegister& zn,
7147
                       const ZRegister& zm) {
7148
  // FMLALT <Zda>.S, <Zn>.H, <Zm>.H
7149
  //  0110 0100 101. .... 1000 01.. .... ....
7150
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7151

7152
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7153
  VIXL_ASSERT(zda.IsLaneSizeS());
7154
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7155

7156
  Emit(0x64a08400 | Rd(zda) | Rn(zn) | Rm(zm));
7157
}
7158

7159
void Assembler::fmlalt(const ZRegister& zda,
7160
                       const ZRegister& zn,
7161
                       const ZRegister& zm,
7162
                       int index) {
7163
  // FMLALT <Zda>.S, <Zn>.H, <Zm>.H
7164
  //  0110 0100 101. .... 1000 01.. .... ....
7165
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7166

7167
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7168
  VIXL_ASSERT(zda.IsLaneSizeS());
7169
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7170
  VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
7171
  Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
7172
                     (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
7173

7174
  Emit(0x64a04400 | Rd(zda) | Rn(zn) | zm_and_idx);
7175
}
7176

7177
void Assembler::fmlslb(const ZRegister& zda,
7178
                       const ZRegister& zn,
7179
                       const ZRegister& zm) {
7180
  // FMLSLB <Zda>.S, <Zn>.H, <Zm>.H
7181
  //  0110 0100 101. .... 1010 00.. .... ....
7182
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7183

7184
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7185
  VIXL_ASSERT(zda.IsLaneSizeS());
7186
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7187

7188
  Emit(0x64a0a000 | Rd(zda) | Rn(zn) | Rm(zm));
7189
}
7190

7191
void Assembler::fmlslb(const ZRegister& zda,
7192
                       const ZRegister& zn,
7193
                       const ZRegister& zm,
7194
                       int index) {
7195
  // FMLSLB <Zda>.S, <Zn>.H, <Zm>.H
7196
  //  0110 0100 101. .... 1010 00.. .... ....
7197
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7198

7199
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7200
  VIXL_ASSERT(zda.IsLaneSizeS());
7201
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7202
  VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
7203
  Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
7204
                     (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
7205

7206
  Emit(0x64a06000 | Rd(zda) | Rn(zn) | zm_and_idx);
7207
}
7208

7209
void Assembler::fmlslt(const ZRegister& zda,
7210
                       const ZRegister& zn,
7211
                       const ZRegister& zm) {
7212
  // FMLSLT <Zda>.S, <Zn>.H, <Zm>.H
7213
  //  0110 0100 101. .... 1010 01.. .... ....
7214
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7215

7216
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7217
  VIXL_ASSERT(zda.IsLaneSizeS());
7218
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7219

7220
  Emit(0x64a0a400 | Rd(zda) | Rn(zn) | Rm(zm));
7221
}
7222

7223
void Assembler::fmlslt(const ZRegister& zda,
7224
                       const ZRegister& zn,
7225
                       const ZRegister& zm,
7226
                       int index) {
7227
  // FMLSLT <Zda>.S, <Zn>.H, <Zm>.H
7228
  //  0110 0100 101. .... 1010 01.. .... ....
7229
  //  o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0>
7230

7231
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7232
  VIXL_ASSERT(zda.IsLaneSizeS());
7233
  VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH());
7234
  VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
7235
  Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
7236
                     (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
7237

7238
  Emit(0x64a06400 | Rd(zda) | Rn(zn) | zm_and_idx);
7239
}
7240

7241
void Assembler::histcnt(const ZRegister& zd,
7242
                        const PRegisterZ& pg,
7243
                        const ZRegister& zn,
7244
                        const ZRegister& zm) {
7245
  // HISTCNT <Zd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
7246
  //  0100 0101 ..1. .... 110. .... .... ....
7247
  //  size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
7248

7249
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7250
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7251
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD());
7252

7253
  Emit(0x4520c000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn) | Rm(zm));
7254
}
7255

7256
void Assembler::histseg(const ZRegister& zd,
7257
                        const ZRegister& zn,
7258
                        const ZRegister& zm) {
7259
  // HISTSEG <Zd>.B, <Zn>.B, <Zm>.B
7260
  //  0100 0101 ..1. .... 1010 00.. .... ....
7261
  //  size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0>
7262

7263
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7264
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7265
  VIXL_ASSERT(zd.IsLaneSizeB());
7266

7267
  Emit(0x4520a000 | Rd(zd) | Rn(zn) | Rm(zm));
7268
}
7269

7270
void Assembler::match(const PRegisterWithLaneSize& pd,
7271
                      const PRegisterZ& pg,
7272
                      const ZRegister& zn,
7273
                      const ZRegister& zm) {
7274
  // MATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
7275
  //  0100 0101 ..1. .... 100. .... ...0 ....
7276
  //  size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0>
7277

7278
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7279
  VIXL_ASSERT(AreSameLaneSize(pd, zn, zm));
7280
  VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH());
7281

7282
  Emit(0x45208000 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
7283
}
7284

7285
void Assembler::mla(const ZRegister& zda,
7286
                    const ZRegister& zn,
7287
                    const ZRegister& zm,
7288
                    int index) {
7289
  // MLA <Zda>.D, <Zn>.D, <Zm>.D[<imm>]
7290
  //  0100 0100 111. .... 0000 10.. .... ....
7291
  //  size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0>
7292

7293
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7294
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
7295

7296
  Instr synthesised_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
7297
                                           zm,
7298
                                           index,
7299
                                           0x44200800,
7300
                                           0x44a00800,
7301
                                           0x44e00800);
7302

7303
  Emit(synthesised_op | Rd(zda) | Rn(zn));
7304
}
7305

7306
void Assembler::mls(const ZRegister& zda,
7307
                    const ZRegister& zn,
7308
                    const ZRegister& zm,
7309
                    int index) {
7310
  // MLS <Zda>.D, <Zn>.D, <Zm>.D[<imm>]
7311
  //  0100 0100 111. .... 0000 11.. .... ....
7312
  //  size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0>
7313

7314
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7315
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
7316

7317
  Instr synthesised_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
7318
                                           zm,
7319
                                           index,
7320
                                           0x44200c00,
7321
                                           0x44a00c00,
7322
                                           0x44e00c00);
7323

7324
  Emit(synthesised_op | Rd(zda) | Rn(zn));
7325
}
7326

7327
void Assembler::mul(const ZRegister& zd,
7328
                    const ZRegister& zn,
7329
                    const ZRegister& zm,
7330
                    int index) {
7331
  // MUL <Zd>.D, <Zn>.D, <Zm>.D[<imm>]
7332
  //  0100 0100 111. .... 1111 10.. .... ....
7333
  //  size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0>
7334

7335
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7336
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7337

7338
  Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(),
7339
                                           zm,
7340
                                           index,
7341
                                           0x4420f800,
7342
                                           0x44a0f800,
7343
                                           0x44e0f800);
7344

7345
  Emit(synthesised_op | Rd(zd) | Rn(zn));
7346
}
7347

7348
void Assembler::mul(const ZRegister& zd,
7349
                    const ZRegister& zn,
7350
                    const ZRegister& zm) {
7351
  // MUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
7352
  //  0000 0100 ..1. .... 0110 00.. .... ....
7353
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
7354

7355
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7356
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7357

7358
  Emit(0x04206000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7359
}
7360

7361
void Assembler::nbsl(const ZRegister& zd,
7362
                     const ZRegister& zn,
7363
                     const ZRegister& zm,
7364
                     const ZRegister& zk) {
7365
  // NBSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D
7366
  //  0000 0100 111. .... 0011 11.. .... ....
7367
  //  opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0>
7368

7369
  USE(zn);
7370
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7371
  VIXL_ASSERT(zd.Is(zn));
7372
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk));
7373
  VIXL_ASSERT(zd.IsLaneSizeD());
7374

7375
  Emit(0x04e03c00 | Rd(zd) | Rm(zm) | Rn(zk));
7376
}
7377

7378
void Assembler::nmatch(const PRegisterWithLaneSize& pd,
7379
                       const PRegisterZ& pg,
7380
                       const ZRegister& zn,
7381
                       const ZRegister& zm) {
7382
  // NMATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T>
7383
  //  0100 0101 ..1. .... 100. .... ...1 ....
7384
  //  size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0>
7385

7386
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7387
  VIXL_ASSERT(AreSameLaneSize(pd, zn, zm));
7388
  VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH());
7389

7390
  Emit(0x45208010 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm));
7391
}
7392

7393
void Assembler::pmul(const ZRegister& zd,
7394
                     const ZRegister& zn,
7395
                     const ZRegister& zm) {
7396
  // PMUL <Zd>.B, <Zn>.B, <Zm>.B
7397
  //  0000 0100 001. .... 0110 01.. .... ....
7398
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
7399

7400
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7401

7402
  Emit(0x04206400 | Rd(zd) | Rn(zn) | Rm(zm));
7403
}
7404

7405
void Assembler::pmullb(const ZRegister& zd,
7406
                       const ZRegister& zn,
7407
                       const ZRegister& zm) {
7408
  // PMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7409
  //  0100 0101 ..0. .... 0110 10.. .... ....
7410
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7411

7412
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7413
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7414
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS());
7415
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
7416
  // SVEPmull128 is not supported
7417
  VIXL_ASSERT(!zd.IsLaneSizeQ());
7418

7419
  Emit(0x45006800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7420
}
7421

7422
void Assembler::pmullt(const ZRegister& zd,
7423
                       const ZRegister& zn,
7424
                       const ZRegister& zm) {
7425
  // PMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7426
  //  0100 0101 ..0. .... 0110 11.. .... ....
7427
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7428

7429
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7430
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7431
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS());
7432
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
7433
  // SVEPmull128 is not supported
7434
  VIXL_ASSERT(!zd.IsLaneSizeQ());
7435

7436
  Emit(0x45006c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7437
}
7438

7439
void Assembler::raddhnb(const ZRegister& zd,
7440
                        const ZRegister& zn,
7441
                        const ZRegister& zm) {
7442
  // RADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7443
  //  0100 0101 ..1. .... 0110 10.. .... ....
7444
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
7445

7446
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7447
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7448
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
7449
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
7450

7451
  Emit(0x45206800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
7452
}
7453

7454
void Assembler::raddhnt(const ZRegister& zd,
7455
                        const ZRegister& zn,
7456
                        const ZRegister& zm) {
7457
  // RADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7458
  //  0100 0101 ..1. .... 0110 11.. .... ....
7459
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
7460

7461
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7462
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7463
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
7464
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
7465

7466
  Emit(0x45206c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
7467
}
7468

7469
#define VIXL_SVE_SHR_LIST(V) \
7470
  V(rshrnb, 0x45201800)      \
7471
  V(rshrnt, 0x45201c00)      \
7472
  V(shrnb, 0x45201000)       \
7473
  V(shrnt, 0x45201400)       \
7474
  V(sqrshrnb, 0x45202800)    \
7475
  V(sqrshrnt, 0x45202c00)    \
7476
  V(sqrshrunb, 0x45200800)   \
7477
  V(sqrshrunt, 0x45200c00)   \
7478
  V(sqshrnb, 0x45202000)     \
7479
  V(sqshrnt, 0x45202400)     \
7480
  V(sqshrunb, 0x45200000)    \
7481
  V(sqshrunt, 0x45200400)    \
7482
  V(uqrshrnb, 0x45203800)    \
7483
  V(uqrshrnt, 0x45203c00)    \
7484
  V(uqshrnb, 0x45203000)     \
7485
  V(uqshrnt, 0x45203400)
7486

7487
#define VIXL_DEFINE_ASM_FUNC(MNE, X)                                         \
7488
  void Assembler::MNE(const ZRegister& zd, const ZRegister& zn, int shift) { \
7489
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));                                 \
7490
    VIXL_ASSERT(!zd.IsLaneSizeD() && !zd.IsLaneSizeQ());                     \
7491
    VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));   \
7492
    Instr encoded_imm =                                                      \
7493
        EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());         \
7494
    SVEBitwiseShiftImmediate(zd, zn, encoded_imm, X);                        \
7495
  }
7496
VIXL_SVE_SHR_LIST(VIXL_DEFINE_ASM_FUNC)
7497
#undef VIXL_DEFINE_ASM_FUNC
7498

7499
void Assembler::rsubhnb(const ZRegister& zd,
7500
                        const ZRegister& zn,
7501
                        const ZRegister& zm) {
7502
  // RSUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7503
  //  0100 0101 ..1. .... 0111 10.. .... ....
7504
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
7505

7506
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7507
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7508
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
7509
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
7510

7511
  Emit(0x45207800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
7512
}
7513

7514
void Assembler::rsubhnt(const ZRegister& zd,
7515
                        const ZRegister& zn,
7516
                        const ZRegister& zm) {
7517
  // RSUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7518
  //  0100 0101 ..1. .... 0111 11.. .... ....
7519
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
7520

7521
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7522
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7523
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
7524
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
7525

7526
  Emit(0x45207c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
7527
}
7528

7529
void Assembler::saba(const ZRegister& zda,
7530
                     const ZRegister& zn,
7531
                     const ZRegister& zm) {
7532
  // SABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
7533
  //  0100 0101 ..0. .... 1111 10.. .... ....
7534
  //  size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0>
7535

7536
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7537
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
7538

7539
  Emit(0x4500f800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7540
}
7541

7542
void Assembler::sabalb(const ZRegister& zda,
7543
                       const ZRegister& zn,
7544
                       const ZRegister& zm) {
7545
  // SABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7546
  //  0100 0101 ..0. .... 1100 00.. .... ....
7547
  //  size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7548

7549
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7550
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7551
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7552
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
7553

7554
  Emit(0x4500c000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7555
}
7556

7557
void Assembler::sabalt(const ZRegister& zda,
7558
                       const ZRegister& zn,
7559
                       const ZRegister& zm) {
7560
  // SABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7561
  //  0100 0101 ..0. .... 1100 01.. .... ....
7562
  //  size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7563

7564
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7565
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7566
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7567
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
7568

7569
  Emit(0x4500c400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7570
}
7571

7572
void Assembler::sabdlb(const ZRegister& zd,
7573
                       const ZRegister& zn,
7574
                       const ZRegister& zm) {
7575
  // SABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7576
  //  0100 0101 ..0. .... 0011 00.. .... ....
7577
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
7578
  //  Zd<4:0>
7579

7580
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7581
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7582
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7583
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7584

7585
  Emit(0x45003000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7586
}
7587

7588
void Assembler::sabdlt(const ZRegister& zd,
7589
                       const ZRegister& zn,
7590
                       const ZRegister& zm) {
7591
  // SABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7592
  //  0100 0101 ..0. .... 0011 01.. .... ....
7593
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
7594
  //  Zd<4:0>
7595

7596
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7597
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7598
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7599
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7600

7601
  Emit(0x45003400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7602
}
7603

7604
void Assembler::sadalp(const ZRegister& zda,
7605
                       const PRegisterM& pg,
7606
                       const ZRegister& zn) {
7607
  // SADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb>
7608
  //  0100 0100 ..00 0100 101. .... .... ....
7609
  //  size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0>
7610

7611
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7612
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7613
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
7614

7615
  Emit(0x4404a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn));
7616
}
7617

7618
void Assembler::saddlb(const ZRegister& zd,
7619
                       const ZRegister& zn,
7620
                       const ZRegister& zm) {
7621
  // SADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7622
  //  0100 0101 ..0. .... 0000 00.. .... ....
7623
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
7624
  //  Zd<4:0>
7625

7626
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7627
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7628
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7629
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7630

7631
  Emit(0x45000000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7632
}
7633

7634
void Assembler::saddlbt(const ZRegister& zd,
7635
                        const ZRegister& zn,
7636
                        const ZRegister& zm) {
7637
  // SADDLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7638
  //  0100 0101 ..0. .... 1000 00.. .... ....
7639
  //  size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0>
7640

7641
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7642
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7643
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7644
  VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS());
7645

7646
  Emit(0x45008000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7647
}
7648

7649
void Assembler::saddlt(const ZRegister& zd,
7650
                       const ZRegister& zn,
7651
                       const ZRegister& zm) {
7652
  // SADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7653
  //  0100 0101 ..0. .... 0000 01.. .... ....
7654
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
7655
  //  Zd<4:0>
7656

7657
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7658
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7659
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7660
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7661

7662
  Emit(0x45000400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7663
}
7664

7665
void Assembler::saddwb(const ZRegister& zd,
7666
                       const ZRegister& zn,
7667
                       const ZRegister& zm) {
7668
  // SADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
7669
  //  0100 0101 ..0. .... 0100 00.. .... ....
7670
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7671

7672
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7673
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
7674
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
7675
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7676

7677
  Emit(0x45004000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7678
}
7679

7680
void Assembler::saddwt(const ZRegister& zd,
7681
                       const ZRegister& zn,
7682
                       const ZRegister& zm) {
7683
  // SADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
7684
  //  0100 0101 ..0. .... 0100 01.. .... ....
7685
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7686

7687
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7688
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
7689
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
7690
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
7691

7692
  Emit(0x45004400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7693
}
7694

7695
void Assembler::sbclb(const ZRegister& zda,
7696
                      const ZRegister& zn,
7697
                      const ZRegister& zm) {
7698
  // SBCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
7699
  //  0100 0101 1.0. .... 1101 00.. .... ....
7700
  //  size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0>
7701

7702
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7703
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
7704
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
7705

7706
  Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0;
7707
  Emit(0x4580d000 | sz | Rd(zda) | Rn(zn) | Rm(zm));
7708
}
7709

7710
void Assembler::sbclt(const ZRegister& zda,
7711
                      const ZRegister& zn,
7712
                      const ZRegister& zm) {
7713
  // SBCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
7714
  //  0100 0101 1.0. .... 1101 01.. .... ....
7715
  //  size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0>
7716

7717
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7718
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
7719
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
7720

7721
  Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0;
7722
  Emit(0x4580d400 | sz | Rd(zda) | Rn(zn) | Rm(zm));
7723
}
7724

7725
void Assembler::shadd(const ZRegister& zd,
7726
                      const PRegisterM& pg,
7727
                      const ZRegister& zn,
7728
                      const ZRegister& zm) {
7729
  // SHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7730
  //  0100 0100 ..01 0000 100. .... .... ....
7731
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7732

7733
  USE(zn);
7734
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7735
  VIXL_ASSERT(zd.Is(zn));
7736
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7737

7738
  Emit(0x44108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7739
}
7740

7741
void Assembler::shsub(const ZRegister& zd,
7742
                      const PRegisterM& pg,
7743
                      const ZRegister& zn,
7744
                      const ZRegister& zm) {
7745
  // SHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7746
  //  0100 0100 ..01 0010 100. .... .... ....
7747
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7748

7749
  USE(zn);
7750
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7751
  VIXL_ASSERT(zd.Is(zn));
7752
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7753

7754
  Emit(0x44128000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7755
}
7756

7757
void Assembler::shsubr(const ZRegister& zd,
7758
                       const PRegisterM& pg,
7759
                       const ZRegister& zn,
7760
                       const ZRegister& zm) {
7761
  // SHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7762
  //  0100 0100 ..01 0110 100. .... .... ....
7763
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7764

7765
  USE(zn);
7766
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7767
  VIXL_ASSERT(zd.Is(zn));
7768
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7769

7770
  Emit(0x44168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7771
}
7772

7773
void Assembler::sli(const ZRegister& zd, const ZRegister& zn, int shift) {
7774
  // SLI <Zd>.<T>, <Zn>.<T>, #<const>
7775
  //  0100 0101 ..0. .... 1111 01.. .... ....
7776
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0>
7777

7778
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7779
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
7780
  Instr encoded_imm =
7781
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
7782

7783
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f400);
7784
}
7785

7786
void Assembler::smaxp(const ZRegister& zd,
7787
                      const PRegisterM& pg,
7788
                      const ZRegister& zn,
7789
                      const ZRegister& zm) {
7790
  // SMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7791
  //  0100 0100 ..01 0100 101. .... .... ....
7792
  //  size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7793

7794
  USE(zn);
7795
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7796
  VIXL_ASSERT(zd.Is(zn));
7797
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7798

7799
  Emit(0x4414a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7800
}
7801

7802
void Assembler::sminp(const ZRegister& zd,
7803
                      const PRegisterM& pg,
7804
                      const ZRegister& zn,
7805
                      const ZRegister& zm) {
7806
  // SMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7807
  //  0100 0100 ..01 0110 101. .... .... ....
7808
  //  size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7809

7810
  USE(zn);
7811
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7812
  VIXL_ASSERT(zd.Is(zn));
7813
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7814

7815
  Emit(0x4416a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7816
}
7817

7818
#define VIXL_SVE_MULL_INDEX_LIST(V) \
7819
  V(smullb, 0x44a0c000)             \
7820
  V(smullt, 0x44a0c400)             \
7821
  V(umullb, 0x44a0d000)             \
7822
  V(umullt, 0x44a0d400)             \
7823
  V(smlalb, 0x44a08000)             \
7824
  V(smlalt, 0x44a08400)             \
7825
  V(smlslb, 0x44a0a000)             \
7826
  V(smlslt, 0x44a0a400)             \
7827
  V(umlalb, 0x44a09000)             \
7828
  V(umlalt, 0x44a09400)             \
7829
  V(umlslb, 0x44a0b000)             \
7830
  V(umlslt, 0x44a0b400)             \
7831
  V(sqdmullb, 0x44a0e000)           \
7832
  V(sqdmullt, 0x44a0e400)
7833

7834
#define VIXL_DEFINE_ASM_FUNC(MNE, OP)                                     \
7835
  void Assembler::MNE(const ZRegister& zda,                               \
7836
                      const ZRegister& zn,                                \
7837
                      const ZRegister& zm,                                \
7838
                      int index) {                                        \
7839
    VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));                              \
7840
    VIXL_ASSERT(AreSameLaneSize(zn, zm));                                 \
7841
    VIXL_ASSERT(zda.IsLaneSizeD() || zda.IsLaneSizeS());                  \
7842
    VIXL_ASSERT(zda.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); \
7843
    Instr zm_with_index = SVEMulLongIndexHelper(zm, index);               \
7844
    Emit(OP | SVESize(zda) | Rd(zda) | Rn(zn) | zm_with_index);           \
7845
  }
7846
VIXL_SVE_MULL_INDEX_LIST(VIXL_DEFINE_ASM_FUNC)
7847
#undef VIXL_DEFINE_ASM_FuNC
7848

7849
void Assembler::smlalb(const ZRegister& zda,
7850
                       const ZRegister& zn,
7851
                       const ZRegister& zm) {
7852
  // SMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7853
  //  0100 0100 ..0. .... 0100 00.. .... ....
7854
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7855

7856
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7857
  VIXL_ASSERT(!zda.IsLaneSizeB());
7858
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7859
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7860

7861
  Emit(0x44004000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7862
}
7863

7864
void Assembler::smlalt(const ZRegister& zda,
7865
                       const ZRegister& zn,
7866
                       const ZRegister& zm) {
7867
  // SMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7868
  //  0100 0100 ..0. .... 0100 01.. .... ....
7869
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7870

7871
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7872
  VIXL_ASSERT(!zda.IsLaneSizeB());
7873
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7874
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7875

7876
  Emit(0x44004400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7877
}
7878

7879
void Assembler::smlslb(const ZRegister& zda,
7880
                       const ZRegister& zn,
7881
                       const ZRegister& zm) {
7882
  // SMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7883
  //  0100 0100 ..0. .... 0101 00.. .... ....
7884
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7885

7886
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7887
  VIXL_ASSERT(!zda.IsLaneSizeB());
7888
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7889
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7890

7891
  Emit(0x44005000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7892
}
7893

7894
void Assembler::smlslt(const ZRegister& zda,
7895
                       const ZRegister& zn,
7896
                       const ZRegister& zm) {
7897
  // SMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7898
  //  0100 0100 ..0. .... 0101 01.. .... ....
7899
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
7900

7901
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7902
  VIXL_ASSERT(!zda.IsLaneSizeB());
7903
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7904
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
7905

7906
  Emit(0x44005400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
7907
}
7908

7909
void Assembler::smulh(const ZRegister& zd,
7910
                      const ZRegister& zn,
7911
                      const ZRegister& zm) {
7912
  // SMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
7913
  //  0000 0100 ..1. .... 0110 10.. .... ....
7914
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
7915

7916
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7917
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7918

7919
  Emit(0x04206800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7920
}
7921

7922
void Assembler::smullb(const ZRegister& zd,
7923
                       const ZRegister& zn,
7924
                       const ZRegister& zm) {
7925
  // SMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7926
  //  0100 0101 ..0. .... 0111 00.. .... ....
7927
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7928

7929
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7930
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7931
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
7932
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
7933

7934
  Emit(0x45007000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7935
}
7936

7937
void Assembler::smullt(const ZRegister& zd,
7938
                       const ZRegister& zn,
7939
                       const ZRegister& zm) {
7940
  // SMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
7941
  //  0100 0101 ..0. .... 0111 01.. .... ....
7942
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
7943

7944
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7945
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
7946
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
7947
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
7948

7949
  Emit(0x45007400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
7950
}
7951

7952
void Assembler::sqabs(const ZRegister& zd,
7953
                      const PRegisterM& pg,
7954
                      const ZRegister& zn) {
7955
  // SQABS <Zd>.<T>, <Pg>/M, <Zn>.<T>
7956
  //  0100 0100 ..00 1000 101. .... .... ....
7957
  //  size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
7958

7959
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7960
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
7961

7962
  Emit(0x4408a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
7963
}
7964

7965
void Assembler::sqadd(const ZRegister& zd,
7966
                      const PRegisterM& pg,
7967
                      const ZRegister& zn,
7968
                      const ZRegister& zm) {
7969
  // SQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
7970
  //  0100 0100 ..01 1000 100. .... .... ....
7971
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
7972

7973
  USE(zn);
7974
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7975
  VIXL_ASSERT(zd.Is(zn));
7976
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7977

7978
  Emit(0x44188000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
7979
}
7980

7981
void Assembler::sqcadd(const ZRegister& zd,
7982
                       const ZRegister& zn,
7983
                       const ZRegister& zm,
7984
                       int rot) {
7985
  // SQCADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const>
7986
  //  0100 0101 ..00 0001 1101 1... .... ....
7987
  //  size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0>
7988

7989
  USE(zn);
7990
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
7991
  VIXL_ASSERT(zd.Is(zn));
7992
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
7993
  VIXL_ASSERT((rot == 90) || (rot == 270));
7994

7995
  Instr rotate_bit = (rot == 90) ? 0 : (1 << 10);
7996
  Emit(0x4501d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm));
7997
}
7998

7999
// This prototype maps to 2 instruction encodings:
8000
//  sqdmlalb_z_zzzi_d
8001
//  sqdmlalb_z_zzzi_s
8002
void Assembler::sqdmlalb(const ZRegister& zda,
8003
                         const ZRegister& zn,
8004
                         const ZRegister& zm,
8005
                         int index) {
8006
  // SQDMLALB <Zda>.D, <Zn>.S, <Zm>.S[<imm>]
8007
  //  0100 0100 111. .... 0010 .0.. .... ....
8008
  //  size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0>
8009

8010
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8011
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8012
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
8013
  VIXL_ASSERT(index >= 0);
8014

8015
  Instr zm_and_idx = 0;
8016
  if (zm.IsLaneSizeH()) {
8017
    // Zm<18:16> | i3h<20:19> | i3l<11>
8018
    VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
8019
    zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
8020
                 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
8021
  } else {
8022
    // Zm<19:16> | i2h<20> | i2l<11>
8023
    VIXL_ASSERT(zm.IsLaneSizeS());
8024
    VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3));
8025
    zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) |
8026
                 Rx<19, 16>(zm);
8027
  }
8028

8029
  Emit(0x44202000 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn));
8030
}
8031

8032
void Assembler::sqdmlalb(const ZRegister& zda,
8033
                         const ZRegister& zn,
8034
                         const ZRegister& zm) {
8035
  // SQDMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8036
  //  0100 0100 ..0. .... 0110 00.. .... ....
8037
  //  size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0>
8038

8039
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8040
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8041
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8042
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8043

8044
  Emit(0x44006000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8045
}
8046

8047
void Assembler::sqdmlalbt(const ZRegister& zda,
8048
                          const ZRegister& zn,
8049
                          const ZRegister& zm) {
8050
  // SQDMLALBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8051
  //  0100 0100 ..0. .... 0000 10.. .... ....
8052
  //  size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0>
8053

8054
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8055
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8056
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8057
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8058

8059
  Emit(0x44000800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8060
}
8061

8062
// This prototype maps to 2 instruction encodings:
8063
//  sqdmlalt_z_zzzi_d
8064
//  sqdmlalt_z_zzzi_s
8065
void Assembler::sqdmlalt(const ZRegister& zda,
8066
                         const ZRegister& zn,
8067
                         const ZRegister& zm,
8068
                         int index) {
8069
  // SQDMLALT <Zda>.D, <Zn>.S, <Zm>.S[<imm>]
8070
  //  0100 0100 111. .... 0010 .1.. .... ....
8071
  //  size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0>
8072

8073
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8074
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8075
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
8076
  VIXL_ASSERT(index >= 0);
8077

8078
  Instr zm_and_idx = 0;
8079
  if (zm.IsLaneSizeH()) {
8080
    // Zm<18:16> | i3h<20:19> | i3l<11>
8081
    VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
8082
    zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
8083
                 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
8084
  } else {
8085
    // Zm<19:16> | i2h<20> | i2l<11>
8086
    VIXL_ASSERT(zm.IsLaneSizeS());
8087
    VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3));
8088
    zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) |
8089
                 Rx<19, 16>(zm);
8090
  }
8091

8092
  Emit(0x44202400 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn));
8093
}
8094

8095
void Assembler::sqdmlalt(const ZRegister& zda,
8096
                         const ZRegister& zn,
8097
                         const ZRegister& zm) {
8098
  // SQDMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8099
  //  0100 0100 ..0. .... 0110 01.. .... ....
8100
  //  size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0>
8101

8102
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8103
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8104
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8105
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8106

8107
  Emit(0x44006400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8108
}
8109

8110
// This prototype maps to 2 instruction encodings:
8111
//  sqdmlslb_z_zzzi_d
8112
//  sqdmlslb_z_zzzi_s
8113
void Assembler::sqdmlslb(const ZRegister& zda,
8114
                         const ZRegister& zn,
8115
                         const ZRegister& zm,
8116
                         int index) {
8117
  // SQDMLSLB <Zda>.D, <Zn>.S, <Zm>.S[<imm>]
8118
  //  0100 0100 111. .... 0011 .0.. .... ....
8119
  //  size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0>
8120

8121
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8122
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8123
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
8124
  VIXL_ASSERT(index >= 0);
8125

8126
  Instr zm_and_idx = 0;
8127
  if (zm.IsLaneSizeH()) {
8128
    // Zm<18:16> | i3h<20:19> | i3l<11>
8129
    VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
8130
    zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
8131
                 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
8132
  } else {
8133
    // Zm<19:16> | i2h<20> | i2l<11>
8134
    VIXL_ASSERT(zm.IsLaneSizeS());
8135
    VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3));
8136
    zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) |
8137
                 Rx<19, 16>(zm);
8138
  }
8139

8140
  Emit(0x44203000 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn));
8141
}
8142

8143
void Assembler::sqdmlslb(const ZRegister& zda,
8144
                         const ZRegister& zn,
8145
                         const ZRegister& zm) {
8146
  // SQDMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8147
  //  0100 0100 ..0. .... 0110 10.. .... ....
8148
  //  size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0>
8149

8150
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8151
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8152
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8153
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8154

8155
  Emit(0x44006800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8156
}
8157

8158
void Assembler::sqdmlslbt(const ZRegister& zda,
8159
                          const ZRegister& zn,
8160
                          const ZRegister& zm) {
8161
  // SQDMLSLBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8162
  //  0100 0100 ..0. .... 0000 11.. .... ....
8163
  //  size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0>
8164

8165
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8166
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8167
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8168
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8169

8170
  Emit(0x44000c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8171
}
8172

8173
// This prototype maps to 2 instruction encodings:
8174
//  sqdmlslt_z_zzzi_d
8175
//  sqdmlslt_z_zzzi_s
8176
void Assembler::sqdmlslt(const ZRegister& zda,
8177
                         const ZRegister& zn,
8178
                         const ZRegister& zm,
8179
                         int index) {
8180
  // SQDMLSLT <Zda>.D, <Zn>.S, <Zm>.S[<imm>]
8181
  //  0100 0100 111. .... 0011 .1.. .... ....
8182
  //  size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0>
8183

8184
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8185
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8186
  VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD());
8187
  VIXL_ASSERT(index >= 0);
8188

8189
  Instr zm_and_idx = 0;
8190
  if (zm.IsLaneSizeH()) {
8191
    // Zm<18:16> | i3h<20:19> | i3l<11>
8192
    VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7));
8193
    zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) |
8194
                 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm);
8195
  } else {
8196
    // Zm<19:16> | i2h<20> | i2l<11>
8197
    VIXL_ASSERT(zm.IsLaneSizeS());
8198
    VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3));
8199
    zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) |
8200
                 Rx<19, 16>(zm);
8201
  }
8202

8203
  Emit(0x44203400 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn));
8204
}
8205

8206
void Assembler::sqdmlslt(const ZRegister& zda,
8207
                         const ZRegister& zn,
8208
                         const ZRegister& zm) {
8209
  // SQDMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8210
  //  0100 0100 ..0. .... 0110 11.. .... ....
8211
  //  size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0>
8212

8213
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8214
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8215
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8216
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
8217

8218
  Emit(0x44006c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8219
}
8220

8221
void Assembler::sqdmulh(const ZRegister& zd,
8222
                        const ZRegister& zn,
8223
                        const ZRegister& zm,
8224
                        int index) {
8225
  // SQDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>]
8226
  //  0100 0100 111. .... 1111 00.. .... ....
8227
  //  size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0>
8228

8229
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8230
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8231

8232
  Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(),
8233
                                           zm,
8234
                                           index,
8235
                                           0x4420f000,
8236
                                           0x44a0f000,
8237
                                           0x44e0f000);
8238

8239
  Emit(synthesised_op | Rd(zd) | Rn(zn));
8240
}
8241

8242
void Assembler::sqdmulh(const ZRegister& zd,
8243
                        const ZRegister& zn,
8244
                        const ZRegister& zm) {
8245
  // SQDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
8246
  //  0000 0100 ..1. .... 0111 00.. .... ....
8247
  //  size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0>
8248

8249
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8250
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8251

8252
  Emit(0x04207000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8253
}
8254

8255
void Assembler::sqdmullb(const ZRegister& zd,
8256
                         const ZRegister& zn,
8257
                         const ZRegister& zm) {
8258
  // SQDMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8259
  //  0100 0101 ..0. .... 0110 00.. .... ....
8260
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8261

8262
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8263
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8264
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
8265
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
8266

8267
  Emit(0x45006000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8268
}
8269

8270
void Assembler::sqdmullt(const ZRegister& zd,
8271
                         const ZRegister& zn,
8272
                         const ZRegister& zm) {
8273
  // SQDMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8274
  //  0100 0101 ..0. .... 0110 01.. .... ....
8275
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8276

8277
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8278
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8279
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
8280
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
8281

8282
  Emit(0x45006400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8283
}
8284

8285
void Assembler::sqneg(const ZRegister& zd,
8286
                      const PRegisterM& pg,
8287
                      const ZRegister& zn) {
8288
  // SQNEG <Zd>.<T>, <Pg>/M, <Zn>.<T>
8289
  //  0100 0100 ..00 1001 101. .... .... ....
8290
  //  size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
8291

8292
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8293
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
8294

8295
  Emit(0x4409a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
8296
}
8297

8298
void Assembler::sqrdcmlah(const ZRegister& zda,
8299
                          const ZRegister& zn,
8300
                          const ZRegister& zm,
8301
                          int index,
8302
                          int rot) {
8303
  // SQRDCMLAH <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const>
8304
  //  0100 0100 101. .... 0111 .... .... ....
8305
  //  size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0>
8306

8307
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8308
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8309
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
8310

8311
  Instr rotate_bit = (rot / 90) << 10;
8312
  Emit(0x44a07000 | SVEMulComplexIndexHelper(zm, index) | rotate_bit | Rd(zda) |
8313
       Rn(zn));
8314
}
8315

8316
void Assembler::sqrdcmlah(const ZRegister& zda,
8317
                          const ZRegister& zn,
8318
                          const ZRegister& zm,
8319
                          int rot) {
8320
  // SQRDCMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const>
8321
  //  0100 0100 ..0. .... 0011 .... .... ....
8322
  //  size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0>
8323

8324
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8325
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8326
  VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
8327

8328
  Instr rotate_bit = (rot / 90) << 10;
8329
  Emit(0x44003000 | rotate_bit | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8330
}
8331

8332
// This prototype maps to 3 instruction encodings:
8333
//  sqrdmlah_z_zzzi_d
8334
//  sqrdmlah_z_zzzi_h
8335
//  sqrdmlah_z_zzzi_s
8336
void Assembler::sqrdmlah(const ZRegister& zda,
8337
                         const ZRegister& zn,
8338
                         const ZRegister& zm,
8339
                         int index) {
8340
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8341
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8342

8343
  Instr op_h = 0x44201000;
8344
  Instr op_s = op_h | (1 << 23);
8345
  Instr op_d = op_h | (3 << 22);
8346
  // The encoding of opcode, index, Zm, and size are synthesized in this
8347
  // variable.
8348
  Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
8349
                                           zm,
8350
                                           index,
8351
                                           op_h,
8352
                                           op_s,
8353
                                           op_d);
8354

8355
  Emit(synthesized_op | Rd(zda) | Rn(zn));
8356
}
8357

8358
void Assembler::sqrdmlah(const ZRegister& zda,
8359
                         const ZRegister& zn,
8360
                         const ZRegister& zm) {
8361
  // SQRDMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
8362
  //  0100 0100 ..0. .... 0111 00.. .... ....
8363
  //  size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0>
8364

8365
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8366
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8367

8368
  Emit(0x44007000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8369
}
8370

8371
// This prototype maps to 3 instruction encodings:
8372
//  sqrdmlsh_z_zzzi_d
8373
//  sqrdmlsh_z_zzzi_h
8374
//  sqrdmlsh_z_zzzi_s
8375
void Assembler::sqrdmlsh(const ZRegister& zda,
8376
                         const ZRegister& zn,
8377
                         const ZRegister& zm,
8378
                         int index) {
8379
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8380
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8381

8382
  Instr op_h = 0x44201400;
8383
  Instr op_s = op_h | (1 << 23);
8384
  Instr op_d = op_h | (3 << 22);
8385
  // The encoding of opcode, index, Zm, and size are synthesized in this
8386
  // variable.
8387
  Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(),
8388
                                           zm,
8389
                                           index,
8390
                                           op_h,
8391
                                           op_s,
8392
                                           op_d);
8393

8394
  Emit(synthesized_op | Rd(zda) | Rn(zn));
8395
}
8396

8397
void Assembler::sqrdmlsh(const ZRegister& zda,
8398
                         const ZRegister& zn,
8399
                         const ZRegister& zm) {
8400
  // SQRDMLSH <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
8401
  //  0100 0100 ..0. .... 0111 01.. .... ....
8402
  //  size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0>
8403

8404
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8405
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8406

8407
  Emit(0x44007400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
8408
}
8409

8410
void Assembler::sqrdmulh(const ZRegister& zd,
8411
                         const ZRegister& zn,
8412
                         const ZRegister& zm,
8413
                         int index) {
8414
  // SQRDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>]
8415
  //  0100 0100 111. .... 1111 01.. .... ....
8416
  //  size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0>
8417

8418
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8419
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8420

8421
  Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(),
8422
                                           zm,
8423
                                           index,
8424
                                           0x4420f400,
8425
                                           0x44a0f400,
8426
                                           0x44e0f400);
8427

8428
  Emit(synthesised_op | Rd(zd) | Rn(zn));
8429
}
8430

8431
void Assembler::sqrdmulh(const ZRegister& zd,
8432
                         const ZRegister& zn,
8433
                         const ZRegister& zm) {
8434
  // SQRDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
8435
  //  0000 0100 ..1. .... 0111 01.. .... ....
8436
  //  size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0>
8437

8438
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8439
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8440

8441
  Emit(0x04207400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8442
}
8443

8444
void Assembler::sqrshl(const ZRegister& zd,
8445
                       const PRegisterM& pg,
8446
                       const ZRegister& zn,
8447
                       const ZRegister& zm) {
8448
  // SQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8449
  //  0100 0100 ..00 1010 100. .... .... ....
8450
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8451
  //  Zdn<4:0>
8452

8453
  USE(zn);
8454
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8455
  VIXL_ASSERT(zd.Is(zn));
8456
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8457

8458
  Emit(0x440a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8459
}
8460

8461
void Assembler::sqrshlr(const ZRegister& zd,
8462
                        const PRegisterM& pg,
8463
                        const ZRegister& zn,
8464
                        const ZRegister& zm) {
8465
  // SQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8466
  //  0100 0100 ..00 1110 100. .... .... ....
8467
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8468
  //  Zdn<4:0>
8469

8470
  USE(zn);
8471
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8472
  VIXL_ASSERT(zd.Is(zn));
8473
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8474

8475
  Emit(0x440e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8476
}
8477

8478
void Assembler::sqshl(const ZRegister& zd,
8479
                      const PRegisterM& pg,
8480
                      const ZRegister& zn,
8481
                      int shift) {
8482
  // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
8483
  //  0000 0100 ..00 0110 100. .... .... ....
8484
  //  tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> |
8485
  //  imm3<7:5> | Zdn<4:0>
8486

8487
  USE(zn);
8488
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8489
  VIXL_ASSERT(zd.Is(zn));
8490
  Instr encoded_imm =
8491
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
8492
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04068000);
8493
}
8494

8495
void Assembler::sqshl(const ZRegister& zd,
8496
                      const PRegisterM& pg,
8497
                      const ZRegister& zn,
8498
                      const ZRegister& zm) {
8499
  // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8500
  //  0100 0100 ..00 1000 100. .... .... ....
8501
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8502
  //  Zdn<4:0>
8503

8504
  USE(zn);
8505
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8506
  VIXL_ASSERT(zd.Is(zn));
8507
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8508

8509
  Emit(0x44088000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8510
}
8511

8512
void Assembler::sqshlr(const ZRegister& zd,
8513
                       const PRegisterM& pg,
8514
                       const ZRegister& zn,
8515
                       const ZRegister& zm) {
8516
  // SQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8517
  //  0100 0100 ..00 1100 100. .... .... ....
8518
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8519
  //  Zdn<4:0>
8520

8521
  USE(zn);
8522
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8523
  VIXL_ASSERT(zd.Is(zn));
8524
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8525

8526
  Emit(0x440c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8527
}
8528

8529
void Assembler::sqshlu(const ZRegister& zd,
8530
                       const PRegisterM& pg,
8531
                       const ZRegister& zn,
8532
                       int shift) {
8533
  // SQSHLU <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
8534
  //  0000 0100 ..00 1111 100. .... .... ....
8535
  //  tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> |
8536
  //  imm3<7:5> | Zdn<4:0>
8537

8538
  USE(zn);
8539
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8540
  VIXL_ASSERT(zd.Is(zn));
8541

8542
  Instr encoded_imm =
8543
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
8544
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040f8000);
8545
}
8546

8547
void Assembler::sqsub(const ZRegister& zd,
8548
                      const PRegisterM& pg,
8549
                      const ZRegister& zn,
8550
                      const ZRegister& zm) {
8551
  // SQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8552
  //  0100 0100 ..01 1010 100. .... .... ....
8553
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
8554

8555
  USE(zn);
8556
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8557
  VIXL_ASSERT(zd.Is(zn));
8558
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8559

8560
  Emit(0x441a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8561
}
8562

8563
void Assembler::sqsubr(const ZRegister& zd,
8564
                       const PRegisterM& pg,
8565
                       const ZRegister& zn,
8566
                       const ZRegister& zm) {
8567
  // SQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8568
  //  0100 0100 ..01 1110 100. .... .... ....
8569
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
8570

8571
  USE(zn);
8572
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8573
  VIXL_ASSERT(zd.Is(zn));
8574
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8575

8576
  Emit(0x441e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8577
}
8578

8579
void Assembler::sqxtnb(const ZRegister& zd, const ZRegister& zn) {
8580
  // SQXTNB <Zd>.<T>, <Zn>.<Tb>
8581
  //  0100 0101 0.1. .000 0100 00.. .... ....
8582
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
8583

8584
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8585
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
8586
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
8587

8588
  // XTN instructions look like immediate shifts with zero shift distance.
8589
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
8590
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45204000);
8591
}
8592

8593
void Assembler::sqxtnt(const ZRegister& zd, const ZRegister& zn) {
8594
  // SQXTNT <Zd>.<T>, <Zn>.<Tb>
8595
  //  0100 0101 0.1. .000 0100 01.. .... ....
8596
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
8597

8598
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8599
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
8600
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
8601

8602
  // XTN instructions look like immediate shifts with zero shift distance.
8603
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
8604
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45204400);
8605
}
8606

8607
void Assembler::sqxtunb(const ZRegister& zd, const ZRegister& zn) {
8608
  // SQXTUNB <Zd>.<T>, <Zn>.<Tb>
8609
  //  0100 0101 0.1. .000 0101 00.. .... ....
8610
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
8611

8612
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8613
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
8614
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
8615

8616
  // XTN instructions look like immediate shifts with zero shift distance.
8617
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
8618
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45205000);
8619
}
8620

8621
void Assembler::sqxtunt(const ZRegister& zd, const ZRegister& zn) {
8622
  // SQXTUNT <Zd>.<T>, <Zn>.<Tb>
8623
  //  0100 0101 0.1. .000 0101 01.. .... ....
8624
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
8625

8626
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8627
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
8628
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
8629

8630
  // XTN instructions look like immediate shifts with zero shift distance.
8631
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
8632
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45205400);
8633
}
8634

8635
void Assembler::srhadd(const ZRegister& zd,
8636
                       const PRegisterM& pg,
8637
                       const ZRegister& zn,
8638
                       const ZRegister& zm) {
8639
  // SRHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8640
  //  0100 0100 ..01 0100 100. .... .... ....
8641
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
8642

8643
  USE(zn);
8644
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8645
  VIXL_ASSERT(zd.Is(zn));
8646
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8647

8648
  Emit(0x44148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8649
}
8650

8651
void Assembler::sri(const ZRegister& zd, const ZRegister& zn, int shift) {
8652
  // SRI <Zd>.<T>, <Zn>.<T>, #<const>
8653
  //  0100 0101 ..0. .... 1111 00.. .... ....
8654
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0>
8655

8656
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8657
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
8658
  Instr encoded_imm =
8659
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
8660

8661
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f000);
8662
}
8663

8664
void Assembler::srshl(const ZRegister& zd,
8665
                      const PRegisterM& pg,
8666
                      const ZRegister& zn,
8667
                      const ZRegister& zm) {
8668
  // SRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8669
  //  0100 0100 ..00 0010 100. .... .... ....
8670
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8671
  //  Zdn<4:0>
8672

8673
  USE(zn);
8674
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8675
  VIXL_ASSERT(zd.Is(zn));
8676
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8677

8678
  Emit(0x44028000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8679
}
8680

8681
void Assembler::srshlr(const ZRegister& zd,
8682
                       const PRegisterM& pg,
8683
                       const ZRegister& zn,
8684
                       const ZRegister& zm) {
8685
  // SRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8686
  //  0100 0100 ..00 0110 100. .... .... ....
8687
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
8688
  //  Zdn<4:0>
8689

8690
  USE(zn);
8691
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8692
  VIXL_ASSERT(zd.Is(zn));
8693
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8694

8695
  Emit(0x44068000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8696
}
8697

8698
void Assembler::srshr(const ZRegister& zd,
8699
                      const PRegisterM& pg,
8700
                      const ZRegister& zn,
8701
                      int shift) {
8702
  // SRSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
8703
  //  0000 0100 ..00 1100 100. .... .... ....
8704
  //  tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> |
8705
  //  imm3<7:5> | Zdn<4:0>
8706

8707
  USE(zn);
8708
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8709
  VIXL_ASSERT(zd.Is(zn));
8710
  Instr encoded_imm =
8711
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
8712
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040c8000);
8713
}
8714

8715
void Assembler::srsra(const ZRegister& zda, const ZRegister& zn, int shift) {
8716
  // SRSRA <Zda>.<T>, <Zn>.<T>, #<const>
8717
  //  0100 0101 ..0. .... 1110 10.. .... ....
8718
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> |
8719
  //  Zda<4:0>
8720

8721
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8722
  VIXL_ASSERT(AreSameLaneSize(zda, zn));
8723
  Instr encoded_imm =
8724
      EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits());
8725

8726
  SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e800);
8727
}
8728

8729
void Assembler::sshllb(const ZRegister& zd, const ZRegister& zn, int shift) {
8730
  // SSHLLB <Zd>.<T>, <Zn>.<Tb>, #<const>
8731
  //  0100 0101 0.0. .... 1010 00.. .... ....
8732
  //  tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8733

8734
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8735
  VIXL_ASSERT(!zd.IsLaneSizeB());
8736
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8737

8738
  Instr encoded_imm =
8739
      EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits());
8740
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a000);
8741
}
8742

8743
void Assembler::sshllt(const ZRegister& zd, const ZRegister& zn, int shift) {
8744
  // SSHLLT <Zd>.<T>, <Zn>.<Tb>, #<const>
8745
  //  0100 0101 0.0. .... 1010 01.. .... ....
8746
  //  tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8747

8748
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8749
  VIXL_ASSERT(!zd.IsLaneSizeB());
8750
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8751

8752
  Instr encoded_imm =
8753
      EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits());
8754
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a400);
8755
}
8756

8757
void Assembler::ssra(const ZRegister& zda, const ZRegister& zn, int shift) {
8758
  // SSRA <Zda>.<T>, <Zn>.<T>, #<const>
8759
  //  0100 0101 ..0. .... 1110 00.. .... ....
8760
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> |
8761
  //  Zda<4:0>
8762

8763
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8764
  VIXL_ASSERT(AreSameLaneSize(zda, zn));
8765
  Instr encoded_imm =
8766
      EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits());
8767

8768
  SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e000);
8769
}
8770

8771
void Assembler::ssublb(const ZRegister& zd,
8772
                       const ZRegister& zn,
8773
                       const ZRegister& zm) {
8774
  // SSUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8775
  //  0100 0101 ..0. .... 0001 00.. .... ....
8776
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
8777
  //  Zd<4:0>
8778

8779
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8780
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8781
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8782
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
8783

8784
  Emit(0x45001000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8785
}
8786

8787
void Assembler::ssublbt(const ZRegister& zd,
8788
                        const ZRegister& zn,
8789
                        const ZRegister& zm) {
8790
  // SSUBLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8791
  //  0100 0101 ..0. .... 1000 10.. .... ....
8792
  //  size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0>
8793

8794
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8795
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8796
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8797
  VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS());
8798

8799
  Emit(0x45008800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8800
}
8801

8802
void Assembler::ssublt(const ZRegister& zd,
8803
                       const ZRegister& zn,
8804
                       const ZRegister& zm) {
8805
  // SSUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8806
  //  0100 0101 ..0. .... 0001 01.. .... ....
8807
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
8808
  //  Zd<4:0>
8809

8810
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8811
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8812
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8813
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
8814

8815
  Emit(0x45001400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8816
}
8817

8818
void Assembler::ssubltb(const ZRegister& zd,
8819
                        const ZRegister& zn,
8820
                        const ZRegister& zm) {
8821
  // SSUBLTB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8822
  //  0100 0101 ..0. .... 1000 11.. .... ....
8823
  //  size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0>
8824

8825
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8826
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8827
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
8828
  VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS());
8829

8830
  Emit(0x45008c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8831
}
8832

8833
void Assembler::ssubwb(const ZRegister& zd,
8834
                       const ZRegister& zn,
8835
                       const ZRegister& zm) {
8836
  // SSUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
8837
  //  0100 0101 ..0. .... 0101 00.. .... ....
8838
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8839

8840
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8841
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
8842
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
8843
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
8844

8845
  Emit(0x45005000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8846
}
8847

8848
void Assembler::ssubwt(const ZRegister& zd,
8849
                       const ZRegister& zn,
8850
                       const ZRegister& zm) {
8851
  // SSUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
8852
  //  0100 0101 ..0. .... 0101 01.. .... ....
8853
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
8854

8855
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8856
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
8857
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
8858
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
8859

8860
  Emit(0x45005400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8861
}
8862

8863
#if 0
8864
// This prototype maps to 2 instruction encodings:
8865
//  stnt1b_z_p_ar_d_64_unscaled
8866
//  stnt1b_z_p_ar_s_x32_unscaled
8867
void Assembler::stnt1b(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) {
8868
  // STNT1B { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}]
8869
  //  1110 0100 000. .... 001. .... .... ....
8870
  //  msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0>
8871

8872
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8873

8874
  Emit(0xe4002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm));
8875
}
8876

8877
void Assembler::stnt1d(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) {
8878
  // STNT1D { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}]
8879
  //  1110 0101 100. .... 001. .... .... ....
8880
  //  msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0>
8881

8882
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8883

8884
  Emit(0xe5802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm));
8885
}
8886

8887
// This prototype maps to 2 instruction encodings:
8888
//  stnt1h_z_p_ar_d_64_unscaled
8889
//  stnt1h_z_p_ar_s_x32_unscaled
8890
void Assembler::stnt1h(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) {
8891
  // STNT1H { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}]
8892
  //  1110 0100 100. .... 001. .... .... ....
8893
  //  msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0>
8894

8895
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8896

8897
  Emit(0xe4802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm));
8898
}
8899

8900
// This prototype maps to 2 instruction encodings:
8901
//  stnt1w_z_p_ar_d_64_unscaled
8902
//  stnt1w_z_p_ar_s_x32_unscaled
8903
void Assembler::stnt1w(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) {
8904
  // STNT1W { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}]
8905
  //  1110 0101 000. .... 001. .... .... ....
8906
  //  msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0>
8907

8908
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8909

8910
  Emit(0xe5002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm));
8911
}
8912
#endif
8913

8914
void Assembler::subhnb(const ZRegister& zd,
8915
                       const ZRegister& zn,
8916
                       const ZRegister& zm) {
8917
  // SUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8918
  //  0100 0101 ..1. .... 0111 00.. .... ....
8919
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
8920

8921
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8922
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8923
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
8924
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
8925

8926
  Emit(0x45207000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
8927
}
8928

8929
void Assembler::subhnt(const ZRegister& zd,
8930
                       const ZRegister& zn,
8931
                       const ZRegister& zm) {
8932
  // SUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
8933
  //  0100 0101 ..1. .... 0111 01.. .... ....
8934
  //  size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0>
8935

8936
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8937
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
8938
  VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2));
8939
  VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes);
8940

8941
  Emit(0x45207400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm));
8942
}
8943

8944
void Assembler::suqadd(const ZRegister& zd,
8945
                       const PRegisterM& pg,
8946
                       const ZRegister& zn,
8947
                       const ZRegister& zm) {
8948
  // SUQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
8949
  //  0100 0100 ..01 1100 100. .... .... ....
8950
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
8951

8952
  USE(zn);
8953
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8954
  VIXL_ASSERT(zd.Is(zn));
8955
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8956

8957
  Emit(0x441c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
8958
}
8959

8960
void Assembler::tbl(const ZRegister& zd,
8961
                    const ZRegister& zn1,
8962
                    const ZRegister& zn2,
8963
                    const ZRegister& zm) {
8964
  // TBL <Zd>.<T>, { <Zn1>.<T>, <Zn2>.<T> }, <Zm>.<T>
8965
  //  0000 0101 ..1. .... 0010 10.. .... ....
8966
  //  size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0>
8967

8968
  USE(zn2);
8969
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8970
  VIXL_ASSERT(AreConsecutive(zn1, zn2));
8971
  VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2, zm));
8972

8973
  Emit(0x05202800 | SVESize(zd) | Rd(zd) | Rn(zn1) | Rn(zn2) | Rm(zm));
8974
}
8975

8976
void Assembler::tbx(const ZRegister& zd,
8977
                    const ZRegister& zn,
8978
                    const ZRegister& zm) {
8979
  // TBX <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
8980
  //  0000 0101 ..1. .... 0010 11.. .... ....
8981
  //  size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0>
8982

8983
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8984
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
8985

8986
  Emit(0x05202c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
8987
}
8988

8989
void Assembler::uaba(const ZRegister& zda,
8990
                     const ZRegister& zn,
8991
                     const ZRegister& zm) {
8992
  // UABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T>
8993
  //  0100 0101 ..0. .... 1111 11.. .... ....
8994
  //  size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0>
8995

8996
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
8997
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
8998

8999
  Emit(0x4500fc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9000
}
9001

9002
void Assembler::uabalb(const ZRegister& zda,
9003
                       const ZRegister& zn,
9004
                       const ZRegister& zm) {
9005
  // UABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9006
  //  0100 0101 ..0. .... 1100 10.. .... ....
9007
  //  size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9008

9009
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9010
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9011
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9012
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
9013

9014
  Emit(0x4500c800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9015
}
9016

9017
void Assembler::uabalt(const ZRegister& zda,
9018
                       const ZRegister& zn,
9019
                       const ZRegister& zm) {
9020
  // UABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9021
  //  0100 0101 ..0. .... 1100 11.. .... ....
9022
  //  size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9023

9024
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9025
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9026
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9027
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
9028

9029
  Emit(0x4500cc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9030
}
9031

9032
void Assembler::uabdlb(const ZRegister& zd,
9033
                       const ZRegister& zn,
9034
                       const ZRegister& zm) {
9035
  // UABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9036
  //  0100 0101 ..0. .... 0011 10.. .... ....
9037
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9038
  //  Zd<4:0>
9039

9040
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9041
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9042
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9043
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9044

9045
  Emit(0x45003800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9046
}
9047

9048
void Assembler::uabdlt(const ZRegister& zd,
9049
                       const ZRegister& zn,
9050
                       const ZRegister& zm) {
9051
  // UABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9052
  //  0100 0101 ..0. .... 0011 11.. .... ....
9053
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9054
  //  Zd<4:0>
9055

9056
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9057
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9058
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9059
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9060

9061
  Emit(0x45003c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9062
}
9063

9064
void Assembler::uadalp(const ZRegister& zda,
9065
                       const PRegisterM& pg,
9066
                       const ZRegister& zn) {
9067
  // UADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb>
9068
  //  0100 0100 ..00 0101 101. .... .... ....
9069
  //  size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0>
9070

9071
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9072
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9073
  VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes);
9074

9075
  Emit(0x4405a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn));
9076
}
9077

9078
void Assembler::uaddlb(const ZRegister& zd,
9079
                       const ZRegister& zn,
9080
                       const ZRegister& zm) {
9081
  // UADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9082
  //  0100 0101 ..0. .... 0000 10.. .... ....
9083
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9084
  //  Zd<4:0>
9085

9086
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9087
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9088
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9089
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9090

9091
  Emit(0x45000800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9092
}
9093

9094
void Assembler::uaddlt(const ZRegister& zd,
9095
                       const ZRegister& zn,
9096
                       const ZRegister& zm) {
9097
  // UADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9098
  //  0100 0101 ..0. .... 0000 11.. .... ....
9099
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9100
  //  Zd<4:0>
9101

9102
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9103
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9104
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9105
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9106

9107
  Emit(0x45000c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9108
}
9109

9110
void Assembler::uaddwb(const ZRegister& zd,
9111
                       const ZRegister& zn,
9112
                       const ZRegister& zm) {
9113
  // UADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
9114
  //  0100 0101 ..0. .... 0100 10.. .... ....
9115
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9116

9117
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9118
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
9119
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
9120
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9121

9122
  Emit(0x45004800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9123
}
9124

9125
void Assembler::uaddwt(const ZRegister& zd,
9126
                       const ZRegister& zn,
9127
                       const ZRegister& zm) {
9128
  // UADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
9129
  //  0100 0101 ..0. .... 0100 11.. .... ....
9130
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9131

9132
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9133
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
9134
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
9135
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9136

9137
  Emit(0x45004c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9138
}
9139

9140
void Assembler::uhadd(const ZRegister& zd,
9141
                      const PRegisterM& pg,
9142
                      const ZRegister& zn,
9143
                      const ZRegister& zm) {
9144
  // UHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9145
  //  0100 0100 ..01 0001 100. .... .... ....
9146
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9147

9148
  USE(zn);
9149
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9150
  VIXL_ASSERT(zd.Is(zn));
9151
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9152

9153
  Emit(0x44118000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9154
}
9155

9156
void Assembler::uhsub(const ZRegister& zd,
9157
                      const PRegisterM& pg,
9158
                      const ZRegister& zn,
9159
                      const ZRegister& zm) {
9160
  // UHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9161
  //  0100 0100 ..01 0011 100. .... .... ....
9162
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9163

9164
  USE(zn);
9165
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9166
  VIXL_ASSERT(zd.Is(zn));
9167
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9168

9169
  Emit(0x44138000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9170
}
9171

9172
void Assembler::uhsubr(const ZRegister& zd,
9173
                       const PRegisterM& pg,
9174
                       const ZRegister& zn,
9175
                       const ZRegister& zm) {
9176
  // UHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9177
  //  0100 0100 ..01 0111 100. .... .... ....
9178
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9179

9180
  USE(zn);
9181
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9182
  VIXL_ASSERT(zd.Is(zn));
9183
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9184

9185
  Emit(0x44178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9186
}
9187

9188
void Assembler::umaxp(const ZRegister& zd,
9189
                      const PRegisterM& pg,
9190
                      const ZRegister& zn,
9191
                      const ZRegister& zm) {
9192
  // UMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9193
  //  0100 0100 ..01 0101 101. .... .... ....
9194
  //  size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9195

9196
  USE(zn);
9197
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9198
  VIXL_ASSERT(zd.Is(zn));
9199
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9200

9201
  Emit(0x4415a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9202
}
9203

9204
void Assembler::uminp(const ZRegister& zd,
9205
                      const PRegisterM& pg,
9206
                      const ZRegister& zn,
9207
                      const ZRegister& zm) {
9208
  // UMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9209
  //  0100 0100 ..01 0111 101. .... .... ....
9210
  //  size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9211

9212
  USE(zn);
9213
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9214
  VIXL_ASSERT(zd.Is(zn));
9215
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9216

9217
  Emit(0x4417a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9218
}
9219

9220
void Assembler::umlalb(const ZRegister& zda,
9221
                       const ZRegister& zn,
9222
                       const ZRegister& zm) {
9223
  // UMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9224
  //  0100 0100 ..0. .... 0100 10.. .... ....
9225
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9226

9227
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9228
  VIXL_ASSERT(!zda.IsLaneSizeB());
9229
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9230
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9231

9232
  Emit(0x44004800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9233
}
9234

9235
void Assembler::umlalt(const ZRegister& zda,
9236
                       const ZRegister& zn,
9237
                       const ZRegister& zm) {
9238
  // UMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9239
  //  0100 0100 ..0. .... 0100 11.. .... ....
9240
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9241

9242
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9243
  VIXL_ASSERT(!zda.IsLaneSizeB());
9244
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9245
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9246

9247
  Emit(0x44004c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9248
}
9249

9250
void Assembler::umlslb(const ZRegister& zda,
9251
                       const ZRegister& zn,
9252
                       const ZRegister& zm) {
9253
  // UMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9254
  //  0100 0100 ..0. .... 0101 10.. .... ....
9255
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9256

9257
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9258
  VIXL_ASSERT(!zda.IsLaneSizeB());
9259
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9260
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9261

9262
  Emit(0x44005800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9263
}
9264

9265
void Assembler::umlslt(const ZRegister& zda,
9266
                       const ZRegister& zn,
9267
                       const ZRegister& zm) {
9268
  // UMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9269
  //  0100 0100 ..0. .... 0101 11.. .... ....
9270
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0>
9271

9272
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9273
  VIXL_ASSERT(!zda.IsLaneSizeB());
9274
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9275
  VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9276

9277
  Emit(0x44005c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9278
}
9279

9280
void Assembler::umulh(const ZRegister& zd,
9281
                      const ZRegister& zn,
9282
                      const ZRegister& zm) {
9283
  // UMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
9284
  //  0000 0100 ..1. .... 0110 11.. .... ....
9285
  //  size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0>
9286

9287
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9288
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9289

9290
  Emit(0x04206c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9291
}
9292

9293
void Assembler::umullb(const ZRegister& zd,
9294
                       const ZRegister& zn,
9295
                       const ZRegister& zm) {
9296
  // UMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9297
  //  0100 0101 ..0. .... 0111 10.. .... ....
9298
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9299

9300
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9301
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9302
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
9303
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
9304

9305
  Emit(0x45007800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9306
}
9307

9308
void Assembler::umullt(const ZRegister& zd,
9309
                       const ZRegister& zn,
9310
                       const ZRegister& zm) {
9311
  // UMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9312
  //  0100 0101 ..0. .... 0111 11.. .... ....
9313
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9314

9315
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9316
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9317
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ());
9318
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
9319

9320
  Emit(0x45007c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9321
}
9322

9323
void Assembler::uqadd(const ZRegister& zd,
9324
                      const PRegisterM& pg,
9325
                      const ZRegister& zn,
9326
                      const ZRegister& zm) {
9327
  // UQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9328
  //  0100 0100 ..01 1001 100. .... .... ....
9329
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9330

9331
  USE(zn);
9332
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9333
  VIXL_ASSERT(zd.Is(zn));
9334
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9335

9336
  Emit(0x44198000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9337
}
9338

9339
void Assembler::uqrshl(const ZRegister& zd,
9340
                       const PRegisterM& pg,
9341
                       const ZRegister& zn,
9342
                       const ZRegister& zm) {
9343
  // UQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9344
  //  0100 0100 ..00 1011 100. .... .... ....
9345
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9346
  //  Zdn<4:0>
9347

9348
  USE(zn);
9349
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9350
  VIXL_ASSERT(zd.Is(zn));
9351
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9352

9353
  Emit(0x440b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9354
}
9355

9356
void Assembler::uqrshlr(const ZRegister& zd,
9357
                        const PRegisterM& pg,
9358
                        const ZRegister& zn,
9359
                        const ZRegister& zm) {
9360
  // UQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9361
  //  0100 0100 ..00 1111 100. .... .... ....
9362
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9363
  //  Zdn<4:0>
9364

9365
  USE(zn);
9366
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9367
  VIXL_ASSERT(zd.Is(zn));
9368
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9369

9370
  Emit(0x440f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9371
}
9372

9373
void Assembler::uqshl(const ZRegister& zd,
9374
                      const PRegisterM& pg,
9375
                      const ZRegister& zn,
9376
                      int shift) {
9377
  // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
9378
  //  0000 0100 ..00 0111 100. .... .... ....
9379
  //  tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> |
9380
  //  imm3<7:5> | Zdn<4:0>
9381

9382
  USE(zn);
9383
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9384
  VIXL_ASSERT(zd.Is(zn));
9385
  Instr encoded_imm =
9386
      EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits());
9387
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04078000);
9388
}
9389

9390
void Assembler::uqshl(const ZRegister& zd,
9391
                      const PRegisterM& pg,
9392
                      const ZRegister& zn,
9393
                      const ZRegister& zm) {
9394
  // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9395
  //  0100 0100 ..00 1001 100. .... .... ....
9396
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9397
  //  Zdn<4:0>
9398

9399
  USE(zn);
9400
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9401
  VIXL_ASSERT(zd.Is(zn));
9402
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9403

9404
  Emit(0x44098000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9405
}
9406

9407
void Assembler::uqshlr(const ZRegister& zd,
9408
                       const PRegisterM& pg,
9409
                       const ZRegister& zn,
9410
                       const ZRegister& zm) {
9411
  // UQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9412
  //  0100 0100 ..00 1101 100. .... .... ....
9413
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9414
  //  Zdn<4:0>
9415

9416
  USE(zn);
9417
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9418
  VIXL_ASSERT(zd.Is(zn));
9419
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9420

9421
  Emit(0x440d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9422
}
9423

9424
void Assembler::uqsub(const ZRegister& zd,
9425
                      const PRegisterM& pg,
9426
                      const ZRegister& zn,
9427
                      const ZRegister& zm) {
9428
  // UQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9429
  //  0100 0100 ..01 1011 100. .... .... ....
9430
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9431

9432
  USE(zn);
9433
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9434
  VIXL_ASSERT(zd.Is(zn));
9435
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9436

9437
  Emit(0x441b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9438
}
9439

9440
void Assembler::uqsubr(const ZRegister& zd,
9441
                       const PRegisterM& pg,
9442
                       const ZRegister& zn,
9443
                       const ZRegister& zm) {
9444
  // UQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9445
  //  0100 0100 ..01 1111 100. .... .... ....
9446
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9447

9448
  USE(zn);
9449
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9450
  VIXL_ASSERT(zd.Is(zn));
9451
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9452

9453
  Emit(0x441f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9454
}
9455

9456
void Assembler::uqxtnb(const ZRegister& zd, const ZRegister& zn) {
9457
  // UQXTNB <Zd>.<T>, <Zn>.<Tb>
9458
  //  0100 0101 0.1. .000 0100 10.. .... ....
9459
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
9460

9461
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9462
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
9463
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
9464

9465
  // XTN instructions look like immediate shifts with zero shift distance.
9466
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
9467
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45204800);
9468
}
9469

9470
void Assembler::uqxtnt(const ZRegister& zd, const ZRegister& zn) {
9471
  // UQXTNT <Zd>.<T>, <Zn>.<Tb>
9472
  //  0100 0101 0.1. .000 0100 11.. .... ....
9473
  //  tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0>
9474

9475
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9476
  VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB());
9477
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2));
9478

9479
  // XTN instructions look like immediate shifts with zero shift distance.
9480
  Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits());
9481
  SVEBitwiseShiftImmediate(zd, zn, size, 0x45204c00);
9482
}
9483

9484
void Assembler::urecpe(const ZRegister& zd,
9485
                       const PRegisterM& pg,
9486
                       const ZRegister& zn) {
9487
  // URECPE <Zd>.S, <Pg>/M, <Zn>.S
9488
  //  0100 0100 ..00 0000 101. .... .... ....
9489
  //  size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
9490

9491
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9492
  VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS());
9493

9494
  Emit(0x4400a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
9495
}
9496

9497
void Assembler::urhadd(const ZRegister& zd,
9498
                       const PRegisterM& pg,
9499
                       const ZRegister& zn,
9500
                       const ZRegister& zm) {
9501
  // URHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9502
  //  0100 0100 ..01 0101 100. .... .... ....
9503
  //  size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9504

9505
  USE(zn);
9506
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9507
  VIXL_ASSERT(zd.Is(zn));
9508
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9509

9510
  Emit(0x44158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9511
}
9512

9513
void Assembler::urshl(const ZRegister& zd,
9514
                      const PRegisterM& pg,
9515
                      const ZRegister& zn,
9516
                      const ZRegister& zm) {
9517
  // URSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9518
  //  0100 0100 ..00 0011 100. .... .... ....
9519
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9520
  //  Zdn<4:0>
9521

9522
  USE(zn);
9523
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9524
  VIXL_ASSERT(zd.Is(zn));
9525
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9526

9527
  Emit(0x44038000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9528
}
9529

9530
void Assembler::urshlr(const ZRegister& zd,
9531
                       const PRegisterM& pg,
9532
                       const ZRegister& zn,
9533
                       const ZRegister& zm) {
9534
  // URSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9535
  //  0100 0100 ..00 0111 100. .... .... ....
9536
  //  size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> |
9537
  //  Zdn<4:0>
9538

9539
  USE(zn);
9540
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9541
  VIXL_ASSERT(zd.Is(zn));
9542
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9543

9544
  Emit(0x44078000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9545
}
9546

9547
void Assembler::urshr(const ZRegister& zd,
9548
                      const PRegisterM& pg,
9549
                      const ZRegister& zn,
9550
                      int shift) {
9551
  // URSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const>
9552
  //  0000 0100 ..00 1101 100. .... .... ....
9553
  //  tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> |
9554
  //  imm3<7:5> | Zdn<4:0>
9555

9556
  USE(zn);
9557
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9558
  VIXL_ASSERT(zd.Is(zn));
9559
  Instr encoded_imm =
9560
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
9561
  SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040d8000);
9562
}
9563

9564
void Assembler::ursqrte(const ZRegister& zd,
9565
                        const PRegisterM& pg,
9566
                        const ZRegister& zn) {
9567
  // URSQRTE <Zd>.S, <Pg>/M, <Zn>.S
9568
  //  0100 0100 ..00 0001 101. .... .... ....
9569
  //  size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0>
9570

9571
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9572
  VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS());
9573

9574
  Emit(0x4401a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn));
9575
}
9576

9577
void Assembler::ursra(const ZRegister& zda, const ZRegister& zn, int shift) {
9578
  // URSRA <Zda>.<T>, <Zn>.<T>, #<const>
9579
  //  0100 0101 ..0. .... 1110 11.. .... ....
9580
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> |
9581
  //  Zda<4:0>
9582

9583
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9584
  VIXL_ASSERT(AreSameLaneSize(zda, zn));
9585
  Instr encoded_imm =
9586
      EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits());
9587

9588
  SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500ec00);
9589
}
9590

9591
void Assembler::ushllb(const ZRegister& zd, const ZRegister& zn, int shift) {
9592
  // USHLLB <Zd>.<T>, <Zn>.<Tb>, #<const>
9593
  //  0100 0101 0.0. .... 1010 10.. .... ....
9594
  //  tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9595

9596
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9597
  VIXL_ASSERT(!zd.IsLaneSizeB());
9598
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9599

9600
  Instr encoded_imm =
9601
      EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits());
9602
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a800);
9603
}
9604

9605
void Assembler::ushllt(const ZRegister& zd, const ZRegister& zn, int shift) {
9606
  // USHLLT <Zd>.<T>, <Zn>.<Tb>, #<const>
9607
  //  0100 0101 0.0. .... 1010 11.. .... ....
9608
  //  tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9609

9610
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9611
  VIXL_ASSERT(!zd.IsLaneSizeB());
9612
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9613

9614
  Instr encoded_imm =
9615
      EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits());
9616
  SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500ac00);
9617
}
9618

9619
void Assembler::usqadd(const ZRegister& zd,
9620
                       const PRegisterM& pg,
9621
                       const ZRegister& zn,
9622
                       const ZRegister& zm) {
9623
  // USQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>
9624
  //  0100 0100 ..01 1101 100. .... .... ....
9625
  //  size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0>
9626

9627
  USE(zn);
9628
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9629
  VIXL_ASSERT(zd.Is(zn));
9630
  VIXL_ASSERT(AreSameLaneSize(zd, zn, zm));
9631

9632
  Emit(0x441d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm));
9633
}
9634

9635
void Assembler::usra(const ZRegister& zda, const ZRegister& zn, int shift) {
9636
  // USRA <Zda>.<T>, <Zn>.<T>, #<const>
9637
  //  0100 0101 ..0. .... 1110 01.. .... ....
9638
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> |
9639
  //  Zda<4:0>
9640

9641
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9642
  VIXL_ASSERT(AreSameLaneSize(zda, zn));
9643
  Instr encoded_imm =
9644
      EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits());
9645

9646
  SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e400);
9647
}
9648

9649
void Assembler::usublb(const ZRegister& zd,
9650
                       const ZRegister& zn,
9651
                       const ZRegister& zm) {
9652
  // USUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9653
  //  0100 0101 ..0. .... 0001 10.. .... ....
9654
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9655
  //  Zd<4:0>
9656

9657
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9658
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9659
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9660
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9661

9662
  Emit(0x45001800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9663
}
9664

9665
void Assembler::usublt(const ZRegister& zd,
9666
                       const ZRegister& zn,
9667
                       const ZRegister& zm) {
9668
  // USUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
9669
  //  0100 0101 ..0. .... 0001 11.. .... ....
9670
  //  size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> |
9671
  //  Zd<4:0>
9672

9673
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9674
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
9675
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2));
9676
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9677

9678
  Emit(0x45001c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9679
}
9680

9681
void Assembler::usubwb(const ZRegister& zd,
9682
                       const ZRegister& zn,
9683
                       const ZRegister& zm) {
9684
  // USUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
9685
  //  0100 0101 ..0. .... 0101 10.. .... ....
9686
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9687

9688
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9689
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
9690
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
9691
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9692

9693
  Emit(0x45005800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9694
}
9695

9696
void Assembler::usubwt(const ZRegister& zd,
9697
                       const ZRegister& zn,
9698
                       const ZRegister& zm) {
9699
  // USUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb>
9700
  //  0100 0101 ..0. .... 0101 11.. .... ....
9701
  //  size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>
9702

9703
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9704
  VIXL_ASSERT(AreSameLaneSize(zd, zn));
9705
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2));
9706
  VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes);
9707

9708
  Emit(0x45005c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
9709
}
9710

9711
void Assembler::whilege(const PRegisterWithLaneSize& pd,
9712
                        const Register& rn,
9713
                        const Register& rm) {
9714
  // WHILEGE <Pd>.<T>, <R><n>, <R><m>
9715
  //  0010 0101 ..1. .... 000. 00.. ...0 ....
9716
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> |
9717
  //  Pd<3:0>
9718

9719
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9720
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
9721
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
9722

9723
  Emit(0x25200000 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
9724
}
9725

9726
void Assembler::whilegt(const PRegisterWithLaneSize& pd,
9727
                        const Register& rn,
9728
                        const Register& rm) {
9729
  // WHILEGT <Pd>.<T>, <R><n>, <R><m>
9730
  //  0010 0101 ..1. .... 000. 00.. ...1 ....
9731
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> |
9732
  //  Pd<3:0>
9733

9734
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9735
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
9736
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
9737

9738
  Emit(0x25200010 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
9739
}
9740

9741
void Assembler::whilehi(const PRegisterWithLaneSize& pd,
9742
                        const Register& rn,
9743
                        const Register& rm) {
9744
  // WHILEHI <Pd>.<T>, <R><n>, <R><m>
9745
  //  0010 0101 ..1. .... 000. 10.. ...1 ....
9746
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> |
9747
  //  Pd<3:0>
9748

9749
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9750
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
9751
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
9752

9753
  Emit(0x25200810 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
9754
}
9755

9756
void Assembler::whilehs(const PRegisterWithLaneSize& pd,
9757
                        const Register& rn,
9758
                        const Register& rm) {
9759
  // WHILEHS <Pd>.<T>, <R><n>, <R><m>
9760
  //  0010 0101 ..1. .... 000. 10.. ...0 ....
9761
  //  size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> |
9762
  //  Pd<3:0>
9763

9764
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9765
  VIXL_ASSERT(AreSameSizeAndType(rn, rm));
9766
  const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000;
9767

9768
  Emit(0x25200800 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm));
9769
}
9770

9771
void Assembler::whilerw(const PRegisterWithLaneSize& pd,
9772
                        const Register& rn,
9773
                        const Register& rm) {
9774
  // WHILERW <Pd>.<T>, <Xn>, <Xm>
9775
  //  0010 0101 ..1. .... 0011 00.. ...1 ....
9776
  //  size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0>
9777

9778
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9779
  VIXL_ASSERT(rn.IsX() && rm.IsX());
9780

9781
  Emit(0x25203010 | SVESize(pd) | Pd(pd) | Rn(rn) | Rm(rm));
9782
}
9783

9784
void Assembler::whilewr(const PRegisterWithLaneSize& pd,
9785
                        const Register& rn,
9786
                        const Register& rm) {
9787
  // WHILEWR <Pd>.<T>, <Xn>, <Xm>
9788
  //  0010 0101 ..1. .... 0011 00.. ...0 ....
9789
  //  size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0>
9790

9791
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9792
  VIXL_ASSERT(rn.IsX() && rm.IsX());
9793

9794
  Emit(0x25203000 | SVESize(pd) | Pd(pd) | Rn(rn) | Rm(rm));
9795
}
9796

9797
void Assembler::xar(const ZRegister& zd,
9798
                    const ZRegister& zn,
9799
                    const ZRegister& zm,
9800
                    int shift) {
9801
  // XAR <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<const>
9802
  //  0000 0100 ..1. .... 0011 01.. .... ....
9803
  //  tszh<23:22> | tszl<20:19> | imm3<18:16> | Zm<9:5> | Zdn<4:0>
9804

9805
  USE(zn);
9806
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
9807
  VIXL_ASSERT(zd.Is(zn));
9808
  VIXL_ASSERT(AreSameLaneSize(zd, zm));
9809

9810
  Instr encoded_imm =
9811
      EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits());
9812
  SVEBitwiseShiftImmediate(zd, zm, encoded_imm, 0x04203400);
9813
}
9814

9815
void Assembler::fmmla(const ZRegister& zda,
9816
                      const ZRegister& zn,
9817
                      const ZRegister& zm) {
9818
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9819
  VIXL_ASSERT((CPUHas(CPUFeatures::kSVEF32MM) && zda.IsLaneSizeS()) ||
9820
              (CPUHas(CPUFeatures::kSVEF64MM) && zda.IsLaneSizeD()));
9821
  VIXL_ASSERT(AreSameLaneSize(zda, zn, zm));
9822

9823
  Emit(0x6420e400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm));
9824
}
9825

9826
void Assembler::smmla(const ZRegister& zda,
9827
                      const ZRegister& zn,
9828
                      const ZRegister& zm) {
9829
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9830
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9831
  VIXL_ASSERT(zda.IsLaneSizeS());
9832
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9833

9834
  Emit(0x45009800 | Rd(zda) | Rn(zn) | Rm(zm));
9835
}
9836

9837
void Assembler::usmmla(const ZRegister& zda,
9838
                       const ZRegister& zn,
9839
                       const ZRegister& zm) {
9840
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9841
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9842
  VIXL_ASSERT(zda.IsLaneSizeS());
9843
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9844

9845
  Emit(0x45809800 | Rd(zda) | Rn(zn) | Rm(zm));
9846
}
9847

9848
void Assembler::ummla(const ZRegister& zda,
9849
                      const ZRegister& zn,
9850
                      const ZRegister& zm) {
9851
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9852
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9853
  VIXL_ASSERT(zda.IsLaneSizeS());
9854
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9855

9856
  Emit(0x45c09800 | Rd(zda) | Rn(zn) | Rm(zm));
9857
}
9858

9859
void Assembler::usdot(const ZRegister& zda,
9860
                      const ZRegister& zn,
9861
                      const ZRegister& zm) {
9862
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9863
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9864
  VIXL_ASSERT(zda.IsLaneSizeS());
9865
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9866

9867
  Emit(0x44807800 | Rd(zda) | Rn(zn) | Rm(zm));
9868
}
9869

9870
void Assembler::usdot(const ZRegister& zda,
9871
                      const ZRegister& zn,
9872
                      const ZRegister& zm,
9873
                      int index) {
9874
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9875
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9876
  VIXL_ASSERT(zda.IsLaneSizeS());
9877
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9878
  VIXL_ASSERT(zm.GetCode() <= 7);
9879
  VIXL_ASSERT(IsUint2(index));
9880

9881
  Emit(0x44a01800 | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn));
9882
}
9883

9884
void Assembler::sudot(const ZRegister& zda,
9885
                      const ZRegister& zn,
9886
                      const ZRegister& zm,
9887
                      int index) {
9888
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
9889
  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM));
9890
  VIXL_ASSERT(zda.IsLaneSizeS());
9891
  VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB());
9892
  VIXL_ASSERT(zm.GetCode() <= 7);
9893
  VIXL_ASSERT(IsUint2(index));
9894

9895
  Emit(0x44a01c00 | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn));
9896
}
9897

9898
}  // namespace aarch64
9899
}  // namespace vixl
9900

9901
Product

Resources

Company
