CoCalc -- ARM64PrivateLinkage.cpp

GitHub Repository: PojavLauncherTeam/openj9
Path: blob/master/runtime/compiler/aarch64/codegen/ARM64PrivateLinkage.cpp
⁶⁰⁰⁴ views
1
/*******************************************************************************
2
 * Copyright (c) 2019, 2022 IBM Corp. and others
3
 *
4
 * This program and the accompanying materials are made available under
5
 * the terms of the Eclipse Public License 2.0 which accompanies this
6
 * distribution and is available at https://www.eclipse.org/legal/epl-2.0/
7
 * or the Apache License, Version 2.0 which accompanies this distribution and
8
 * is available at https://www.apache.org/licenses/LICENSE-2.0.
9
 *
10
 * This Source Code may also be made available under the following
11
 * Secondary Licenses when the conditions for such availability set
12
 * forth in the Eclipse Public License, v. 2.0 are satisfied: GNU
13
 * General Public License, version 2 with the GNU Classpath
14
 * Exception [1] and GNU General Public License, version 2 with the
15
 * OpenJDK Assembly Exception [2].
16
 *
17
 * [1] https://www.gnu.org/software/classpath/license.html
18
 * [2] http://openjdk.java.net/legal/assembly-exception.html
19
 *
20
 * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
21
 *******************************************************************************/
22

23
#include <algorithm>
24
#include <iterator>
25

26
#include "codegen/ARM64Instruction.hpp"
27
#include "codegen/ARM64OutOfLineCodeSection.hpp"
28
#include "codegen/ARM64PrivateLinkage.hpp"
29
#include "codegen/CallSnippet.hpp"
30
#include "codegen/CodeGenerator.hpp"
31
#include "codegen/CodeGeneratorUtils.hpp"
32
#include "codegen/ConstantDataSnippet.hpp"
33
#include "codegen/GCStackAtlas.hpp"
34
#include "codegen/GenerateInstructions.hpp"
35
#include "codegen/Linkage_inlines.hpp"
36
#include "codegen/Machine.hpp"
37
#include "codegen/MemoryReference.hpp"
38
#include "codegen/RealRegister.hpp"
39
#include "codegen/Register.hpp"
40
#include "codegen/StackCheckFailureSnippet.hpp"
41
#include "compile/Compilation.hpp"
42
#include "env/CompilerEnv.hpp"
43
#include "env/J2IThunk.hpp"
44
#include "env/PersistentCHTable.hpp"
45
#include "env/StackMemoryRegion.hpp"
46
#include "il/Node_inlines.hpp"
47
#include "il/ParameterSymbol.hpp"
48
#include "il/ResolvedMethodSymbol.hpp"
49
#include "il/SymbolReference.hpp"
50
#include "infra/Assert.hpp"
51
#include "infra/List.hpp"
52
#include "runtime/Runtime.hpp"
53

54
#define MIN_PROFILED_CALL_FREQUENCY (.075f)
55
#define MAX_PROFILED_CALL_FREQUENCY (.90f)
56

57
uint32_t J9::ARM64::PrivateLinkage::_globalRegisterNumberToRealRegisterMap[] =
58
   {
59
   // GPRs
60
   TR::RealRegister::x15,
61
   TR::RealRegister::x14,
62
   TR::RealRegister::x13,
63
   TR::RealRegister::x12,
64
   TR::RealRegister::x11,
65
   TR::RealRegister::x10,
66
   TR::RealRegister::x9,
67
   TR::RealRegister::x8, // indirect result location register
68
   TR::RealRegister::x18, // platform register
69
   // callee-saved registers
70
   TR::RealRegister::x28,
71
   TR::RealRegister::x27,
72
   TR::RealRegister::x26,
73
   TR::RealRegister::x25,
74
   TR::RealRegister::x24,
75
   TR::RealRegister::x23,
76
   TR::RealRegister::x22,
77
   TR::RealRegister::x21,
78
   // parameter registers
79
   TR::RealRegister::x7,
80
   TR::RealRegister::x6,
81
   TR::RealRegister::x5,
82
   TR::RealRegister::x4,
83
   TR::RealRegister::x3,
84
   TR::RealRegister::x2,
85
   TR::RealRegister::x1,
86
   TR::RealRegister::x0,
87

88
   // FPRs
89
   TR::RealRegister::v31,
90
   TR::RealRegister::v30,
91
   TR::RealRegister::v29,
92
   TR::RealRegister::v28,
93
   TR::RealRegister::v27,
94
   TR::RealRegister::v26,
95
   TR::RealRegister::v25,
96
   TR::RealRegister::v24,
97
   TR::RealRegister::v23,
98
   TR::RealRegister::v22,
99
   TR::RealRegister::v21,
100
   TR::RealRegister::v20,
101
   TR::RealRegister::v19,
102
   TR::RealRegister::v18,
103
   TR::RealRegister::v17,
104
   TR::RealRegister::v16,
105
   // callee-saved registers
106
   TR::RealRegister::v15,
107
   TR::RealRegister::v14,
108
   TR::RealRegister::v13,
109
   TR::RealRegister::v12,
110
   TR::RealRegister::v11,
111
   TR::RealRegister::v10,
112
   TR::RealRegister::v9,
113
   TR::RealRegister::v8,
114
   // parameter registers
115
   TR::RealRegister::v7,
116
   TR::RealRegister::v6,
117
   TR::RealRegister::v5,
118
   TR::RealRegister::v4,
119
   TR::RealRegister::v3,
120
   TR::RealRegister::v2,
121
   TR::RealRegister::v1,
122
   TR::RealRegister::v0
123
   };
124

125
J9::ARM64::PrivateLinkage::PrivateLinkage(TR::CodeGenerator *cg)
126
   : J9::PrivateLinkage(cg),
127
   _interpretedMethodEntryPoint(NULL),
128
   _jittedMethodEntryPoint(NULL)
129
   {
130
   int32_t i;
131

132
   _properties._properties = 0;
133

134
   _properties._registerFlags[TR::RealRegister::NoReg] = 0;
135
   _properties._registerFlags[TR::RealRegister::x0]    = IntegerArgument|IntegerReturn;
136
   _properties._registerFlags[TR::RealRegister::x1]    = IntegerArgument;
137
   _properties._registerFlags[TR::RealRegister::x2]    = IntegerArgument;
138
   _properties._registerFlags[TR::RealRegister::x3]    = IntegerArgument;
139
   _properties._registerFlags[TR::RealRegister::x4]    = IntegerArgument;
140
   _properties._registerFlags[TR::RealRegister::x5]    = IntegerArgument;
141
   _properties._registerFlags[TR::RealRegister::x6]    = IntegerArgument;
142
   _properties._registerFlags[TR::RealRegister::x7]    = IntegerArgument;
143

144
   for (i = TR::RealRegister::x8; i <= TR::RealRegister::x15; i++)
145
      _properties._registerFlags[i] = 0; // x8 - x15 volatile
146

147
   _properties._registerFlags[TR::RealRegister::x16]   = ARM64_Reserved; // IP0
148
   _properties._registerFlags[TR::RealRegister::x17]   = ARM64_Reserved; // IP1
149

150
   _properties._registerFlags[TR::RealRegister::x18]   = 0;
151

152
   _properties._registerFlags[TR::RealRegister::x19]   = Preserved|ARM64_Reserved; // vmThread
153
   _properties._registerFlags[TR::RealRegister::x20]   = Preserved|ARM64_Reserved; // Java SP
154

155
   for (i = TR::RealRegister::x21; i <= TR::RealRegister::x28; i++)
156
      _properties._registerFlags[i] = Preserved; // x21 - x28 Preserved
157

158
   _properties._registerFlags[TR::RealRegister::x29]   = ARM64_Reserved; // FP
159
   _properties._registerFlags[TR::RealRegister::lr]    = ARM64_Reserved; // LR
160
   _properties._registerFlags[TR::RealRegister::sp]    = ARM64_Reserved;
161
   _properties._registerFlags[TR::RealRegister::xzr]   = ARM64_Reserved;
162

163
   _properties._registerFlags[TR::RealRegister::v0]    = FloatArgument|FloatReturn;
164
   _properties._registerFlags[TR::RealRegister::v1]    = FloatArgument;
165
   _properties._registerFlags[TR::RealRegister::v2]    = FloatArgument;
166
   _properties._registerFlags[TR::RealRegister::v3]    = FloatArgument;
167
   _properties._registerFlags[TR::RealRegister::v4]    = FloatArgument;
168
   _properties._registerFlags[TR::RealRegister::v5]    = FloatArgument;
169
   _properties._registerFlags[TR::RealRegister::v6]    = FloatArgument;
170
   _properties._registerFlags[TR::RealRegister::v7]    = FloatArgument;
171

172
   for (i = TR::RealRegister::v8; i <= TR::RealRegister::LastFPR; i++)
173
      _properties._registerFlags[i] = 0; // v8 - v31 volatile
174

175
   _properties._numIntegerArgumentRegisters  = 8;
176
   _properties._firstIntegerArgumentRegister = 0;
177
   _properties._numFloatArgumentRegisters    = 8;
178
   _properties._firstFloatArgumentRegister   = 8;
179

180
   _properties._argumentRegisters[0]  = TR::RealRegister::x0;
181
   _properties._argumentRegisters[1]  = TR::RealRegister::x1;
182
   _properties._argumentRegisters[2]  = TR::RealRegister::x2;
183
   _properties._argumentRegisters[3]  = TR::RealRegister::x3;
184
   _properties._argumentRegisters[4]  = TR::RealRegister::x4;
185
   _properties._argumentRegisters[5]  = TR::RealRegister::x5;
186
   _properties._argumentRegisters[6]  = TR::RealRegister::x6;
187
   _properties._argumentRegisters[7]  = TR::RealRegister::x7;
188
   _properties._argumentRegisters[8]  = TR::RealRegister::v0;
189
   _properties._argumentRegisters[9]  = TR::RealRegister::v1;
190
   _properties._argumentRegisters[10] = TR::RealRegister::v2;
191
   _properties._argumentRegisters[11] = TR::RealRegister::v3;
192
   _properties._argumentRegisters[12] = TR::RealRegister::v4;
193
   _properties._argumentRegisters[13] = TR::RealRegister::v5;
194
   _properties._argumentRegisters[14] = TR::RealRegister::v6;
195
   _properties._argumentRegisters[15] = TR::RealRegister::v7;
196

197
   std::copy(std::begin(_globalRegisterNumberToRealRegisterMap), std::end(_globalRegisterNumberToRealRegisterMap), std::begin(_properties._allocationOrder));
198

199
   _properties._firstIntegerReturnRegister = 0;
200
   _properties._firstFloatReturnRegister   = 1;
201

202
   _properties._returnRegisters[0]  = TR::RealRegister::x0;
203
   _properties._returnRegisters[1]  = TR::RealRegister::v0;
204

205
   _properties._numAllocatableIntegerRegisters = 25;
206
   _properties._numAllocatableFloatRegisters   = 32;
207

208
   _properties._preservedRegisterMapForGC   = 0x1fe40000;
209
   _properties._methodMetaDataRegister      = TR::RealRegister::x19;
210
   _properties._stackPointerRegister        = TR::RealRegister::x20;
211
   _properties._framePointerRegister        = TR::RealRegister::x29;
212
   _properties._computedCallTargetRegister  = TR::RealRegister::x8;
213
   _properties._vtableIndexArgumentRegister = TR::RealRegister::x9;
214
   _properties._j9methodArgumentRegister    = TR::RealRegister::x0;
215

216
   // Volatile GPR (0-15, 18) + FPR (0-31) + VFT Reg
217
   _properties._numberOfDependencyGPRegisters = 17 + 32 + 1;
218
   setOffsetToFirstParm(0);
219
   _properties._offsetToFirstLocal            = -8;
220
   }
221

222
TR::ARM64LinkageProperties& J9::ARM64::PrivateLinkage::getProperties()
223
   {
224
   return _properties;
225
   }
226

227
uint32_t J9::ARM64::PrivateLinkage::getRightToLeft()
228
   {
229
   return getProperties().getRightToLeft();
230
   }
231

232
intptr_t
233
J9::ARM64::PrivateLinkage::entryPointFromCompiledMethod()
234
   {
235
   return reinterpret_cast<intptr_t>(getJittedMethodEntryPoint()->getBinaryEncoding());
236
   }
237

238
intptr_t
239
J9::ARM64::PrivateLinkage::entryPointFromInterpretedMethod()
240
   {
241
   return reinterpret_cast<intptr_t>(getInterpretedMethodEntryPoint()->getBinaryEncoding());
242
   }
243

244
void J9::ARM64::PrivateLinkage::alignLocalReferences(uint32_t &stackIndex)
245
   {
246
   TR::Compilation *comp = self()->comp();
247
   TR::GCStackAtlas *atlas = self()->cg()->getStackAtlas();
248
   const int32_t localObjectAlignment = TR::Compiler->om.getObjectAlignmentInBytes();
249
   const uint8_t pointerSize = TR::Compiler->om.sizeofReferenceAddress();
250

251
   if (comp->useCompressedPointers())
252
      {
253
      if (comp->getOption(TR_TraceCG))
254
         {
255
         traceMsg(comp,"\nLOCAL OBJECT ALIGNMENT: stack offset before alignment: %d,", stackIndex);
256
         }
257

258
      // stackIndex in mapCompactedStack is calculated using only local reference sizes and does not include the padding
259
      stackIndex -= pointerSize * atlas->getNumberOfPaddingSlots();
260

261
      if (comp->getOption(TR_TraceCG))
262
         {
263
         traceMsg(comp," with padding: %d,", stackIndex);
264
         }
265
      // If there are any local objects we have to make sure they are aligned properly
266
      // when compressed pointers are used.  Otherwise, pointer compression may clobber
267
      // part of the pointer.
268
      //
269
      // Each auto's GC index will have already been aligned, so just the starting stack
270
      // offset needs to be aligned.
271
      //
272
      uint32_t unalignedStackIndex = stackIndex;
273
      stackIndex &= ~(localObjectAlignment - 1);
274
      uint32_t paddingBytes = unalignedStackIndex - stackIndex;
275
      if (paddingBytes > 0)
276
         {
277
         TR_ASSERT_FATAL((paddingBytes & (pointerSize - 1)) == 0, "Padding bytes should be a multiple of the slot/pointer size");
278
         uint32_t paddingSlots = paddingBytes / pointerSize;
279
         atlas->setNumberOfSlotsMapped(atlas->getNumberOfSlotsMapped() + paddingSlots);
280
         }
281
      }
282
   }
283

284
void J9::ARM64::PrivateLinkage::mapStack(TR::ResolvedMethodSymbol *method)
285
   {
286
   if (self()->cg()->getLocalsIG() && self()->cg()->getSupportsCompactedLocals())
287
      {
288
      mapCompactedStack(method);
289
      return;
290
      }
291

292
   const TR::ARM64LinkageProperties& linkageProperties = getProperties();
293
   int32_t firstLocalOffset = linkageProperties.getOffsetToFirstLocal();
294
   uint32_t stackIndex = firstLocalOffset;
295
   int32_t lowGCOffset = stackIndex;
296

297
   TR::GCStackAtlas *atlas = cg()->getStackAtlas();
298

299
   // Map all garbage collected references together so can concisely represent
300
   // stack maps. They must be mapped so that the GC map index in each local
301
   // symbol is honoured.
302
   //
303
   uint32_t numberOfLocalSlotsMapped = atlas->getNumberOfSlotsMapped() - atlas->getNumberOfParmSlotsMapped();
304

305
   stackIndex -= numberOfLocalSlotsMapped * TR::Compiler->om.sizeofReferenceAddress();
306

307
   if (comp()->useCompressedPointers())
308
      {
309
      // If there are any local objects we have to make sure they are aligned properly
310
      // when compressed pointers are used.  Otherwise, pointer compression may clobber
311
      // part of the pointer.
312
      //
313
      // Each auto's GC index will have already been aligned, so just the starting stack
314
      // offset needs to be aligned.
315
      //
316
      uint32_t unalignedStackIndex = stackIndex;
317
      stackIndex &= ~(TR::Compiler->om.getObjectAlignmentInBytes() - 1);
318
      uint32_t paddingBytes = unalignedStackIndex - stackIndex;
319
      if (paddingBytes > 0)
320
         {
321
         TR_ASSERT((paddingBytes & (TR::Compiler->om.sizeofReferenceAddress() - 1)) == 0, "Padding bytes should be a multiple of the slot/pointer size");
322
         uint32_t paddingSlots = paddingBytes / TR::Compiler->om.sizeofReferenceAddress();
323
         atlas->setNumberOfSlotsMapped(atlas->getNumberOfSlotsMapped() + paddingSlots);
324
         }
325
      }
326

327
   ListIterator<TR::AutomaticSymbol> automaticIterator(&method->getAutomaticList());
328
   TR::AutomaticSymbol *localCursor;
329
   int32_t firstLocalGCIndex = atlas->getNumberOfParmSlotsMapped();
330

331
   // Map local references to set the stack position correct according to the GC map index
332
   //
333
   for (localCursor = automaticIterator.getFirst(); localCursor; localCursor = automaticIterator.getNext())
334
      {
335
      if (localCursor->getGCMapIndex() >= 0)
336
         {
337
         localCursor->setOffset(stackIndex + TR::Compiler->om.sizeofReferenceAddress() * (localCursor->getGCMapIndex() - firstLocalGCIndex));
338
         if (localCursor->getGCMapIndex() == atlas->getIndexOfFirstInternalPointer())
339
            {
340
            atlas->setOffsetOfFirstInternalPointer(localCursor->getOffset() - firstLocalOffset);
341
            }
342
         }
343
      }
344

345
   method->setObjectTempSlots((lowGCOffset - stackIndex) / TR::Compiler->om.sizeofReferenceAddress());
346
   lowGCOffset = stackIndex;
347

348
   // Now map the rest of the locals
349
   //
350
   automaticIterator.reset();
351
   localCursor = automaticIterator.getFirst();
352

353
   while (localCursor != NULL)
354
      {
355
      if (localCursor->getGCMapIndex() < 0 &&
356
          localCursor->getSize() != 8)
357
         {
358
         mapSingleAutomatic(localCursor, stackIndex);
359
         }
360

361
      localCursor = automaticIterator.getNext();
362
      }
363

364
   automaticIterator.reset();
365
   localCursor = automaticIterator.getFirst();
366

367
   while (localCursor != NULL)
368
      {
369
      if (localCursor->getGCMapIndex() < 0 &&
370
          localCursor->getSize() == 8)
371
         {
372
         stackIndex -= (stackIndex & 0x4)?4:0;
373
         mapSingleAutomatic(localCursor, stackIndex);
374
         }
375

376
      localCursor = automaticIterator.getNext();
377
      }
378

379
   method->setLocalMappingCursor(stackIndex);
380

381
   mapIncomingParms(method);
382

383
   atlas->setLocalBaseOffset(lowGCOffset - firstLocalOffset);
384
   atlas->setParmBaseOffset(atlas->getParmBaseOffset() + getOffsetToFirstParm() - firstLocalOffset);
385
   }
386

387
void J9::ARM64::PrivateLinkage::mapSingleAutomatic(TR::AutomaticSymbol *p, uint32_t &stackIndex)
388
   {
389
   mapSingleAutomatic(p, p->getRoundedSize(), stackIndex);
390
   }
391

392
void J9::ARM64::PrivateLinkage::mapSingleAutomatic(TR::AutomaticSymbol *p, uint32_t size, uint32_t &stackIndex)
393
   {
394
   /*
395
    * Align stack-allocated objects that don't have GC map index > 0.
396
    */
397
   if (comp()->useCompressedPointers() && p->isLocalObject() && (p->getGCMapIndex() == -1))
398
      {
399
      int32_t roundup = TR::Compiler->om.getObjectAlignmentInBytes() - 1;
400

401
      size = (size  + roundup) & (~roundup);
402
      }
403

404
   p->setOffset(stackIndex -= size);
405
   }
406

407
static void lockRegister(TR::RealRegister *regToAssign)
408
   {
409
   regToAssign->setState(TR::RealRegister::Locked);
410
   regToAssign->setAssignedRegister(regToAssign);
411
   }
412

413
void J9::ARM64::PrivateLinkage::initARM64RealRegisterLinkage()
414
   {
415
   TR::Machine *machine = cg()->machine();
416
   TR::RealRegister *reg;
417
   int icount;
418

419
   reg = machine->getRealRegister(TR::RealRegister::RegNum::x16); // IP0
420
   lockRegister(reg);
421

422
   reg = machine->getRealRegister(TR::RealRegister::RegNum::x17); // IP1
423
   lockRegister(reg);
424

425
   reg = machine->getRealRegister(TR::RealRegister::RegNum::x19); // vmThread
426
   lockRegister(reg);
427

428
   reg = machine->getRealRegister(TR::RealRegister::RegNum::x20); // Java SP
429
   lockRegister(reg);
430

431
   reg = machine->getRealRegister(TR::RealRegister::RegNum::x29); // FP
432
   lockRegister(reg);
433

434
   reg = machine->getRealRegister(TR::RealRegister::RegNum::lr); // LR
435
   lockRegister(reg);
436

437
   reg = machine->getRealRegister(TR::RealRegister::RegNum::sp); // SP
438
   lockRegister(reg);
439

440
   // assign "maximum" weight to registers x0-x15
441
   for (icount = TR::RealRegister::x0; icount <= TR::RealRegister::x15; icount++)
442
      machine->getRealRegister((TR::RealRegister::RegNum)icount)->setWeight(0xf000);
443

444
   // assign "maximum" weight to registers x21-x28
445
   for (icount = TR::RealRegister::x21; icount <= TR::RealRegister::x28; icount++)
446
      machine->getRealRegister((TR::RealRegister::RegNum)icount)->setWeight(0xf000);
447

448
   // assign "maximum" weight to registers v0-v31
449
   for (icount = TR::RealRegister::v0; icount <= TR::RealRegister::v31; icount++)
450
      machine->getRealRegister((TR::RealRegister::RegNum)icount)->setWeight(0xf000);
451
   }
452

453

454
void
455
J9::ARM64::PrivateLinkage::setParameterLinkageRegisterIndex(TR::ResolvedMethodSymbol *method)
456
   {
457
   ListIterator<TR::ParameterSymbol> paramIterator(&(method->getParameterList()));
458
   TR::ParameterSymbol *paramCursor = paramIterator.getFirst();
459
   int32_t numIntArgs = 0, numFloatArgs = 0;
460
   const TR::ARM64LinkageProperties& properties = getProperties();
461

462
   while ( (paramCursor!=NULL) &&
463
           ( (numIntArgs < properties.getNumIntArgRegs()) ||
464
             (numFloatArgs < properties.getNumFloatArgRegs()) ) )
465
      {
466
      int32_t index = -1;
467

468
      switch (paramCursor->getDataType())
469
         {
470
         case TR::Int8:
471
         case TR::Int16:
472
         case TR::Int32:
473
         case TR::Int64:
474
         case TR::Address:
475
            if (numIntArgs < properties.getNumIntArgRegs())
476
               {
477
               index = numIntArgs;
478
               }
479
            numIntArgs++;
480
            break;
481

482
         case TR::Float:
483
         case TR::Double:
484
            if (numFloatArgs < properties.getNumFloatArgRegs())
485
               {
486
               index = numFloatArgs;
487
               }
488
            numFloatArgs++;
489
            break;
490
         }
491

492
      paramCursor->setLinkageRegisterIndex(index);
493
      paramCursor = paramIterator.getNext();
494
      }
495
   }
496

497

498
int32_t
499
J9::ARM64::PrivateLinkage::calculatePreservedRegisterSaveSize(
500
      uint32_t &registerSaveDescription,
501
      uint32_t &numGPRsSaved)
502
   {
503
   TR::Machine *machine = cg()->machine();
504

505
   TR::RealRegister::RegNum firstPreservedGPR = TR::RealRegister::x21;
506
   TR::RealRegister::RegNum lastPreservedGPR = TR::RealRegister::x28;
507

508
   // Create a bit vector of preserved registers that have been modified
509
   // in this method.
510
   //
511
   for (int32_t i = firstPreservedGPR; i <= lastPreservedGPR; i++)
512
      {
513
      if (machine->getRealRegister((TR::RealRegister::RegNum)i)->getHasBeenAssignedInMethod())
514
         {
515
         registerSaveDescription |= 1 << (i-1);
516
         numGPRsSaved++;
517
         }
518
      }
519

520
   return numGPRsSaved*8;
521
   }
522

523
/**
524
 * @brief Generates instructions for initializing local variable and internal pointer slots in prologue
525
 *
526
 * @param[in] cursor                          : instruction cursor
527
 * @param[in] numSlotsToBeInitialized         : number of slots to be initialized
528
 * @param[in] offsetToFirstSlotFromAdjustedSP : offset to first slot from adjusted Java SP
529
 * @param[in] zeroReg                         : zero register (x31)
530
 * @param[in] baseReg                         : base register (x10)
531
 * @param[in] javaSP                          : Java SP register (x20)
532
 * @param[in] cg                              : Code Generator
533
 *
534
 * @return instruction cursor
535
 */
536
static TR::Instruction* initializeLocals(TR::Instruction *cursor, uint32_t numSlotsToBeInitialized, int32_t offsetToFirstSlotFromAdjustedSP,
537
                              TR::RealRegister *zeroReg, TR::RealRegister *baseReg, TR::RealRegister *javaSP, TR::CodeGenerator *cg)
538
   {
539
   auto loopCount = numSlotsToBeInitialized / 2;
540
   // stp instruction has 7bit immediate offset which is scaled by 8 for 64bit registers.
541
   // If the offset to the last 2 slots cleared by stp instruction does not fit in imm7,
542
   // we use x10 as base register.
543
   const bool isImm7OffsetOverflow = (loopCount > 0) &&
544
         !constantIsImm7((offsetToFirstSlotFromAdjustedSP + (loopCount - 1) * 2 * TR::Compiler->om.sizeofReferenceAddress()) >> 3);
545

546
   auto offset = offsetToFirstSlotFromAdjustedSP;
547
   if (isImm7OffsetOverflow)
548
      {
549
      if (!constantIsImm7(offset >> 3))
550
         {
551
         // If offset does not fit in imm7, update baseReg and reset offset to 0
552
         if (constantIsUnsignedImm12(offset))
553
            {
554
            cursor = generateTrg1Src1ImmInstruction(cg, TR::InstOpCode::addimmx, NULL, baseReg, javaSP, offset, cursor);
555
            }
556
         else
557
            {
558
            cursor = loadConstant32(cg, NULL, offset, baseReg, cursor);
559
            cursor = generateTrg1Src2Instruction(cg, TR::InstOpCode::addx, NULL, baseReg, javaSP, baseReg, cursor);
560
            }
561
         offset = 0;
562
         }
563
      else
564
         {
565
         // mov baseReg, javaSP
566
         cursor = generateTrg1Src2Instruction(cg, TR::InstOpCode::orrx, NULL, baseReg, zeroReg, javaSP, cursor);
567
         }
568

569

570
      for (int32_t i = 0; i < loopCount; i++)
571
         {
572
         if (!constantIsImm7(offset >> 3))
573
            {
574
            // If offset does not fit in imm7, update baseReg and reset offset to 0
575
            cursor = generateTrg1Src1ImmInstruction(cg, TR::InstOpCode::addimmx, NULL, baseReg, baseReg, offset, cursor);
576
            offset = 0;
577
            }
578
         TR::MemoryReference *localMR = TR::MemoryReference::createWithDisplacement(cg, baseReg, offset);
579
         cursor = generateMemSrc2Instruction(cg, TR::InstOpCode::stpoffx, NULL, localMR, zeroReg, zeroReg, cursor);
580
         offset += (TR::Compiler->om.sizeofReferenceAddress() * 2);
581
         }
582
      if (numSlotsToBeInitialized % 2)
583
         {
584
         // clear residue
585
         TR::MemoryReference *localMR = TR::MemoryReference::createWithDisplacement(cg, baseReg, offset);
586
         cursor = generateMemSrc1Instruction(cg, TR::InstOpCode::strimmx, NULL, localMR, zeroReg, cursor);
587
         }
588
      }
589
   else
590
      {
591
      for (int32_t i = 0; i < loopCount; i++, offset += (TR::Compiler->om.sizeofReferenceAddress() * 2))
592
         {
593
         TR::MemoryReference *localMR = TR::MemoryReference::createWithDisplacement(cg, javaSP, offset);
594
         cursor = generateMemSrc2Instruction(cg, TR::InstOpCode::stpoffx, NULL, localMR, zeroReg, zeroReg, cursor);
595
         }
596
      if (numSlotsToBeInitialized % 2)
597
         {
598
         // clear residue
599
         TR::MemoryReference *localMR = TR::MemoryReference::createWithDisplacement(cg, javaSP, offset);
600
         cursor = generateMemSrc1Instruction(cg, TR::InstOpCode::strimmx, NULL, localMR, zeroReg, cursor);
601
         }
602
      }
603

604
   return cursor;
605
   }
606

607
void J9::ARM64::PrivateLinkage::createPrologue(TR::Instruction *cursor)
608
   {
609

610
   // Prologues are emitted post-RA so it is fine to use real registers directly
611
   // in instructions
612
   //
613
   TR::ARM64LinkageProperties& properties = getProperties();
614
   TR::Machine *machine = cg()->machine();
615
   TR::RealRegister *vmThread = machine->getRealRegister(properties.getMethodMetaDataRegister());   // x19
616
   TR::RealRegister *javaSP = machine->getRealRegister(properties.getStackPointerRegister());       // x20
617

618
   TR::Instruction *beforeInterpreterMethodEntryPointInstruction = cursor;
619

620
   // --------------------------------------------------------------------------
621
   // Create the entry point when transitioning from an interpreted method.
622
   // Parameters are passed on the stack, so load them into the appropriate
623
   // linkage registers expected by the JITed method entry point.
624
   //
625
   cursor = loadStackParametersToLinkageRegisters(cursor);
626

627
   TR::Instruction *beforeJittedMethodEntryPointInstruction = cursor;
628

629
   // Entry breakpoint
630
   //
631
   if (comp()->getOption(TR_EntryBreakPoints))
632
      {
633
      cursor = generateExceptionInstruction(cg(), TR::InstOpCode::brkarm64, NULL, 0, cursor);
634
      }
635

636
   // --------------------------------------------------------------------------
637
   // Determine the bitvector of registers to preserve in the prologue
638
   //
639
   uint32_t registerSaveDescription = 0;
640
   uint32_t numGPRsSaved = 0;
641

642
   uint32_t preservedRegisterSaveSize = calculatePreservedRegisterSaveSize(registerSaveDescription, numGPRsSaved);
643

644
   // Offset between the entry JavaSP of a method and the first mapped local.  This covers
645
   // the space needed to preserve the RA.  It is a negative (or zero) offset.
646
   //
647
   int32_t firstLocalOffset = properties.getOffsetToFirstLocal();
648

649
   // The localMappingCursor is a negative-offset mapping of locals (autos and spills) to
650
   // the stack relative to the entry JavaSP of a method.  It includes the offset to the
651
   // first mapped local.
652
   //
653
   TR::ResolvedMethodSymbol *bodySymbol = comp()->getJittedMethodSymbol();
654
   int32_t localsSize = -(int32_t)(bodySymbol->getLocalMappingCursor());
655

656
   // Size of the frame needed to handle the argument storage requirements of any method
657
   // call in the current method.
658
   //
659
   // The offset to the first parm is the offset between the entry JavaSP and the first
660
   // mapped parameter.  It is a positive (or zero) offset.
661
   //
662
   int32_t outgoingArgsSize = cg()->getLargestOutgoingArgSize() + getOffsetToFirstParm();
663

664
   int32_t frameSizeIncludingReturnAddress = preservedRegisterSaveSize + localsSize + outgoingArgsSize;
665

666
   // Align the frame to 16 bytes
667
   //
668
   int32_t alignedFrameSizeIncludingReturnAddress = (frameSizeIncludingReturnAddress + 15) & ~15;
669

670
   // The frame size maintained by the code generator does not include the RA
671
   //
672
   cg()->setFrameSizeInBytes(alignedFrameSizeIncludingReturnAddress + firstLocalOffset);
673

674
   // --------------------------------------------------------------------------
675
   // Encode register save description (RSD)
676
   //
677
   int32_t preservedRegisterOffsetFromJavaBP = (alignedFrameSizeIncludingReturnAddress - outgoingArgsSize + firstLocalOffset);
678

679
   TR_ASSERT_FATAL(preservedRegisterOffsetFromJavaBP >= 0, "expecting a positive preserved register area offset");
680

681
   // Frame size is too large for the RSD word in the metadata
682
   //
683
   if (preservedRegisterOffsetFromJavaBP > 0xffff)
684
      {
685
      comp()->failCompilation<TR::CompilationInterrupted>("Overflowed or underflowed bounds of regSaveOffset in calculateFrameSize.");
686
      }
687

688
   registerSaveDescription |= (preservedRegisterOffsetFromJavaBP & 0xffff);
689

690
   cg()->setRegisterSaveDescription(registerSaveDescription);
691

692
   // In FSD, we must save linkage regs to the incoming argument area because
693
   // the stack overflow check doesn't preserve them.
694
   bool parmsHaveBeenStored = false;
695
   if (comp()->getOption(TR_FullSpeedDebug))
696
      {
697
      cursor = saveParametersToStack(cursor);
698
      parmsHaveBeenStored = true;
699
      }
700

701
   // --------------------------------------------------------------------------
702
   // Store return address (RA)
703
   //
704
   TR::MemoryReference *returnAddressMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, firstLocalOffset);
705
   cursor = generateMemSrc1Instruction(cg(), TR::InstOpCode::sturx, NULL, returnAddressMR, machine->getRealRegister(TR::RealRegister::lr), cursor);
706

707
   // --------------------------------------------------------------------------
708
   // Speculatively adjust Java SP with the needed frame size.
709
   // This includes the preserved RA slot.
710
   //
711
   if (constantIsUnsignedImm12(alignedFrameSizeIncludingReturnAddress))
712
      {
713
      cursor = generateTrg1Src1ImmInstruction(cg(), TR::InstOpCode::subimmx, NULL, javaSP, javaSP, alignedFrameSizeIncludingReturnAddress, cursor);
714
      }
715
   else
716
      {
717
      TR::RealRegister *x9Reg = machine->getRealRegister(TR::RealRegister::RegNum::x9);
718

719
      if (constantIsUnsignedImm16(alignedFrameSizeIncludingReturnAddress))
720
         {
721
         // x9 will contain the aligned frame size
722
         //
723
         cursor = loadConstant32(cg(), NULL, alignedFrameSizeIncludingReturnAddress, x9Reg, cursor);
724
         cursor = generateTrg1Src2Instruction(cg(), TR::InstOpCode::subx, NULL, javaSP, javaSP, x9Reg, cursor);
725
         }
726
      else
727
         {
728
         TR_ASSERT_FATAL(0, "Large frame size not supported in prologue yet");
729
         }
730
      }
731

732
   // --------------------------------------------------------------------------
733
   // Perform javaSP overflow check
734
   //
735
   if (!comp()->isDLT())
736
      {
737
      //    if (javaSP < vmThread->SOM)
738
      //       goto stackOverflowSnippetLabel
739
      //
740
      // stackOverflowRestartLabel:
741
      //
742
      TR::MemoryReference *somMR = TR::MemoryReference::createWithDisplacement(cg(), vmThread, cg()->getStackLimitOffset());
743
      TR::RealRegister *somReg = machine->getRealRegister(TR::RealRegister::RegNum::x10);
744
      cursor = generateTrg1MemInstruction(cg(), TR::InstOpCode::ldrimmx, NULL, somReg, somMR, cursor);
745

746
      TR::RealRegister *zeroReg = machine->getRealRegister(TR::RealRegister::xzr);
747
      cursor = generateTrg1Src2Instruction(cg(), TR::InstOpCode::subsx, NULL, zeroReg, javaSP, somReg, cursor);
748

749
      TR::LabelSymbol *stackOverflowSnippetLabel = generateLabelSymbol(cg());
750
      cursor = generateConditionalBranchInstruction(cg(), TR::InstOpCode::b_cond, NULL, stackOverflowSnippetLabel, TR::CC_LS, cursor);
751

752
      TR::LabelSymbol *stackOverflowRestartLabel = generateLabelSymbol(cg());
753
      cursor = generateLabelInstruction(cg(), TR::InstOpCode::label, NULL, stackOverflowRestartLabel, cursor);
754

755
      cg()->addSnippet(new (cg()->trHeapMemory()) TR::ARM64StackCheckFailureSnippet(cg(), NULL, stackOverflowRestartLabel, stackOverflowSnippetLabel));
756
      }
757
   else
758
      {
759
      // If StackCheckFailureSnippet is not added to the end of the snippet list and no data snippets exist,
760
      // we might have a HelperCallSnippet at the end of the method.
761
      // HelperCallSnippets add a GCMap to the instruction next to the `bl` instruction to the helper,
762
      // and if a HelperCallSnippet is at the end of the method, GCMap is added to the address beyond the range of the method.
763
      // To avoid that, we add a dummy ConstantDataSnippet. (Data snippets are emitted after normal snippets.)
764
      if (!cg()->hasDataSnippets())
765
         {
766
         auto snippet = cg()->findOrCreate4ByteConstant(NULL, 0);
767
         snippet->setReloType(TR_NoRelocation);
768
         }
769
      }
770

771
   // --------------------------------------------------------------------------
772
   // Preserve GPRs
773
   //
774
   // javaSP has been adjusted, so preservedRegs start at offset outgoingArgSize
775
   // relative to the javaSP
776
   //
777
   // Registers are preserved in order from x21 (low memory) -> x28 (high memory)
778
   //
779
   if (numGPRsSaved)
780
      {
781
      TR::RealRegister::RegNum firstPreservedGPR = TR::RealRegister::x21;
782
      TR::RealRegister::RegNum lastPreservedGPR = TR::RealRegister::x28;
783

784
      int32_t preservedRegisterOffsetFromJavaSP = outgoingArgsSize;
785

786
      for (TR::RealRegister::RegNum regIndex = firstPreservedGPR; regIndex <= lastPreservedGPR; regIndex=(TR::RealRegister::RegNum)((uint32_t)regIndex+1))
787
         {
788
         TR::RealRegister *preservedRealReg = machine->getRealRegister(regIndex);
789
         if (preservedRealReg->getHasBeenAssignedInMethod())
790
            {
791
            TR::MemoryReference *preservedRegMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, preservedRegisterOffsetFromJavaSP);
792
            cursor = generateMemSrc1Instruction(cg(), TR::InstOpCode::strimmx, NULL, preservedRegMR, preservedRealReg, cursor);
793
            preservedRegisterOffsetFromJavaSP += 8;
794
            numGPRsSaved--;
795
            }
796
         }
797

798
      TR_ASSERT_FATAL(numGPRsSaved == 0, "preserved register mismatch in prologue");
799
      }
800

801
   // --------------------------------------------------------------------------
802
   // Initialize locals
803
   //
804
   TR::GCStackAtlas *atlas = cg()->getStackAtlas();
805
   if (atlas)
806
      {
807
      // The GC stack maps are conservative in that they all say that
808
      // collectable locals are live. This means that these locals must be
809
      // cleared out in case a GC happens before they are allocated a valid
810
      // value.
811
      // The atlas contains the number of locals that need to be cleared. They
812
      // are all mapped together starting at GC index 0.
813
      //
814
      uint32_t numLocalsToBeInitialized = atlas->getNumberOfSlotsToBeInitialized();
815
      if (numLocalsToBeInitialized > 0 || atlas->getInternalPointerMap())
816
         {
817
         // The LocalBaseOffset and firstLocalOffset are either negative or zero values
818
         //
819
         int32_t initializedLocalsOffsetFromAdjustedJavaSP = alignedFrameSizeIncludingReturnAddress + atlas->getLocalBaseOffset() + firstLocalOffset;
820

821
         TR::RealRegister *zeroReg = machine->getRealRegister(TR::RealRegister::RegNum::xzr);
822
         TR::RealRegister *baseReg = machine->getRealRegister(TR::RealRegister::RegNum::x10);
823

824
         cursor = initializeLocals(cursor, numLocalsToBeInitialized, initializedLocalsOffsetFromAdjustedJavaSP, 
825
                              zeroReg, baseReg, javaSP, cg());
826

827
         if (atlas->getInternalPointerMap())
828
            {
829
            // Total number of slots to be initialized is number of pinning arrays +
830
            // number of derived internal pointer stack slots
831
            //
832
            int32_t numSlotsToBeInitialized = atlas->getNumberOfDistinctPinningArrays() + atlas->getInternalPointerMap()->getNumInternalPointers();
833
            int32_t offsetToFirstInternalPointerFromAdjustedJavaSP = alignedFrameSizeIncludingReturnAddress + atlas->getOffsetOfFirstInternalPointer() + firstLocalOffset;
834

835
            cursor = initializeLocals(cursor, numSlotsToBeInitialized, offsetToFirstInternalPointerFromAdjustedJavaSP, 
836
                              zeroReg, baseReg, javaSP, cg());
837
            }
838
         }
839
      }
840

841
   // Adjust final offsets on locals and parm symbols now that the frame size is known.
842
   // These offsets are relative to the javaSP which has been adjusted downward to
843
   // accommodate the frame of this method.
844
   //
845
   ListIterator<TR::AutomaticSymbol> automaticIterator(&bodySymbol->getAutomaticList());
846
   TR::AutomaticSymbol *localCursor = automaticIterator.getFirst();
847

848
   while (localCursor != NULL)
849
      {
850
      localCursor->setOffset(localCursor->getOffset() + alignedFrameSizeIncludingReturnAddress);
851
      localCursor = automaticIterator.getNext();
852
      }
853

854
   ListIterator<TR::ParameterSymbol> parameterIterator(&bodySymbol->getParameterList());
855
   TR::ParameterSymbol *parmCursor = parameterIterator.getFirst();
856
   while (parmCursor != NULL)
857
      {
858
      parmCursor->setParameterOffset(parmCursor->getParameterOffset() + alignedFrameSizeIncludingReturnAddress);
859
      parmCursor = parameterIterator.getNext();
860
      }
861

862
   // Ensure arguments reside where the method body expects them to be (either in registers or
863
   // on the stack).  This state is influenced by global register assignment.
864
   //
865
   cursor = copyParametersToHomeLocation(cursor, parmsHaveBeenStored);
866

867
   // Set the instructions for method entry points
868
   setInterpretedMethodEntryPoint(beforeInterpreterMethodEntryPointInstruction->getNext());
869
   setJittedMethodEntryPoint(beforeJittedMethodEntryPointInstruction->getNext());
870
   }
871

872
void J9::ARM64::PrivateLinkage::createEpilogue(TR::Instruction *cursor)
873
   {
874
   const TR::ARM64LinkageProperties& properties = getProperties();
875
   TR::Machine *machine = cg()->machine();
876
   TR::Node *lastNode = cursor->getNode();
877
   TR::ResolvedMethodSymbol *bodySymbol = comp()->getJittedMethodSymbol();
878
   TR::RealRegister *javaSP = machine->getRealRegister(properties.getStackPointerRegister()); // x20
879

880
   // restore preserved GPRs
881
   int32_t preservedRegisterOffsetFromJavaSP = cg()->getLargestOutgoingArgSize() + getOffsetToFirstParm(); // outgoingArgsSize
882
   TR::RealRegister::RegNum firstPreservedGPR = TR::RealRegister::x21;
883
   TR::RealRegister::RegNum lastPreservedGPR = TR::RealRegister::x28;
884
   for (TR::RealRegister::RegNum r = firstPreservedGPR; r <= lastPreservedGPR; r = (TR::RealRegister::RegNum)((uint32_t)r+1))
885
      {
886
      TR::RealRegister *rr = machine->getRealRegister(r);
887
      if (rr->getHasBeenAssignedInMethod())
888
         {
889
         TR::MemoryReference *preservedRegMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, preservedRegisterOffsetFromJavaSP);
890
         cursor = generateTrg1MemInstruction(cg(), TR::InstOpCode::ldrimmx, lastNode, rr, preservedRegMR, cursor);
891
         preservedRegisterOffsetFromJavaSP += 8;
892
         }
893
      }
894

895
   // remove space for preserved registers
896
   int32_t firstLocalOffset = properties.getOffsetToFirstLocal();
897

898
   uint32_t alignedFrameSizeIncludingReturnAddress = cg()->getFrameSizeInBytes() - firstLocalOffset;
899
   if (constantIsUnsignedImm12(alignedFrameSizeIncludingReturnAddress))
900
      {
901
      cursor = generateTrg1Src1ImmInstruction(cg(), TR::InstOpCode::addimmx, lastNode, javaSP, javaSP, alignedFrameSizeIncludingReturnAddress, cursor);
902
      }
903
   else
904
      {
905
      TR::RealRegister *x9Reg = machine->getRealRegister(TR::RealRegister::RegNum::x9);
906
      cursor = loadConstant32(cg(), lastNode, alignedFrameSizeIncludingReturnAddress, x9Reg, cursor);
907
      cursor = generateTrg1Src2Instruction(cg(), TR::InstOpCode::addx, lastNode, javaSP, javaSP, x9Reg, cursor);
908
      }
909

910
   // restore return address
911
   TR::RealRegister *lr = machine->getRealRegister(TR::RealRegister::lr);
912
   if (machine->getLinkRegisterKilled())
913
      {
914
      TR::MemoryReference *returnAddressMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, firstLocalOffset);
915
      cursor = generateTrg1MemInstruction(cg(), TR::InstOpCode::ldurx, lastNode, lr, returnAddressMR, cursor);
916
      }
917

918
   // return
919
   generateRegBranchInstruction(cg(), TR::InstOpCode::ret, lastNode, lr, cursor);
920
   }
921

922
void J9::ARM64::PrivateLinkage::pushOutgoingMemArgument(TR::Register *argReg, int32_t offset, TR::InstOpCode::Mnemonic opCode, TR::ARM64MemoryArgument &memArg)
923
   {
924
   const TR::ARM64LinkageProperties& properties = self()->getProperties();
925
   TR::RealRegister *javaSP = cg()->machine()->getRealRegister(properties.getStackPointerRegister()); // x20
926

927
   TR::MemoryReference *result = TR::MemoryReference::createWithDisplacement(cg(), javaSP, offset);
928
   memArg.argRegister = argReg;
929
   memArg.argMemory = result;
930
   memArg.opCode = opCode;
931
   }
932

933
int32_t J9::ARM64::PrivateLinkage::buildArgs(TR::Node *callNode,
934
   TR::RegisterDependencyConditions *dependencies)
935
   {
936
   return buildPrivateLinkageArgs(callNode, dependencies, TR_Private);
937
   }
938

939
int32_t J9::ARM64::PrivateLinkage::buildPrivateLinkageArgs(TR::Node *callNode,
940
   TR::RegisterDependencyConditions *dependencies,
941
   TR_LinkageConventions linkage)
942
   {
943
   TR_ASSERT(linkage == TR_Private || linkage == TR_Helper || linkage == TR_CHelper, "Unexpected linkage convention");
944

945
   const TR::ARM64LinkageProperties& properties = getProperties();
946
   TR::ARM64MemoryArgument *pushToMemory = NULL;
947
   TR::Register *tempReg;
948
   int32_t argIndex = 0;
949
   int32_t numMemArgs = 0;
950
   int32_t memArgSize = 0;
951
   int32_t firstExplicitArg = 0;
952
   int32_t from, to, step;
953
   int32_t argSize = -getOffsetToFirstParm();
954
   int32_t totalSize = 0;
955
   int32_t multiplier;
956

957
   uint32_t numIntegerArgs = 0;
958
   uint32_t numFloatArgs = 0;
959

960
   TR::Node *child;
961
   TR::DataType childType;
962
   TR::DataType resType = callNode->getType();
963

964
   uint32_t firstArgumentChild = callNode->getFirstArgumentIndex();
965

966
   TR::MethodSymbol *callSymbol = callNode->getSymbol()->castToMethodSymbol();
967

968
   bool isHelperCall = linkage == TR_Helper || linkage == TR_CHelper;
969
   bool rightToLeft = isHelperCall &&
970
                      //we want the arguments for induceOSR to be passed from left to right as in any other non-helper call
971
                      !callNode->getSymbolReference()->isOSRInductionHelper();
972

973
   if (rightToLeft)
974
      {
975
      from = callNode->getNumChildren() - 1;
976
      to   = firstArgumentChild;
977
      step = -1;
978
      }
979
   else
980
      {
981
      from = firstArgumentChild;
982
      to   = callNode->getNumChildren() - 1;
983
      step = 1;
984
      }
985

986
   uint32_t numIntArgRegs = properties.getNumIntArgRegs();
987
   uint32_t numFloatArgRegs = properties.getNumFloatArgRegs();
988

989
   TR::RealRegister::RegNum specialArgReg = TR::RealRegister::NoReg;
990
   switch (callSymbol->getMandatoryRecognizedMethod())
991
      {
992
      // Node: special long args are still only passed in one GPR
993
      case TR::java_lang_invoke_ComputedCalls_dispatchJ9Method:
994
         specialArgReg = getProperties().getJ9MethodArgumentRegister();
995
         // Other args go in memory
996
         numIntArgRegs   = 0;
997
         numFloatArgRegs = 0;
998
         break;
999
      case TR::java_lang_invoke_ComputedCalls_dispatchVirtual:
1000
      case TR::com_ibm_jit_JITHelpers_dispatchVirtual:
1001
         specialArgReg = getProperties().getVTableIndexArgumentRegister();
1002
         break;
1003
      }
1004
   if (specialArgReg != TR::RealRegister::NoReg)
1005
      {
1006
      if (comp()->getOption(TR_TraceCG))
1007
         {
1008
         traceMsg(comp(), "Special arg %s in %s\n",
1009
            comp()->getDebug()->getName(callNode->getChild(from)),
1010
            comp()->getDebug()->getName(cg()->machine()->getRealRegister(specialArgReg)));
1011
         }
1012
      // Skip the special arg in the first loop
1013
      from += step;
1014
      }
1015

1016
   // C helpers have an implicit first argument (the VM thread) that we have to account for
1017
   if (linkage == TR_CHelper)
1018
      {
1019
      TR_ASSERT(numIntArgRegs > 0, "This code doesn't handle passing this implicit arg on the stack");
1020
      numIntegerArgs++;
1021
      totalSize += TR::Compiler->om.sizeofReferenceAddress();
1022
      }
1023

1024
   for (int32_t i = from; (rightToLeft && i >= to) || (!rightToLeft && i <= to); i += step)
1025
      {
1026
      child = callNode->getChild(i);
1027
      childType = child->getDataType();
1028

1029
      switch (childType)
1030
         {
1031
         case TR::Int8:
1032
         case TR::Int16:
1033
         case TR::Int32:
1034
         case TR::Int64:
1035
         case TR::Address:
1036
            multiplier = (childType == TR::Int64) ? 2 : 1;
1037
            if (numIntegerArgs >= numIntArgRegs)
1038
               {
1039
               numMemArgs++;
1040
               memArgSize += TR::Compiler->om.sizeofReferenceAddress() * multiplier;
1041
               }
1042
            numIntegerArgs++;
1043
            totalSize += TR::Compiler->om.sizeofReferenceAddress() * multiplier;
1044
            break;
1045
         case TR::Float:
1046
         case TR::Double:
1047
            multiplier = (childType == TR::Double) ? 2 : 1;
1048
            if (numFloatArgs >= numFloatArgRegs)
1049
               {
1050
               numMemArgs++;
1051
               memArgSize += TR::Compiler->om.sizeofReferenceAddress() * multiplier;
1052
               }
1053
            numFloatArgs++;
1054
            totalSize += TR::Compiler->om.sizeofReferenceAddress() * multiplier;
1055
            break;
1056
         default:
1057
            TR_ASSERT(false, "Argument type %s is not supported\n", childType.toString());
1058
         }
1059
      }
1060

1061
   // From here, down, any new stack allocations will expire / die when the function returns
1062
   TR::StackMemoryRegion stackMemoryRegion(*trMemory());
1063

1064
   if (numMemArgs > 0)
1065
      {
1066
      pushToMemory = new (trStackMemory()) TR::ARM64MemoryArgument[numMemArgs];
1067
      }
1068

1069
   if (specialArgReg)
1070
      from -= step;  // we do want to process special args in the following loop
1071

1072
   numIntegerArgs = 0;
1073
   numFloatArgs = 0;
1074

1075
   // C helpers have an implicit first argument (the VM thread) that we have to account for
1076
   if (linkage == TR_CHelper)
1077
      {
1078
      TR_ASSERT(numIntArgRegs > 0, "This code doesn't handle passing this implicit arg on the stack");
1079
      TR::Register *vmThreadArgRegister = cg()->allocateRegister();
1080
      generateMovInstruction(cg(), callNode, vmThreadArgRegister, cg()->getMethodMetaDataRegister());
1081
      dependencies->addPreCondition(vmThreadArgRegister, properties.getIntegerArgumentRegister(numIntegerArgs));
1082
      if (resType.getDataType() == TR::NoType)
1083
         dependencies->addPostCondition(vmThreadArgRegister, properties.getIntegerArgumentRegister(numIntegerArgs));
1084
      numIntegerArgs++;
1085
      firstExplicitArg = 1;
1086
      }
1087

1088
   // Helper linkage preserves all argument registers except the return register
1089
   // TODO: C helper linkage does not, this code needs to make sure argument registers are killed in post dependencies
1090
   for (int32_t i = from; (rightToLeft && i >= to) || (!rightToLeft && i <= to); i += step)
1091
      {
1092
      TR::Register *argRegister;
1093
      TR::InstOpCode::Mnemonic op;
1094
      bool isSpecialArg = (i == from && specialArgReg != TR::RealRegister::NoReg);
1095

1096
      child = callNode->getChild(i);
1097
      childType = child->getDataType();
1098

1099
      switch (childType)
1100
         {
1101
         case TR::Int8:
1102
         case TR::Int16:
1103
         case TR::Int32:
1104
         case TR::Int64:
1105
         case TR::Address:
1106
            if (childType == TR::Address)
1107
               {
1108
               argRegister = pushAddressArg(child);
1109
               }
1110
            else if (childType == TR::Int64)
1111
               {
1112
               argRegister = pushLongArg(child);
1113
               }
1114
            else
1115
               {
1116
               argRegister = pushIntegerWordArg(child);
1117
               }
1118
            if (isSpecialArg)
1119
               {
1120
               if (specialArgReg == properties.getIntegerReturnRegister(0))
1121
                  {
1122
                  TR::Register *resultReg;
1123
                  if (resType.isAddress())
1124
                     resultReg = cg()->allocateCollectedReferenceRegister();
1125
                  else
1126
                     resultReg = cg()->allocateRegister();
1127
                  dependencies->addPreCondition(argRegister, specialArgReg);
1128
                  dependencies->addPostCondition(resultReg, properties.getIntegerReturnRegister(0));
1129
                  }
1130
               else
1131
                  {
1132
                  TR::addDependency(dependencies, argRegister, specialArgReg, TR_GPR, cg());
1133
                  }
1134
               }
1135
            else
1136
               {
1137
               argSize += TR::Compiler->om.sizeofReferenceAddress() * ((childType == TR::Int64) ? 2 : 1);
1138
               if (numIntegerArgs < numIntArgRegs)
1139
                  {
1140
                  if (!cg()->canClobberNodesRegister(child, 0))
1141
                     {
1142
                     if (argRegister->containsCollectedReference())
1143
                        tempReg = cg()->allocateCollectedReferenceRegister();
1144
                     else
1145
                        tempReg = cg()->allocateRegister();
1146
                     generateMovInstruction(cg(), callNode, tempReg, argRegister);
1147
                     argRegister = tempReg;
1148
                     }
1149
                  if (numIntegerArgs == firstExplicitArg)
1150
                     {
1151
                     // the first integer argument
1152
                     TR::Register *resultReg;
1153
                     if (resType.isAddress())
1154
                        resultReg = cg()->allocateCollectedReferenceRegister();
1155
                     else
1156
                        resultReg = cg()->allocateRegister();
1157
                     dependencies->addPreCondition(argRegister, properties.getIntegerArgumentRegister(numIntegerArgs));
1158
                     dependencies->addPostCondition(resultReg, TR::RealRegister::x0);
1159
                     if (firstExplicitArg == 1)
1160
                        dependencies->addPostCondition(argRegister, properties.getIntegerArgumentRegister(numIntegerArgs));
1161
                     }
1162
                  else
1163
                     {
1164
                     TR::addDependency(dependencies, argRegister, properties.getIntegerArgumentRegister(numIntegerArgs), TR_GPR, cg());
1165
                     }
1166
                  }
1167
               else // numIntegerArgs >= numIntArgRegs
1168
                  {
1169
                  op = ((childType == TR::Address) || (childType == TR::Int64)) ? TR::InstOpCode::strimmx : TR::InstOpCode::strimmw;
1170
                  pushOutgoingMemArgument(argRegister, totalSize - argSize, op, pushToMemory[argIndex++]);
1171
                  }
1172
               numIntegerArgs++;
1173
               }
1174
            break;
1175
         case TR::Float:
1176
         case TR::Double:
1177
            if (childType == TR::Float)
1178
               {
1179
               argSize += TR::Compiler->om.sizeofReferenceAddress();
1180
               argRegister = pushFloatArg(child);
1181
               }
1182
            else
1183
               {
1184
               argSize += TR::Compiler->om.sizeofReferenceAddress() * 2;
1185
               argRegister = pushDoubleArg(child);
1186
               }
1187
            if (numFloatArgs < numFloatArgRegs)
1188
               {
1189
               if (!cg()->canClobberNodesRegister(child, 0))
1190
                  {
1191
                  tempReg = cg()->allocateRegister(TR_FPR);
1192
                  op = (childType == TR::Float) ? TR::InstOpCode::fmovs : TR::InstOpCode::fmovd;
1193
                  generateTrg1Src1Instruction(cg(), op, callNode, tempReg, argRegister);
1194
                  argRegister = tempReg;
1195
                  }
1196
               if (numFloatArgs == 0 && resType.isFloatingPoint())
1197
                  {
1198
                  TR::Register *resultReg;
1199
                  if (resType.getDataType() == TR::Float)
1200
                     resultReg = cg()->allocateSinglePrecisionRegister();
1201
                  else
1202
                     resultReg = cg()->allocateRegister(TR_FPR);
1203
                  dependencies->addPreCondition(argRegister, TR::RealRegister::v0);
1204
                  dependencies->addPostCondition(resultReg, TR::RealRegister::v0);
1205
                  }
1206
               else
1207
                  TR::addDependency(dependencies, argRegister, properties.getFloatArgumentRegister(numFloatArgs), TR_FPR, cg());
1208
               }
1209
            else // numFloatArgs >= numFloatArgRegs
1210
               {
1211
               op = (childType == TR::Float) ? TR::InstOpCode::vstrimms : TR::InstOpCode::vstrimmd;
1212
               pushOutgoingMemArgument(argRegister, totalSize - argSize, op, pushToMemory[argIndex++]);
1213
               }
1214
            numFloatArgs++;
1215
            break;
1216
         }
1217
      }
1218

1219
   for (int32_t i = TR::RealRegister::FirstGPR; i <= TR::RealRegister::LastGPR; ++i)
1220
      {
1221
      TR::RealRegister::RegNum realReg = (TR::RealRegister::RegNum)i;
1222
      if (properties.getPreserved(realReg) || (properties.getRegisterFlags(realReg) & ARM64_Reserved))
1223
         continue;
1224
      if (realReg == specialArgReg)
1225
         continue; // already added deps above.  No need to add them here.
1226
      if (callSymbol->isComputed() && i == getProperties().getComputedCallTargetRegister())
1227
         continue;
1228
      if (!dependencies->searchPreConditionRegister(realReg))
1229
         {
1230
         if (realReg == properties.getIntegerArgumentRegister(0) && callNode->getDataType() == TR::Address)
1231
            {
1232
            dependencies->addPreCondition(cg()->allocateRegister(), TR::RealRegister::x0);
1233
            dependencies->addPostCondition(cg()->allocateCollectedReferenceRegister(), TR::RealRegister::x0);
1234
            }
1235
         else
1236
            {
1237
            // Helper linkage preserves all registers that are not argument registers, so we don't need to spill them.
1238
            if (linkage != TR_Helper)
1239
               TR::addDependency(dependencies, NULL, realReg, TR_GPR, cg());
1240
            }
1241
         }
1242
      }
1243

1244
   if (callNode->getType().isFloatingPoint() && numFloatArgs == 0)
1245
      {
1246
      //add return floating-point register dependency
1247
      TR::addDependency(dependencies, NULL, (TR::RealRegister::RegNum)getProperties().getFloatReturnRegister(), TR_FPR, cg());
1248
      }
1249

1250
   // Helper linkage preserves all registers that are not argument registers, so we don't need to spill them.
1251
   if (linkage != TR_Helper)
1252
      {
1253
      for (int32_t i = TR::RealRegister::FirstFPR; i <= TR::RealRegister::LastFPR; ++i)
1254
         {
1255
         TR::RealRegister::RegNum realReg = (TR::RealRegister::RegNum)i;
1256
         if (properties.getPreserved(realReg))
1257
            continue;
1258
         if (!dependencies->searchPreConditionRegister(realReg))
1259
            {
1260
            TR::addDependency(dependencies, NULL, realReg, TR_FPR, cg());
1261
            }
1262
         }
1263
      }
1264

1265
   /* Spills all vector registers */
1266
   if ((linkage != TR_Helper) && killsVectorRegisters())
1267
      {
1268
      TR::Register *tmpReg = cg()->allocateRegister();
1269
      dependencies->addPostCondition(tmpReg, TR::RealRegister::KillVectorRegs);
1270
      cg()->stopUsingRegister(tmpReg);
1271
      }
1272

1273
   if (numMemArgs > 0)
1274
      {
1275
      for (argIndex = 0; argIndex < numMemArgs; argIndex++)
1276
         {
1277
         TR::Register *aReg = pushToMemory[argIndex].argRegister;
1278
         generateMemSrc1Instruction(cg(), pushToMemory[argIndex].opCode, callNode, pushToMemory[argIndex].argMemory, aReg);
1279
         cg()->stopUsingRegister(aReg);
1280
         }
1281
      }
1282

1283
   return totalSize;
1284
   }
1285

1286
void J9::ARM64::PrivateLinkage::buildDirectCall(TR::Node *callNode,
1287
   TR::SymbolReference *callSymRef,
1288
   TR::RegisterDependencyConditions *dependencies,
1289
   const TR::ARM64LinkageProperties &pp,
1290
   uint32_t argSize)
1291
   {
1292
   TR::Instruction *gcPoint;
1293
   TR::MethodSymbol *callSymbol = callSymRef->getSymbol()->castToMethodSymbol();
1294

1295
   TR_J9VMBase *fej9 = (TR_J9VMBase *)(comp()->fe());
1296

1297
   if (callSymRef->getReferenceNumber() >= TR_ARM64numRuntimeHelpers)
1298
      fej9->reserveTrampolineIfNecessary(comp(), callSymRef, false);
1299

1300
   bool forceUnresolvedDispatch = !fej9->isResolvedDirectDispatchGuaranteed(comp());
1301

1302
   if (callSymbol->isJITInternalNative() ||
1303
       (!callSymRef->isUnresolved() && !callSymbol->isInterpreted() &&
1304
        ((forceUnresolvedDispatch && callSymbol->isHelper()) || !forceUnresolvedDispatch)))
1305
      {
1306
      bool isMyself = comp()->isRecursiveMethodTarget(callSymbol);
1307

1308
      gcPoint = generateImmSymInstruction(cg(), TR::InstOpCode::bl, callNode,
1309
         isMyself ? 0 : (uintptr_t)callSymbol->getMethodAddress(),
1310
         dependencies,
1311
         callSymRef ? callSymRef : callNode->getSymbolReference(),
1312
         NULL);
1313
      }
1314
   else
1315
      {
1316
      TR::LabelSymbol *label = generateLabelSymbol(cg());
1317
      TR::Snippet *snippet;
1318

1319
      if (callSymRef->isUnresolved() || comp()->compileRelocatableCode())
1320
         {
1321
         snippet = new (trHeapMemory()) TR::ARM64UnresolvedCallSnippet(cg(), callNode, label, argSize);
1322
         }
1323
      else
1324
         {
1325
         snippet = new (trHeapMemory()) TR::ARM64CallSnippet(cg(), callNode, label, argSize);
1326
         snippet->gcMap().setGCRegisterMask(pp.getPreservedRegisterMapForGC());
1327
         }
1328

1329
      cg()->addSnippet(snippet);
1330
      gcPoint = generateImmSymInstruction(cg(), TR::InstOpCode::bl, callNode,
1331
         0, dependencies,
1332
         new (trHeapMemory()) TR::SymbolReference(comp()->getSymRefTab(), label),
1333
         snippet);
1334

1335
      // Nop is necessary due to confusion when resolving shared slots at a transition
1336
      if (callSymRef->isOSRInductionHelper())
1337
         cg()->generateNop(callNode);
1338

1339
      }
1340

1341
   gcPoint->ARM64NeedsGCMap(cg(), callSymbol->getLinkageConvention() == TR_Helper ? 0xffffffff : pp.getPreservedRegisterMapForGC());
1342
   }
1343

1344
TR::Register *J9::ARM64::PrivateLinkage::buildDirectDispatch(TR::Node *callNode)
1345
   {
1346
   TR::SymbolReference *callSymRef = callNode->getSymbolReference();
1347
   const TR::ARM64LinkageProperties &pp = getProperties();
1348
   // Extra post dependency for killing vector registers (see KillVectorRegs)
1349
   const int extraPostReg = killsVectorRegisters() ? 1 : 0;
1350
   TR::RegisterDependencyConditions *dependencies =
1351
      new (trHeapMemory()) TR::RegisterDependencyConditions(
1352
         pp.getNumberOfDependencyGPRegisters(),
1353
         pp.getNumberOfDependencyGPRegisters() + extraPostReg, trMemory());
1354

1355
   int32_t argSize = buildArgs(callNode, dependencies);
1356

1357
   buildDirectCall(callNode, callSymRef, dependencies, pp, argSize);
1358
   cg()->machine()->setLinkRegisterKilled(true);
1359

1360
   TR::Register *retReg;
1361
   switch(callNode->getOpCodeValue())
1362
      {
1363
      case TR::icall:
1364
         retReg = dependencies->searchPostConditionRegister(
1365
                     pp.getIntegerReturnRegister());
1366
         break;
1367
      case TR::lcall:
1368
      case TR::acall:
1369
         retReg = dependencies->searchPostConditionRegister(
1370
                     pp.getLongReturnRegister());
1371
         break;
1372
      case TR::fcall:
1373
      case TR::dcall:
1374
         retReg = dependencies->searchPostConditionRegister(
1375
                     pp.getFloatReturnRegister());
1376
         break;
1377
      case TR::call:
1378
         retReg = NULL;
1379
         break;
1380
      default:
1381
         retReg = NULL;
1382
         TR_ASSERT_FATAL(false, "Unsupported direct call Opcode.");
1383
      }
1384

1385
   callNode->setRegister(retReg);
1386

1387
   dependencies->stopUsingDepRegs(cg(), retReg);
1388
   return retReg;
1389
   }
1390

1391
/**
1392
 * @brief Gets profiled call site information
1393
 *
1394
 * @param[in] cg:           code generator
1395
 * @param[in] callNode:     node for call
1396
 * @param[in] maxStaticPIC: maximum number of static PICs
1397
 * @param[out] values:      list of PIC items
1398
 * @returns true if any call site information is returned
1399
 */
1400
static bool getProfiledCallSiteInfo(TR::CodeGenerator *cg, TR::Node *callNode, uint32_t maxStaticPICs, TR_ScratchList<J9::ARM64PICItem> &values)
1401
   {
1402
   TR::Compilation *comp = cg->comp();
1403
   TR_J9VMBase *fej9 = (TR_J9VMBase *)(cg->fe());
1404
   if (comp->compileRelocatableCode())
1405
      return false;
1406

1407
   TR::SymbolReference *methodSymRef = callNode->getSymbolReference();
1408
   TR::MethodSymbol    *methodSymbol = methodSymRef->getSymbol()->castToMethodSymbol();
1409

1410
   if (!methodSymbol->isVirtual() && !methodSymbol->isInterface())
1411
      return false;
1412

1413
   TR_AddressInfo *info = static_cast<TR_AddressInfo*>(TR_ValueProfileInfoManager::getProfiledValueInfo(callNode, comp, AddressInfo));
1414
   if (!info)
1415
      {
1416
      if (comp->getOption(TR_TraceCG))
1417
         {
1418
         traceMsg(comp, "Profiled target not found for node %p\n", callNode);
1419
         }
1420
      return false;
1421
      }
1422
   static const bool tracePIC = feGetEnv("TR_TracePIC") != NULL;
1423
   if (tracePIC)
1424
      {
1425
      traceMsg(comp, "Value profile info for callNode %p in %s\n", callNode, comp->signature());
1426
      info->getProfiler()->dumpInfo(comp->getOutFile());
1427
      traceMsg(comp, "\n");
1428
      }
1429
   uint32_t totalFreq = info->getTotalFrequency();
1430
   if (totalFreq == 0 || info->getTopProbability() < MIN_PROFILED_CALL_FREQUENCY)
1431
      {
1432
      if (comp->getOption(TR_TraceCG))
1433
         {
1434
         traceMsg(comp, "Profiled target with enough frequency not found for node %p\n", callNode);
1435
         }
1436
      return false;
1437
      }
1438

1439
   TR_ScratchList<TR_ExtraAddressInfo> allValues(comp->trMemory());
1440
   info->getSortedList(comp, &allValues);
1441

1442
   TR_ResolvedMethod   *owningMethod = methodSymRef->getOwningMethod(comp);
1443
   TR_OpaqueClassBlock *callSiteMethodClass;
1444

1445
   if (methodSymbol->isVirtual())
1446
       callSiteMethodClass = methodSymRef->getSymbol()->getResolvedMethodSymbol()->getResolvedMethod()->classOfMethod();
1447

1448
   ListIterator<TR_ExtraAddressInfo> valuesIt(&allValues);
1449

1450
   uint32_t             numStaticPics = 0;
1451
   TR_ExtraAddressInfo *profiledInfo;
1452
   for (profiledInfo = valuesIt.getFirst();  numStaticPics < maxStaticPICs && profiledInfo != NULL; profiledInfo = valuesIt.getNext())
1453
      {
1454
      float freq = (float)profiledInfo->_frequency / totalFreq;
1455
      if (freq < MIN_PROFILED_CALL_FREQUENCY)
1456
         break;
1457

1458
      TR_OpaqueClassBlock *clazz = (TR_OpaqueClassBlock *)profiledInfo->_value;
1459
      if (comp->getPersistentInfo()->isObsoleteClass(clazz, fej9))
1460
         continue;
1461

1462
      TR_ResolvedMethod *method;
1463

1464
      if (methodSymbol->isVirtual())
1465
         {
1466
         TR_ASSERT_FATAL(callSiteMethodClass, "Expecting valid callSiteMethodClass for virtual call");
1467
         if (!cg->isProfiledClassAndCallSiteCompatible(clazz, callSiteMethodClass))
1468
            continue;
1469

1470
         method = owningMethod->getResolvedVirtualMethod(comp, clazz, methodSymRef->getOffset());
1471
         }
1472
      else
1473
         {
1474
         method = owningMethod->getResolvedInterfaceMethod(comp, clazz, methodSymRef->getCPIndex());
1475
         }
1476

1477
      if (!method || method->isInterpreted())
1478
         continue;
1479

1480
      values.add(new (comp->trStackMemory()) J9::ARM64PICItem(clazz, method, freq));
1481
      ++numStaticPics;
1482
      }
1483

1484
   return numStaticPics > 0;
1485
   }
1486

1487
/**
1488
 * @brief Generates instruction sequence for static PIC call
1489
 *
1490
 * @param[in] cg:             code generator
1491
 * @param[in] callNode:       node for call
1492
 * @param[in] profiledClass:  class suggested by interpreter profiler
1493
 * @param[in] profiledMethod: method suggested by interpreter profiler
1494
 * @param[in] vftReg:         register containing VFT
1495
 * @param[in] tempReg:        temporary register
1496
 * @param[in] missLabel:      label for cache miss
1497
 * @param[in] regMapForGC:    register map for GC
1498
 * @returns instruction making direct call to the method
1499
 */
1500
static TR::Instruction* buildStaticPICCall(TR::CodeGenerator *cg, TR::Node *callNode, TR_OpaqueClassBlock *profiledClass, TR_ResolvedMethod *profiledMethod,
1501
                                             TR::Register *vftReg, TR::Register *tempReg, TR::LabelSymbol *missLabel, uint32_t regMapForGC)
1502
   {
1503
   TR::Compilation *comp = cg->comp();
1504
   TR::SymbolReference *methodSymRef = callNode->getSymbolReference();
1505
   TR::SymbolReference *profiledMethodSymRef = comp->getSymRefTab()->findOrCreateMethodSymbol(methodSymRef->getOwningMethodIndex(),
1506
                                                                                             -1,
1507
                                                                                             profiledMethod,
1508
                                                                                             TR::MethodSymbol::Virtual);
1509

1510
   TR_J9VMBase *fej9 = (TR_J9VMBase *)(cg->fe());
1511
   if (comp->compileRelocatableCode())
1512
      {
1513
      loadAddressConstantInSnippet(cg, callNode, reinterpret_cast<intptr_t>(profiledClass), tempReg, TR_ClassPointer);
1514
      }
1515
   else
1516
      {
1517
      bool isUnloadAssumptionRequired = fej9->isUnloadAssumptionRequired(profiledClass, comp->getCurrentMethod());
1518

1519
      if (isUnloadAssumptionRequired)
1520
         {
1521
         loadAddressConstantInSnippet(cg, callNode, reinterpret_cast<intptr_t>(profiledClass), tempReg, TR_NoRelocation, true); 
1522
         }
1523
      else
1524
         {
1525
         loadAddressConstant(cg, callNode, reinterpret_cast<intptr_t>(profiledClass), tempReg, NULL, true);
1526
         }
1527
      }
1528
   generateCompareInstruction(cg, callNode, vftReg, tempReg, true);
1529

1530
   generateConditionalBranchInstruction(cg, TR::InstOpCode::b_cond, callNode, missLabel, TR::CC_NE);
1531

1532
   TR::Instruction *gcPoint = generateImmSymInstruction(cg, TR::InstOpCode::bl, callNode, (uintptr_t)profiledMethod->startAddressForJittedMethod(),
1533
                                                                                  NULL, profiledMethodSymRef, NULL);
1534
   gcPoint->ARM64NeedsGCMap(cg, regMapForGC);
1535
   fej9->reserveTrampolineIfNecessary(comp, profiledMethodSymRef, false);
1536
   return gcPoint;
1537
   }
1538

1539
/**
1540
 * @brief Generates instruction sequence for virtual call
1541
 *
1542
 * @param[in] cg:          code generator
1543
 * @param[in] callNode:    node for the virtual call
1544
 * @param[in] vftReg:      register containing VFT
1545
 * @param[in] x9:          x9 register
1546
 * @param[in] regMapForGC: register map for GC
1547
 */
1548
static void buildVirtualCall(TR::CodeGenerator *cg, TR::Node *callNode, TR::Register *vftReg, TR::Register *x9, uint32_t regMapForGC)
1549
   {
1550
   int32_t offset = callNode->getSymbolReference()->getOffset();
1551
   TR_ASSERT_FATAL(offset < 0, "Unexpected positive offset for virtual call");
1552

1553
   // jitVTableIndex() in oti/JITInterface.hpp assumes the instruction sequence below
1554
   if (offset >= -65536)
1555
      {
1556
      generateTrg1ImmInstruction(cg, TR::InstOpCode::movnx, callNode, x9, ~offset & 0xFFFF);
1557
      }
1558
   else
1559
      {
1560
      generateTrg1ImmInstruction(cg, TR::InstOpCode::movzx, callNode, x9, offset & 0xFFFF);
1561
      generateTrg1ImmInstruction(cg, TR::InstOpCode::movkx, callNode, x9,
1562
                                 (((offset >> 16) & 0xFFFF) | TR::MOV_LSL16));
1563
      generateTrg1Src1ImmInstruction(cg, TR::InstOpCode::sbfmx, callNode, x9, x9, 0x1F); // sxtw x9, w9
1564
      }
1565
   TR::MemoryReference *tempMR = TR::MemoryReference::createWithIndexReg(cg, vftReg, x9);
1566
   generateTrg1MemInstruction(cg, TR::InstOpCode::ldroffx, callNode, x9, tempMR);
1567
   TR::Instruction *gcPoint = generateRegBranchInstruction(cg, TR::InstOpCode::blr, callNode, x9);
1568
   gcPoint->ARM64NeedsGCMap(cg, regMapForGC);
1569
   }
1570

1571
/**
1572
 * @brief Generates instruction sequence for interface call
1573
 *
1574
 * @param[in] cg:          code generator
1575
 * @param[in] callNode:    node for the interface call
1576
 * @param[in] vftReg:      vft register
1577
 * @param[in] tmpReg:      temporary register
1578
 * @param[in] ifcSnippet:  interface call snippet
1579
 * @param[in] regMapForGC: register map for GC
1580
 */
1581
static void buildInterfaceCall(TR::CodeGenerator *cg, TR::Node *callNode, TR::Register *vftReg, TR::Register *tmpReg, TR::ARM64InterfaceCallSnippet *ifcSnippet, uint32_t regMapForGC)
1582
   {
1583
   /*
1584
    *  Generating following instruction sequence.
1585
    *
1586
    *  ldrx tmpReg, L_firstClassCacheSlot
1587
    *  cmpx vftReg, tmpReg
1588
    *  ldrx tmpReg, L_firstBranchAddressCacheSlot
1589
    *  beq  hitLabel
1590
    *  ldrx tmpReg, L_secondClassCacheSlot
1591
    *  cmpx vftReg, tmpReg
1592
    *  bne  snippetLabel
1593
    *  ldrx tmpReg, L_secondBranchAddressCacheSlot
1594
    * hitLabel:
1595
    *  blr  tmpReg
1596
    * doneLabel:
1597
    */
1598

1599
   TR::LabelSymbol *ifcSnippetLabel = ifcSnippet->getSnippetLabel();
1600
   TR::LabelSymbol *firstClassCacheSlotLabel = ifcSnippet->getFirstClassCacheSlotLabel();
1601
   generateTrg1ImmSymInstruction(cg, TR::InstOpCode::ldrx, callNode, tmpReg, 0, firstClassCacheSlotLabel);
1602

1603
   TR::LabelSymbol *hitLabel = generateLabelSymbol(cg);
1604
   generateCompareInstruction(cg, callNode, vftReg, tmpReg, true);
1605
   TR::LabelSymbol *firstBranchAddressCacheSlotLabel = ifcSnippet->getFirstBranchAddressCacheSlotLabel();
1606

1607
   generateTrg1ImmSymInstruction(cg, TR::InstOpCode::ldrx, callNode, tmpReg, 0, firstBranchAddressCacheSlotLabel);
1608
   generateConditionalBranchInstruction(cg, TR::InstOpCode::b_cond, callNode, hitLabel, TR::CC_EQ);
1609

1610
   TR::LabelSymbol *secondClassCacheSlotLabel = ifcSnippet->getSecondClassCacheSlotLabel();
1611

1612
   generateTrg1ImmSymInstruction(cg, TR::InstOpCode::ldrx, callNode, tmpReg, 0, secondClassCacheSlotLabel);
1613
   generateCompareInstruction(cg, callNode, vftReg, tmpReg, true);
1614
   TR::Instruction *gcPoint = generateConditionalBranchInstruction(cg, TR::InstOpCode::b_cond, callNode, ifcSnippetLabel, TR::CC_NE);
1615
   gcPoint->ARM64NeedsGCMap(cg, regMapForGC);
1616
   TR::LabelSymbol *secondBranchAddressCacheSlotLabel = ifcSnippet->getSecondBranchAddressCacheSlotLabel();
1617

1618
   generateTrg1ImmSymInstruction(cg, TR::InstOpCode::ldrx, callNode, tmpReg, 0, secondBranchAddressCacheSlotLabel);
1619
   generateLabelInstruction(cg, TR::InstOpCode::label, callNode, hitLabel);
1620
   gcPoint = generateRegBranchInstruction(cg, TR::InstOpCode::blr, callNode, tmpReg);
1621
   gcPoint->ARM64NeedsGCMap(cg, regMapForGC);
1622
   }
1623

1624
static TR::Register *evaluateUpToVftChild(TR::Node *callNode, TR::CodeGenerator *cg)
1625
   {
1626
   TR::Register *vftReg = NULL;
1627
   if (callNode->getFirstArgumentIndex() == 1)
1628
      {
1629
      TR::Node *child = callNode->getFirstChild();
1630
      vftReg = cg->evaluate(child);
1631
      cg->decReferenceCount(child);
1632
      }
1633
   TR_ASSERT_FATAL(vftReg != NULL, "Failed to find vft child.");
1634
   return vftReg;
1635
   }
1636

1637
void J9::ARM64::PrivateLinkage::buildVirtualDispatch(TR::Node *callNode,
1638
   TR::RegisterDependencyConditions *dependencies,
1639
   uint32_t argSize)
1640
   {
1641
   TR::Register *x0 = dependencies->searchPreConditionRegister(TR::RealRegister::x0);
1642
   TR::Register *x9 = dependencies->searchPreConditionRegister(TR::RealRegister::x9);
1643

1644
   TR::SymbolReference *methodSymRef = callNode->getSymbolReference();
1645
   TR::MethodSymbol *methodSymbol = methodSymRef->getSymbol()->castToMethodSymbol();
1646
   TR::LabelSymbol *doneLabel = generateLabelSymbol(cg());
1647
   uint32_t regMapForGC = getProperties().getPreservedRegisterMapForGC();
1648
   void *thunk = NULL;
1649

1650
   TR::Instruction *gcPoint;
1651

1652
   TR_J9VMBase *fej9 = (TR_J9VMBase *)(comp()->fe());
1653

1654
   // Computed calls
1655
   //
1656
   if (methodSymbol->isComputed())
1657
      {
1658
      TR::Register *vftReg = evaluateUpToVftChild(callNode, cg());
1659
      TR::addDependency(dependencies, vftReg, getProperties().getComputedCallTargetRegister(), TR_GPR, cg());
1660

1661
      switch (methodSymbol->getMandatoryRecognizedMethod())
1662
         {
1663
         case TR::java_lang_invoke_ComputedCalls_dispatchVirtual:
1664
         case TR::com_ibm_jit_JITHelpers_dispatchVirtual:
1665
            {
1666
            // Need a j2i thunk for the method that will ultimately be dispatched by this handle call
1667
            char *j2iSignature = fej9->getJ2IThunkSignatureForDispatchVirtual(methodSymbol->getMethod()->signatureChars(), methodSymbol->getMethod()->signatureLength(), comp());
1668
            int32_t signatureLen = strlen(j2iSignature);
1669
            thunk = fej9->getJ2IThunk(j2iSignature, signatureLen, comp());
1670
            if (!thunk)
1671
               {
1672
               thunk = fej9->setJ2IThunk(j2iSignature, signatureLen,
1673
                                         TR::ARM64CallSnippet::generateVIThunk(fej9->getEquivalentVirtualCallNodeForDispatchVirtual(callNode, comp()), argSize, cg()), comp());
1674
               }
1675
            }
1676
         default:
1677
            if (fej9->needsInvokeExactJ2IThunk(callNode, comp()))
1678
               {
1679
               comp()->getPersistentInfo()->getInvokeExactJ2IThunkTable()->addThunk(
1680
                  TR::ARM64CallSnippet::generateInvokeExactJ2IThunk(callNode, argSize, cg(), methodSymbol->getMethod()->signatureChars()), fej9);
1681
               }
1682
            break;
1683
         }
1684

1685
      TR::Instruction *gcPoint = generateRegBranchInstruction(cg(), TR::InstOpCode::blr, callNode, vftReg, dependencies);
1686
      gcPoint->ARM64NeedsGCMap(cg(), regMapForGC);
1687

1688
      return;
1689
      }
1690

1691
   // Virtual and interface calls
1692
   //
1693
   TR_ASSERT_FATAL(methodSymbol->isVirtual() || methodSymbol->isInterface(), "Unexpected method type");
1694

1695
   thunk = fej9->getJ2IThunk(methodSymbol->getMethod(), comp());
1696
   if (!thunk)
1697
      thunk = fej9->setJ2IThunk(methodSymbol->getMethod(), TR::ARM64CallSnippet::generateVIThunk(callNode, argSize, cg()), comp());
1698

1699
   bool callIsSafe = methodSymRef != comp()->getSymRefTab()->findObjectNewInstanceImplSymbol();
1700

1701
   // evaluate vftReg because it is required for implicit NULLCHK
1702
   TR::Register *vftReg = evaluateUpToVftChild(callNode, cg());
1703
   TR::addDependency(dependencies, vftReg, TR::RealRegister::NoReg, TR_GPR, cg());
1704

1705
   if (methodSymbol->isVirtual())
1706
      {
1707
      TR::MemoryReference *tempMR;
1708
      if (methodSymRef->isUnresolved() || comp()->compileRelocatableCode())
1709
         {
1710
         TR::LabelSymbol *vcSnippetLabel = generateLabelSymbol(cg());
1711
         TR::ARM64VirtualUnresolvedSnippet *vcSnippet =
1712
            new (trHeapMemory())
1713
            TR::ARM64VirtualUnresolvedSnippet(cg(), callNode, vcSnippetLabel, argSize, doneLabel, (uint8_t *)thunk);
1714
         cg()->addSnippet(vcSnippet);
1715

1716

1717
         // The following instructions are modified by _virtualUnresolvedHelper
1718
         // in aarch64/runtime/PicBuilder.spp to load the vTable index in x9
1719

1720
         // This `b` instruction is modified to movzx x9, lower 16bit of offset
1721
         generateLabelInstruction(cg(), TR::InstOpCode::b, callNode, vcSnippetLabel);
1722
         generateTrg1ImmInstruction(cg(), TR::InstOpCode::movkx, callNode, x9, TR::MOV_LSL16);
1723
         generateTrg1Src1ImmInstruction(cg(), TR::InstOpCode::sbfmx, callNode, x9, x9, 0x1F); // sxtw x9, w9
1724
         tempMR = TR::MemoryReference::createWithIndexReg(cg(), vftReg, x9);
1725
         generateTrg1MemInstruction(cg(), TR::InstOpCode::ldroffx, callNode, x9, tempMR);
1726
         gcPoint = generateRegBranchInstruction(cg(), TR::InstOpCode::blr, callNode, x9);
1727
         gcPoint->ARM64NeedsGCMap(cg(), regMapForGC);
1728
         generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, doneLabel, dependencies);
1729

1730
         return;
1731
         }
1732

1733
      // Handle guarded devirtualization next
1734
      //
1735
      if (callIsSafe)
1736
         {
1737
         TR::ResolvedMethodSymbol *resolvedMethodSymbol = methodSymRef->getSymbol()->getResolvedMethodSymbol();
1738
         TR_ResolvedMethod       *resolvedMethod = resolvedMethodSymbol->getResolvedMethod();
1739

1740
         if (comp()->performVirtualGuardNOPing() &&
1741
             comp()->isVirtualGuardNOPingRequired() &&
1742
             !resolvedMethod->isInterpreted() &&
1743
             !callNode->isTheVirtualCallNodeForAGuardedInlinedCall())
1744
            {
1745
            TR_VirtualGuard *virtualGuard = NULL;
1746

1747
            if (!resolvedMethod->virtualMethodIsOverridden() &&
1748
                !resolvedMethod->isAbstract())
1749
               {
1750
               if (comp()->getOption(TR_TraceCG))
1751
                  {
1752
                  traceMsg(comp(), "Creating TR_NonoverriddenGuard for node %p\n", callNode);
1753
                  }
1754
               virtualGuard = TR_VirtualGuard::createGuardedDevirtualizationGuard(TR_NonoverriddenGuard, comp(), callNode);
1755
               }
1756
            else
1757
               {
1758
               TR_DevirtualizedCallInfo *devirtualizedCallInfo = comp()->findDevirtualizedCall(callNode);
1759
               TR_OpaqueClassBlock      *refinedThisClass = devirtualizedCallInfo ? devirtualizedCallInfo->_thisType : NULL;
1760
               TR_OpaqueClassBlock      *thisClass = refinedThisClass ? refinedThisClass : resolvedMethod->containingClass();
1761

1762
               TR_PersistentCHTable     *chTable = comp()->getPersistentInfo()->getPersistentCHTable();
1763
               /* Devirtualization is not currently supported for AOT compilations */
1764
               if (thisClass && TR::Compiler->cls.isAbstractClass(comp(), thisClass) && !comp()->compileRelocatableCode())
1765
                  {
1766
                  TR_ResolvedMethod *calleeMethod = chTable->findSingleAbstractImplementer(thisClass, methodSymRef->getOffset(), methodSymRef->getOwningMethod(comp()), comp());
1767
                  if (calleeMethod &&
1768
                      (comp()->isRecursiveMethodTarget(calleeMethod) ||
1769
                       !calleeMethod->isInterpreted() ||
1770
                       calleeMethod->isJITInternalNative()))
1771
                     {
1772
                     if (comp()->getOption(TR_TraceCG))
1773
                        {
1774
                        traceMsg(comp(), "Creating TR_AbstractGuard for node %p\n", callNode);
1775
                        }
1776
                     resolvedMethod = calleeMethod;
1777
                     virtualGuard = TR_VirtualGuard::createGuardedDevirtualizationGuard(TR_AbstractGuard, comp(), callNode);
1778
                     }
1779
                  }
1780
               else if (refinedThisClass &&
1781
                        resolvedMethod->virtualMethodIsOverridden() &&
1782
                        !chTable->isOverriddenInThisHierarchy(resolvedMethod, refinedThisClass, methodSymRef->getOffset(), comp()))
1783
                  {
1784
                  TR_ResolvedMethod *calleeMethod = methodSymRef->getOwningMethod(comp())->getResolvedVirtualMethod(comp(), refinedThisClass, methodSymRef->getOffset());
1785
                  if (calleeMethod &&
1786
                      (comp()->isRecursiveMethodTarget(calleeMethod) ||
1787
                       !calleeMethod->isInterpreted() ||
1788
                       calleeMethod->isJITInternalNative()))
1789
                     {
1790
                     if (comp()->getOption(TR_TraceCG))
1791
                        {
1792
                        traceMsg(comp(), "Creating TR_HierarchyGuard for node %p\n", callNode);
1793
                        }
1794
                     resolvedMethod = calleeMethod;
1795
                     virtualGuard = TR_VirtualGuard::createGuardedDevirtualizationGuard(TR_HierarchyGuard, comp(), callNode);
1796
                     }
1797
                  }
1798
               }
1799

1800
            // If we have a virtual call guard generate a direct call
1801
            // in the inline path and the virtual call out of line.
1802
            // If the guard is later patched we'll go out of line path.
1803
            //
1804
            if (virtualGuard)
1805
               {
1806
               TR::LabelSymbol *virtualCallLabel = generateLabelSymbol(cg());
1807
               generateVirtualGuardNOPInstruction(cg(), callNode, virtualGuard->addNOPSite(), NULL, virtualCallLabel);
1808

1809
               if (comp()->getOption(TR_EnableHCR))
1810
                  {
1811
                  if (cg()->supportsMergingGuards())
1812
                     {
1813
                     virtualGuard->setMergedWithHCRGuard();
1814
                     }
1815
                  else
1816
                     {
1817
                     TR_VirtualGuard *HCRGuard = TR_VirtualGuard::createGuardedDevirtualizationGuard(TR_HCRGuard, comp(), callNode);
1818
                     generateVirtualGuardNOPInstruction(cg(), callNode, HCRGuard->addNOPSite(), NULL, virtualCallLabel);
1819
                     }
1820
                  }
1821
               if (resolvedMethod != resolvedMethodSymbol->getResolvedMethod())
1822
                  {
1823
                  methodSymRef = comp()->getSymRefTab()->findOrCreateMethodSymbol(methodSymRef->getOwningMethodIndex(),
1824
                                                                                  -1,
1825
                                                                                  resolvedMethod,
1826
                                                                                  TR::MethodSymbol::Virtual);
1827
                  methodSymbol = methodSymRef->getSymbol()->castToMethodSymbol();
1828
                  resolvedMethodSymbol = methodSymbol->getResolvedMethodSymbol();
1829
                  resolvedMethod = resolvedMethodSymbol->getResolvedMethod();
1830
                  }
1831
               uintptr_t methodAddress = comp()->isRecursiveMethodTarget(resolvedMethod) ? 0 : (uintptr_t)resolvedMethod->startAddressForJittedMethod();
1832
               TR::Instruction *gcPoint = generateImmSymInstruction(cg(), TR::InstOpCode::bl, callNode, methodAddress, NULL, methodSymRef, NULL);
1833
               generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, doneLabel, dependencies);
1834
               gcPoint->ARM64NeedsGCMap(cg(), regMapForGC);
1835

1836
               fej9->reserveTrampolineIfNecessary(comp(), methodSymRef, false);
1837

1838
               // Out of line virtual call
1839
               //
1840
               TR_ARM64OutOfLineCodeSection *virtualCallOOL = new (trHeapMemory()) TR_ARM64OutOfLineCodeSection(virtualCallLabel, doneLabel, cg());
1841

1842
               virtualCallOOL->swapInstructionListsWithCompilation();
1843
               TR::Instruction *OOLLabelInstr = generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, virtualCallLabel);
1844

1845
               // XXX: Temporary fix, OOL instruction stream does not pick up live locals or monitors correctly.
1846
               TR_ASSERT(!OOLLabelInstr->getLiveLocals() && !OOLLabelInstr->getLiveMonitors(), "Expecting first OOL instruction to not have live locals/monitors info");
1847
               OOLLabelInstr->setLiveLocals(gcPoint->getLiveLocals());
1848
               OOLLabelInstr->setLiveMonitors(gcPoint->getLiveMonitors());
1849

1850
               buildVirtualCall(cg(), callNode, vftReg, x9, regMapForGC);
1851

1852
               generateLabelInstruction(cg(), TR::InstOpCode::b, callNode, doneLabel);
1853
               virtualCallOOL->swapInstructionListsWithCompilation();
1854
               cg()->getARM64OutOfLineCodeSectionList().push_front(virtualCallOOL);
1855

1856
               return;
1857
               }
1858
            }
1859
         }
1860
      }
1861

1862
   // Profile-driven virtual and interface calls
1863
   //
1864
   // If the top value dominates everything else, generate a single static
1865
   // PIC call inline and a virtual call or dynamic PIC call out of line.
1866
   //
1867
   // Otherwise generate a reasonable amount of static PIC calls and a
1868
   // virtual call or dynamic PIC call all inline.
1869
   //
1870
   if (callIsSafe && !callNode->isTheVirtualCallNodeForAGuardedInlinedCall() && !comp()->getOption(TR_DisableInterpreterProfiling))
1871
      {
1872
      static uint32_t  maxVirtualStaticPICs = comp()->getOptions()->getMaxStaticPICSlots(comp()->getMethodHotness());
1873
      static uint32_t  maxInterfaceStaticPICs = comp()->getOptions()->getNumInterfaceCallCacheSlots();
1874

1875
      TR_ScratchList<J9::ARM64PICItem> values(cg()->trMemory());
1876
      const uint32_t maxStaticPICs = methodSymbol->isInterface() ? maxInterfaceStaticPICs : maxVirtualStaticPICs;
1877

1878
      if (getProfiledCallSiteInfo(cg(), callNode, maxStaticPICs, values))
1879
         {
1880
         ListIterator<J9::ARM64PICItem>  i(&values);
1881
         J9::ARM64PICItem               *pic = i.getFirst();
1882

1883
         // If this value is dominant, optimize exclusively for it
1884
         if (pic->_frequency > MAX_PROFILED_CALL_FREQUENCY)
1885
            {
1886
            if (comp()->getOption(TR_TraceCG))
1887
               {
1888
               traceMsg(comp(), "Found dominant profiled target, frequency = %f\n", pic->_frequency);
1889
               }
1890
            TR::LabelSymbol *slowCallLabel = generateLabelSymbol(cg());
1891

1892
            TR::Instruction *gcPoint = buildStaticPICCall(cg(), callNode, pic->_clazz, pic->_method,
1893
                                                            vftReg, x9, slowCallLabel, regMapForGC);
1894
            generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, doneLabel, dependencies);
1895

1896
            // Out of line virtual/interface call
1897
            //
1898
            TR_ARM64OutOfLineCodeSection *slowCallOOL = new (trHeapMemory()) TR_ARM64OutOfLineCodeSection(slowCallLabel, doneLabel, cg());
1899

1900
            slowCallOOL->swapInstructionListsWithCompilation();
1901
            TR::Instruction *OOLLabelInstr = generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, slowCallLabel);
1902

1903
            // XXX: Temporary fix, OOL instruction stream does not pick up live locals or monitors correctly.
1904
            TR_ASSERT_FATAL(!OOLLabelInstr->getLiveLocals() && !OOLLabelInstr->getLiveMonitors(), "Expecting first OOL instruction to not have live locals/monitors info");
1905
            OOLLabelInstr->setLiveLocals(gcPoint->getLiveLocals());
1906
            OOLLabelInstr->setLiveMonitors(gcPoint->getLiveMonitors());
1907

1908
            TR::LabelSymbol *doneOOLLabel = generateLabelSymbol(cg());
1909

1910
            if (methodSymbol->isInterface())
1911
               {
1912
               TR::LabelSymbol               *ifcSnippetLabel = generateLabelSymbol(cg());
1913
               TR::LabelSymbol               *firstClassCacheSlotLabel = generateLabelSymbol(cg());
1914
               TR::LabelSymbol               *firstBranchAddressCacheSlotLabel = generateLabelSymbol(cg());
1915
               TR::LabelSymbol               *secondClassCacheSlotLabel = generateLabelSymbol(cg());
1916
               TR::LabelSymbol               *secondBranchAddressCacheSlotLabel = generateLabelSymbol(cg());
1917
               TR::ARM64InterfaceCallSnippet *ifcSnippet = new (trHeapMemory()) TR::ARM64InterfaceCallSnippet(cg(), callNode, ifcSnippetLabel,
1918
                                                                                                              argSize, doneOOLLabel, firstClassCacheSlotLabel, secondClassCacheSlotLabel,
1919
                                                                                                              firstBranchAddressCacheSlotLabel, secondBranchAddressCacheSlotLabel, static_cast<uint8_t *>(thunk));
1920
               cg()->addSnippet(ifcSnippet);
1921
               buildInterfaceCall(cg(), callNode, vftReg, x9, ifcSnippet, regMapForGC);
1922
               }
1923
            else
1924
               {
1925
               buildVirtualCall(cg(), callNode, vftReg, x9, regMapForGC);
1926
               }
1927

1928
            generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, doneOOLLabel);
1929
            generateLabelInstruction(cg(), TR::InstOpCode::b, callNode, doneLabel);
1930
            slowCallOOL->swapInstructionListsWithCompilation();
1931
            cg()->getARM64OutOfLineCodeSectionList().push_front(slowCallOOL);
1932

1933
            return;
1934
            }
1935
         else
1936
            {
1937
             if (comp()->getOption(TR_TraceCG))
1938
               {
1939
               traceMsg(comp(), "Generating %d static PIC calls\n", values.getSize());
1940
               }
1941
            // Build multiple static PIC calls
1942
            while (pic)
1943
               {
1944
               TR::LabelSymbol *nextLabel = generateLabelSymbol(cg());
1945

1946
               buildStaticPICCall(cg(), callNode, pic->_clazz, pic->_method,
1947
                                  vftReg, x9, nextLabel, regMapForGC);
1948
               generateLabelInstruction(cg(), TR::InstOpCode::b, callNode, doneLabel);
1949
               generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, nextLabel);
1950
               pic = i.getNext();
1951
               }
1952
            // Regular virtual/interface call will be built below
1953
            }
1954
         }
1955
      }
1956

1957
   // Finally, regular virtual and interface calls
1958
   //
1959
   if (methodSymbol->isInterface())
1960
      {
1961
      // interface calls
1962
      // ToDo: Inline interface dispatch
1963

1964
      TR::LabelSymbol *ifcSnippetLabel = generateLabelSymbol(cg());
1965
      TR::LabelSymbol *firstClassCacheSlotLabel = generateLabelSymbol(cg());
1966
      TR::LabelSymbol *firstBranchAddressCacheSlotLabel = generateLabelSymbol(cg());
1967
      TR::LabelSymbol *secondClassCacheSlotLabel = generateLabelSymbol(cg());
1968
      TR::LabelSymbol *secondBranchAddressCacheSlotLabel = generateLabelSymbol(cg());
1969
      TR::ARM64InterfaceCallSnippet *ifcSnippet =
1970
         new (trHeapMemory())
1971
         TR::ARM64InterfaceCallSnippet(cg(), callNode, ifcSnippetLabel, argSize, doneLabel, firstClassCacheSlotLabel, firstBranchAddressCacheSlotLabel, secondClassCacheSlotLabel, secondBranchAddressCacheSlotLabel, static_cast<uint8_t *>(thunk));
1972
      cg()->addSnippet(ifcSnippet);
1973

1974
      buildInterfaceCall(cg(), callNode, vftReg, x9, ifcSnippet, regMapForGC);
1975
      }
1976
   else
1977
      {
1978
      buildVirtualCall(cg(), callNode, vftReg, x9, regMapForGC);
1979
      }
1980
   generateLabelInstruction(cg(), TR::InstOpCode::label, callNode, doneLabel, dependencies);
1981
   }
1982

1983
TR::Register *J9::ARM64::PrivateLinkage::buildIndirectDispatch(TR::Node *callNode)
1984
   {
1985
   const TR::ARM64LinkageProperties &pp = getProperties();
1986
   TR::RealRegister *sp = cg()->machine()->getRealRegister(pp.getStackPointerRegister());
1987

1988
   // Extra post dependency for killing vector registers (see KillVectorRegs)
1989
   const int extraPostReg = killsVectorRegisters() ? 1 : 0;
1990
   TR::RegisterDependencyConditions *dependencies =
1991
      new (trHeapMemory()) TR::RegisterDependencyConditions(
1992
         pp.getNumberOfDependencyGPRegisters(),
1993
         pp.getNumberOfDependencyGPRegisters() + extraPostReg, trMemory());
1994

1995
   int32_t argSize = buildArgs(callNode, dependencies);
1996

1997
   buildVirtualDispatch(callNode, dependencies, argSize);
1998
   cg()->machine()->setLinkRegisterKilled(true);
1999

2000
   TR::Register *retReg;
2001
   switch(callNode->getOpCodeValue())
2002
      {
2003
      case TR::icalli:
2004
         retReg = dependencies->searchPostConditionRegister(
2005
                     pp.getIntegerReturnRegister());
2006
         break;
2007
      case TR::lcalli:
2008
      case TR::acalli:
2009
         retReg = dependencies->searchPostConditionRegister(
2010
                     pp.getLongReturnRegister());
2011
         break;
2012
      case TR::fcalli:
2013
      case TR::dcalli:
2014
         retReg = dependencies->searchPostConditionRegister(
2015
                     pp.getFloatReturnRegister());
2016
         break;
2017
      case TR::calli:
2018
         retReg = NULL;
2019
         break;
2020
      default:
2021
         retReg = NULL;
2022
         TR_ASSERT_FATAL(false, "Unsupported indirect call Opcode.");
2023
      }
2024

2025
   callNode->setRegister(retReg);
2026

2027
   dependencies->stopUsingDepRegs(cg(), retReg);
2028
   return retReg;
2029
   }
2030

2031
TR::Instruction *
2032
J9::ARM64::PrivateLinkage::loadStackParametersToLinkageRegisters(TR::Instruction *cursor)
2033
   {
2034
   TR::Machine *machine = cg()->machine();
2035
   TR::ARM64LinkageProperties& properties = getProperties();
2036
   TR::RealRegister *javaSP = machine->getRealRegister(properties.getStackPointerRegister());       // x20
2037

2038
   TR::ResolvedMethodSymbol *bodySymbol = comp()->getJittedMethodSymbol();
2039
   ListIterator<TR::ParameterSymbol> parmIterator(&(bodySymbol->getParameterList()));
2040
   TR::ParameterSymbol *parmCursor;
2041

2042
   // Copy from stack all parameters that belong in linkage regs
2043
   //
2044
   for (parmCursor = parmIterator.getFirst();
2045
        parmCursor != NULL;
2046
        parmCursor = parmIterator.getNext())
2047
      {
2048
      if (parmCursor->isParmPassedInRegister())
2049
         {
2050
         int8_t lri = parmCursor->getLinkageRegisterIndex();
2051
         TR::RealRegister *linkageReg;
2052
         TR::InstOpCode::Mnemonic op;
2053
         TR::DataType dataType = parmCursor->getDataType();
2054

2055
         if (dataType == TR::Double || dataType == TR::Float)
2056
            {
2057
            linkageReg = machine->getRealRegister(properties.getFloatArgumentRegister(lri));
2058
            op = (dataType == TR::Double) ? TR::InstOpCode::vldrimmd : TR::InstOpCode::vldrimms;
2059
            }
2060
         else
2061
            {
2062
            linkageReg = machine->getRealRegister(properties.getIntegerArgumentRegister(lri));
2063
            op = (dataType == TR::Int64 || dataType == TR::Address) ? TR::InstOpCode::ldrimmx : TR::InstOpCode::ldrimmw;
2064
            }
2065

2066
         TR::MemoryReference *stackMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, parmCursor->getParameterOffset());
2067
         cursor = generateTrg1MemInstruction(cg(), op, NULL, linkageReg, stackMR, cursor);
2068
         }
2069
      }
2070

2071
    return cursor;
2072
    }
2073

2074
TR::Instruction *
2075
J9::ARM64::PrivateLinkage::saveParametersToStack(TR::Instruction *cursor)
2076
   {
2077
   TR::Machine *machine = cg()->machine();
2078
   TR::ARM64LinkageProperties& properties = getProperties();
2079
   TR::RealRegister *javaSP = machine->getRealRegister(properties.getStackPointerRegister());       // x20
2080

2081
   TR::ResolvedMethodSymbol *bodySymbol = comp()->getJittedMethodSymbol();
2082
   ListIterator<TR::ParameterSymbol> parmIterator(&(bodySymbol->getParameterList()));
2083
   TR::ParameterSymbol *parmCursor;
2084

2085
   // Store to stack all parameters passed in linkage registers
2086
   //
2087
   for (parmCursor = parmIterator.getFirst();
2088
        parmCursor != NULL;
2089
        parmCursor = parmIterator.getNext())
2090
      {
2091
      if (parmCursor->isParmPassedInRegister())
2092
         {
2093
         int8_t lri = parmCursor->getLinkageRegisterIndex();
2094
         TR::RealRegister *linkageReg;
2095
         TR::InstOpCode::Mnemonic op;
2096

2097
         if (parmCursor->getDataType() == TR::Double || parmCursor->getDataType() == TR::Float)
2098
            {
2099
            linkageReg = machine->getRealRegister(properties.getFloatArgumentRegister(lri));
2100
            op = (parmCursor->getDataType() == TR::Double) ? TR::InstOpCode::vstrimmd : TR::InstOpCode::vstrimms;
2101
            }
2102
         else
2103
            {
2104
            linkageReg = machine->getRealRegister(properties.getIntegerArgumentRegister(lri));
2105
            op = TR::InstOpCode::strimmx;
2106
            }
2107

2108
         TR::MemoryReference *stackMR = TR::MemoryReference::createWithDisplacement(cg(), javaSP, parmCursor->getParameterOffset());
2109
         cursor = generateMemSrc1Instruction(cg(), op, NULL, stackMR, linkageReg, cursor);
2110
         }
2111
      }
2112

2113
   return cursor;
2114
   }
2115

2116
void J9::ARM64::PrivateLinkage::performPostBinaryEncoding()
2117
   {
2118
   // --------------------------------------------------------------------------
2119
   // Encode the size of the interpreter entry area into the linkage info word
2120
   //
2121
   TR_ASSERT_FATAL(cg()->getReturnTypeInfoInstruction(),
2122
                   "Expecting the return type info instruction to be created");
2123

2124
   TR::ARM64ImmInstruction *linkageInfoWordInstruction = cg()->getReturnTypeInfoInstruction();
2125
   uint32_t linkageInfoWord = linkageInfoWordInstruction->getSourceImmediate();
2126

2127
   intptr_t jittedMethodEntryAddress = reinterpret_cast<intptr_t>(getJittedMethodEntryPoint()->getBinaryEncoding());
2128
   intptr_t interpretedMethodEntryAddress = reinterpret_cast<intptr_t>(getInterpretedMethodEntryPoint()->getBinaryEncoding());
2129

2130
   linkageInfoWord = (static_cast<uint32_t>(jittedMethodEntryAddress - interpretedMethodEntryAddress) << 16) | linkageInfoWord;
2131
   linkageInfoWordInstruction->setSourceImmediate(linkageInfoWord);
2132

2133
   *(uint32_t *)(linkageInfoWordInstruction->getBinaryEncoding()) = linkageInfoWord;
2134

2135
   // Set recompilation info
2136
   //
2137
   TR::Recompilation *recomp = comp()->getRecompilationInfo();
2138
   if (recomp != NULL && recomp->couldBeCompiledAgain())
2139
      {
2140
      J9::PrivateLinkage::LinkageInfo *lkInfo = J9::PrivateLinkage::LinkageInfo::get(cg()->getCodeStart());
2141
      if (recomp->useSampling())
2142
         lkInfo->setSamplingMethodBody();
2143
      else
2144
         lkInfo->setCountingMethodBody();
2145
      }
2146
   }
2147

2148
int32_t J9::ARM64::HelperLinkage::buildArgs(TR::Node *callNode,
2149
   TR::RegisterDependencyConditions *dependencies)
2150
   {
2151
   return buildPrivateLinkageArgs(callNode, dependencies, _helperLinkage);
2152
   }
2153

2154
Product

Resources

Company
