X86InsertPrefetch.cpp

//===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass applies cache prefetch instructions based on a profile. The pass
// assumes DiscriminateMemOps ran immediately before, to ensure debug info
// matches the one used at profile generation time. The profile is encoded in
// afdo format (text or binary). It contains prefetch hints recommendations.
// Each recommendation is made in terms of debug info locations, a type (i.e.
// nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a
// memory operand (see X86DiscriminateMemOps). The prefetch will be made for
// a location at that memory operand + the delta specified in the
// recommendation.
//
//===----------------------------------------------------------------------===//

#include "X86.h"
#include "X86InstrBuilder.h"
#include "X86InstrInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86Subtarget.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Module.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Transforms/IPO/SampleProfile.h"
using namespace llvm;
using namespace sampleprof;

static cl::opt<std::string>
    PrefetchHintsFile("prefetch-hints-file",
                      cl::desc("Path to the prefetch hints profile. See also "
                               "-x86-discriminate-memops"),
                      cl::Hidden);
namespace {

class X86InsertPrefetch : public MachineFunctionPass {
  void getAnalysisUsage(AnalysisUsage &AU) const override;
  bool doInitialization(Module &) override;

  bool runOnMachineFunction(MachineFunction &MF) override;
  struct PrefetchInfo {
    unsigned InstructionID;
    int64_t Delta;
  };
  typedef SmallVectorImpl<PrefetchInfo> Prefetches;
  bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI,
                        Prefetches &prefetches) const;

public:
  static char ID;
  X86InsertPrefetch(const std::string &PrefetchHintsFilename);
  StringRef getPassName() const override {
    return "X86 Insert Cache Prefetches";
  }

private:
  std::string Filename;
  std::unique_ptr<SampleProfileReader> Reader;
};

using PrefetchHints = SampleRecord::CallTargetMap;

// Return any prefetching hints for the specified MachineInstruction. The hints
// are returned as pairs (name, delta).
ErrorOr<const PrefetchHints &>
getPrefetchHints(const FunctionSamples *TopSamples, const MachineInstr &MI) {
  if (const auto &Loc = MI.getDebugLoc())
    if (const auto *Samples = TopSamples->findFunctionSamples(Loc))
      return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc),
                                          Loc->getBaseDiscriminator());
  return std::error_code();
}

// The prefetch instruction can't take memory operands involving vector
// registers.
bool IsMemOpCompatibleWithPrefetch(const MachineInstr &MI, int Op) {
  Register BaseReg = MI.getOperand(Op + X86::AddrBaseReg).getReg();
  Register IndexReg = MI.getOperand(Op + X86::AddrIndexReg).getReg();
  return (BaseReg == 0 ||
          X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) ||
          X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg)) &&
         (IndexReg == 0 ||
          X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg) ||
          X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg));
}

} // end anonymous namespace

//===----------------------------------------------------------------------===//
//            Implementation
//===----------------------------------------------------------------------===//

char X86InsertPrefetch::ID = 0;

X86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename)
    : MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {}

/// Return true if the provided MachineInstruction has cache prefetch hints. In
/// that case, the prefetch hints are stored, in order, in the Prefetches
/// vector.
bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
                                         const MachineInstr &MI,
                                         Prefetches &Prefetches) const {
  assert(Prefetches.empty() &&
         "Expected caller passed empty PrefetchInfo vector.");

  // There is no point to match prefetch hints if the profile is using MD5.
  if (FunctionSamples::UseMD5)
    return false;

  static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
      {"_nta_", X86::PREFETCHNTA},
      {"_t0_", X86::PREFETCHT0},
      {"_t1_", X86::PREFETCHT1},
      {"_t2_", X86::PREFETCHT2},
  };
  static const char *SerializedPrefetchPrefix = "__prefetch";

  auto T = getPrefetchHints(TopSamples, MI);
  if (!T)
    return false;
  int16_t max_index = -1;
  // Convert serialized prefetch hints into PrefetchInfo objects, and populate
  // the Prefetches vector.
  for (const auto &S_V : *T) {
    StringRef Name = S_V.first.stringRef();
    if (Name.consume_front(SerializedPrefetchPrefix)) {
      int64_t D = static_cast<int64_t>(S_V.second);
      unsigned IID = 0;
      for (const auto &HintType : HintTypes) {
        if (Name.consume_front(HintType.first)) {
          IID = HintType.second;
          break;
        }
      }
      if (IID == 0)
        return false;
      uint8_t index = 0;
      Name.consumeInteger(10, index);

      if (index >= Prefetches.size())
        Prefetches.resize(index + 1);
      Prefetches[index] = {IID, D};
      max_index = std::max(max_index, static_cast<int16_t>(index));
    }
  }
  assert(max_index + 1 >= 0 &&
         "Possible overflow: max_index + 1 should be positive.");
  assert(static_cast<size_t>(max_index + 1) == Prefetches.size() &&
         "The number of prefetch hints received should match the number of "
         "PrefetchInfo objects returned");
  return !Prefetches.empty();
}

bool X86InsertPrefetch::doInitialization(Module &M) {
  if (Filename.empty())
    return false;

  LLVMContext &Ctx = M.getContext();
  // TODO: Propagate virtual file system into LLVM targets.
  auto FS = vfs::getRealFileSystem();
  ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr =
      SampleProfileReader::create(Filename, Ctx, *FS);
  if (std::error_code EC = ReaderOrErr.getError()) {
    std::string Msg = "Could not open profile: " + EC.message();
    Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg,
                                             DiagnosticSeverity::DS_Warning));
    return false;
  }
  Reader = std::move(ReaderOrErr.get());
  Reader->read();
  return true;
}

void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.setPreservesAll();
  MachineFunctionPass::getAnalysisUsage(AU);
}

bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
  if (!Reader)
    return false;
  const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction());
  if (!Samples)
    return false;

  bool Changed = false;

  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
  SmallVector<PrefetchInfo, 4> Prefetches;
  for (auto &MBB : MF) {
    for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) {
      auto Current = MI;
      ++MI;

      int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags);
      if (Offset < 0)
        continue;
      unsigned Bias = X86II::getOperandBias(Current->getDesc());
      int MemOpOffset = Offset + Bias;
      // FIXME(mtrofin): ORE message when the recommendation cannot be taken.
      if (!IsMemOpCompatibleWithPrefetch(*Current, MemOpOffset))
        continue;
      Prefetches.clear();
      if (!findPrefetchInfo(Samples, *Current, Prefetches))
        continue;
      assert(!Prefetches.empty() &&
             "The Prefetches vector should contain at least a value if "
             "findPrefetchInfo returned true.");
      for (auto &PrefInfo : Prefetches) {
        unsigned PFetchInstrID = PrefInfo.InstructionID;
        int64_t Delta = PrefInfo.Delta;
        const MCInstrDesc &Desc = TII->get(PFetchInstrID);
        MachineInstr *PFetch =
            MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true);
        MachineInstrBuilder MIB(MF, PFetch);

        static_assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 &&
                          X86::AddrIndexReg == 2 && X86::AddrDisp == 3 &&
                          X86::AddrSegmentReg == 4,
                      "Unexpected change in X86 operand offset order.");

        // This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc.
        // FIXME(mtrofin): consider adding a:
        //     MachineInstrBuilder::set(unsigned offset, op).
        MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg())
            .addImm(
                Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm())
            .addReg(
                Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg())
            .addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() +
                    Delta)
            .addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg)
                        .getReg());

        if (!Current->memoperands_empty()) {
          MachineMemOperand *CurrentOp = *(Current->memoperands_begin());
          MIB.addMemOperand(MF.getMachineMemOperand(
              CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize()));
        }

        // Insert before Current. This is because Current may clobber some of
        // the registers used to describe the input memory operand.
        MBB.insert(Current, PFetch);
        Changed = true;
      }
    }
  }
  return Changed;
}

FunctionPass *llvm::createX86InsertPrefetchPass() {
  return new X86InsertPrefetch(PrefetchHintsFile);
}