BPFPreserveStaticOffset.cpp

//===------ BPFPreserveStaticOffset.cpp -----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// TLDR: replaces llvm.preserve.static.offset + GEP + load / store
//           with llvm.bpf.getelementptr.and.load / store
//
// This file implements BPFPreserveStaticOffsetPass transformation.
// This transformation address two BPF verifier specific issues:
//
// (a) Access to the fields of some structural types is allowed only
//     using load and store instructions with static immediate offsets.
//
//     Examples of such types are `struct __sk_buff` and `struct
//     bpf_sock_ops`.  This is so because offsets of the fields of
//     these structures do not match real offsets in the running
//     kernel. During BPF program load LDX and STX instructions
//     referring to the fields of these types are rewritten so that
//     offsets match real offsets. For this rewrite to happen field
//     offsets have to be encoded as immediate operands of the
//     instructions.
//
//     See kernel/bpf/verifier.c:convert_ctx_access function in the
//     Linux kernel source tree for details.
//
// (b) Pointers to context parameters of BPF programs must not be
//     modified before access.
//
//     During BPF program verification a tag PTR_TO_CTX is tracked for
//     register values. In case if register with such tag is modified
//     BPF program is not allowed to read or write memory using this
//     register. See kernel/bpf/verifier.c:check_mem_access function
//     in the Linux kernel source tree for details.
//
// The following sequence of the IR instructions:
//
//   %x = getelementptr %ptr, %constant_offset
//   %y = load %x
//
// Is translated as a single machine instruction:
//
//   LDW %ptr, %constant_offset
//
// In order for cases (a) and (b) to work the sequence %x-%y above has
// to be preserved by the IR passes.
//
// However, several optimization passes might sink `load` instruction
// or hoist `getelementptr` instruction so that the instructions are
// no longer in sequence. Examples of such passes are:
// SimplifyCFGPass, InstCombinePass, GVNPass.
// After such modification the verifier would reject the BPF program.
//
// To avoid this issue the patterns like (load/store (getelementptr ...))
// are replaced by calls to BPF specific intrinsic functions:
// - llvm.bpf.getelementptr.and.load
// - llvm.bpf.getelementptr.and.store
//
// These calls are lowered back to (load/store (getelementptr ...))
// by BPFCheckAndAdjustIR pass right before the translation from IR to
// machine instructions.
//
// The transformation is split into the following steps:
// - When IR is generated from AST the calls to intrinsic function
//   llvm.preserve.static.offset are inserted.
// - BPFPreserveStaticOffsetPass is executed as early as possible
//   with AllowPatial set to true, this handles marked GEP chains
//   with constant offsets.
// - BPFPreserveStaticOffsetPass is executed at ScalarOptimizerLateEPCallback
//   with AllowPatial set to false, this handles marked GEP chains
//   with offsets that became constant after loop unrolling, e.g.
//   to handle the following code:
//
// struct context { int x[4]; } __attribute__((preserve_static_offset));
//
//   struct context *ctx = ...;
// #pragma clang loop unroll(full)
//   for (int i = 0; i < 4; ++i)
//     foo(ctx->x[i]);
//
// The early BPFPreserveStaticOffsetPass run is necessary to allow
// additional GVN / CSE opportunities after functions inlining.
// The relative order of optimization applied to function:
// - early stage (1)
// - ...
// - function inlining (2)
// - ...
// - loop unrolling
// - ...
// - ScalarOptimizerLateEPCallback (3)
//
// When function A is inlined into function B all optimizations for A
// are already done, while some passes remain for B. In case if
// BPFPreserveStaticOffsetPass is done at (3) but not done at (1)
// the code after (2) would contain a mix of
// (load (gep %p)) and (get.and.load %p) usages:
// - the (load (gep %p)) would come from the calling function;
// - the (get.and.load %p) would come from the callee function.
// Thus clobbering CSE / GVN passes done after inlining.

#include "BPF.h"
#include "BPFCORE.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsBPF.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"

#define DEBUG_TYPE "bpf-preserve-static-offset"

using namespace llvm;

static const unsigned GepAndLoadFirstIdxArg = 6;
static const unsigned GepAndStoreFirstIdxArg = 7;

static bool isIntrinsicCall(Value *I, Intrinsic::ID Id) {
  if (auto *Call = dyn_cast<CallInst>(I))
    if (Function *Func = Call->getCalledFunction())
      return Func->getIntrinsicID() == Id;
  return false;
}

static bool isPreserveStaticOffsetCall(Value *I) {
  return isIntrinsicCall(I, Intrinsic::preserve_static_offset);
}

static CallInst *isGEPAndLoad(Value *I) {
  if (isIntrinsicCall(I, Intrinsic::bpf_getelementptr_and_load))
    return cast<CallInst>(I);
  return nullptr;
}

static CallInst *isGEPAndStore(Value *I) {
  if (isIntrinsicCall(I, Intrinsic::bpf_getelementptr_and_store))
    return cast<CallInst>(I);
  return nullptr;
}

template <class T = Instruction>
static DILocation *mergeDILocations(SmallVector<T *> &Insns) {
  DILocation *Merged = (*Insns.begin())->getDebugLoc();
  for (T *I : Insns)
    Merged = DILocation::getMergedLocation(Merged, I->getDebugLoc());
  return Merged;
}

static CallInst *makeIntrinsicCall(Module *M,
                                   Intrinsic::BPFIntrinsics Intrinsic,
                                   ArrayRef<Type *> Types,
                                   ArrayRef<Value *> Args) {

  Function *Fn = Intrinsic::getDeclaration(M, Intrinsic, Types);
  return CallInst::Create(Fn, Args);
}

static void setParamElementType(CallInst *Call, unsigned ArgNo, Type *Type) {
  LLVMContext &C = Call->getContext();
  Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ElementType, Type));
}

static void setParamReadNone(CallInst *Call, unsigned ArgNo) {
  LLVMContext &C = Call->getContext();
  Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ReadNone));
}

static void setParamReadOnly(CallInst *Call, unsigned ArgNo) {
  LLVMContext &C = Call->getContext();
  Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ReadOnly));
}

static void setParamWriteOnly(CallInst *Call, unsigned ArgNo) {
  LLVMContext &C = Call->getContext();
  Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::WriteOnly));
}

namespace {
struct GEPChainInfo {
  bool InBounds;
  Type *SourceElementType;
  SmallVector<Value *> Indices;
  SmallVector<GetElementPtrInst *> Members;

  GEPChainInfo() { reset(); }

  void reset() {
    InBounds = true;
    SourceElementType = nullptr;
    Indices.clear();
    Members.clear();
  }
};
} // Anonymous namespace

template <class T = std::disjunction<LoadInst, StoreInst>>
static void fillCommonArgs(LLVMContext &C, SmallVector<Value *> &Args,
                           GEPChainInfo &GEP, T *Insn) {
  Type *Int8Ty = Type::getInt8Ty(C);
  Type *Int1Ty = Type::getInt1Ty(C);
  // Implementation of Align guarantees that ShiftValue < 64
  unsigned AlignShiftValue = Log2_64(Insn->getAlign().value());
  Args.push_back(GEP.Members[0]->getPointerOperand());
  Args.push_back(ConstantInt::get(Int1Ty, Insn->isVolatile()));
  Args.push_back(ConstantInt::get(Int8Ty, (unsigned)Insn->getOrdering()));
  Args.push_back(ConstantInt::get(Int8Ty, (unsigned)Insn->getSyncScopeID()));
  Args.push_back(ConstantInt::get(Int8Ty, AlignShiftValue));
  Args.push_back(ConstantInt::get(Int1Ty, GEP.InBounds));
  Args.append(GEP.Indices.begin(), GEP.Indices.end());
}

static Instruction *makeGEPAndLoad(Module *M, GEPChainInfo &GEP,
                                   LoadInst *Load) {
  SmallVector<Value *> Args;
  fillCommonArgs(M->getContext(), Args, GEP, Load);
  CallInst *Call = makeIntrinsicCall(M, Intrinsic::bpf_getelementptr_and_load,
                                     {Load->getType()}, Args);
  setParamElementType(Call, 0, GEP.SourceElementType);
  Call->applyMergedLocation(mergeDILocations(GEP.Members), Load->getDebugLoc());
  Call->setName((*GEP.Members.rbegin())->getName());
  if (Load->isUnordered()) {
    Call->setOnlyReadsMemory();
    Call->setOnlyAccessesArgMemory();
    setParamReadOnly(Call, 0);
  }
  for (unsigned I = GepAndLoadFirstIdxArg; I < Args.size(); ++I)
    Call->addParamAttr(I, Attribute::ImmArg);
  Call->setAAMetadata(Load->getAAMetadata());
  return Call;
}

static Instruction *makeGEPAndStore(Module *M, GEPChainInfo &GEP,
                                    StoreInst *Store) {
  SmallVector<Value *> Args;
  Args.push_back(Store->getValueOperand());
  fillCommonArgs(M->getContext(), Args, GEP, Store);
  CallInst *Call =
      makeIntrinsicCall(M, Intrinsic::bpf_getelementptr_and_store,
                        {Store->getValueOperand()->getType()}, Args);
  setParamElementType(Call, 1, GEP.SourceElementType);
  if (Store->getValueOperand()->getType()->isPointerTy())
    setParamReadNone(Call, 0);
  Call->applyMergedLocation(mergeDILocations(GEP.Members),
                            Store->getDebugLoc());
  if (Store->isUnordered()) {
    Call->setOnlyWritesMemory();
    Call->setOnlyAccessesArgMemory();
    setParamWriteOnly(Call, 1);
  }
  for (unsigned I = GepAndStoreFirstIdxArg; I < Args.size(); ++I)
    Call->addParamAttr(I, Attribute::ImmArg);
  Call->setAAMetadata(Store->getAAMetadata());
  return Call;
}

static unsigned getOperandAsUnsigned(CallInst *Call, unsigned ArgNo) {
  if (auto *Int = dyn_cast<ConstantInt>(Call->getOperand(ArgNo)))
    return Int->getValue().getZExtValue();
  std::string Report;
  raw_string_ostream ReportS(Report);
  ReportS << "Expecting ConstantInt as argument #" << ArgNo << " of " << *Call
          << "\n";
  report_fatal_error(StringRef(Report));
}

static GetElementPtrInst *reconstructGEP(CallInst *Call, int Delta) {
  SmallVector<Value *> Indices;
  Indices.append(Call->data_operands_begin() + 6 + Delta,
                 Call->data_operands_end());
  Type *GEPPointeeType = Call->getParamElementType(Delta);
  auto *GEP =
      GetElementPtrInst::Create(GEPPointeeType, Call->getOperand(Delta),
                                ArrayRef<Value *>(Indices), Call->getName());
  GEP->setIsInBounds(getOperandAsUnsigned(Call, 5 + Delta));
  return GEP;
}

template <class T = std::disjunction<LoadInst, StoreInst>>
static void reconstructCommon(CallInst *Call, GetElementPtrInst *GEP, T *Insn,
                              int Delta) {
  Insn->setVolatile(getOperandAsUnsigned(Call, 1 + Delta));
  Insn->setOrdering((AtomicOrdering)getOperandAsUnsigned(Call, 2 + Delta));
  Insn->setSyncScopeID(getOperandAsUnsigned(Call, 3 + Delta));
  unsigned AlignShiftValue = getOperandAsUnsigned(Call, 4 + Delta);
  Insn->setAlignment(Align(1ULL << AlignShiftValue));
  GEP->setDebugLoc(Call->getDebugLoc());
  Insn->setDebugLoc(Call->getDebugLoc());
  Insn->setAAMetadata(Call->getAAMetadata());
}

std::pair<GetElementPtrInst *, LoadInst *>
BPFPreserveStaticOffsetPass::reconstructLoad(CallInst *Call) {
  GetElementPtrInst *GEP = reconstructGEP(Call, 0);
  Type *ReturnType = Call->getFunctionType()->getReturnType();
  auto *Load = new LoadInst(ReturnType, GEP, "",
                            /* These would be set in reconstructCommon */
                            false, Align(1));
  reconstructCommon(Call, GEP, Load, 0);
  return std::pair{GEP, Load};
}

std::pair<GetElementPtrInst *, StoreInst *>
BPFPreserveStaticOffsetPass::reconstructStore(CallInst *Call) {
  GetElementPtrInst *GEP = reconstructGEP(Call, 1);
  auto *Store = new StoreInst(Call->getOperand(0), GEP,
                              /* These would be set in reconstructCommon */
                              false, Align(1));
  reconstructCommon(Call, GEP, Store, 1);
  return std::pair{GEP, Store};
}

static bool isZero(Value *V) {
  auto *CI = dyn_cast<ConstantInt>(V);
  return CI && CI->isZero();
}

// Given a chain of GEP instructions collect information necessary to
// merge this chain as a single GEP instruction of form:
//   getelementptr %<type>, ptr %p, i32 0, <field_idx1>, <field_idx2>, ...
static bool foldGEPChainAsStructAccess(SmallVector<GetElementPtrInst *> &GEPs,
                                       GEPChainInfo &Info) {
  if (GEPs.empty())
    return false;

  if (!all_of(GEPs, [=](GetElementPtrInst *GEP) {
        return GEP->hasAllConstantIndices();
      }))
    return false;

  GetElementPtrInst *First = GEPs[0];
  Info.InBounds = First->isInBounds();
  Info.SourceElementType = First->getSourceElementType();
  Type *ResultElementType = First->getResultElementType();
  Info.Indices.append(First->idx_begin(), First->idx_end());
  Info.Members.push_back(First);

  for (auto *Iter = GEPs.begin() + 1; Iter != GEPs.end(); ++Iter) {
    GetElementPtrInst *GEP = *Iter;
    if (!isZero(*GEP->idx_begin())) {
      Info.reset();
      return false;
    }
    if (!GEP->getSourceElementType() ||
        GEP->getSourceElementType() != ResultElementType) {
      Info.reset();
      return false;
    }
    Info.InBounds &= GEP->isInBounds();
    Info.Indices.append(GEP->idx_begin() + 1, GEP->idx_end());
    Info.Members.push_back(GEP);
    ResultElementType = GEP->getResultElementType();
  }

  return true;
}

// Given a chain of GEP instructions collect information necessary to
// merge this chain as a single GEP instruction of form:
//   getelementptr i8, ptr %p, i64 %offset
static bool foldGEPChainAsU8Access(SmallVector<GetElementPtrInst *> &GEPs,
                                   GEPChainInfo &Info) {
  if (GEPs.empty())
    return false;

  GetElementPtrInst *First = GEPs[0];
  const DataLayout &DL = First->getDataLayout();
  LLVMContext &C = First->getContext();
  Type *PtrTy = First->getType()->getScalarType();
  APInt Offset(DL.getIndexTypeSizeInBits(PtrTy), 0);
  for (GetElementPtrInst *GEP : GEPs) {
    if (!GEP->accumulateConstantOffset(DL, Offset)) {
      Info.reset();
      return false;
    }
    Info.InBounds &= GEP->isInBounds();
    Info.Members.push_back(GEP);
  }
  Info.SourceElementType = Type::getInt8Ty(C);
  Info.Indices.push_back(ConstantInt::get(C, Offset));

  return true;
}

static void reportNonStaticGEPChain(Instruction *Insn) {
  auto Msg = DiagnosticInfoUnsupported(
      *Insn->getFunction(),
      Twine("Non-constant offset in access to a field of a type marked "
            "with preserve_static_offset might be rejected by BPF verifier")
          .concat(Insn->getDebugLoc()
                      ? ""
                      : " (pass -g option to get exact location)"),
      Insn->getDebugLoc(), DS_Warning);
  Insn->getContext().diagnose(Msg);
}

static bool allZeroIndices(SmallVector<GetElementPtrInst *> &GEPs) {
  return GEPs.empty() || all_of(GEPs, [=](GetElementPtrInst *GEP) {
           return GEP->hasAllZeroIndices();
         });
}

static bool tryToReplaceWithGEPBuiltin(Instruction *LoadOrStoreTemplate,
                                       SmallVector<GetElementPtrInst *> &GEPs,
                                       Instruction *InsnToReplace) {
  GEPChainInfo GEPChain;
  if (!foldGEPChainAsStructAccess(GEPs, GEPChain) &&
      !foldGEPChainAsU8Access(GEPs, GEPChain)) {
    return false;
  }
  Module *M = InsnToReplace->getModule();
  if (auto *Load = dyn_cast<LoadInst>(LoadOrStoreTemplate)) {
    Instruction *Replacement = makeGEPAndLoad(M, GEPChain, Load);
    Replacement->insertBefore(InsnToReplace);
    InsnToReplace->replaceAllUsesWith(Replacement);
  }
  if (auto *Store = dyn_cast<StoreInst>(LoadOrStoreTemplate)) {
    Instruction *Replacement = makeGEPAndStore(M, GEPChain, Store);
    Replacement->insertBefore(InsnToReplace);
  }
  return true;
}

// Check if U->getPointerOperand() == I
static bool isPointerOperand(Value *I, User *U) {
  if (auto *L = dyn_cast<LoadInst>(U))
    return L->getPointerOperand() == I;
  if (auto *S = dyn_cast<StoreInst>(U))
    return S->getPointerOperand() == I;
  if (auto *GEP = dyn_cast<GetElementPtrInst>(U))
    return GEP->getPointerOperand() == I;
  if (auto *Call = isGEPAndLoad(U))
    return Call->getArgOperand(0) == I;
  if (auto *Call = isGEPAndStore(U))
    return Call->getArgOperand(1) == I;
  return false;
}

static bool isInlineableCall(User *U) {
  if (auto *Call = dyn_cast<CallInst>(U))
    return Call->hasFnAttr(Attribute::InlineHint);
  return false;
}

static void rewriteAccessChain(Instruction *Insn,
                               SmallVector<GetElementPtrInst *> &GEPs,
                               SmallVector<Instruction *> &Visited,
                               bool AllowPatial, bool &StillUsed);

static void rewriteUses(Instruction *Insn,
                        SmallVector<GetElementPtrInst *> &GEPs,
                        SmallVector<Instruction *> &Visited, bool AllowPatial,
                        bool &StillUsed) {
  for (User *U : Insn->users()) {
    auto *UI = dyn_cast<Instruction>(U);
    if (UI && (isPointerOperand(Insn, UI) || isPreserveStaticOffsetCall(UI) ||
               isInlineableCall(UI)))
      rewriteAccessChain(UI, GEPs, Visited, AllowPatial, StillUsed);
    else
      LLVM_DEBUG({
        llvm::dbgs() << "unsupported usage in BPFPreserveStaticOffsetPass:\n";
        llvm::dbgs() << "  Insn: " << *Insn << "\n";
        llvm::dbgs() << "  User: " << *U << "\n";
      });
  }
}

// A DFS traversal of GEP chain trees starting from Root.
//
// Recursion descends through GEP instructions and
// llvm.preserve.static.offset calls. Recursion stops at any other
// instruction. If load or store instruction is reached it is replaced
// by a call to `llvm.bpf.getelementptr.and.load` or
// `llvm.bpf.getelementptr.and.store` intrinsic.
// If `llvm.bpf.getelementptr.and.load/store` is reached the accumulated
// GEPs are merged into the intrinsic call.
// If nested calls to `llvm.preserve.static.offset` are encountered these
// calls are marked for deletion.
//
// Parameters description:
// - Insn - current position in the tree
// - GEPs - GEP instructions for the current branch
// - Visited - a list of visited instructions in DFS order,
//   order is important for unused instruction deletion.
// - AllowPartial - when true GEP chains that can't be folded are
//   not reported, otherwise diagnostic message is show for such chains.
// - StillUsed - set to true if one of the GEP chains could not be
//   folded, makes sense when AllowPartial is false, means that root
//   preserve.static.offset call is still in use and should remain
//   until the next run of this pass.
static void rewriteAccessChain(Instruction *Insn,
                               SmallVector<GetElementPtrInst *> &GEPs,
                               SmallVector<Instruction *> &Visited,
                               bool AllowPatial, bool &StillUsed) {
  auto MarkAndTraverseUses = [&]() {
    Visited.push_back(Insn);
    rewriteUses(Insn, GEPs, Visited, AllowPatial, StillUsed);
  };
  auto TryToReplace = [&](Instruction *LoadOrStore) {
    // Do nothing for (preserve.static.offset (load/store ..)) or for
    // GEPs with zero indices. Such constructs lead to zero offset and
    // are simplified by other passes.
    if (allZeroIndices(GEPs))
      return;
    if (tryToReplaceWithGEPBuiltin(LoadOrStore, GEPs, Insn)) {
      Visited.push_back(Insn);
      return;
    }
    if (!AllowPatial)
      reportNonStaticGEPChain(Insn);
    StillUsed = true;
  };
  if (isa<LoadInst>(Insn) || isa<StoreInst>(Insn)) {
    TryToReplace(Insn);
  } else if (isGEPAndLoad(Insn)) {
    auto [GEP, Load] =
        BPFPreserveStaticOffsetPass::reconstructLoad(cast<CallInst>(Insn));
    GEPs.push_back(GEP);
    TryToReplace(Load);
    GEPs.pop_back();
    delete Load;
    delete GEP;
  } else if (isGEPAndStore(Insn)) {
    // This  case can't be merged with the above because
    // `delete Load` / `delete Store` wants a concrete type,
    // destructor of Instruction is protected.
    auto [GEP, Store] =
        BPFPreserveStaticOffsetPass::reconstructStore(cast<CallInst>(Insn));
    GEPs.push_back(GEP);
    TryToReplace(Store);
    GEPs.pop_back();
    delete Store;
    delete GEP;
  } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Insn)) {
    GEPs.push_back(GEP);
    MarkAndTraverseUses();
    GEPs.pop_back();
  } else if (isPreserveStaticOffsetCall(Insn)) {
    MarkAndTraverseUses();
  } else if (isInlineableCall(Insn)) {
    // Preserve preserve.static.offset call for parameters of
    // functions that might be inlined. These would be removed on a
    // second pass after inlining.
    // Might happen when a pointer to a preserve_static_offset
    // structure is passed as parameter of a function that would be
    // inlined inside a loop that would be unrolled.
    if (AllowPatial)
      StillUsed = true;
  } else {
    SmallString<128> Buf;
    raw_svector_ostream BufStream(Buf);
    BufStream << *Insn;
    report_fatal_error(
        Twine("Unexpected rewriteAccessChain Insn = ").concat(Buf));
  }
}

static void removeMarkerCall(Instruction *Marker) {
  Marker->replaceAllUsesWith(Marker->getOperand(0));
  Marker->eraseFromParent();
}

static bool rewriteAccessChain(Instruction *Marker, bool AllowPatial,
                               SmallPtrSetImpl<Instruction *> &RemovedMarkers) {
  SmallVector<GetElementPtrInst *> GEPs;
  SmallVector<Instruction *> Visited;
  bool StillUsed = false;
  rewriteUses(Marker, GEPs, Visited, AllowPatial, StillUsed);
  // Check if Visited instructions could be removed, iterate in
  // reverse to unblock instructions higher in the chain.
  for (auto V = Visited.rbegin(); V != Visited.rend(); ++V) {
    if (isPreserveStaticOffsetCall(*V)) {
      removeMarkerCall(*V);
      RemovedMarkers.insert(*V);
    } else if ((*V)->use_empty()) {
      (*V)->eraseFromParent();
    }
  }
  return StillUsed;
}

static std::vector<Instruction *>
collectPreserveStaticOffsetCalls(Function &F) {
  std::vector<Instruction *> Calls;
  for (Instruction &Insn : instructions(F))
    if (isPreserveStaticOffsetCall(&Insn))
      Calls.push_back(&Insn);
  return Calls;
}

bool isPreserveArrayIndex(Value *V) {
  return isIntrinsicCall(V, Intrinsic::preserve_array_access_index);
}

bool isPreserveStructIndex(Value *V) {
  return isIntrinsicCall(V, Intrinsic::preserve_struct_access_index);
}

bool isPreserveUnionIndex(Value *V) {
  return isIntrinsicCall(V, Intrinsic::preserve_union_access_index);
}

static void removePAICalls(Instruction *Marker) {
  auto IsPointerOperand = [](Value *Op, User *U) {
    if (auto *GEP = dyn_cast<GetElementPtrInst>(U))
      return GEP->getPointerOperand() == Op;
    if (isPreserveStaticOffsetCall(U) || isPreserveArrayIndex(U) ||
        isPreserveStructIndex(U) || isPreserveUnionIndex(U))
      return cast<CallInst>(U)->getArgOperand(0) == Op;
    return false;
  };

  SmallVector<Value *, 32> WorkList;
  WorkList.push_back(Marker);
  do {
    Value *V = WorkList.pop_back_val();
    for (User *U : V->users())
      if (IsPointerOperand(V, U))
        WorkList.push_back(U);
    auto *Call = dyn_cast<CallInst>(V);
    if (!Call)
      continue;
    if (isPreserveArrayIndex(V))
      BPFCoreSharedInfo::removeArrayAccessCall(Call);
    else if (isPreserveStructIndex(V))
      BPFCoreSharedInfo::removeStructAccessCall(Call);
    else if (isPreserveUnionIndex(V))
      BPFCoreSharedInfo::removeUnionAccessCall(Call);
  } while (!WorkList.empty());
}

// Look for sequences:
// - llvm.preserve.static.offset -> getelementptr... -> load
// - llvm.preserve.static.offset -> getelementptr... -> store
// And replace those with calls to intrinsics:
// - llvm.bpf.getelementptr.and.load
// - llvm.bpf.getelementptr.and.store
static bool rewriteFunction(Function &F, bool AllowPartial) {
  LLVM_DEBUG(dbgs() << "********** BPFPreserveStaticOffsetPass (AllowPartial="
                    << AllowPartial << ") ************\n");

  auto MarkerCalls = collectPreserveStaticOffsetCalls(F);
  SmallPtrSet<Instruction *, 16> RemovedMarkers;

  LLVM_DEBUG(dbgs() << "There are " << MarkerCalls.size()
                    << " preserve.static.offset calls\n");

  if (MarkerCalls.empty())
    return false;

  for (auto *Call : MarkerCalls)
    removePAICalls(Call);

  for (auto *Call : MarkerCalls) {
    if (RemovedMarkers.contains(Call))
      continue;
    bool StillUsed = rewriteAccessChain(Call, AllowPartial, RemovedMarkers);
    if (!StillUsed || !AllowPartial)
      removeMarkerCall(Call);
  }

  return true;
}

PreservedAnalyses
llvm::BPFPreserveStaticOffsetPass::run(Function &F,
                                       FunctionAnalysisManager &AM) {
  return rewriteFunction(F, AllowPartial) ? PreservedAnalyses::none()
                                          : PreservedAnalyses::all();
}