Path: blob/main/contrib/llvm-project/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
35294 views
//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//1//2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.3// See https://llvm.org/LICENSE.txt for license information.4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception5//6//===----------------------------------------------------------------------===//7//8// This pass abstracted struct/union member accesses in order to support9// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program10// which can run on different kernels. In particular, if bpf program tries to11// access a particular kernel data structure member, the details of the12// intermediate member access will be remembered so bpf loader can do13// necessary adjustment right before program loading.14//15// For example,16//17// struct s {18// int a;19// int b;20// };21// struct t {22// struct s c;23// int d;24// };25// struct t e;26//27// For the member access e.c.b, the compiler will generate code28// &e + 429//30// The compile-once run-everywhere instead generates the following code31// r = 432// &e + r33// The "4" in "r = 4" can be changed based on a particular kernel version.34// For example, on a particular kernel version, if struct s is changed to35//36// struct s {37// int new_field;38// int a;39// int b;40// }41//42// By repeating the member access on the host, the bpf loader can43// adjust "r = 4" as "r = 8".44//45// This feature relies on the following three intrinsic calls:46// addr = preserve_array_access_index(base, dimension, index)47// addr = preserve_union_access_index(base, di_index)48// !llvm.preserve.access.index <union_ditype>49// addr = preserve_struct_access_index(base, gep_index, di_index)50// !llvm.preserve.access.index <struct_ditype>51//52// Bitfield member access needs special attention. User cannot take the53// address of a bitfield acceess. To facilitate kernel verifier54// for easy bitfield code optimization, a new clang intrinsic is introduced:55// uint32_t __builtin_preserve_field_info(member_access, info_kind)56// In IR, a chain with two (or more) intrinsic calls will be generated:57// ...58// addr = preserve_struct_access_index(base, 1, 1) !struct s59// uint32_t result = bpf_preserve_field_info(addr, info_kind)60//61// Suppose the info_kind is FIELD_SIGNEDNESS,62// The above two IR intrinsics will be replaced with63// a relocatable insn:64// signness = /* signness of member_access */65// and signness can be changed by bpf loader based on the66// types on the host.67//68// User can also test whether a field exists or not with69// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)70// The field will be always available (result = 1) during initial71// compilation, but bpf loader can patch with the correct value72// on the target host where the member_access may or may not be available73//74//===----------------------------------------------------------------------===//7576#include "BPF.h"77#include "BPFCORE.h"78#include "BPFTargetMachine.h"79#include "llvm/BinaryFormat/Dwarf.h"80#include "llvm/DebugInfo/BTF/BTF.h"81#include "llvm/IR/DebugInfoMetadata.h"82#include "llvm/IR/GlobalVariable.h"83#include "llvm/IR/Instruction.h"84#include "llvm/IR/Instructions.h"85#include "llvm/IR/IntrinsicsBPF.h"86#include "llvm/IR/Module.h"87#include "llvm/IR/PassManager.h"88#include "llvm/IR/Type.h"89#include "llvm/IR/User.h"90#include "llvm/IR/Value.h"91#include "llvm/IR/ValueHandle.h"92#include "llvm/Pass.h"93#include "llvm/Transforms/Utils/BasicBlockUtils.h"94#include <stack>9596#define DEBUG_TYPE "bpf-abstract-member-access"9798namespace llvm {99constexpr StringRef BPFCoreSharedInfo::AmaAttr;100uint32_t BPFCoreSharedInfo::SeqNum;101102Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB,103Instruction *Input,104Instruction *Before) {105Function *Fn = Intrinsic::getDeclaration(106M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()});107Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()),108BPFCoreSharedInfo::SeqNum++);109110auto *NewInst = CallInst::Create(Fn, {SeqNumVal, Input});111NewInst->insertBefore(Before);112return NewInst;113}114} // namespace llvm115116using namespace llvm;117118namespace {119class BPFAbstractMemberAccess final {120public:121BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {}122123bool run(Function &F);124125struct CallInfo {126uint32_t Kind;127uint32_t AccessIndex;128MaybeAlign RecordAlignment;129MDNode *Metadata;130WeakTrackingVH Base;131};132typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack;133134private:135enum : uint32_t {136BPFPreserveArrayAI = 1,137BPFPreserveUnionAI = 2,138BPFPreserveStructAI = 3,139BPFPreserveFieldInfoAI = 4,140};141142TargetMachine *TM;143const DataLayout *DL = nullptr;144Module *M = nullptr;145146static std::map<std::string, GlobalVariable *> GEPGlobals;147// A map to link preserve_*_access_index intrinsic calls.148std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;149// A map to hold all the base preserve_*_access_index intrinsic calls.150// The base call is not an input of any other preserve_*151// intrinsics.152std::map<CallInst *, CallInfo> BaseAICalls;153// A map to hold <AnonRecord, TypeDef> relationships154std::map<DICompositeType *, DIDerivedType *> AnonRecords;155156void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty);157void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy);158void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy);159void ResetMetadata(struct CallInfo &CInfo);160161bool doTransformation(Function &F);162163void traceAICall(CallInst *Call, CallInfo &ParentInfo);164void traceBitCast(BitCastInst *BitCast, CallInst *Parent,165CallInfo &ParentInfo);166void traceGEP(GetElementPtrInst *GEP, CallInst *Parent,167CallInfo &ParentInfo);168void collectAICallChains(Function &F);169170bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo);171bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI,172const MDNode *ChildMeta);173bool removePreserveAccessIndexIntrinsic(Function &F);174bool HasPreserveFieldInfoCall(CallInfoStack &CallStack);175void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,176uint32_t &StartBitOffset, uint32_t &EndBitOffset);177uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,178uint32_t AccessIndex, uint32_t PatchImm,179MaybeAlign RecordAlignment);180181Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,182std::string &AccessKey, MDNode *&BaseMeta);183MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo,184std::string &AccessKey, bool &IsInt32Ret);185bool transformGEPChain(CallInst *Call, CallInfo &CInfo);186};187188std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals;189} // End anonymous namespace190191bool BPFAbstractMemberAccess::run(Function &F) {192LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");193194M = F.getParent();195if (!M)196return false;197198// Bail out if no debug info.199if (M->debug_compile_units().empty())200return false;201202// For each argument/return/local_variable type, trace the type203// pattern like '[derived_type]* [composite_type]' to check204// and remember (anon record -> typedef) relations where the205// anon record is defined as206// typedef [const/volatile/restrict]* [anon record]207DISubprogram *SP = F.getSubprogram();208if (SP && SP->isDefinition()) {209for (DIType *Ty: SP->getType()->getTypeArray())210CheckAnonRecordType(nullptr, Ty);211for (const DINode *DN : SP->getRetainedNodes()) {212if (const auto *DV = dyn_cast<DILocalVariable>(DN))213CheckAnonRecordType(nullptr, DV->getType());214}215}216217DL = &M->getDataLayout();218return doTransformation(F);219}220221void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {222if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) {223if (AnonRecords.find(Ty) != AnonRecords.end()) {224if (AnonRecords[Ty] != nullptr)225CInfo.Metadata = AnonRecords[Ty];226}227}228}229230void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,231DICompositeType *CTy) {232if (!CTy->getName().empty() || !ParentTy ||233ParentTy->getTag() != dwarf::DW_TAG_typedef)234return;235236if (AnonRecords.find(CTy) == AnonRecords.end()) {237AnonRecords[CTy] = ParentTy;238return;239}240241// Two or more typedef's may point to the same anon record.242// If this is the case, set the typedef DIType to be nullptr243// to indicate the duplication case.244DIDerivedType *CurrTy = AnonRecords[CTy];245if (CurrTy == ParentTy)246return;247AnonRecords[CTy] = nullptr;248}249250void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,251DIDerivedType *DTy) {252DIType *BaseType = DTy->getBaseType();253if (!BaseType)254return;255256unsigned Tag = DTy->getTag();257if (Tag == dwarf::DW_TAG_pointer_type)258CheckAnonRecordType(nullptr, BaseType);259else if (Tag == dwarf::DW_TAG_typedef)260CheckAnonRecordType(DTy, BaseType);261else262CheckAnonRecordType(ParentTy, BaseType);263}264265void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy,266DIType *Ty) {267if (!Ty)268return;269270if (auto *CTy = dyn_cast<DICompositeType>(Ty))271return CheckCompositeType(ParentTy, CTy);272else if (auto *DTy = dyn_cast<DIDerivedType>(Ty))273return CheckDerivedType(ParentTy, DTy);274}275276static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) {277if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&278Tag != dwarf::DW_TAG_volatile_type &&279Tag != dwarf::DW_TAG_restrict_type &&280Tag != dwarf::DW_TAG_member)281return false;282if (Tag == dwarf::DW_TAG_typedef && !skipTypedef)283return false;284return true;285}286287static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) {288while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {289if (!SkipDIDerivedTag(DTy->getTag(), skipTypedef))290break;291Ty = DTy->getBaseType();292}293return Ty;294}295296static const DIType * stripQualifiers(const DIType *Ty) {297while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {298if (!SkipDIDerivedTag(DTy->getTag(), true))299break;300Ty = DTy->getBaseType();301}302return Ty;303}304305static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) {306DINodeArray Elements = CTy->getElements();307uint32_t DimSize = 1;308for (uint32_t I = StartDim; I < Elements.size(); ++I) {309if (auto *Element = dyn_cast_or_null<DINode>(Elements[I]))310if (Element->getTag() == dwarf::DW_TAG_subrange_type) {311const DISubrange *SR = cast<DISubrange>(Element);312auto *CI = SR->getCount().dyn_cast<ConstantInt *>();313DimSize *= CI->getSExtValue();314}315}316317return DimSize;318}319320static Type *getBaseElementType(const CallInst *Call) {321// Element type is stored in an elementtype() attribute on the first param.322return Call->getParamElementType(0);323}324325static uint64_t getConstant(const Value *IndexValue) {326const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue);327assert(CV);328return CV->getValue().getZExtValue();329}330331/// Check whether a call is a preserve_*_access_index intrinsic call or not.332bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,333CallInfo &CInfo) {334if (!Call)335return false;336337const auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand());338if (!GV)339return false;340if (GV->getName().starts_with("llvm.preserve.array.access.index")) {341CInfo.Kind = BPFPreserveArrayAI;342CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);343if (!CInfo.Metadata)344report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic");345CInfo.AccessIndex = getConstant(Call->getArgOperand(2));346CInfo.Base = Call->getArgOperand(0);347CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call));348return true;349}350if (GV->getName().starts_with("llvm.preserve.union.access.index")) {351CInfo.Kind = BPFPreserveUnionAI;352CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);353if (!CInfo.Metadata)354report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic");355ResetMetadata(CInfo);356CInfo.AccessIndex = getConstant(Call->getArgOperand(1));357CInfo.Base = Call->getArgOperand(0);358return true;359}360if (GV->getName().starts_with("llvm.preserve.struct.access.index")) {361CInfo.Kind = BPFPreserveStructAI;362CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);363if (!CInfo.Metadata)364report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic");365ResetMetadata(CInfo);366CInfo.AccessIndex = getConstant(Call->getArgOperand(2));367CInfo.Base = Call->getArgOperand(0);368CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call));369return true;370}371if (GV->getName().starts_with("llvm.bpf.preserve.field.info")) {372CInfo.Kind = BPFPreserveFieldInfoAI;373CInfo.Metadata = nullptr;374// Check validity of info_kind as clang did not check this.375uint64_t InfoKind = getConstant(Call->getArgOperand(1));376if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND)377report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");378CInfo.AccessIndex = InfoKind;379return true;380}381if (GV->getName().starts_with("llvm.bpf.preserve.type.info")) {382CInfo.Kind = BPFPreserveFieldInfoAI;383CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);384if (!CInfo.Metadata)385report_fatal_error("Missing metadata for llvm.preserve.type.info intrinsic");386uint64_t Flag = getConstant(Call->getArgOperand(1));387if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG)388report_fatal_error("Incorrect flag for llvm.bpf.preserve.type.info intrinsic");389if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE)390CInfo.AccessIndex = BTF::TYPE_EXISTENCE;391else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH)392CInfo.AccessIndex = BTF::TYPE_MATCH;393else394CInfo.AccessIndex = BTF::TYPE_SIZE;395return true;396}397if (GV->getName().starts_with("llvm.bpf.preserve.enum.value")) {398CInfo.Kind = BPFPreserveFieldInfoAI;399CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);400if (!CInfo.Metadata)401report_fatal_error("Missing metadata for llvm.preserve.enum.value intrinsic");402uint64_t Flag = getConstant(Call->getArgOperand(2));403if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG)404report_fatal_error("Incorrect flag for llvm.bpf.preserve.enum.value intrinsic");405if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE)406CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE;407else408CInfo.AccessIndex = BTF::ENUM_VALUE;409return true;410}411412return false;413}414415static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,416uint32_t GEPIndex) {417uint32_t Dimension = 1;418if (DimensionIndex > 0)419Dimension = getConstant(Call->getArgOperand(DimensionIndex));420421Constant *Zero =422ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0);423SmallVector<Value *, 4> IdxList;424for (unsigned I = 0; I < Dimension; ++I)425IdxList.push_back(Zero);426IdxList.push_back(Call->getArgOperand(GEPIndex));427428auto *GEP = GetElementPtrInst::CreateInBounds(getBaseElementType(Call),429Call->getArgOperand(0), IdxList,430"", Call->getIterator());431Call->replaceAllUsesWith(GEP);432Call->eraseFromParent();433}434435void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) {436replaceWithGEP(Call, 1, 2);437}438439void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) {440replaceWithGEP(Call, 0, 1);441}442443void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) {444Call->replaceAllUsesWith(Call->getArgOperand(0));445Call->eraseFromParent();446}447448bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) {449std::vector<CallInst *> PreserveArrayIndexCalls;450std::vector<CallInst *> PreserveUnionIndexCalls;451std::vector<CallInst *> PreserveStructIndexCalls;452bool Found = false;453454for (auto &BB : F)455for (auto &I : BB) {456auto *Call = dyn_cast<CallInst>(&I);457CallInfo CInfo;458if (!IsPreserveDIAccessIndexCall(Call, CInfo))459continue;460461Found = true;462if (CInfo.Kind == BPFPreserveArrayAI)463PreserveArrayIndexCalls.push_back(Call);464else if (CInfo.Kind == BPFPreserveUnionAI)465PreserveUnionIndexCalls.push_back(Call);466else467PreserveStructIndexCalls.push_back(Call);468}469470// do the following transformation:471// . addr = preserve_array_access_index(base, dimension, index)472// is transformed to473// addr = GEP(base, dimenion's zero's, index)474// . addr = preserve_union_access_index(base, di_index)475// is transformed to476// addr = base, i.e., all usages of "addr" are replaced by "base".477// . addr = preserve_struct_access_index(base, gep_index, di_index)478// is transformed to479// addr = GEP(base, 0, gep_index)480for (CallInst *Call : PreserveArrayIndexCalls)481BPFCoreSharedInfo::removeArrayAccessCall(Call);482for (CallInst *Call : PreserveStructIndexCalls)483BPFCoreSharedInfo::removeStructAccessCall(Call);484for (CallInst *Call : PreserveUnionIndexCalls)485BPFCoreSharedInfo::removeUnionAccessCall(Call);486487return Found;488}489490/// Check whether the access index chain is valid. We check491/// here because there may be type casts between two492/// access indexes. We want to ensure memory access still valid.493bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,494uint32_t ParentAI,495const MDNode *ChildType) {496if (!ChildType)497return true; // preserve_field_info, no type comparison needed.498499const DIType *PType = stripQualifiers(cast<DIType>(ParentType));500const DIType *CType = stripQualifiers(cast<DIType>(ChildType));501502// Child is a derived/pointer type, which is due to type casting.503// Pointer type cannot be in the middle of chain.504if (isa<DIDerivedType>(CType))505return false;506507// Parent is a pointer type.508if (const auto *PtrTy = dyn_cast<DIDerivedType>(PType)) {509if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type)510return false;511return stripQualifiers(PtrTy->getBaseType()) == CType;512}513514// Otherwise, struct/union/array types515const auto *PTy = dyn_cast<DICompositeType>(PType);516const auto *CTy = dyn_cast<DICompositeType>(CType);517assert(PTy && CTy && "ParentType or ChildType is null or not composite");518519uint32_t PTyTag = PTy->getTag();520assert(PTyTag == dwarf::DW_TAG_array_type ||521PTyTag == dwarf::DW_TAG_structure_type ||522PTyTag == dwarf::DW_TAG_union_type);523524uint32_t CTyTag = CTy->getTag();525assert(CTyTag == dwarf::DW_TAG_array_type ||526CTyTag == dwarf::DW_TAG_structure_type ||527CTyTag == dwarf::DW_TAG_union_type);528529// Multi dimensional arrays, base element should be the same530if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag)531return PTy->getBaseType() == CTy->getBaseType();532533DIType *Ty;534if (PTyTag == dwarf::DW_TAG_array_type)535Ty = PTy->getBaseType();536else537Ty = dyn_cast<DIType>(PTy->getElements()[ParentAI]);538539return dyn_cast<DICompositeType>(stripQualifiers(Ty)) == CTy;540}541542void BPFAbstractMemberAccess::traceAICall(CallInst *Call,543CallInfo &ParentInfo) {544for (User *U : Call->users()) {545Instruction *Inst = dyn_cast<Instruction>(U);546if (!Inst)547continue;548549if (auto *BI = dyn_cast<BitCastInst>(Inst)) {550traceBitCast(BI, Call, ParentInfo);551} else if (auto *CI = dyn_cast<CallInst>(Inst)) {552CallInfo ChildInfo;553554if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&555IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,556ChildInfo.Metadata)) {557AIChain[CI] = std::make_pair(Call, ParentInfo);558traceAICall(CI, ChildInfo);559} else {560BaseAICalls[Call] = ParentInfo;561}562} else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {563if (GI->hasAllZeroIndices())564traceGEP(GI, Call, ParentInfo);565else566BaseAICalls[Call] = ParentInfo;567} else {568BaseAICalls[Call] = ParentInfo;569}570}571}572573void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,574CallInst *Parent,575CallInfo &ParentInfo) {576for (User *U : BitCast->users()) {577Instruction *Inst = dyn_cast<Instruction>(U);578if (!Inst)579continue;580581if (auto *BI = dyn_cast<BitCastInst>(Inst)) {582traceBitCast(BI, Parent, ParentInfo);583} else if (auto *CI = dyn_cast<CallInst>(Inst)) {584CallInfo ChildInfo;585if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&586IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,587ChildInfo.Metadata)) {588AIChain[CI] = std::make_pair(Parent, ParentInfo);589traceAICall(CI, ChildInfo);590} else {591BaseAICalls[Parent] = ParentInfo;592}593} else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {594if (GI->hasAllZeroIndices())595traceGEP(GI, Parent, ParentInfo);596else597BaseAICalls[Parent] = ParentInfo;598} else {599BaseAICalls[Parent] = ParentInfo;600}601}602}603604void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,605CallInfo &ParentInfo) {606for (User *U : GEP->users()) {607Instruction *Inst = dyn_cast<Instruction>(U);608if (!Inst)609continue;610611if (auto *BI = dyn_cast<BitCastInst>(Inst)) {612traceBitCast(BI, Parent, ParentInfo);613} else if (auto *CI = dyn_cast<CallInst>(Inst)) {614CallInfo ChildInfo;615if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&616IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,617ChildInfo.Metadata)) {618AIChain[CI] = std::make_pair(Parent, ParentInfo);619traceAICall(CI, ChildInfo);620} else {621BaseAICalls[Parent] = ParentInfo;622}623} else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {624if (GI->hasAllZeroIndices())625traceGEP(GI, Parent, ParentInfo);626else627BaseAICalls[Parent] = ParentInfo;628} else {629BaseAICalls[Parent] = ParentInfo;630}631}632}633634void BPFAbstractMemberAccess::collectAICallChains(Function &F) {635AIChain.clear();636BaseAICalls.clear();637638for (auto &BB : F)639for (auto &I : BB) {640CallInfo CInfo;641auto *Call = dyn_cast<CallInst>(&I);642if (!IsPreserveDIAccessIndexCall(Call, CInfo) ||643AIChain.find(Call) != AIChain.end())644continue;645646traceAICall(Call, CInfo);647}648}649650/// Get the start and the end of storage offset for \p MemberTy.651void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,652Align RecordAlignment,653uint32_t &StartBitOffset,654uint32_t &EndBitOffset) {655uint32_t MemberBitSize = MemberTy->getSizeInBits();656uint32_t MemberBitOffset = MemberTy->getOffsetInBits();657658if (RecordAlignment > 8) {659// If the Bits are within an aligned 8-byte, set the RecordAlignment660// to 8, other report the fatal error.661if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64)662report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "663"requiring too big alignment");664RecordAlignment = Align(8);665}666667uint32_t AlignBits = RecordAlignment.value() * 8;668if (MemberBitSize > AlignBits)669report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "670"bitfield size greater than record alignment");671672StartBitOffset = MemberBitOffset & ~(AlignBits - 1);673if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize))674report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "675"cross alignment boundary");676EndBitOffset = StartBitOffset + AlignBits;677}678679uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,680DICompositeType *CTy,681uint32_t AccessIndex,682uint32_t PatchImm,683MaybeAlign RecordAlignment) {684if (InfoKind == BTF::FIELD_EXISTENCE)685return 1;686687uint32_t Tag = CTy->getTag();688if (InfoKind == BTF::FIELD_BYTE_OFFSET) {689if (Tag == dwarf::DW_TAG_array_type) {690auto *EltTy = stripQualifiers(CTy->getBaseType());691PatchImm += AccessIndex * calcArraySize(CTy, 1) *692(EltTy->getSizeInBits() >> 3);693} else if (Tag == dwarf::DW_TAG_structure_type) {694auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);695if (!MemberTy->isBitField()) {696PatchImm += MemberTy->getOffsetInBits() >> 3;697} else {698unsigned SBitOffset, NextSBitOffset;699GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset,700NextSBitOffset);701PatchImm += SBitOffset >> 3;702}703}704return PatchImm;705}706707if (InfoKind == BTF::FIELD_BYTE_SIZE) {708if (Tag == dwarf::DW_TAG_array_type) {709auto *EltTy = stripQualifiers(CTy->getBaseType());710return calcArraySize(CTy, 1) * (EltTy->getSizeInBits() >> 3);711} else {712auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);713uint32_t SizeInBits = MemberTy->getSizeInBits();714if (!MemberTy->isBitField())715return SizeInBits >> 3;716717unsigned SBitOffset, NextSBitOffset;718GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset,719NextSBitOffset);720SizeInBits = NextSBitOffset - SBitOffset;721if (SizeInBits & (SizeInBits - 1))722report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info");723return SizeInBits >> 3;724}725}726727if (InfoKind == BTF::FIELD_SIGNEDNESS) {728const DIType *BaseTy;729if (Tag == dwarf::DW_TAG_array_type) {730// Signedness only checked when final array elements are accessed.731if (CTy->getElements().size() != 1)732report_fatal_error("Invalid array expression for llvm.bpf.preserve.field.info");733BaseTy = stripQualifiers(CTy->getBaseType());734} else {735auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);736BaseTy = stripQualifiers(MemberTy->getBaseType());737}738739// Only basic types and enum types have signedness.740const auto *BTy = dyn_cast<DIBasicType>(BaseTy);741while (!BTy) {742const auto *CompTy = dyn_cast<DICompositeType>(BaseTy);743// Report an error if the field expression does not have signedness.744if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)745report_fatal_error("Invalid field expression for llvm.bpf.preserve.field.info");746BaseTy = stripQualifiers(CompTy->getBaseType());747BTy = dyn_cast<DIBasicType>(BaseTy);748}749uint32_t Encoding = BTy->getEncoding();750return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);751}752753if (InfoKind == BTF::FIELD_LSHIFT_U64) {754// The value is loaded into a value with FIELD_BYTE_SIZE size,755// and then zero or sign extended to U64.756// FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations757// to extract the original value.758const Triple &Triple = TM->getTargetTriple();759DIDerivedType *MemberTy = nullptr;760bool IsBitField = false;761uint32_t SizeInBits;762763if (Tag == dwarf::DW_TAG_array_type) {764auto *EltTy = stripQualifiers(CTy->getBaseType());765SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits();766} else {767MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);768SizeInBits = MemberTy->getSizeInBits();769IsBitField = MemberTy->isBitField();770}771772if (!IsBitField) {773if (SizeInBits > 64)774report_fatal_error("too big field size for llvm.bpf.preserve.field.info");775return 64 - SizeInBits;776}777778unsigned SBitOffset, NextSBitOffset;779GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset);780if (NextSBitOffset - SBitOffset > 64)781report_fatal_error("too big field size for llvm.bpf.preserve.field.info");782783unsigned OffsetInBits = MemberTy->getOffsetInBits();784if (Triple.getArch() == Triple::bpfel)785return SBitOffset + 64 - OffsetInBits - SizeInBits;786else787return OffsetInBits + 64 - NextSBitOffset;788}789790if (InfoKind == BTF::FIELD_RSHIFT_U64) {791DIDerivedType *MemberTy = nullptr;792bool IsBitField = false;793uint32_t SizeInBits;794if (Tag == dwarf::DW_TAG_array_type) {795auto *EltTy = stripQualifiers(CTy->getBaseType());796SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits();797} else {798MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);799SizeInBits = MemberTy->getSizeInBits();800IsBitField = MemberTy->isBitField();801}802803if (!IsBitField) {804if (SizeInBits > 64)805report_fatal_error("too big field size for llvm.bpf.preserve.field.info");806return 64 - SizeInBits;807}808809unsigned SBitOffset, NextSBitOffset;810GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset);811if (NextSBitOffset - SBitOffset > 64)812report_fatal_error("too big field size for llvm.bpf.preserve.field.info");813814return 64 - SizeInBits;815}816817llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");818}819820bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) {821// This is called in error return path, no need to maintain CallStack.822while (CallStack.size()) {823auto StackElem = CallStack.top();824if (StackElem.second.Kind == BPFPreserveFieldInfoAI)825return true;826CallStack.pop();827}828return false;829}830831/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base832/// pointer of the first preserve_*_access_index call, and construct the access833/// string, which will be the name of a global variable.834Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,835CallInfo &CInfo,836std::string &AccessKey,837MDNode *&TypeMeta) {838Value *Base = nullptr;839std::string TypeName;840CallInfoStack CallStack;841842// Put the access chain into a stack with the top as the head of the chain.843while (Call) {844CallStack.push(std::make_pair(Call, CInfo));845CInfo = AIChain[Call].second;846Call = AIChain[Call].first;847}848849// The access offset from the base of the head of chain is also850// calculated here as all debuginfo types are available.851852// Get type name and calculate the first index.853// We only want to get type name from typedef, structure or union.854// If user wants a relocation like855// int *p; ... __builtin_preserve_access_index(&p[4]) ...856// or857// int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ...858// we will skip them.859uint32_t FirstIndex = 0;860uint32_t PatchImm = 0; // AccessOffset or the requested field info861uint32_t InfoKind = BTF::FIELD_BYTE_OFFSET;862while (CallStack.size()) {863auto StackElem = CallStack.top();864Call = StackElem.first;865CInfo = StackElem.second;866867if (!Base)868Base = CInfo.Base;869870DIType *PossibleTypeDef = stripQualifiers(cast<DIType>(CInfo.Metadata),871false);872DIType *Ty = stripQualifiers(PossibleTypeDef);873if (CInfo.Kind == BPFPreserveUnionAI ||874CInfo.Kind == BPFPreserveStructAI) {875// struct or union type. If the typedef is in the metadata, always876// use the typedef.877TypeName = std::string(PossibleTypeDef->getName());878TypeMeta = PossibleTypeDef;879PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3);880break;881}882883assert(CInfo.Kind == BPFPreserveArrayAI);884885// Array entries will always be consumed for accumulative initial index.886CallStack.pop();887888// BPFPreserveArrayAI889uint64_t AccessIndex = CInfo.AccessIndex;890891DIType *BaseTy = nullptr;892bool CheckElemType = false;893if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) {894// array type895assert(CTy->getTag() == dwarf::DW_TAG_array_type);896897898FirstIndex += AccessIndex * calcArraySize(CTy, 1);899BaseTy = stripQualifiers(CTy->getBaseType());900CheckElemType = CTy->getElements().size() == 1;901} else {902// pointer type903auto *DTy = cast<DIDerivedType>(Ty);904assert(DTy->getTag() == dwarf::DW_TAG_pointer_type);905906BaseTy = stripQualifiers(DTy->getBaseType());907CTy = dyn_cast<DICompositeType>(BaseTy);908if (!CTy) {909CheckElemType = true;910} else if (CTy->getTag() != dwarf::DW_TAG_array_type) {911FirstIndex += AccessIndex;912CheckElemType = true;913} else {914FirstIndex += AccessIndex * calcArraySize(CTy, 0);915}916}917918if (CheckElemType) {919auto *CTy = dyn_cast<DICompositeType>(BaseTy);920if (!CTy) {921if (HasPreserveFieldInfoCall(CallStack))922report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic");923return nullptr;924}925926unsigned CTag = CTy->getTag();927if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) {928TypeName = std::string(CTy->getName());929} else {930if (HasPreserveFieldInfoCall(CallStack))931report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic");932return nullptr;933}934TypeMeta = CTy;935PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3);936break;937}938}939assert(TypeName.size());940AccessKey += std::to_string(FirstIndex);941942// Traverse the rest of access chain to complete offset calculation943// and access key construction.944while (CallStack.size()) {945auto StackElem = CallStack.top();946CInfo = StackElem.second;947CallStack.pop();948949if (CInfo.Kind == BPFPreserveFieldInfoAI) {950InfoKind = CInfo.AccessIndex;951if (InfoKind == BTF::FIELD_EXISTENCE)952PatchImm = 1;953break;954}955956// If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,957// the action will be extracting field info.958if (CallStack.size()) {959auto StackElem2 = CallStack.top();960CallInfo CInfo2 = StackElem2.second;961if (CInfo2.Kind == BPFPreserveFieldInfoAI) {962InfoKind = CInfo2.AccessIndex;963assert(CallStack.size() == 1);964}965}966967// Access Index968uint64_t AccessIndex = CInfo.AccessIndex;969AccessKey += ":" + std::to_string(AccessIndex);970971MDNode *MDN = CInfo.Metadata;972// At this stage, it cannot be pointer type.973auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN)));974PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm,975CInfo.RecordAlignment);976}977978// Access key is the979// "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" +980// access string,981// uniquely identifying one relocation.982// The prefix "llvm." indicates this is a temporary global, which should983// not be emitted to ELF file.984AccessKey = "llvm." + TypeName + ":" + std::to_string(InfoKind) + ":" +985std::to_string(PatchImm) + "$" + AccessKey;986987return Base;988}989990MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call,991CallInfo &CInfo,992std::string &AccessKey,993bool &IsInt32Ret) {994DIType *Ty = stripQualifiers(cast<DIType>(CInfo.Metadata), false);995assert(!Ty->getName().empty());996997int64_t PatchImm;998std::string AccessStr("0");999if (CInfo.AccessIndex == BTF::TYPE_EXISTENCE ||1000CInfo.AccessIndex == BTF::TYPE_MATCH) {1001PatchImm = 1;1002} else if (CInfo.AccessIndex == BTF::TYPE_SIZE) {1003// typedef debuginfo type has size 0, get the eventual base type.1004DIType *BaseTy = stripQualifiers(Ty, true);1005PatchImm = BaseTy->getSizeInBits() / 8;1006} else {1007// ENUM_VALUE_EXISTENCE and ENUM_VALUE1008IsInt32Ret = false;10091010// The argument could be a global variable or a getelementptr with base to1011// a global variable depending on whether the clang option `opaque-options`1012// is set or not.1013const GlobalVariable *GV =1014cast<GlobalVariable>(Call->getArgOperand(1)->stripPointerCasts());1015assert(GV->hasInitializer());1016const ConstantDataArray *DA = cast<ConstantDataArray>(GV->getInitializer());1017assert(DA->isString());1018StringRef ValueStr = DA->getAsString();10191020// ValueStr format: <EnumeratorStr>:<Value>1021size_t Separator = ValueStr.find_first_of(':');1022StringRef EnumeratorStr = ValueStr.substr(0, Separator);10231024// Find enumerator index in the debuginfo1025DIType *BaseTy = stripQualifiers(Ty, true);1026const auto *CTy = cast<DICompositeType>(BaseTy);1027assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type);1028int EnumIndex = 0;1029for (const auto Element : CTy->getElements()) {1030const auto *Enum = cast<DIEnumerator>(Element);1031if (Enum->getName() == EnumeratorStr) {1032AccessStr = std::to_string(EnumIndex);1033break;1034}1035EnumIndex++;1036}10371038if (CInfo.AccessIndex == BTF::ENUM_VALUE) {1039StringRef EValueStr = ValueStr.substr(Separator + 1);1040PatchImm = std::stoll(std::string(EValueStr));1041} else {1042PatchImm = 1;1043}1044}10451046AccessKey = "llvm." + Ty->getName().str() + ":" +1047std::to_string(CInfo.AccessIndex) + std::string(":") +1048std::to_string(PatchImm) + std::string("$") + AccessStr;10491050return Ty;1051}10521053/// Call/Kind is the base preserve_*_access_index() call. Attempts to do1054/// transformation to a chain of relocable GEPs.1055bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,1056CallInfo &CInfo) {1057std::string AccessKey;1058MDNode *TypeMeta;1059Value *Base = nullptr;1060bool IsInt32Ret;10611062IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI;1063if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) {1064TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret);1065} else {1066Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta);1067if (!Base)1068return false;1069}10701071BasicBlock *BB = Call->getParent();1072GlobalVariable *GV;10731074if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) {1075IntegerType *VarType;1076if (IsInt32Ret)1077VarType = Type::getInt32Ty(BB->getContext()); // 32bit return value1078else1079VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value10801081GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage,1082nullptr, AccessKey);1083GV->addAttribute(BPFCoreSharedInfo::AmaAttr);1084GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta);1085GEPGlobals[AccessKey] = GV;1086} else {1087GV = GEPGlobals[AccessKey];1088}10891090if (CInfo.Kind == BPFPreserveFieldInfoAI) {1091// Load the global variable which represents the returned field info.1092LoadInst *LDInst;1093if (IsInt32Ret)1094LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "",1095Call->getIterator());1096else1097LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "",1098Call->getIterator());10991100Instruction *PassThroughInst =1101BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call);1102Call->replaceAllUsesWith(PassThroughInst);1103Call->eraseFromParent();1104return true;1105}11061107// For any original GEP Call and Base %2 like1108// %4 = bitcast %struct.net_device** %dev1 to i64*1109// it is transformed to:1110// %6 = load llvm.sk_buff:0:50$0:0:0:2:01111// %7 = bitcast %struct.sk_buff* %2 to i8*1112// %8 = getelementptr i8, i8* %7, %61113// %9 = bitcast i8* %8 to i64*1114// using %9 instead of %41115// The original Call inst is removed.11161117// Load the global variable.1118auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "",1119Call->getIterator());11201121// Generate a BitCast1122auto *BCInst =1123new BitCastInst(Base, PointerType::getUnqual(BB->getContext()));1124BCInst->insertBefore(Call);11251126// Generate a GetElementPtr1127auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()),1128BCInst, LDInst);1129GEP->insertBefore(Call);11301131// Generate a BitCast1132auto *BCInst2 = new BitCastInst(GEP, Call->getType());1133BCInst2->insertBefore(Call);11341135// For the following code,1136// Block0:1137// ...1138// if (...) goto Block1 else ...1139// Block1:1140// %6 = load llvm.sk_buff:0:50$0:0:0:2:01141// %7 = bitcast %struct.sk_buff* %2 to i8*1142// %8 = getelementptr i8, i8* %7, %61143// ...1144// goto CommonExit1145// Block2:1146// ...1147// if (...) goto Block3 else ...1148// Block3:1149// %6 = load llvm.bpf_map:0:40$0:0:0:2:01150// %7 = bitcast %struct.sk_buff* %2 to i8*1151// %8 = getelementptr i8, i8* %7, %61152// ...1153// goto CommonExit1154// CommonExit1155// SimplifyCFG may generate:1156// Block0:1157// ...1158// if (...) goto Block_Common else ...1159// Block2:1160// ...1161// if (...) goto Block_Common else ...1162// Block_Common:1163// PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0]1164// %6 = load PHI1165// %7 = bitcast %struct.sk_buff* %2 to i8*1166// %8 = getelementptr i8, i8* %7, %61167// ...1168// goto CommonExit1169// For the above code, we cannot perform proper relocation since1170// "load PHI" has two possible relocations.1171//1172// To prevent above tail merging, we use __builtin_bpf_passthrough()1173// where one of its parameters is a seq_num. Since two1174// __builtin_bpf_passthrough() funcs will always have different seq_num,1175// tail merging cannot happen. The __builtin_bpf_passthrough() will be1176// removed in the beginning of Target IR passes.1177//1178// This approach is also used in other places when global var1179// representing a relocation is used.1180Instruction *PassThroughInst =1181BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call);1182Call->replaceAllUsesWith(PassThroughInst);1183Call->eraseFromParent();11841185return true;1186}11871188bool BPFAbstractMemberAccess::doTransformation(Function &F) {1189bool Transformed = false;11901191// Collect PreserveDIAccessIndex Intrinsic call chains.1192// The call chains will be used to generate the access1193// patterns similar to GEP.1194collectAICallChains(F);11951196for (auto &C : BaseAICalls)1197Transformed = transformGEPChain(C.first, C.second) || Transformed;11981199return removePreserveAccessIndexIntrinsic(F) || Transformed;1200}12011202PreservedAnalyses1203BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) {1204return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none()1205: PreservedAnalyses::all();1206}120712081209