Path: blob/main/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
1693 views
//! This module defines Pulley-specific machine instruction types.12use core::marker::PhantomData;34use crate::binemit::{Addend, CodeOffset, Reloc};5use crate::ir::types::{self, F32, F64, I8, I8X16, I16, I32, I64, I128};6use crate::ir::{self, MemFlags, Type};7use crate::isa::FunctionAlignment;8use crate::isa::pulley_shared::abi::PulleyMachineDeps;9use crate::{CodegenError, CodegenResult, settings};10use crate::{machinst::*, trace};11use alloc::string::{String, ToString};12use regalloc2::RegClass;13use smallvec::SmallVec;1415pub mod regs;16pub use self::regs::*;17pub mod args;18pub use self::args::*;19pub mod emit;20pub use self::emit::*;2122//=============================================================================23// Instructions (top level): definition2425pub use crate::isa::pulley_shared::lower::isle::generated_code::MInst as Inst;26pub use crate::isa::pulley_shared::lower::isle::generated_code::RawInst;2728impl From<RawInst> for Inst {29fn from(raw: RawInst) -> Inst {30Inst::Raw { raw }31}32}3334use super::PulleyTargetKind;3536mod generated {37use super::*;38use crate::isa::pulley_shared::lower::isle::generated_code::RawInst;3940include!(concat!(env!("OUT_DIR"), "/pulley_inst_gen.rs"));41}4243/// Out-of-line data for return-calls, to keep the size of `Inst` down.44#[derive(Clone, Debug)]45pub struct ReturnCallInfo<T> {46/// Where this call is going.47pub dest: T,4849/// The size of the argument area for this return-call, potentially smaller50/// than that of the caller, but never larger.51pub new_stack_arg_size: u32,5253/// The in-register arguments and their constraints.54pub uses: CallArgList,55}5657impl Inst {58/// Generic constructor for a load (zero-extending where appropriate).59pub fn gen_load(dst: Writable<Reg>, mem: Amode, ty: Type, flags: MemFlags) -> Inst {60if ty.is_vector() {61assert_eq!(ty.bytes(), 16);62Inst::VLoad {63dst: dst.map(|r| VReg::new(r).unwrap()),64mem,65ty,66flags,67}68} else if ty.is_int() {69assert!(ty.bytes() <= 8);70Inst::XLoad {71dst: dst.map(|r| XReg::new(r).unwrap()),72mem,73ty,74flags,75}76} else {77Inst::FLoad {78dst: dst.map(|r| FReg::new(r).unwrap()),79mem,80ty,81flags,82}83}84}8586/// Generic constructor for a store.87pub fn gen_store(mem: Amode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {88if ty.is_vector() {89assert_eq!(ty.bytes(), 16);90Inst::VStore {91mem,92src: VReg::new(from_reg).unwrap(),93ty,94flags,95}96} else if ty.is_int() {97assert!(ty.bytes() <= 8);98Inst::XStore {99mem,100src: XReg::new(from_reg).unwrap(),101ty,102flags,103}104} else {105Inst::FStore {106mem,107src: FReg::new(from_reg).unwrap(),108ty,109flags,110}111}112}113}114115fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {116match inst {117Inst::Args { args } => {118for ArgPair { vreg, preg } in args {119collector.reg_fixed_def(vreg, *preg);120}121}122Inst::Rets { rets } => {123for RetPair { vreg, preg } in rets {124collector.reg_fixed_use(vreg, *preg);125}126}127128Inst::DummyUse { reg } => {129collector.reg_use(reg);130}131132Inst::Nop => {}133134Inst::TrapIf { cond, code: _ } => {135cond.get_operands(collector);136}137138Inst::GetSpecial { dst, reg } => {139collector.reg_def(dst);140// Note that this is explicitly ignored as this is only used for141// special registers that don't participate in register allocation142// such as the stack pointer, frame pointer, etc.143assert!(reg.is_special());144}145146Inst::LoadExtNameNear { dst, .. } | Inst::LoadExtNameFar { dst, .. } => {147collector.reg_def(dst);148}149150Inst::Call { info } => {151let CallInfo {152uses,153defs,154dest,155try_call_info,156clobbers,157..158} = &mut **info;159160// Pulley supports having the first few integer arguments in any161// register, so flag that with `reg_use` here.162let PulleyCall { args, .. } = dest;163for arg in args {164collector.reg_use(arg);165}166167// Remaining arguments (and return values) are all in fixed168// registers according to Pulley's ABI, however.169for CallArgPair { vreg, preg } in uses {170collector.reg_fixed_use(vreg, *preg);171}172for CallRetPair { vreg, location } in defs {173match location {174RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),175RetLocation::Stack(..) => collector.any_def(vreg),176}177}178collector.reg_clobbers(*clobbers);179if let Some(try_call_info) = try_call_info {180try_call_info.collect_operands(collector);181}182}183Inst::IndirectCallHost { info } => {184let CallInfo {185uses,186defs,187try_call_info,188clobbers,189..190} = &mut **info;191for CallArgPair { vreg, preg } in uses {192collector.reg_fixed_use(vreg, *preg);193}194for CallRetPair { vreg, location } in defs {195match location {196RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),197RetLocation::Stack(..) => collector.any_def(vreg),198}199}200collector.reg_clobbers(*clobbers);201if let Some(try_call_info) = try_call_info {202try_call_info.collect_operands(collector);203}204}205Inst::IndirectCall { info } => {206collector.reg_use(&mut info.dest);207let CallInfo {208uses,209defs,210try_call_info,211clobbers,212..213} = &mut **info;214for CallArgPair { vreg, preg } in uses {215collector.reg_fixed_use(vreg, *preg);216}217for CallRetPair { vreg, location } in defs {218match location {219RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),220RetLocation::Stack(..) => collector.any_def(vreg),221}222}223collector.reg_clobbers(*clobbers);224if let Some(try_call_info) = try_call_info {225try_call_info.collect_operands(collector);226}227}228Inst::ReturnCall { info } => {229for CallArgPair { vreg, preg } in &mut info.uses {230collector.reg_fixed_use(vreg, *preg);231}232}233Inst::ReturnIndirectCall { info } => {234// Use a fixed location of where to store the value to235// return-call-to. Using a fixed location prevents this register236// from being allocated to a callee-saved register which will get237// clobbered during the register restores just before the238// return-call.239//240// Also note that `x15` is specifically the last caller-saved241// register and, at this time, the only non-argument caller-saved242// register. This register allocation constraint is why it's not an243// argument register.244collector.reg_fixed_use(&mut info.dest, regs::x15());245246for CallArgPair { vreg, preg } in &mut info.uses {247collector.reg_fixed_use(vreg, *preg);248}249}250251Inst::Jump { .. } => {}252253Inst::BrIf {254cond,255taken: _,256not_taken: _,257} => {258cond.get_operands(collector);259}260261Inst::LoadAddr { dst, mem } => {262collector.reg_def(dst);263mem.get_operands(collector);264}265266Inst::XLoad {267dst,268mem,269ty: _,270flags: _,271} => {272collector.reg_def(dst);273mem.get_operands(collector);274}275276Inst::XStore {277mem,278src,279ty: _,280flags: _,281} => {282mem.get_operands(collector);283collector.reg_use(src);284}285286Inst::FLoad {287dst,288mem,289ty: _,290flags: _,291} => {292collector.reg_def(dst);293mem.get_operands(collector);294}295296Inst::FStore {297mem,298src,299ty: _,300flags: _,301} => {302mem.get_operands(collector);303collector.reg_use(src);304}305306Inst::VLoad {307dst,308mem,309ty: _,310flags: _,311} => {312collector.reg_def(dst);313mem.get_operands(collector);314}315316Inst::VStore {317mem,318src,319ty: _,320flags: _,321} => {322mem.get_operands(collector);323collector.reg_use(src);324}325326Inst::BrTable { idx, .. } => {327collector.reg_use(idx);328}329330Inst::Raw { raw } => generated::get_operands(raw, collector),331332Inst::EmitIsland { .. } => {}333334Inst::LabelAddress { dst, label: _ } => {335collector.reg_def(dst);336}337}338}339340/// A newtype over a Pulley instruction that also carries a phantom type341/// parameter describing whether we are targeting 32- or 64-bit Pulley bytecode.342///343/// Implements `Deref`, `DerefMut`, and `From`/`Into` for `Inst` to allow for344/// seamless conversion between `Inst` and `InstAndKind`.345#[derive(Clone, Debug)]346pub struct InstAndKind<P>347where348P: PulleyTargetKind,349{350inst: Inst,351kind: PhantomData<P>,352}353354impl<P> From<Inst> for InstAndKind<P>355where356P: PulleyTargetKind,357{358fn from(inst: Inst) -> Self {359Self {360inst,361kind: PhantomData,362}363}364}365366impl<P> From<RawInst> for InstAndKind<P>367where368P: PulleyTargetKind,369{370fn from(inst: RawInst) -> Self {371Self {372inst: inst.into(),373kind: PhantomData,374}375}376}377378impl<P> From<InstAndKind<P>> for Inst379where380P: PulleyTargetKind,381{382fn from(inst: InstAndKind<P>) -> Self {383inst.inst384}385}386387impl<P> core::ops::Deref for InstAndKind<P>388where389P: PulleyTargetKind,390{391type Target = Inst;392393fn deref(&self) -> &Self::Target {394&self.inst395}396}397398impl<P> core::ops::DerefMut for InstAndKind<P>399where400P: PulleyTargetKind,401{402fn deref_mut(&mut self) -> &mut Self::Target {403&mut self.inst404}405}406407impl<P> MachInst for InstAndKind<P>408where409P: PulleyTargetKind,410{411type LabelUse = LabelUse;412type ABIMachineSpec = PulleyMachineDeps<P>;413414const TRAP_OPCODE: &'static [u8] = TRAP_OPCODE;415416fn gen_dummy_use(reg: Reg) -> Self {417Inst::DummyUse { reg }.into()418}419420fn canonical_type_for_rc(rc: RegClass) -> Type {421match rc {422regalloc2::RegClass::Int => I64,423regalloc2::RegClass::Float => F64,424regalloc2::RegClass::Vector => I8X16,425}426}427428fn is_safepoint(&self) -> bool {429match self.inst {430Inst::Raw {431raw: RawInst::Trap { .. },432}433| Inst::Call { .. }434| Inst::IndirectCall { .. }435| Inst::IndirectCallHost { .. } => true,436_ => false,437}438}439440fn get_operands(&mut self, collector: &mut impl OperandVisitor) {441pulley_get_operands(self, collector);442}443444fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {445match self.inst {446Inst::Raw {447raw: RawInst::Xmov { dst, src },448} => Some((Writable::from_reg(*dst.to_reg()), *src)),449_ => None,450}451}452453fn is_included_in_clobbers(&self) -> bool {454!self.is_args()455}456457fn is_trap(&self) -> bool {458match self.inst {459Inst::Raw {460raw: RawInst::Trap { .. },461} => true,462_ => false,463}464}465466fn is_args(&self) -> bool {467match self.inst {468Inst::Args { .. } => true,469_ => false,470}471}472473fn is_term(&self) -> MachTerminator {474match &self.inst {475Inst::Raw {476raw: RawInst::Ret { .. },477}478| Inst::Rets { .. } => MachTerminator::Ret,479Inst::Jump { .. } => MachTerminator::Branch,480Inst::BrIf { .. } => MachTerminator::Branch,481Inst::BrTable { .. } => MachTerminator::Branch,482Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => MachTerminator::RetCall,483Inst::Call { info } if info.try_call_info.is_some() => MachTerminator::Branch,484Inst::IndirectCall { info } if info.try_call_info.is_some() => MachTerminator::Branch,485Inst::IndirectCallHost { info } if info.try_call_info.is_some() => {486MachTerminator::Branch487}488_ => MachTerminator::None,489}490}491492fn is_mem_access(&self) -> bool {493todo!()494}495496fn call_type(&self) -> CallType {497match &self.inst {498Inst::Call { .. } | Inst::IndirectCall { .. } | Inst::IndirectCallHost { .. } => {499CallType::Regular500}501502Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => CallType::TailCall,503504_ => CallType::None,505}506}507508fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self {509match ty {510ir::types::I8 | ir::types::I16 | ir::types::I32 | ir::types::I64 => RawInst::Xmov {511dst: WritableXReg::try_from(to_reg).unwrap(),512src: XReg::new(from_reg).unwrap(),513}514.into(),515ir::types::F32 | ir::types::F64 => RawInst::Fmov {516dst: WritableFReg::try_from(to_reg).unwrap(),517src: FReg::new(from_reg).unwrap(),518}519.into(),520_ if ty.is_vector() => RawInst::Vmov {521dst: WritableVReg::try_from(to_reg).unwrap(),522src: VReg::new(from_reg).unwrap(),523}524.into(),525_ => panic!("don't know how to generate a move for type {ty}"),526}527}528529fn gen_nop(_preferred_size: usize) -> Self {530todo!()531}532533fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {534match ty {535I8 => Ok((&[RegClass::Int], &[I8])),536I16 => Ok((&[RegClass::Int], &[I16])),537I32 => Ok((&[RegClass::Int], &[I32])),538I64 => Ok((&[RegClass::Int], &[I64])),539F32 => Ok((&[RegClass::Float], &[F32])),540F64 => Ok((&[RegClass::Float], &[F64])),541I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),542_ if ty.is_vector() => {543debug_assert!(ty.bits() <= 512);544545// Here we only need to return a SIMD type with the same size as `ty`.546// We use these types for spills and reloads, so prefer types with lanes <= 31547// since that fits in the immediate field of `vsetivli`.548const SIMD_TYPES: [[Type; 1]; 6] = [549[types::I8X2],550[types::I8X4],551[types::I8X8],552[types::I8X16],553[types::I16X16],554[types::I32X16],555];556let idx = (ty.bytes().ilog2() - 1) as usize;557let ty = &SIMD_TYPES[idx][..];558559Ok((&[RegClass::Vector], ty))560}561_ => Err(CodegenError::Unsupported(format!(562"Unexpected SSA-value type: {ty}"563))),564}565}566567fn gen_jump(label: MachLabel) -> Self {568Inst::Jump { label }.into()569}570571fn worst_case_size() -> CodeOffset {572// `VShuffle { dst, src1, src2, imm }` is 22 bytes:573// 3-byte opcode574// dst, src1, src2575// 16-byte immediate57622577}578579fn ref_type_regclass(_settings: &settings::Flags) -> RegClass {580RegClass::Int581}582583fn function_alignment() -> FunctionAlignment {584FunctionAlignment {585minimum: 1,586preferred: 1,587}588}589}590591const TRAP_OPCODE: &'static [u8] = &[592pulley_interpreter::opcode::Opcode::ExtendedOp as u8,593((pulley_interpreter::opcode::ExtendedOpcode::Trap as u16) >> 0) as u8,594((pulley_interpreter::opcode::ExtendedOpcode::Trap as u16) >> 8) as u8,595];596597#[test]598fn test_trap_encoding() {599let mut dst = std::vec::Vec::new();600pulley_interpreter::encode::trap(&mut dst);601assert_eq!(dst, TRAP_OPCODE);602}603604//=============================================================================605// Pretty-printing of instructions.606607pub fn reg_name(reg: Reg) -> String {608match reg.to_real_reg() {609Some(real) => {610let n = real.hw_enc();611match (real.class(), n) {612(RegClass::Int, 63) => format!("sp"),613(RegClass::Int, 62) => format!("lr"),614(RegClass::Int, 61) => format!("fp"),615(RegClass::Int, 60) => format!("tmp0"),616(RegClass::Int, 59) => format!("tmp1"),617618(RegClass::Int, _) => format!("x{n}"),619(RegClass::Float, _) => format!("f{n}"),620(RegClass::Vector, _) => format!("v{n}"),621}622}623None => {624format!("{reg:?}")625}626}627}628629fn pretty_print_try_call(info: &TryCallInfo) -> String {630format!(631"; jump {:?}; catch [{}]",632info.continuation,633info.pretty_print_dests()634)635}636637impl Inst {638fn print_with_state<P>(&self, _state: &mut EmitState<P>) -> String639where640P: PulleyTargetKind,641{642use core::fmt::Write;643644let format_reg = |reg: Reg| -> String { reg_name(reg) };645646match self {647Inst::Args { args } => {648let mut s = "args".to_string();649for arg in args {650let preg = format_reg(arg.preg);651let def = format_reg(arg.vreg.to_reg());652write!(&mut s, " {def}={preg}").unwrap();653}654s655}656Inst::Rets { rets } => {657let mut s = "rets".to_string();658for ret in rets {659let preg = format_reg(ret.preg);660let vreg = format_reg(ret.vreg);661write!(&mut s, " {vreg}={preg}").unwrap();662}663s664}665666Inst::DummyUse { reg } => {667let reg = format_reg(*reg);668format!("dummy_use {reg}")669}670671Inst::TrapIf { cond, code } => {672format!("trap_{cond} // code = {code:?}")673}674675Inst::Nop => format!("nop"),676677Inst::GetSpecial { dst, reg } => {678let dst = format_reg(*dst.to_reg());679let reg = format_reg(**reg);680format!("xmov {dst}, {reg}")681}682683Inst::LoadExtNameNear { dst, name, offset } => {684let dst = format_reg(*dst.to_reg());685format!("{dst} = load_ext_name_near {name:?}, {offset}")686}687688Inst::LoadExtNameFar { dst, name, offset } => {689let dst = format_reg(*dst.to_reg());690format!("{dst} = load_ext_name_far {name:?}, {offset}")691}692693Inst::Call { info } => {694let try_call = info695.try_call_info696.as_ref()697.map(|tci| pretty_print_try_call(tci))698.unwrap_or_default();699format!("call {info:?}{try_call}")700}701702Inst::IndirectCall { info } => {703let callee = format_reg(*info.dest);704let try_call = info705.try_call_info706.as_ref()707.map(|tci| pretty_print_try_call(tci))708.unwrap_or_default();709format!("indirect_call {callee}, {info:?}{try_call}")710}711712Inst::ReturnCall { info } => {713format!("return_call {info:?}")714}715716Inst::ReturnIndirectCall { info } => {717let callee = format_reg(*info.dest);718format!("return_indirect_call {callee}, {info:?}")719}720721Inst::IndirectCallHost { info } => {722let try_call = info723.try_call_info724.as_ref()725.map(|tci| pretty_print_try_call(tci))726.unwrap_or_default();727format!("indirect_call_host {info:?}{try_call}")728}729730Inst::Jump { label } => format!("jump {}", label.to_string()),731732Inst::BrIf {733cond,734taken,735not_taken,736} => {737let taken = taken.to_string();738let not_taken = not_taken.to_string();739format!("br_{cond}, {taken}; jump {not_taken}")740}741742Inst::LoadAddr { dst, mem } => {743let dst = format_reg(*dst.to_reg());744let mem = mem.to_string();745format!("{dst} = load_addr {mem}")746}747748Inst::XLoad {749dst,750mem,751ty,752flags,753} => {754let dst = format_reg(*dst.to_reg());755let ty = ty.bits();756let mem = mem.to_string();757format!("{dst} = xload{ty} {mem} // flags ={flags}")758}759760Inst::XStore {761mem,762src,763ty,764flags,765} => {766let ty = ty.bits();767let mem = mem.to_string();768let src = format_reg(**src);769format!("xstore{ty} {mem}, {src} // flags = {flags}")770}771772Inst::FLoad {773dst,774mem,775ty,776flags,777} => {778let dst = format_reg(*dst.to_reg());779let ty = ty.bits();780let mem = mem.to_string();781format!("{dst} = fload{ty} {mem} // flags ={flags}")782}783784Inst::FStore {785mem,786src,787ty,788flags,789} => {790let ty = ty.bits();791let mem = mem.to_string();792let src = format_reg(**src);793format!("fstore{ty} {mem}, {src} // flags = {flags}")794}795796Inst::VLoad {797dst,798mem,799ty,800flags,801} => {802let dst = format_reg(*dst.to_reg());803let ty = ty.bits();804let mem = mem.to_string();805format!("{dst} = vload{ty} {mem} // flags ={flags}")806}807808Inst::VStore {809mem,810src,811ty,812flags,813} => {814let ty = ty.bits();815let mem = mem.to_string();816let src = format_reg(**src);817format!("vstore{ty} {mem}, {src} // flags = {flags}")818}819820Inst::BrTable {821idx,822default,823targets,824} => {825let idx = format_reg(**idx);826format!("br_table {idx} {default:?} {targets:?}")827}828Inst::Raw { raw } => generated::print(raw),829830Inst::EmitIsland { space_needed } => format!("emit_island {space_needed}"),831832Inst::LabelAddress { dst, label } => {833let dst = format_reg(dst.to_reg().to_reg());834format!("label_address {dst}, {label:?}")835}836}837}838}839840/// Different forms of label references for different instruction formats.841#[derive(Clone, Copy, Debug, PartialEq, Eq)]842pub enum LabelUse {843/// A PC-relative `jump`/`call`/etc... instruction with an `i32` relative844/// target.845///846/// The relative distance to the destination is added to the 4 bytes at the847/// label site.848PcRel,849}850851impl MachInstLabelUse for LabelUse {852/// Alignment for veneer code. Pulley instructions don't require any853/// particular alignment.854const ALIGN: CodeOffset = 1;855856/// Maximum PC-relative range (positive), inclusive.857fn max_pos_range(self) -> CodeOffset {858match self {859Self::PcRel => 0x7fff_ffff,860}861}862863/// Maximum PC-relative range (negative).864fn max_neg_range(self) -> CodeOffset {865match self {866Self::PcRel => 0x8000_0000,867}868}869870/// Size of window into code needed to do the patch.871fn patch_size(self) -> CodeOffset {872match self {873Self::PcRel => 4,874}875}876877/// Perform the patch.878fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {879let use_relative = (label_offset as i64) - (use_offset as i64);880debug_assert!(use_relative <= self.max_pos_range() as i64);881debug_assert!(use_relative >= -(self.max_neg_range() as i64));882let pc_rel = i32::try_from(use_relative).unwrap() as u32;883match self {884Self::PcRel => {885let buf: &mut [u8; 4] = buffer.try_into().unwrap();886let addend = u32::from_le_bytes(*buf);887trace!(888"patching label use @ {use_offset:#x} \889to label {label_offset:#x} via \890PC-relative offset {pc_rel:#x} \891adding in {addend:#x}"892);893let value = pc_rel.wrapping_add(addend);894*buf = value.to_le_bytes();895}896}897}898899/// Is a veneer supported for this label reference type?900fn supports_veneer(self) -> bool {901match self {902Self::PcRel => false,903}904}905906/// How large is the veneer, if supported?907fn veneer_size(self) -> CodeOffset {908match self {909Self::PcRel => 0,910}911}912913fn worst_case_veneer_size() -> CodeOffset {9140915}916917/// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return918/// an offset and label-use for the veneer's use of the original label.919fn generate_veneer(920self,921_buffer: &mut [u8],922_veneer_offset: CodeOffset,923) -> (CodeOffset, LabelUse) {924match self {925Self::PcRel => panic!("veneer not supported for {self:?}"),926}927}928929fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {930match (reloc, addend) {931(Reloc::PulleyPcRel, 0) => Some(LabelUse::PcRel),932_ => None,933}934}935}936937938