Path: blob/main/crates/cranelift/src/compiler/component.rs
1692 views
//! Compilation support for the component model.12use crate::{3TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT,4compiler::{Abi, Compiler},5};6use anyhow::Result;7use cranelift_codegen::ir::condcodes::IntCC;8use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};9use cranelift_codegen::isa::{CallConv, TargetIsa};10use cranelift_frontend::FunctionBuilder;11use wasmtime_environ::{CompiledFunctionBody, component::*};12use wasmtime_environ::{13EntityRef, HostCall, ModuleInternedTypeIndex, PtrSize, TrapSentinel, Tunables, WasmFuncType,14WasmValType,15};16use wasmtime_environ::{FuncKey, fact::PREPARE_CALL_FIXED_PARAMS};1718struct TrampolineCompiler<'a> {19compiler: &'a Compiler,20isa: &'a (dyn TargetIsa + 'static),21builder: FunctionBuilder<'a>,22component: &'a Component,23types: &'a ComponentTypesBuilder,24offsets: VMComponentOffsets<u8>,25abi: Abi,26block0: ir::Block,27signature: ModuleInternedTypeIndex,28tunables: &'a Tunables,29}3031/// What host functions can be called, used in `translate_hostcall` below.32enum HostCallee {33/// Call a host-lowered function specified by this index.34Lowering(LoweredIndex),35/// Call a host libcall, specified by this accessor.36Libcall(GetLibcallFn),37}3839type GetLibcallFn =40fn(&dyn TargetIsa, &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex);4142impl From<LoweredIndex> for HostCallee {43fn from(index: LoweredIndex) -> HostCallee {44HostCallee::Lowering(index)45}46}4748impl From<GetLibcallFn> for HostCallee {49fn from(f: GetLibcallFn) -> HostCallee {50HostCallee::Libcall(f)51}52}5354/// How to interpret the results of a host function.55enum HostResult {56/// The host function has no results.57None,5859/// The host function returns the sentinel specified which is interpreted60/// and translated to the real return value.61Sentinel(TrapSentinel),6263/// The host function returns a `bool` indicating whether it succeeded or64/// not.65///66/// After the return value is interpreted the host function also filled in67/// `ptr` and `len` with wasm return values which need to be returned.68///69/// If `ptr` and `len` are not specified then this must be used with70/// `WasmArgs::ValRawList` and that ptr/len is used.71MultiValue {72/// The base pointer of the `ValRaw` list on the stack.73ptr: Option<ir::Value>,74/// The length of the `ValRaw` list on the stack.75len: Option<ir::Value>,76},77}7879impl From<TrapSentinel> for HostResult {80fn from(sentinel: TrapSentinel) -> HostResult {81HostResult::Sentinel(sentinel)82}83}8485/// Different means of passing WebAssembly arguments to host calls.86#[derive(Debug, Copy, Clone)]87enum WasmArgs {88/// All wasm arguments to the host are passed directly as values, typically89/// through registers.90InRegisters,9192/// All wasm arguments to the host are passed indirectly by spilling them93/// to the stack as a sequence of contiguous `ValRaw`s.94ValRawList,9596/// The first `n` arguments are passed in registers, but everything after97/// that is spilled to the stack.98InRegistersUpTo(usize),99}100101impl<'a> TrampolineCompiler<'a> {102fn new(103compiler: &'a Compiler,104func_compiler: &'a mut super::FunctionCompiler<'_>,105component: &'a Component,106types: &'a ComponentTypesBuilder,107index: TrampolineIndex,108abi: Abi,109tunables: &'a Tunables,110) -> TrampolineCompiler<'a> {111let isa = &*compiler.isa;112let signature = component.trampolines[index];113let ty = types[signature].unwrap_func();114let func = ir::Function::with_name_signature(115ir::UserFuncName::user(0, 0),116match abi {117Abi::Wasm => crate::wasm_call_signature(isa, ty, &compiler.tunables),118Abi::Array => crate::array_call_signature(isa),119},120);121let (builder, block0) = func_compiler.builder(func);122TrampolineCompiler {123compiler,124isa,125builder,126component,127types,128offsets: VMComponentOffsets::new(isa.pointer_bytes(), component),129abi,130block0,131signature,132tunables,133}134}135136fn translate(&mut self, trampoline: &Trampoline) {137match trampoline {138Trampoline::Transcoder {139op,140from,141from64,142to,143to64,144} => {145match self.abi {146Abi::Wasm => {147self.translate_transcode(*op, *from, *from64, *to, *to64);148}149// Transcoders can only actually be called by Wasm, so let's assert150// that here.151Abi::Array => {152self.builder.ins().trap(TRAP_INTERNAL_ASSERT);153}154}155}156Trampoline::LowerImport {157index,158options,159lower_ty,160} => {161let pointer_type = self.isa.pointer_type();162self.translate_hostcall(163HostCallee::Lowering(*index),164HostResult::MultiValue {165ptr: None,166len: None,167},168WasmArgs::ValRawList,169|me, params| {170let vmctx = params[0];171params.extend([172me.builder.ins().load(173pointer_type,174MemFlags::trusted(),175vmctx,176i32::try_from(me.offsets.lowering_data(*index)).unwrap(),177),178me.index_value(*lower_ty),179me.index_value(*options),180]);181},182);183}184Trampoline::AlwaysTrap => {185if self.tunables.signals_based_traps {186self.builder.ins().trap(TRAP_ALWAYS);187return;188}189self.translate_libcall(190host::trap,191TrapSentinel::Falsy,192WasmArgs::InRegisters,193|me, params| {194let code = wasmtime_environ::Trap::AlwaysTrapAdapter as u8;195params.push(me.builder.ins().iconst(ir::types::I8, i64::from(code)));196},197);198}199Trampoline::ResourceNew(ty) => {200// Currently this only supports resources represented by `i32`201assert_eq!(202self.types[self.signature].unwrap_func().params()[0],203WasmValType::I32204);205self.translate_libcall(206host::resource_new32,207TrapSentinel::NegativeOne,208WasmArgs::InRegisters,209|me, params| {210params.push(me.index_value(*ty));211},212);213}214Trampoline::ResourceRep(ty) => {215// Currently this only supports resources represented by `i32`216assert_eq!(217self.types[self.signature].unwrap_func().returns()[0],218WasmValType::I32219);220self.translate_libcall(221host::resource_rep32,222TrapSentinel::NegativeOne,223WasmArgs::InRegisters,224|me, params| {225params.push(me.index_value(*ty));226},227);228}229Trampoline::ResourceDrop(ty) => {230self.translate_resource_drop(*ty);231}232Trampoline::BackpressureSet { instance } => {233self.translate_libcall(234host::backpressure_set,235TrapSentinel::Falsy,236WasmArgs::InRegisters,237|me, params| {238params.push(me.index_value(*instance));239},240);241}242Trampoline::TaskReturn { results, options } => {243self.translate_libcall(244host::task_return,245TrapSentinel::Falsy,246WasmArgs::ValRawList,247|me, params| {248params.push(me.index_value(*results));249params.push(me.index_value(*options));250},251);252}253Trampoline::TaskCancel { instance } => {254self.translate_libcall(255host::task_cancel,256TrapSentinel::Falsy,257WasmArgs::InRegisters,258|me, params| {259params.push(me.index_value(*instance));260},261);262}263Trampoline::WaitableSetNew { instance } => {264self.translate_libcall(265host::waitable_set_new,266TrapSentinel::NegativeOne,267WasmArgs::InRegisters,268|me, params| {269params.push(me.index_value(*instance));270},271);272}273Trampoline::WaitableSetWait { options } => {274self.translate_libcall(275host::waitable_set_wait,276TrapSentinel::NegativeOne,277WasmArgs::InRegisters,278|me, params| {279params.push(me.index_value(*options));280},281);282}283Trampoline::WaitableSetPoll { options } => {284self.translate_libcall(285host::waitable_set_poll,286TrapSentinel::NegativeOne,287WasmArgs::InRegisters,288|me, params| {289params.push(me.index_value(*options));290},291);292}293Trampoline::WaitableSetDrop { instance } => {294self.translate_libcall(295host::waitable_set_drop,296TrapSentinel::Falsy,297WasmArgs::InRegisters,298|me, params| {299params.push(me.index_value(*instance));300},301);302}303Trampoline::WaitableJoin { instance } => {304self.translate_libcall(305host::waitable_join,306TrapSentinel::Falsy,307WasmArgs::InRegisters,308|me, params| {309params.push(me.index_value(*instance));310},311);312}313Trampoline::Yield { async_ } => {314self.translate_libcall(315host::yield_,316TrapSentinel::NegativeOne,317WasmArgs::InRegisters,318|me, params| {319params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));320},321);322}323Trampoline::SubtaskDrop { instance } => {324self.translate_libcall(325host::subtask_drop,326TrapSentinel::Falsy,327WasmArgs::InRegisters,328|me, params| {329params.push(me.index_value(*instance));330},331);332}333Trampoline::SubtaskCancel { instance, async_ } => {334self.translate_libcall(335host::subtask_cancel,336TrapSentinel::NegativeOne,337WasmArgs::InRegisters,338|me, params| {339params.push(me.index_value(*instance));340params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));341},342);343}344Trampoline::StreamNew { ty } => {345self.translate_libcall(346host::stream_new,347TrapSentinel::NegativeOne,348WasmArgs::InRegisters,349|me, params| {350params.push(me.index_value(*ty));351},352);353}354Trampoline::StreamRead { ty, options } => {355if let Some(info) = self.flat_stream_element_info(*ty).cloned() {356self.translate_libcall(357host::flat_stream_read,358TrapSentinel::NegativeOne,359WasmArgs::InRegisters,360|me, params| {361params.extend([362me.index_value(*ty),363me.index_value(*options),364me.builder365.ins()366.iconst(ir::types::I32, i64::from(info.size32)),367me.builder368.ins()369.iconst(ir::types::I32, i64::from(info.align32)),370]);371},372);373} else {374self.translate_libcall(375host::stream_read,376TrapSentinel::NegativeOne,377WasmArgs::InRegisters,378|me, params| {379params.push(me.index_value(*ty));380params.push(me.index_value(*options));381},382);383}384}385Trampoline::StreamWrite { ty, options } => {386if let Some(info) = self.flat_stream_element_info(*ty).cloned() {387self.translate_libcall(388host::flat_stream_write,389TrapSentinel::NegativeOne,390WasmArgs::InRegisters,391|me, params| {392params.extend([393me.index_value(*ty),394me.index_value(*options),395me.builder396.ins()397.iconst(ir::types::I32, i64::from(info.size32)),398me.builder399.ins()400.iconst(ir::types::I32, i64::from(info.align32)),401]);402},403);404} else {405self.translate_libcall(406host::stream_write,407TrapSentinel::NegativeOne,408WasmArgs::InRegisters,409|me, params| {410params.push(me.index_value(*ty));411params.push(me.index_value(*options));412},413);414}415}416Trampoline::StreamCancelRead { ty, async_ } => {417self.translate_libcall(418host::stream_cancel_read,419TrapSentinel::NegativeOne,420WasmArgs::InRegisters,421|me, params| {422params.push(me.index_value(*ty));423params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));424},425);426}427Trampoline::StreamCancelWrite { ty, async_ } => {428self.translate_libcall(429host::stream_cancel_write,430TrapSentinel::NegativeOne,431WasmArgs::InRegisters,432|me, params| {433params.push(me.index_value(*ty));434params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));435},436);437}438Trampoline::StreamDropReadable { ty } => {439self.translate_libcall(440host::stream_drop_readable,441TrapSentinel::Falsy,442WasmArgs::InRegisters,443|me, params| {444params.push(me.index_value(*ty));445},446);447}448Trampoline::StreamDropWritable { ty } => {449self.translate_libcall(450host::stream_drop_writable,451TrapSentinel::Falsy,452WasmArgs::InRegisters,453|me, params| {454params.push(me.index_value(*ty));455},456);457}458Trampoline::FutureNew { ty } => {459self.translate_libcall(460host::future_new,461TrapSentinel::NegativeOne,462WasmArgs::InRegisters,463|me, params| {464params.push(me.index_value(*ty));465},466);467}468Trampoline::FutureRead { ty, options } => {469self.translate_libcall(470host::future_read,471TrapSentinel::NegativeOne,472WasmArgs::InRegisters,473|me, params| {474params.push(me.index_value(*ty));475params.push(me.index_value(*options));476},477);478}479Trampoline::FutureWrite { ty, options } => {480self.translate_libcall(481host::future_write,482TrapSentinel::NegativeOne,483WasmArgs::InRegisters,484|me, params| {485params.push(me.index_value(*ty));486params.push(me.index_value(*options));487},488);489}490Trampoline::FutureCancelRead { ty, async_ } => {491self.translate_libcall(492host::future_cancel_read,493TrapSentinel::NegativeOne,494WasmArgs::InRegisters,495|me, params| {496params.push(me.index_value(*ty));497params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));498},499);500}501Trampoline::FutureCancelWrite { ty, async_ } => {502self.translate_libcall(503host::future_cancel_write,504TrapSentinel::NegativeOne,505WasmArgs::InRegisters,506|me, params| {507params.push(me.index_value(*ty));508params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));509},510);511}512Trampoline::FutureDropReadable { ty } => {513self.translate_libcall(514host::future_drop_readable,515TrapSentinel::Falsy,516WasmArgs::InRegisters,517|me, params| {518params.push(me.index_value(*ty));519},520);521}522Trampoline::FutureDropWritable { ty } => {523self.translate_libcall(524host::future_drop_writable,525TrapSentinel::Falsy,526WasmArgs::InRegisters,527|me, params| {528params.push(me.index_value(*ty));529},530);531}532Trampoline::ErrorContextNew { ty, options } => {533self.translate_libcall(534host::error_context_new,535TrapSentinel::NegativeOne,536WasmArgs::InRegisters,537|me, params| {538params.push(me.index_value(*ty));539params.push(me.index_value(*options));540},541);542}543Trampoline::ErrorContextDebugMessage { ty, options } => {544self.translate_libcall(545host::error_context_debug_message,546TrapSentinel::Falsy,547WasmArgs::InRegisters,548|me, params| {549params.push(me.index_value(*ty));550params.push(me.index_value(*options));551},552);553}554Trampoline::ErrorContextDrop { ty } => {555self.translate_libcall(556host::error_context_drop,557TrapSentinel::Falsy,558WasmArgs::InRegisters,559|me, params| {560params.push(me.index_value(*ty));561},562);563}564Trampoline::ResourceTransferOwn => {565self.translate_libcall(566host::resource_transfer_own,567TrapSentinel::NegativeOne,568WasmArgs::InRegisters,569|_, _| {},570);571}572Trampoline::ResourceTransferBorrow => {573self.translate_libcall(574host::resource_transfer_borrow,575TrapSentinel::NegativeOne,576WasmArgs::InRegisters,577|_, _| {},578);579}580Trampoline::ResourceEnterCall => {581self.translate_libcall(582host::resource_enter_call,583HostResult::None,584WasmArgs::InRegisters,585|_, _| {},586);587}588Trampoline::ResourceExitCall => {589self.translate_libcall(590host::resource_exit_call,591TrapSentinel::Falsy,592WasmArgs::InRegisters,593|_, _| {},594);595}596Trampoline::PrepareCall { memory } => {597self.translate_libcall(598host::prepare_call,599TrapSentinel::Falsy,600WasmArgs::InRegistersUpTo(PREPARE_CALL_FIXED_PARAMS.len()),601|me, params| {602let vmctx = params[0];603params.push(me.load_optional_memory(vmctx, *memory));604},605);606}607Trampoline::SyncStartCall { callback } => {608let pointer_type = self.isa.pointer_type();609let wasm_func_ty = &self.types[self.signature].unwrap_func();610let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(611&WasmFuncType::new(612Box::new([]),613wasm_func_ty.returns().iter().copied().collect(),614),615&mut self.builder,616&[],617);618let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len));619self.translate_libcall(620host::sync_start,621HostResult::MultiValue {622ptr: Some(values_vec_ptr),623len: Some(values_vec_len),624},625WasmArgs::InRegisters,626|me, params| {627let vmctx = params[0];628params.push(me.load_callback(vmctx, *callback));629params.push(values_vec_ptr);630params.push(values_vec_len);631},632);633}634Trampoline::AsyncStartCall {635callback,636post_return,637} => {638self.translate_libcall(639host::async_start,640TrapSentinel::NegativeOne,641WasmArgs::InRegisters,642|me, params| {643let vmctx = params[0];644params.extend([645me.load_callback(vmctx, *callback),646me.load_post_return(vmctx, *post_return),647]);648},649);650}651Trampoline::FutureTransfer => {652self.translate_libcall(653host::future_transfer,654TrapSentinel::NegativeOne,655WasmArgs::InRegisters,656|_, _| {},657);658}659Trampoline::StreamTransfer => {660self.translate_libcall(661host::stream_transfer,662TrapSentinel::NegativeOne,663WasmArgs::InRegisters,664|_, _| {},665);666}667Trampoline::ErrorContextTransfer => {668self.translate_libcall(669host::error_context_transfer,670TrapSentinel::NegativeOne,671WasmArgs::InRegisters,672|_, _| {},673);674}675Trampoline::ContextGet(i) => {676self.translate_libcall(677host::context_get,678TrapSentinel::NegativeOne,679WasmArgs::InRegisters,680|me, params| {681params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*i)));682},683);684}685Trampoline::ContextSet(i) => {686self.translate_libcall(687host::context_set,688TrapSentinel::Falsy,689WasmArgs::InRegisters,690|me, params| {691params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*i)));692},693);694}695}696}697698/// Determine whether the specified type can be optimized as a stream699/// payload by lifting and lowering with a simple `memcpy`.700///701/// Any type containing only "flat", primitive data (i.e. no pointers or702/// handles) should qualify for this optimization, but it's also okay to703/// conservatively return `None` here; the fallback slow path will always704/// work -- it just won't be as efficient.705fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option<&CanonicalAbiInfo> {706let payload = self.types[self.types[ty].ty].payload;707match payload {708None => Some(&CanonicalAbiInfo::ZERO),709Some(710payload @ (InterfaceType::Bool711| InterfaceType::S8712| InterfaceType::U8713| InterfaceType::S16714| InterfaceType::U16715| InterfaceType::S32716| InterfaceType::U32717| InterfaceType::S64718| InterfaceType::U64719| InterfaceType::Float32720| InterfaceType::Float64721| InterfaceType::Char),722) => Some(self.types.canonical_abi(&payload)),723// TODO: Recursively check for other "flat" types (i.e. those without pointers or handles),724// e.g. `record`s, `variant`s, etc. which contain only flat types.725_ => None,726}727}728729/// Helper function to spill the wasm arguments `args` to this function into730/// a stack-allocated array.731fn store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value) {732let pointer_type = self.isa.pointer_type();733let wasm_func_ty = &self.types[self.signature].unwrap_func();734735match self.abi {736// For the wasm ABI a stack needs to be allocated and these737// arguments are stored onto the stack.738Abi::Wasm => {739let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(740wasm_func_ty,741&mut self.builder,742args,743);744let len = self.builder.ins().iconst(pointer_type, i64::from(len));745(ptr, len)746}747748// For the array ABI all arguments were already in a stack, so749// forward along that pointer/len.750Abi::Array => {751let params = self.builder.func.dfg.block_params(self.block0);752(params[2], params[3])753}754}755}756757/// Convenience wrapper around `translate_hostcall` to enable type inference758/// on the `get_libcall` parameter here.759fn translate_libcall(760&mut self,761get_libcall: GetLibcallFn,762host_result: impl Into<HostResult>,763wasm_args: WasmArgs,764extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>),765) {766self.translate_hostcall(767HostCallee::Libcall(get_libcall),768host_result.into(),769wasm_args,770extra_host_args,771)772}773774/// Translates an invocation of a host function and interpret the result.775///776/// This is intended to be a relatively narrow waist which most intrinsics777/// go through. The configuration supported here is:778///779/// * `host_callee` - what's being called, either a libcall or a lowered780/// function781/// * `host_result` - how to interpret the return value to see if it's a782/// trap783/// * `wasm_args` - how to pass wasm args to the host, either in registers784/// or on the stack785/// * `extra_host_args` - a closure used to push extra arguments just before786/// the wasm arguments are forwarded.787fn translate_hostcall(788&mut self,789host_callee: HostCallee,790host_result: impl Into<HostResult>,791wasm_args: WasmArgs,792extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>),793) {794let pointer_type = self.isa.pointer_type();795let wasm_func_ty = self.types[self.signature].unwrap_func();796797// Load all parameters in an ABI-agnostic fashion, of which the798// `VMComponentContext` will be the first.799let params = self.abi_load_params();800let vmctx = params[0];801let wasm_params = ¶ms[2..];802803// Start building up arguments to the host. The first is always the804// vmctx. After is whatever `extra_host_args` appends, and then finally805// is what `WasmArgs` specifies.806let mut host_args = vec![vmctx];807extra_host_args(self, &mut host_args);808let mut val_raw_ptr = None;809let mut val_raw_len = None;810match wasm_args {811// Wasm params are passed through as values themselves.812WasmArgs::InRegisters => host_args.extend(wasm_params.iter().copied()),813814// Wasm params are spilled and then the ptr/len is passed.815WasmArgs::ValRawList => {816let (ptr, len) = self.store_wasm_arguments(wasm_params);817val_raw_ptr = Some(ptr);818val_raw_len = Some(len);819host_args.push(ptr);820host_args.push(len);821}822823// A mixture of the above two.824WasmArgs::InRegistersUpTo(n) => {825let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(826&WasmFuncType::new(827wasm_func_ty.params().iter().skip(n).copied().collect(),828Box::new([]),829),830&mut self.builder,831&wasm_params[n..],832);833let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len));834835host_args.extend(wasm_params[..n].iter().copied());836host_args.push(values_vec_ptr);837host_args.push(values_vec_len);838}839}840841// Next perform the actual invocation of the host with `host_args`.842let call = match host_callee {843HostCallee::Libcall(get_libcall) => self.call_libcall(vmctx, get_libcall, &host_args),844HostCallee::Lowering(index) => {845// Load host function pointer from the vmcontext and then call that846// indirect function pointer with the list of arguments.847let host_fn = self.builder.ins().load(848pointer_type,849MemFlags::trusted(),850vmctx,851i32::try_from(self.offsets.lowering_callee(index)).unwrap(),852);853let host_sig = {854let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple()));855for param in host_args.iter() {856let ty = self.builder.func.dfg.value_type(*param);857sig.params.push(ir::AbiParam::new(ty));858}859// return value is a bool whether a trap was raised or not860sig.returns.push(ir::AbiParam::new(ir::types::I8));861self.builder.import_signature(sig)862};863self.compiler.call_indirect_host(864&mut self.builder,865HostCall::ComponentLowerImport,866host_sig,867host_fn,868&host_args,869)870}871};872873// Acquire the result of this function (if any) and interpret it874// according to `host_result`.875//876// NOte that all match arms here end with `abi_store_results` which877// accounts for the ABI of this function when storing results.878let result = self.builder.func.dfg.inst_results(call).get(0).copied();879let result_ty = result.map(|v| self.builder.func.dfg.value_type(v));880let expected = wasm_func_ty.returns();881match host_result.into() {882HostResult::Sentinel(TrapSentinel::NegativeOne) => {883assert_eq!(expected.len(), 1);884let (result, result_ty) = (result.unwrap(), result_ty.unwrap());885let result = match (result_ty, expected[0]) {886(ir::types::I64, WasmValType::I32) => {887self.raise_if_negative_one_and_truncate(result)888}889(ir::types::I64, WasmValType::I64) | (ir::types::I32, WasmValType::I32) => {890self.raise_if_negative_one(result)891}892other => panic!("unsupported NegativeOne combo {other:?}"),893};894self.abi_store_results(&[result]);895}896HostResult::Sentinel(TrapSentinel::Falsy) => {897assert_eq!(expected.len(), 0);898self.raise_if_host_trapped(result.unwrap());899self.abi_store_results(&[]);900}901HostResult::Sentinel(_) => todo!("support additional return types if/when necessary"),902HostResult::None => {903assert!(result.is_none());904self.abi_store_results(&[]);905}906907HostResult::MultiValue { ptr, len } => {908let ptr = ptr.or(val_raw_ptr).unwrap();909let len = len.or(val_raw_len).unwrap();910self.raise_if_host_trapped(result.unwrap());911let results = self.compiler.load_values_from_array(912wasm_func_ty.returns(),913&mut self.builder,914ptr,915len,916);917self.abi_store_results(&results);918}919}920}921922fn index_value(&mut self, index: impl EntityRef) -> ir::Value {923self.builder924.ins()925.iconst(ir::types::I32, i64::try_from(index.index()).unwrap())926}927928fn translate_resource_drop(&mut self, resource: TypeResourceTableIndex) {929let args = self.abi_load_params();930let vmctx = args[0];931let caller_vmctx = args[1];932let pointer_type = self.isa.pointer_type();933934// The arguments this shim passes along to the libcall are:935//936// * the vmctx937// * a constant value for this `ResourceDrop` intrinsic938// * the wasm handle index to drop939let mut host_args = Vec::new();940host_args.push(vmctx);941host_args.push(942self.builder943.ins()944.iconst(ir::types::I32, i64::from(resource.as_u32())),945);946host_args.push(args[2]);947948let call = self.call_libcall(vmctx, host::resource_drop, &host_args);949950// Immediately raise a trap if requested by the host951let should_run_destructor =952self.raise_if_negative_one(self.builder.func.dfg.inst_results(call)[0]);953954let resource_ty = self.types[resource].ty;955let resource_def = self956.component957.defined_resource_index(resource_ty)958.map(|idx| {959self.component960.initializers961.iter()962.filter_map(|i| match i {963GlobalInitializer::Resource(r) if r.index == idx => Some(r),964_ => None,965})966.next()967.unwrap()968});969let has_destructor = match resource_def {970Some(def) => def.dtor.is_some(),971None => true,972};973// Synthesize the following:974//975// ...976// brif should_run_destructor, run_destructor_block, return_block977//978// run_destructor_block:979// ;; test may_enter, but only if the component instances980// ;; differ981// flags = load.i32 vmctx+$offset982// masked = band flags, $FLAG_MAY_ENTER983// trapz masked, CANNOT_ENTER_CODE984//985// ;; ============================================================986// ;; this is conditionally emitted based on whether the resource987// ;; has a destructor or not, and can be statically omitted988// ;; because that information is known at compile time here.989// rep = ushr.i64 rep, 1990// rep = ireduce.i32 rep991// dtor = load.ptr vmctx+$offset992// func_addr = load.ptr dtor+$offset993// callee_vmctx = load.ptr dtor+$offset994// call_indirect func_addr, callee_vmctx, vmctx, rep995// ;; ============================================================996//997// jump return_block998//999// return_block:1000// return1001//1002// This will decode `should_run_destructor` and run the destructor1003// funcref if one is specified for this resource. Note that not all1004// resources have destructors, hence the null check.1005self.builder.ensure_inserted_block();1006let current_block = self.builder.current_block().unwrap();1007let run_destructor_block = self.builder.create_block();1008self.builder1009.insert_block_after(run_destructor_block, current_block);1010let return_block = self.builder.create_block();1011self.builder1012.insert_block_after(return_block, run_destructor_block);10131014self.builder.ins().brif(1015should_run_destructor,1016run_destructor_block,1017&[],1018return_block,1019&[],1020);10211022let trusted = ir::MemFlags::trusted().with_readonly();10231024self.builder.switch_to_block(run_destructor_block);10251026// If this is a defined resource within the component itself then a1027// check needs to be emitted for the `may_enter` flag. Note though1028// that this check can be elided if the resource table resides in1029// the same component instance that defined the resource as the1030// component is calling itself.1031if let Some(def) = resource_def {1032if self.types[resource].instance != def.instance {1033let flags = self.builder.ins().load(1034ir::types::I32,1035trusted,1036vmctx,1037i32::try_from(self.offsets.instance_flags(def.instance)).unwrap(),1038);1039let masked = self1040.builder1041.ins()1042.band_imm(flags, i64::from(FLAG_MAY_ENTER));1043self.builder.ins().trapz(masked, TRAP_CANNOT_ENTER);1044}1045}10461047// Conditionally emit destructor-execution code based on whether we1048// statically know that a destructor exists or not.1049if has_destructor {1050let rep = self.builder.ins().ushr_imm(should_run_destructor, 1);1051let rep = self.builder.ins().ireduce(ir::types::I32, rep);1052let index = self.types[resource].ty;1053// NB: despite the vmcontext storing nullable funcrefs for function1054// pointers we know this is statically never null due to the1055// `has_destructor` check above.1056let dtor_func_ref = self.builder.ins().load(1057pointer_type,1058trusted,1059vmctx,1060i32::try_from(self.offsets.resource_destructor(index)).unwrap(),1061);1062if self.compiler.emit_debug_checks {1063self.builder1064.ins()1065.trapz(dtor_func_ref, TRAP_INTERNAL_ASSERT);1066}1067let func_addr = self.builder.ins().load(1068pointer_type,1069trusted,1070dtor_func_ref,1071i32::from(self.offsets.ptr.vm_func_ref_wasm_call()),1072);1073let callee_vmctx = self.builder.ins().load(1074pointer_type,1075trusted,1076dtor_func_ref,1077i32::from(self.offsets.ptr.vm_func_ref_vmctx()),1078);10791080let sig = crate::wasm_call_signature(1081self.isa,1082&self.types[self.signature].unwrap_func(),1083&self.compiler.tunables,1084);1085let sig_ref = self.builder.import_signature(sig);10861087// NB: note that the "caller" vmctx here is the caller of this1088// intrinsic itself, not the `VMComponentContext`. This effectively1089// takes ourselves out of the chain here but that's ok since the1090// caller is only used for store/limits and that same info is1091// stored, but elsewhere, in the component context.1092self.builder.ins().call_indirect(1093sig_ref,1094func_addr,1095&[callee_vmctx, caller_vmctx, rep],1096);1097}1098self.builder.ins().jump(return_block, &[]);1099self.builder.seal_block(run_destructor_block);11001101self.builder.switch_to_block(return_block);1102self.builder.seal_block(return_block);1103self.abi_store_results(&[]);1104}11051106fn load_optional_memory(1107&mut self,1108vmctx: ir::Value,1109memory: Option<RuntimeMemoryIndex>,1110) -> ir::Value {1111match memory {1112Some(idx) => self.load_memory(vmctx, idx),1113None => self.builder.ins().iconst(self.isa.pointer_type(), 0),1114}1115}11161117fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value {1118self.builder.ins().load(1119self.isa.pointer_type(),1120MemFlags::trusted(),1121vmctx,1122i32::try_from(self.offsets.runtime_memory(memory)).unwrap(),1123)1124}11251126fn load_callback(1127&mut self,1128vmctx: ir::Value,1129callback: Option<RuntimeCallbackIndex>,1130) -> ir::Value {1131let pointer_type = self.isa.pointer_type();1132match callback {1133Some(idx) => self.builder.ins().load(1134pointer_type,1135MemFlags::trusted(),1136vmctx,1137i32::try_from(self.offsets.runtime_callback(idx)).unwrap(),1138),1139None => self.builder.ins().iconst(pointer_type, 0),1140}1141}11421143fn load_post_return(1144&mut self,1145vmctx: ir::Value,1146post_return: Option<RuntimePostReturnIndex>,1147) -> ir::Value {1148let pointer_type = self.isa.pointer_type();1149match post_return {1150Some(idx) => self.builder.ins().load(1151pointer_type,1152MemFlags::trusted(),1153vmctx,1154i32::try_from(self.offsets.runtime_post_return(idx)).unwrap(),1155),1156None => self.builder.ins().iconst(pointer_type, 0),1157}1158}11591160/// Loads a host function pointer for a libcall stored at the `offset`1161/// provided in the libcalls array.1162///1163/// The offset is calculated in the `host` module below.1164fn load_libcall(1165&mut self,1166vmctx: ir::Value,1167index: ComponentBuiltinFunctionIndex,1168) -> ir::Value {1169let pointer_type = self.isa.pointer_type();1170// First load the pointer to the builtins structure which is static1171// per-process.1172let builtins_array = self.builder.ins().load(1173pointer_type,1174MemFlags::trusted().with_readonly(),1175vmctx,1176i32::try_from(self.offsets.builtins()).unwrap(),1177);1178// Next load the function pointer at `offset` and return that.1179self.builder.ins().load(1180pointer_type,1181MemFlags::trusted().with_readonly(),1182builtins_array,1183i32::try_from(index.index() * u32::from(self.offsets.ptr.size())).unwrap(),1184)1185}11861187fn abi_load_params(&mut self) -> Vec<ir::Value> {1188let mut block0_params = self.builder.func.dfg.block_params(self.block0).to_vec();1189match self.abi {1190// Wasm and native ABIs pass parameters as normal function1191// parameters.1192Abi::Wasm => block0_params,11931194// The array ABI passes a pointer/length as the 3rd/4th arguments1195// and those are used to load the actual wasm parameters.1196Abi::Array => {1197let results = self.compiler.load_values_from_array(1198self.types[self.signature].unwrap_func().params(),1199&mut self.builder,1200block0_params[2],1201block0_params[3],1202);1203block0_params.truncate(2);1204block0_params.extend(results);1205block0_params1206}1207}1208}12091210fn abi_store_results(&mut self, results: &[ir::Value]) {1211match self.abi {1212// Wasm/native ABIs return values as usual.1213Abi::Wasm => {1214self.builder.ins().return_(results);1215}12161217// The array ABI stores all results in the pointer/length passed1218// as arguments to this function, which contractually are required1219// to have enough space for the results.1220Abi::Array => {1221let block0_params = self.builder.func.dfg.block_params(self.block0);1222let (ptr, len) = (block0_params[2], block0_params[3]);1223self.compiler.store_values_to_array(1224&mut self.builder,1225self.types[self.signature].unwrap_func().returns(),1226results,1227ptr,1228len,1229);1230let true_value = self.builder.ins().iconst(ir::types::I8, 1);1231self.builder.ins().return_(&[true_value]);1232}1233}1234}12351236fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {1237let caller_vmctx = self.builder.func.dfg.block_params(self.block0)[1];1238self.compiler1239.raise_if_host_trapped(&mut self.builder, caller_vmctx, succeeded);1240}12411242fn raise_if_transcode_trapped(&mut self, amount_copied: ir::Value) {1243let pointer_type = self.isa.pointer_type();1244let minus_one = self.builder.ins().iconst(pointer_type, -1);1245let succeeded = self1246.builder1247.ins()1248.icmp(IntCC::NotEqual, amount_copied, minus_one);1249self.raise_if_host_trapped(succeeded);1250}12511252fn raise_if_negative_one_and_truncate(&mut self, ret: ir::Value) -> ir::Value {1253let ret = self.raise_if_negative_one(ret);1254self.builder.ins().ireduce(ir::types::I32, ret)1255}12561257fn raise_if_negative_one(&mut self, ret: ir::Value) -> ir::Value {1258let result_ty = self.builder.func.dfg.value_type(ret);1259let minus_one = self.builder.ins().iconst(result_ty, -1);1260let succeeded = self.builder.ins().icmp(IntCC::NotEqual, ret, minus_one);1261self.raise_if_host_trapped(succeeded);1262ret1263}12641265fn call_libcall(1266&mut self,1267vmctx: ir::Value,1268get_libcall: GetLibcallFn,1269args: &[ir::Value],1270) -> ir::Inst {1271let (host_sig, index) = get_libcall(self.isa, &mut self.builder.func);1272let host_fn = self.load_libcall(vmctx, index);1273self.compiler1274.call_indirect_host(&mut self.builder, index, host_sig, host_fn, args)1275}1276}12771278impl ComponentCompiler for Compiler {1279fn compile_trampoline(1280&self,1281component: &ComponentTranslation,1282types: &ComponentTypesBuilder,1283key: FuncKey,1284tunables: &Tunables,1285_symbol: &str,1286) -> Result<AllCallFunc<CompiledFunctionBody>> {1287let compile = |abi: Abi| -> Result<_> {1288let mut compiler = self.function_compiler();1289let mut c = TrampolineCompiler::new(1290self,1291&mut compiler,1292&component.component,1293types,1294key.unwrap_component_trampoline(),1295abi,1296tunables,1297);12981299// If we are crossing the Wasm-to-native boundary, we need to save the1300// exit FP and return address for stack walking purposes. However, we1301// always debug assert that our vmctx is a component context, regardless1302// whether we are actually crossing that boundary because it should1303// always hold.1304let vmctx = c.builder.block_params(c.block0)[0];1305let pointer_type = self.isa.pointer_type();1306self.debug_assert_vmctx_kind(1307&mut c.builder,1308vmctx,1309wasmtime_environ::component::VMCOMPONENT_MAGIC,1310);1311if let Abi::Wasm = abi {1312let vm_store_context = c.builder.ins().load(1313pointer_type,1314MemFlags::trusted(),1315vmctx,1316i32::try_from(c.offsets.vm_store_context()).unwrap(),1317);1318super::save_last_wasm_exit_fp_and_pc(1319&mut c.builder,1320pointer_type,1321&c.offsets.ptr,1322vm_store_context,1323);1324}13251326c.translate(&component.trampolines[key.unwrap_component_trampoline()]);1327c.builder.finalize();1328compiler.cx.abi = Some(abi);13291330Ok(CompiledFunctionBody {1331code: super::box_dyn_any_compiler_context(Some(compiler.cx)),1332needs_gc_heap: false,1333})1334};13351336Ok(AllCallFunc {1337wasm_call: compile(Abi::Wasm)?,1338array_call: compile(Abi::Array)?,1339})1340}1341}13421343impl TrampolineCompiler<'_> {1344fn translate_transcode(1345&mut self,1346op: Transcode,1347from: RuntimeMemoryIndex,1348from64: bool,1349to: RuntimeMemoryIndex,1350to64: bool,1351) {1352let pointer_type = self.isa.pointer_type();1353let vmctx = self.builder.func.dfg.block_params(self.block0)[0];13541355// Determine the static signature of the host libcall for this transcode1356// operation and additionally calculate the static offset within the1357// transode libcalls array.1358let get_libcall = match op {1359Transcode::Copy(FixedEncoding::Utf8) => host::utf8_to_utf8,1360Transcode::Copy(FixedEncoding::Utf16) => host::utf16_to_utf16,1361Transcode::Copy(FixedEncoding::Latin1) => host::latin1_to_latin1,1362Transcode::Latin1ToUtf16 => host::latin1_to_utf16,1363Transcode::Latin1ToUtf8 => host::latin1_to_utf8,1364Transcode::Utf16ToCompactProbablyUtf16 => host::utf16_to_compact_probably_utf16,1365Transcode::Utf16ToCompactUtf16 => host::utf16_to_compact_utf16,1366Transcode::Utf16ToLatin1 => host::utf16_to_latin1,1367Transcode::Utf16ToUtf8 => host::utf16_to_utf8,1368Transcode::Utf8ToCompactUtf16 => host::utf8_to_compact_utf16,1369Transcode::Utf8ToLatin1 => host::utf8_to_latin1,1370Transcode::Utf8ToUtf16 => host::utf8_to_utf16,1371};13721373// Load the base pointers for the from/to linear memories.1374let from_base = self.load_runtime_memory_base(vmctx, from);1375let to_base = self.load_runtime_memory_base(vmctx, to);13761377let mut args = Vec::new();1378args.push(vmctx);13791380let uses_retptr = match op {1381Transcode::Utf16ToUtf81382| Transcode::Latin1ToUtf81383| Transcode::Utf8ToLatin11384| Transcode::Utf16ToLatin1 => true,1385_ => false,1386};13871388// Most transcoders share roughly the same signature despite doing very1389// different things internally, so most libcalls are lumped together1390// here.1391match op {1392Transcode::Copy(_)1393| Transcode::Latin1ToUtf161394| Transcode::Utf16ToCompactProbablyUtf161395| Transcode::Utf8ToLatin11396| Transcode::Utf16ToLatin11397| Transcode::Utf8ToUtf16 => {1398args.push(self.ptr_param(0, from64, from_base));1399args.push(self.len_param(1, from64));1400args.push(self.ptr_param(2, to64, to_base));1401}14021403Transcode::Utf16ToUtf8 | Transcode::Latin1ToUtf8 => {1404args.push(self.ptr_param(0, from64, from_base));1405args.push(self.len_param(1, from64));1406args.push(self.ptr_param(2, to64, to_base));1407args.push(self.len_param(3, to64));1408}14091410Transcode::Utf8ToCompactUtf16 | Transcode::Utf16ToCompactUtf16 => {1411args.push(self.ptr_param(0, from64, from_base));1412args.push(self.len_param(1, from64));1413args.push(self.ptr_param(2, to64, to_base));1414args.push(self.len_param(3, to64));1415args.push(self.len_param(4, to64));1416}1417};1418if uses_retptr {1419let slot = self1420.builder1421.func1422.create_sized_stack_slot(ir::StackSlotData::new(1423ir::StackSlotKind::ExplicitSlot,1424pointer_type.bytes(),14250,1426));1427args.push(self.builder.ins().stack_addr(pointer_type, slot, 0));1428}1429let call = self.call_libcall(vmctx, get_libcall, &args);1430let mut results = self.builder.func.dfg.inst_results(call).to_vec();1431if uses_retptr {1432results.push(self.builder.ins().load(1433pointer_type,1434ir::MemFlags::trusted(),1435*args.last().unwrap(),14360,1437));1438}1439let mut raw_results = Vec::new();14401441// Like the arguments the results are fairly similar across libcalls, so1442// they're lumped into various buckets here.1443match op {1444Transcode::Copy(_) | Transcode::Latin1ToUtf16 => {1445self.raise_if_host_trapped(results[0]);1446}14471448Transcode::Utf8ToUtf161449| Transcode::Utf16ToCompactProbablyUtf161450| Transcode::Utf8ToCompactUtf161451| Transcode::Utf16ToCompactUtf16 => {1452self.raise_if_transcode_trapped(results[0]);1453raw_results.push(self.cast_from_pointer(results[0], to64));1454}14551456Transcode::Latin1ToUtf81457| Transcode::Utf16ToUtf81458| Transcode::Utf8ToLatin11459| Transcode::Utf16ToLatin1 => {1460self.raise_if_transcode_trapped(results[0]);1461raw_results.push(self.cast_from_pointer(results[0], from64));1462raw_results.push(self.cast_from_pointer(results[1], to64));1463}1464};14651466self.builder.ins().return_(&raw_results);1467}14681469// Helper function to cast an input parameter to the host pointer type.1470fn len_param(&mut self, param: usize, is64: bool) -> ir::Value {1471let val = self.builder.func.dfg.block_params(self.block0)[2 + param];1472self.cast_to_pointer(val, is64)1473}14741475// Helper function to interpret an input parameter as a pointer into1476// linear memory. This will cast the input parameter to the host integer1477// type and then add that value to the base.1478//1479// Note that bounds-checking happens in adapter modules, and this1480// trampoline is simply calling the host libcall.1481fn ptr_param(&mut self, param: usize, is64: bool, base: ir::Value) -> ir::Value {1482let val = self.len_param(param, is64);1483self.builder.ins().iadd(base, val)1484}14851486// Helper function to cast a core wasm input to a host pointer type1487// which will go into the host libcall.1488fn cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {1489let pointer_type = self.isa.pointer_type();1490let host64 = pointer_type == ir::types::I64;1491if is64 == host64 {1492val1493} else if !is64 {1494assert!(host64);1495self.builder.ins().uextend(pointer_type, val)1496} else {1497assert!(!host64);1498self.builder.ins().ireduce(pointer_type, val)1499}1500}15011502// Helper to cast a host pointer integer type to the destination type.1503fn cast_from_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {1504let host64 = self.isa.pointer_type() == ir::types::I64;1505if is64 == host64 {1506val1507} else if !is64 {1508assert!(host64);1509self.builder.ins().ireduce(ir::types::I32, val)1510} else {1511assert!(!host64);1512self.builder.ins().uextend(ir::types::I64, val)1513}1514}15151516fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value {1517let pointer_type = self.isa.pointer_type();1518let from_vmmemory_definition = self.load_memory(vmctx, mem);1519self.builder.ins().load(1520pointer_type,1521MemFlags::trusted(),1522from_vmmemory_definition,1523i32::from(self.offsets.ptr.vmmemory_definition_base()),1524)1525}1526}15271528/// Module with macro-generated contents that will return the signature and1529/// offset for each of the host transcoder functions.1530///1531/// Note that a macro is used here to keep this in sync with the actual1532/// transcoder functions themselves which are also defined via a macro.1533mod host {1534use cranelift_codegen::ir::{self, AbiParam};1535use cranelift_codegen::isa::{CallConv, TargetIsa};1536use wasmtime_environ::component::ComponentBuiltinFunctionIndex;15371538macro_rules! define {1539(1540$(1541$( #[$attr:meta] )*1542$name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;1543)*1544) => {1545$(1546pub(super) fn $name(isa: &dyn TargetIsa, func: &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex) {1547let pointer_type = isa.pointer_type();1548let sig = build_sig(1549isa,1550func,1551&[$( define!(@ty pointer_type $param) ),*],1552&[$( define!(@ty pointer_type $result) ),*],1553);15541555return (sig, ComponentBuiltinFunctionIndex::$name())1556}1557)*1558};15591560(@ty $ptr:ident size) => ($ptr);1561(@ty $ptr:ident ptr_u8) => ($ptr);1562(@ty $ptr:ident ptr_u16) => ($ptr);1563(@ty $ptr:ident ptr_size) => ($ptr);1564(@ty $ptr:ident bool) => (ir::types::I8);1565(@ty $ptr:ident u8) => (ir::types::I8);1566(@ty $ptr:ident u32) => (ir::types::I32);1567(@ty $ptr:ident u64) => (ir::types::I64);1568(@ty $ptr:ident vmctx) => ($ptr);1569}15701571wasmtime_environ::foreach_builtin_component_function!(define);15721573fn build_sig(1574isa: &dyn TargetIsa,1575func: &mut ir::Function,1576params: &[ir::Type],1577returns: &[ir::Type],1578) -> ir::SigRef {1579let mut sig = ir::Signature {1580params: params.iter().map(|ty| AbiParam::new(*ty)).collect(),1581returns: returns.iter().map(|ty| AbiParam::new(*ty)).collect(),1582call_conv: CallConv::triple_default(isa.triple()),1583};15841585// Once we're declaring the signature of a host function we must respect1586// the default ABI of the platform which is where argument extension of1587// params/results may come into play.1588let extension = isa.default_argument_extension();1589for arg in sig.params.iter_mut().chain(sig.returns.iter_mut()) {1590if arg.value_type.is_int() {1591arg.extension = extension;1592}1593}1594func.import_signature(sig)1595}1596}159715981599