Add the concept of "backend intrinsics" to Cranelift

cranelift/codegen/src/ir/extname.rs

@@ -138,6 +138,8 @@ pub enum ExternalName {
     LibCall(LibCall),
     /// A well-known symbol.
     KnownSymbol(KnownSymbol),
+    /// A per-backend intrinsic with meaning on a per-backend basis.
+    BackendIntrinsic(u32),
 }
 
 impl Default for ExternalName {
@@ -206,6 +208,7 @@ impl<'a> fmt::Display for DisplayableExternalName<'a> {
             ExternalName::TestCase(testcase) => testcase.fmt(f),
             ExternalName::LibCall(lc) => write!(f, "%{lc}"),
             ExternalName::KnownSymbol(ks) => write!(f, "%{ks}"),
+            ExternalName::BackendIntrinsic(i) => write!(f, "%intrinsic{i}"),
         }
     }
 }
@@ -224,6 +227,13 @@ impl FromStr for ExternalName {
             return Ok(Self::LibCall(lc));
         }
 
+        // Try to parse as a backend intrinsic
+        if let Some(n) = s.strip_prefix("intrinsic") {
+            if let Ok(n) = n.parse() {
+                return Ok(Self::BackendIntrinsic(n));
+            }
+        }
+
         // Otherwise its a test case name
         Ok(Self::testcase(s.as_bytes()))
     }

cranelift/codegen/src/isa/pulley_shared/abi.rs

@@ -542,6 +542,17 @@ where
     }
 
     fn gen_call(dest: &CallDest, tmp: Writable<Reg>, info: CallInfo<()>) -> SmallVec<[Self::I; 2]> {
+        // Backend intrinsics for the pulley backend are used to implement the
+        // `call_indirect_host` opcode in Pulley.
+        if let CallDest::ExtName(name @ ir::ExternalName::BackendIntrinsic(_), RelocDistance::Far) =
+            dest
+        {
+            return smallvec![Inst::IndirectCallHost {
+                info: Box::new(info.map(|()| name.clone()))
+            }
+            .into()];
+        }
+
         if info.callee_conv == isa::CallConv::Tail || info.callee_conv == isa::CallConv::Fast {
             match &dest {
                 &CallDest::ExtName(ref name, RelocDistance::Near) => smallvec![Inst::Call {

cranelift/codegen/src/isa/pulley_shared/inst.isle

@@ -49,6 +49,10 @@
     ;; An indirect call to an unknown callee.
     (IndirectCall (info BoxCallIndInfo))
 
+    ;; An indirect call out to a host-defined function. The host function
+    ;; pointer is the first "argument" of this function call.
+    (IndirectCallHost (info BoxCallInfo))
+
     ;; Unconditional jumps.
     (Jump (label MachLabel))
 

cranelift/codegen/src/isa/pulley_shared/inst/emit.rs

@@ -217,6 +217,26 @@ fn pulley_emit<P>(
 
         Inst::IndirectCall { .. } => todo!(),
 
+        Inst::IndirectCallHost { info } => {
+            // This call is only usable with `BackendIntrinsic` names where the
+            // intrinsic number is the signature number embedded in the opcode.
+            let signum = match info.dest {
+                ir::ExternalName::BackendIntrinsic(n) => n,
+                _ => unreachable!(),
+            };
+            enc::call_indirect_host(sink, u8::try_from(signum).unwrap());
+
+            if let Some(s) = state.take_stack_map() {
+                let offset = sink.cur_offset();
+                sink.push_user_stack_map(state, offset, s);
+            }
+            sink.add_call_site();
+
+            // If a callee pop is happening here that means that something has
+            // messed up, these are expected to be "very simple" signatures.
+            assert!(info.callee_pop_size == 0);
+        }
+
         Inst::Jump { label } => {
             sink.use_label_at_offset(start_offset + 1, *label, LabelUse::Jump(1));
             sink.add_uncond_branch(start_offset, start_offset + 5, *label);

cranelift/codegen/src/isa/pulley_shared/inst/mod.rs

@@ -91,7 +91,7 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_def(dst);
         }
 
-        Inst::Call { info } => {
+        Inst::Call { info } | Inst::IndirectCallHost { info } => {
             let CallInfo { uses, defs, .. } = &mut **info;
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
@@ -582,6 +582,10 @@ impl Inst {
                 format!("indirect_call {callee}, {info:?}")
             }
 
+            Inst::IndirectCallHost { info } => {
+                format!("indirect_call_host {info:?}")
+            }
+
             Inst::Jump { label } => format!("jump {}", label.to_string()),
 
             Inst::BrIf {

cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif

@@ -0,0 +1,37 @@
+test compile precise-output
+target pulley64
+
+function %call_indirect_host() {
+  fn0 = %intrinsic0() system_v
+block0:
+  call fn0()
+  return
+}
+
+; VCode:
+;   x30 = xconst8 -16
+;   x27 = xadd32 x27, x30
+;   store64 sp+8, x28 // flags =  notrap aligned
+;   store64 sp+0, x29 // flags =  notrap aligned
+;   x29 = xmov x27
+; block0:
+;   indirect_call_host CallInfo { dest: BackendIntrinsic(0), uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65279, 4294967295, 0] }, callee_conv: SystemV, caller_conv: Fast, callee_pop_size: 0 }
+;   x28 = load64_u sp+8 // flags = notrap aligned
+;   x29 = load64_u sp+0 // flags = notrap aligned
+;   x30 = xconst8 16
+;   x27 = xadd32 x27, x30
+;   ret
+;
+; Disassembled:
+;        0: 14 1e f0                        xconst8 spilltmp0, -16
+;        3: 18 7b 7b                        xadd32 sp, sp, spilltmp0
+;        6: 32 1b 08 1c                     store64_offset8 sp, 8, lr
+;        a: 30 1b 1d                        store64 sp, fp
+;        d: 11 1d 1b                        xmov fp, sp
+;       10: 43 03 00 00                     call_indirect_host 0
+;       14: 2b 1c 1b 08                     load64_offset8 lr, sp, 8
+;       18: 28 1d 1b                        load64 fp, sp
+;       1b: 14 1e 10                        xconst8 spilltmp0, 16
+;       1e: 18 7b 7b                        xadd32 sp, sp, spilltmp0
+;       21: 00                              ret
+

cranelift/interpreter/src/step.rs

@@ -381,7 +381,9 @@ where
                     InterpreterFunctionRef::Function(function)
                 }
                 ExternalName::LibCall(libcall) => InterpreterFunctionRef::LibCall(libcall),
-                ExternalName::KnownSymbol(_) => unimplemented!(),
+                ExternalName::KnownSymbol(_) | ExternalName::BackendIntrinsic(_) => {
+                    unimplemented!()
+                }
             };
 
             let make_control_flow = match inst.opcode() {

cranelift/module/src/module.rs

@@ -54,6 +54,9 @@ impl ModuleReloc {
             FinalizedRelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => {
                 ModuleRelocTarget::KnownSymbol(ks)
             }
+            FinalizedRelocTarget::ExternalName(ExternalName::BackendIntrinsic(_)) => {
+                panic!("backend created relocation for own intrinsic")
+            }
             FinalizedRelocTarget::Func(offset) => {
                 ModuleRelocTarget::FunctionOffset(func_id, offset)
             }

crates/cranelift/Cargo.toml

@@ -32,11 +32,12 @@ thiserror = { workspace = true }
 cfg-if = { workspace = true }
 wasmtime-versioned-export-macros = { workspace = true }
 itertools = "0.12"
+pulley-interpreter = { workspace = true, optional = true }
 
 [features]
 all-arch = ["cranelift-codegen/all-arch"]
 host-arch = ["cranelift-codegen/host-arch"]
-pulley = ["cranelift-codegen/pulley"]
+pulley = ["cranelift-codegen/pulley", "dep:pulley-interpreter"]
 trace-log = ["cranelift-codegen/trace-log"]
 component-model = ["wasmtime-environ/component-model"]
 incremental-cache = ["cranelift-codegen/incremental-cache"]

crates/cranelift/src/compiler.rs

@@ -119,6 +119,98 @@ impl Compiler {
             wmemcheck,
         }
     }
+
+    /// Peform an indirect call from Cranelift-generated code to native code in
+    /// Wasmtime itself.
+    ///
+    /// For native platforms this is a simple `call_indirect` instruction but
+    /// for the Pulley backend this is special as it's transitioning from
+    /// Cranelift-generated bytecode to native code on the host. That requires a
+    /// special opcode in the interpreter and is modeled slightly differently in
+    /// Cranelift IR.
+    fn call_indirect_host(
+        &self,
+        builder: &mut FunctionBuilder<'_>,
+        sig: ir::SigRef,
+        addr: Value,
+        args: &[Value],
+    ) -> ir::Inst {
+        let signature = &builder.func.dfg.signatures[sig];
+
+        // When calling the host we should always be using the platform's
+        // default calling convention since it'll be calling Rust code in
+        // Wasmtime itself.
+        assert_eq!(signature.call_conv, self.isa.default_call_conv());
+
+        #[cfg(feature = "pulley")]
+        {
+            use cranelift_codegen::ir::types::{I32, I64, I8};
+
+            // If pulley is enabled, even if we're not targetting it, determine
+            // what pulley signature that the input `signature` maps to. This is
+            // done to ensure that even on native platforms we've always got a
+            // signature listed in pulley for all platform intrinsics. In theory
+            // the set of signatures here doesn't change over time all that
+            // much. If a new signature is added then the `pulley/src/lib.rs`
+            // file and the `for_each_host_signature!` macro need to be updated.
+            // In theory that's all that needs to happen as well...
+            macro_rules! pulley_signum {
+                ($(fn($($args:ident),*) $(-> $ret:ident)?;)*) => {'outer: {
+
+                    let mut ret = 0;
+
+                    $(
+                        let mut params = signature.params.iter().map(|p| p.value_type);
+                        let mut results = signature.returns.iter().map(|p| p.value_type);
+                        if true
+                            $(&& params.next() == Some($args))*
+                            && params.next().is_none()
+                            $(&& results.next() == Some($ret))?
+                            && results.next().is_none()
+                        {
+                            break 'outer ret;
+                        }
+                        ret += 1;
+                    )*
+
+                    let _ = ret;
+                    unimplemented!("no pulley host signature found for {signature:?}");
+                }};
+            }
+
+            let pulley_signum = pulley_interpreter::for_each_host_signature!(pulley_signum);
+
+            let is_pulley = match self.isa.triple().architecture {
+                target_lexicon::Architecture::Pulley32 => true,
+                target_lexicon::Architecture::Pulley64 => true,
+                _ => false,
+            };
+
+            // If this target is actually pulley then a custom `call`
+            // instruction is emitted. This will generate a new function with
+            // the Cranelift-name of a "backend intrinsic" which is how the
+            // Pulley backend models this special opcode that doesn't otherwise
+            // map into the Cranelift set of opcodes.
+            if is_pulley {
+                let mut new_signature = signature.clone();
+                new_signature
+                    .params
+                    .insert(0, ir::AbiParam::new(self.isa.pointer_type()));
+                let new_sig = builder.func.import_signature(new_signature);
+                let name = ir::ExternalName::BackendIntrinsic(pulley_signum);
+                let func = builder.func.import_function(ir::ExtFuncData {
+                    name,
+                    signature: new_sig,
+                    colocated: false,
+                });
+                let mut raw_args = vec![addr];
+                raw_args.extend_from_slice(args);
+                return builder.ins().call(func, &raw_args);
+            }
+        }
+
+        builder.ins().call_indirect(sig, addr, args)
+    }
 }
 
 impl wasmtime_environ::Compiler for Compiler {
@@ -360,7 +452,8 @@ impl wasmtime_environ::Compiler for Compiler {
 
         // Do an indirect call to the callee.
         let callee_signature = builder.func.import_signature(array_call_sig);
-        builder.ins().call_indirect(
+        self.call_indirect_host(
+            &mut builder,
             callee_signature,
             callee,
             &[callee_vmctx, caller_vmctx, args_base, args_len],
@@ -557,9 +650,7 @@ impl wasmtime_environ::Compiler for Compiler {
         // all the same results as the libcall.
         let block_params = builder.block_params(block0).to_vec();
         let host_sig = builder.func.import_signature(host_sig);
-        let call = builder
-            .ins()
-            .call_indirect(host_sig, func_addr, &block_params);
+        let call = self.call_indirect_host(&mut builder, host_sig, func_addr, &block_params);
         let results = builder.func.dfg.inst_results(call).to_vec();
         builder.ins().return_(&results);
         builder.finalize();

crates/cranelift/src/compiler/component.rs

@@ -237,9 +237,8 @@ impl<'a> TrampolineCompiler<'a> {
             i32::try_from(self.offsets.lowering_callee(index)).unwrap(),
         );
         let host_sig = self.builder.import_signature(host_sig);
-        self.builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &callee_args);
+        self.compiler
+            .call_indirect_host(&mut self.builder, host_sig, host_fn, &callee_args);
 
         match self.abi {
             Abi::Wasm => {
@@ -275,9 +274,8 @@ impl<'a> TrampolineCompiler<'a> {
             ir::types::I8,
             i64::from(wasmtime_environ::Trap::AlwaysTrapAdapter as u8),
         );
-        self.builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &[vmctx, code]);
+        self.compiler
+            .call_indirect_host(&mut self.builder, host_sig, host_fn, &[vmctx, code]);
         // debug trap in case execution actually falls through, but this
         // shouldn't ever get hit at runtime.
         self.builder.ins().trap(TRAP_INTERNAL_ASSERT);
@@ -309,10 +307,9 @@ impl<'a> TrampolineCompiler<'a> {
         let (host_sig, offset) = host::resource_new32(self.isa, &mut self.builder.func);
 
         let host_fn = self.load_libcall(vmctx, offset);
-        let call = self
-            .builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &host_args);
+        let call =
+            self.compiler
+                .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let result = self.builder.func.dfg.inst_results(call)[0];
         self.abi_store_results(&[result]);
     }
@@ -343,10 +340,9 @@ impl<'a> TrampolineCompiler<'a> {
         let (host_sig, offset) = host::resource_rep32(self.isa, &mut self.builder.func);
 
         let host_fn = self.load_libcall(vmctx, offset);
-        let call = self
-            .builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &host_args);
+        let call =
+            self.compiler
+                .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let result = self.builder.func.dfg.inst_results(call)[0];
         self.abi_store_results(&[result]);
     }
@@ -373,10 +369,9 @@ impl<'a> TrampolineCompiler<'a> {
 
         let (host_sig, offset) = host::resource_drop(self.isa, &mut self.builder.func);
         let host_fn = self.load_libcall(vmctx, offset);
-        let call = self
-            .builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &host_args);
+        let call =
+            self.compiler
+                .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let should_run_destructor = self.builder.func.dfg.inst_results(call)[0];
 
         let resource_ty = self.types[resource].ty;
@@ -556,10 +551,9 @@ impl<'a> TrampolineCompiler<'a> {
         host_args.extend(args[2..].iter().copied());
         let (host_sig, offset) = get_libcall(self.isa, &mut self.builder.func);
         let host_fn = self.load_libcall(vmctx, offset);
-        let call = self
-            .builder
-            .ins()
-            .call_indirect(host_sig, host_fn, &host_args);
+        let call =
+            self.compiler
+                .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let results = self.builder.func.dfg.inst_results(call).to_vec();
         self.builder.ins().return_(&results);
     }
@@ -786,7 +780,9 @@ impl TrampolineCompiler<'_> {
                 ));
             args.push(self.builder.ins().stack_addr(pointer_type, slot, 0));
         }
-        let call = self.builder.ins().call_indirect(sig, libcall, &args);
+        let call = self
+            .compiler
+            .call_indirect_host(&mut self.builder, sig, libcall, &args);
         let mut results = self.builder.func.dfg.inst_results(call).to_vec();
         if uses_retptr {
             results.push(self.builder.ins().load(