A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: add patchable call instructions. (#12101)
Chris Fallinc00e9ea2ecb7
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rsindex 9604c6904b..8e1365dd85 100644--- a/cranelift/codegen/meta/src/shared/instructions.rs+++ b/cranelift/codegen/meta/src/shared/instructions.rs@@ -3918,4 +3918,31 @@ pub(crate) fn define().other_side_effects(),);++ ig.push(+ Inst::new(+ "patchable_call",+ r#"+ A call instruction that can be turned on/off by patching machine code.++ This call is different than an ordinary call in a few ways:+ - It can only call functions with the `patchable` ABI.+ - As a result of that, the called function cannot have any+ return values.+ - The call emits metadata into the MachBuffer that allows the consumer+ of the machine code to edit the code to either make the call or not.++ Note that the lack of return values is necessary: because the call+ may or may not occur, dynamically, we cannot rely on any values+ actually being defined.+ "#,+ &formats.call,+ )+ .operands_in(vec![+ Operand::new("FN", &entities.func_ref)+ .with_doc("function to call, declared by `function`"),+ Operand::new("args", &entities.varargs).with_doc("call arguments"),+ ])+ .call(),+ );}diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rsindex 7b42e17286..60507594ba 100644--- a/cranelift/codegen/src/inst_predicates.rs+++ b/cranelift/codegen/src/inst_predicates.rs@@ -149,7 +149,11 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool {| Opcode::Fence| Opcode::Debugtrap| Opcode::SequencePoint => true,- Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true,+ Opcode::Call+ | Opcode::CallIndirect+ | Opcode::TryCall+ | Opcode::TryCallIndirect+ | Opcode::PatchableCall => true,op if op.can_trap() => true,_ => false,}diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 1c7bc76fdd..44c9c023f3 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -817,6 +817,9 @@;; An indirect return-call macro instruction.(ReturnCallInd (info BoxReturnCallIndInfo))+ ;; A patchable call instruction.+ (PatchableCall (info BoxCallInfo))+;; A pseudo-instruction that captures register arguments in vregs.(Args(args VecArgPair))@@ -4510,6 +4513,11 @@(rule (return_call_ind_impl info)(SideEffectNoResult.Inst (MInst.ReturnCallInd info)))+;; Helper for creating `MInst.PatchableCall` instructions.+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)+(rule (patchable_call_impl info)+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))+;; Helpers for pinned register manipulation.(decl write_pinned_reg (Reg) SideEffectNoResult)diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 91beb84efb..678ffa5a1b 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -2941,7 +2941,8 @@ impl MachInstEmit for Inst {sink.put4(0xd65f0bff | (op2 << 9)); // reta{key}}}- &Inst::Call { ref info } => {+ &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {+ let is_patchable = matches!(self, Inst::PatchableCall { .. });let user_stack_map = state.take_stack_map();sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);sink.put4(enc_jump26(0b100101, 0));@@ -2955,6 +2956,8 @@ impl MachInstEmit for Inst {Some(state.frame_layout.sp_to_fp()),try_call.exception_handlers(&state.frame_layout),);+ } else if is_patchable {+ sink.add_patchable_call_site(4);} else {sink.add_call_site();}diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex 1ba8ec50ff..f8c8b997cc 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -801,7 +801,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {}Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}Inst::Jump { .. } => {}- Inst::Call { info, .. } => {+ Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {let CallInfo { uses, defs, .. } = &mut **info;for CallArgPair { vreg, preg } in uses {collector.reg_fixed_use(vreg, *preg);@@ -1008,6 +1008,7 @@ impl MachInst for Inst {match self {Inst::Call { .. }| Inst::CallInd { .. }+ | Inst::PatchableCall { .. }| Inst::ElfTlsGetAddr { .. }| Inst::MachOTlsGetAddr { .. } => CallType::Regular,@@ -1092,7 +1093,7 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::Call { .. } | Inst::CallInd { .. } => true,+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,_ => false,}}@@ -1110,6 +1111,10 @@ impl MachInst for Inst {Inst::Nop4}+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {+ smallvec![0x1f, 0x20, 0x03, 0xd5]+ }+fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {match ty {I8 => Ok((&[RegClass::Int], &[I8])),@@ -2594,6 +2599,9 @@ impl Inst {.unwrap_or_default();format!("blr {rn}{try_call}")}+ &Inst::PatchableCall { .. } => {+ format!("bl 0 ; patchable")+ }&Inst::ReturnCall { ref info } => {let mut s = format!("return_call {:?} new_stack_arg_size:{}",diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex 33c06c4c4a..783c015b5a 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -2573,6 +2573,17 @@(_ Unit (emit_side_effect (call_ind_impl info))))output))+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++;; Direct call to an in-range function.+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))+ (let ((abi Sig (abi_sig sig_ref))+ (uses CallArgList (gen_call_args abi args))+ (defs CallRetList (gen_patchable_call_rets))+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))+ (_ Unit (emit_side_effect (patchable_call_impl info))))+ (output_none)))+;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isleindex 901753841c..176346cf13 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle@@ -51,6 +51,9 @@;; An indirect call to an unknown callee.(IndirectCall (info BoxCallIndInfo))+ ;; A patchable direct call to a known callee.+ (PatchableCall (info BoxCallInfo))+;; A direct return-call macro instruction.(ReturnCall (info BoxReturnCallInfo))@@ -749,6 +752,11 @@(rule (indirect_call_host_impl info)(SideEffectNoResult.Inst (MInst.IndirectCallHost info)))+;; Helper for creating `MInst.Call` instructions.+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)+(rule (patchable_call_impl info)+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))+;; Helper for creating `MInst.ReturnCall` instructions.(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)(rule (return_call_impl info)diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rsindex c8de063259..70ee02ffdf 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs+++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs@@ -166,7 +166,10 @@ fn pulley_emit<P>(sink.add_reloc_at_offset(end - size, Reloc::Abs8, &**name, *offset);}- Inst::Call { info } => {+ Inst::Call { info } | Inst::PatchableCall { info } => {+ let is_patchable = matches!(inst, Inst::PatchableCall { .. });+ let start = sink.cur_offset();+// If arguments happen to already be in the right register for the// ABI then remove them from this list. Otherwise emit the// appropriate `Call` instruction depending on how many arguments we@@ -196,6 +199,8 @@ fn pulley_emit<P>(Some(state.frame_layout.sp_to_fp()),try_call.exception_handlers(&state.frame_layout),);+ } else if is_patchable {+ sink.add_patchable_call_site(sink.cur_offset() - start);} else {sink.add_call_site();}diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rsindex 606ec93ae3..bc7082ee6e 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs+++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs@@ -148,7 +148,7 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_def(dst);}- Inst::Call { info } => {+ Inst::Call { info } | Inst::PatchableCall { info } => {let CallInfo {uses,defs,@@ -435,7 +435,8 @@ where}| Inst::Call { .. }| Inst::IndirectCall { .. }- | Inst::IndirectCallHost { .. } => true,+ | Inst::IndirectCallHost { .. }+ | Inst::PatchableCall { .. } => true,_ => false,}}@@ -498,9 +499,10 @@ wherefn call_type(&self) -> CallType {match &self.inst {- Inst::Call { .. } | Inst::IndirectCall { .. } | Inst::IndirectCallHost { .. } => {- CallType::Regular- }+ Inst::Call { .. }+ | Inst::IndirectCall { .. }+ | Inst::IndirectCallHost { .. }+ | Inst::PatchableCall { .. } => CallType::Regular,Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => CallType::TailCall,@@ -533,6 +535,16 @@ wheretodo!()}+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {+ let mut bytes = smallvec::smallvec![];+ let nop = pulley_interpreter::op::Nop {};+ nop.encode(&mut bytes);+ // NOP needs to be a 1-byte opcode so it can be used to+ // overwrite a callsite of any length.+ assert_eq!(bytes.len(), 1);+ bytes+ }+fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {match ty {I8 => Ok((&[RegClass::Int], &[I8])),@@ -712,6 +724,10 @@ impl Inst {format!("indirect_call {callee}, {info:?}{try_call}")}+ Inst::PatchableCall { info } => {+ format!("patchable_call {info:?}")+ }+Inst::ReturnCall { info } => {format!("return_call {info:?}")}diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isleindex 0373d45157..b5d9a87525 100644--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle@@ -171,6 +171,17 @@(_ Unit (emit_side_effect (indirect_call_impl info))))output))+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++;; Direct call to an in-range function.+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))+ (let ((abi Sig (abi_sig sig_ref))+ (uses CallArgList (gen_call_args abi args))+ (defs CallRetList (gen_patchable_call_rets))+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))+ (_ Unit (emit_side_effect (patchable_call_impl info))))+ (output_none)))+;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to a pulley function.diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isleindex 62774b59fe..9a4c29edaf 100644--- a/cranelift/codegen/src/isa/riscv64/inst.isle+++ b/cranelift/codegen/src/isa/riscv64/inst.isle@@ -111,11 +111,15 @@(from_bits u8)(to_bits u8))+ ;; A machine direct-call instruction.(Call (info BoxCallInfo))- ;; A machine indirect-call instruction.+ ;; A machine indirect-call instruction.(CallInd (info BoxCallIndInfo))+ ;; A patchable machine direct-call instruction.+ (PatchableCall (info BoxCallInfo))+;; A direct return-call macro instruction.(ReturnCall (info BoxReturnCallInfo))@@ -3114,6 +3118,11 @@(rule (call_ind_impl info)(SideEffectNoResult.Inst (MInst.CallInd info)))+;; Helper for creating `MInst.PatchableCall` instructions.+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)+(rule (patchable_call_impl info)+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))+;; Helper for creating `MInst.ReturnCall` instructions.(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)(rule (return_call_impl info)diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rsindex 77bd801c4f..a4801081a9 100644--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs@@ -167,6 +167,7 @@ impl Inst {| Inst::Extend { .. }| Inst::Call { .. }| Inst::CallInd { .. }+ | Inst::PatchableCall { .. }| Inst::ReturnCall { .. }| Inst::ReturnCallInd { .. }| Inst::Jal { .. }@@ -1193,9 +1194,12 @@ impl Inst {.for_each(|i| i.emit(sink, emit_info, state));}- &Inst::Call { ref info } => {+ &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {+ let is_patchable = matches!(self, Inst::PatchableCall { .. });+sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0);+ let start = sink.cur_offset();Inst::construct_auipc_and_jalr(Some(writable_link_reg()), writable_link_reg(), 0).into_iter().for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off));@@ -1210,6 +1214,9 @@ impl Inst {Some(state.frame_layout.sp_to_fp()),try_call.exception_handlers(&state.frame_layout),);+ } else if is_patchable {+ let len = sink.cur_offset() - start;+ sink.add_patchable_call_site(len);} else {sink.add_call_site();}diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rsindex 4a3890b093..1d2d5f6fdb 100644--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs@@ -330,7 +330,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_use(rn);collector.reg_def(rd);}- Inst::Call { info, .. } => {+ Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {let CallInfo { uses, defs, .. } = &mut **info;for CallArgPair { vreg, preg } in uses {collector.reg_fixed_use(vreg, *preg);@@ -725,7 +725,7 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::Call { .. } | Inst::CallInd { .. } => true,+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,_ => false,}}@@ -764,9 +764,10 @@ impl MachInst for Inst {fn call_type(&self) -> CallType {match self {- Inst::Call { .. } | Inst::CallInd { .. } | Inst::ElfTlsGetAddr { .. } => {- CallType::Regular- }+ Inst::Call { .. }+ | Inst::CallInd { .. }+ | Inst::PatchableCall { .. }+ | Inst::ElfTlsGetAddr { .. } => CallType::Regular,Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,@@ -810,6 +811,10 @@ impl MachInst for Inst {Inst::Nop4}+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {+ smallvec![0x13, 0x00, 0x00, 0x00]+ }+fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {match ty {I8 => Ok((&[RegClass::Int], &[I8])),@@ -1352,6 +1357,9 @@ impl Inst {.unwrap_or_default();format!("callind {rd}{try_call}")}+ &MInst::PatchableCall { ref info } => {+ format!("call {} ; patchable", info.dest.display(None))+ }&MInst::ReturnCall { ref info } => {let mut s = format!("return_call {:?} new_stack_arg_size:{}",diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex 920e5c2119..e1c5f02266 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -2731,6 +2731,15 @@(_ Unit (emit_side_effect (call_ind_impl info))))output))+;; Patchable direct call to an in-range function.+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))+ (let ((abi Sig (abi_sig sig_ref))+ (uses CallArgList (gen_call_args abi args))+ (defs CallRetList (gen_patchable_call_rets))+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))+ (_ Unit (emit_side_effect (patchable_call_impl info))))+ (output_none)))+;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex 46576a42d4..04d6839ddf 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -901,6 +901,11 @@(link WritableReg)(info BoxCallInfo))+ ;; A patchable machine call instruction.+ (PatchableCall+ (link WritableReg)+ (info BoxCallInfo))+;; A machine tail call instruction (direct or indirect).(ReturnCall(info BoxReturnCallInfo))@@ -2712,6 +2717,11 @@(rule (call_impl reg info)(SideEffectNoResult.Inst (MInst.Call reg info)))+;; Helper for emitting `MInst.PatchableCall` instructions.+(decl patchable_call_impl (WritableReg BoxCallInfo) SideEffectNoResult)+(rule (patchable_call_impl reg info)+ (SideEffectNoResult.Inst (MInst.PatchableCall reg info)))+;; Helper for emitting `MInst.ReturnCall` instructions.(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)(rule (return_call_impl info)diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rsindex d67bd5f56c..1e8ac63932 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs@@ -3297,7 +3297,10 @@ impl Inst {assert_eq!(state.nominal_sp_offset, 0);state.nominal_sp_offset += size;}- &Inst::Call { link, ref info } => {+ &Inst::Call { link, ref info } | &Inst::PatchableCall { link, ref info } => {+ let is_patchable = matches!(self, Inst::PatchableCall { .. });+ let start = sink.cur_offset();+let enc: &[u8] = match &info.dest {CallInstDest::Direct { name } => {let offset = sink.cur_offset() + 2;@@ -3321,6 +3324,8 @@ impl Inst {Some(state.frame_layout.sp_to_fp()),try_call.exception_handlers(&state.frame_layout),);+ } else if is_patchable {+ sink.add_patchable_call_site(sink.cur_offset() - start);} else {sink.add_call_site();}diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rsindex 8c16117ec6..d960e6eba0 100644--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs@@ -222,6 +222,7 @@ impl Inst {| Inst::AllocateArgs { .. }| Inst::Call { .. }| Inst::ReturnCall { .. }+ | Inst::PatchableCall { .. }| Inst::Args { .. }| Inst::Rets { .. }| Inst::Ret { .. }@@ -899,7 +900,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Opecollector.reg_use(rn);}Inst::AllocateArgs { .. } => {}- Inst::Call { link, info, .. } => {+ Inst::Call { link, info, .. } | Inst::PatchableCall { link, info, .. } => {let CallInfo {dest,uses,@@ -1125,14 +1126,16 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::Call { .. } => true,+ Inst::Call { .. } | Inst::PatchableCall { .. } => true,_ => false,}}fn call_type(&self) -> CallType {match self {- Inst::Call { .. } | Inst::ElfTlsGetOffset { .. } => CallType::Regular,+ Inst::Call { .. } | Inst::PatchableCall { .. } | Inst::ElfTlsGetOffset { .. } => {+ CallType::Regular+ }Inst::ReturnCall { .. } => CallType::TailCall,@@ -1161,6 +1164,10 @@ impl MachInst for Inst {}}+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {+ smallvec::smallvec![0x07, 0x07]+ }+fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {match ty {types::I8 => Ok((&[RegClass::Int], &[types::I8])),@@ -3188,7 +3195,8 @@ impl Inst {format!("slgfi {}, {}", show_reg(stack_reg()), size)}}- &Inst::Call { link, ref info } => {+ &Inst::Call { link, ref info } | &Inst::PatchableCall { link, ref info } => {+ let is_patchable = matches!(self, Inst::PatchableCall { .. });state.nominal_sp_offset = 0;let link = link.to_reg();let (opcode, dest) = match &info.dest {@@ -3219,14 +3227,16 @@ impl Inst {} else {"".to_string()};+ let patchable = if is_patchable { " ; patchable" } else { "" };format!(- "{} {}, {}{}{}{}",+ "{} {}, {}{}{}{}{}",opcode,show_reg(link),dest,callee_pop_size,retval_loads,- try_call+ try_call,+ patchable)}&Inst::ReturnCall { ref info } => {diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex da1b648c69..885634f6a1 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -4000,6 +4000,15 @@(_ Unit (emit_side_effect (call_impl (writable_link_reg) info))))output))+;; Patchable direct call to an in-range function.+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))+ (let ((abi Sig (abi_sig sig_ref))+ (_ Unit (abi_emit_call_adjust_stack abi))+ (uses CallArgList (gen_call_args abi (abi_prepare_args abi args)))+ (defs CallRetList (gen_patchable_call_rets))+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))+ (_ Unit (emit_side_effect (patchable_call_impl (writable_link_reg) info))))+ (output_none)));;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex c5c1c7c891..b6272f43eb 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -323,6 +323,9 @@;; A pseudoinstruction that serves as a sequence point.(SequencePoint)+ ;; A patchable call to a known location.+ (PatchableCallKnown (info BoxCallInfo))+;; An instruction assembled outside of cranelift-codegen.(External (inst AssemblerInst))))@@ -1342,6 +1345,10 @@(rule (return_call_unknown info)(SideEffectNoResult.Inst (MInst.ReturnCallUnknown info)))+;; Helper for creating `PatchableCallKnown` instructions.+(decl patchable_call_known (BoxCallInfo) SideEffectNoResult)+(rule (patchable_call_known info)+ (SideEffectNoResult.Inst (MInst.PatchableCallKnown info)));;;; Helpers for emitting stack switches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex e4afbd2a14..251f4a6985 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -408,7 +408,8 @@ pub(crate) fn emit(Inst::addq_mi(rsp, guard_plus_count).emit(sink, info, state);}- Inst::CallKnown { info: call_info } => {+ Inst::CallKnown { info: call_info } | Inst::PatchableCallKnown { info: call_info } => {+ let is_patchable = matches!(inst, Inst::PatchableCallKnown { .. });let stack_map = state.take_stack_map();asm::inst::callq_d::new(0).emit(sink, info, state);@@ -430,6 +431,10 @@ pub(crate) fn emit(Some(state.frame_layout().sp_to_fp()),try_call.exception_handlers(&state.frame_layout()),);+ } else if is_patchable {+ // Patchable call-site of length 5 bytes. The+ // MachBuffer knows how to NOP out the callsite.+ sink.add_patchable_call_site(5);} else {sink.add_call_site();}@@ -438,6 +443,7 @@ pub(crate) fn emit(// callee, to ensure that StackAMode values are always computed from// a consistent SP.if call_info.callee_pop_size > 0 {+ assert!(!is_patchable);let rsp = Writable::from_reg(regs::rsp());let callee_pop_size = i32::try_from(call_info.callee_pop_size).expect("`callee_pop_size` is too large to fit in a 32-bit immediate");diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex c3ee37a958..1d96403d28 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -77,6 +77,7 @@ impl Inst {| Inst::CallUnknown { .. }| Inst::ReturnCallKnown { .. }| Inst::ReturnCallUnknown { .. }+ | Inst::PatchableCallKnown { .. }| Inst::CheckedSRemSeq { .. }| Inst::CheckedSRemSeq8 { .. }| Inst::CvtFloatToSintSeq { .. }@@ -644,6 +645,11 @@ impl PrettyPrint for Inst {s}+ Inst::PatchableCallKnown { info } => {+ let op = ljustify("patchable_call".to_string());+ format!("{op} {:?}", info.dest)+ }+Inst::Rets { rets } => {let mut s = "rets".to_string();for ret in rets {@@ -959,7 +965,7 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_early_def(tmp);}- Inst::CallKnown { info } => {+ Inst::CallKnown { info } | Inst::PatchableCallKnown { info } => {// Probestack is special and is only inserted after// regalloc, so we do not need to represent its ABI to the// register allocator. Assert that we don't alter that@@ -1385,6 +1391,10 @@ impl MachInst for Inst {Inst::nop(std::cmp::min(preferred_size, 9) as u8)}+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {+ smallvec![0x90]+ }+fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {match ty {types::I8 => Ok((&[RegClass::Int], &[types::I8])),@@ -1447,7 +1457,9 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,+ Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::PatchableCallKnown { .. } => {+ true+ }_ => false,}}diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex ebfd43bbf1..018a6fa272 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -3568,6 +3568,17 @@(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))(emit_side_effect (call_unknown info))))+;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;++;; Direct call to an in-range function.+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))+ (let ((abi Sig (abi_sig sig_ref))+ (uses CallArgList (gen_call_args abi args))+ (defs CallRetList (gen_patchable_call_rets))+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))+ (_ Unit (emit_side_effect (patchable_call_known info))))+ (output_none)))+;; Rules for `stack_switch` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; currently, only the Basic model is supporteddiff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rsindex 09b356d699..48dda69840 100644--- a/cranelift/codegen/src/isa/x64/pcc.rs+++ b/cranelift/codegen/src/isa/x64/pcc.rs@@ -213,6 +213,8 @@ pub(crate) fn check(Inst::SequencePoint { .. } => Ok(()),+ Inst::PatchableCallKnown { .. } => Ok(()),+Inst::External { .. } => Ok(()), // TODO: unsure what to do about this!}}diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rsindex da77d12ae9..c003ee18f7 100644--- a/cranelift/codegen/src/machinst/buffer.rs+++ b/cranelift/codegen/src/machinst/buffer.rs@@ -250,6 +250,8 @@ pub struct MachBuffer<I: VCodeInst> {traps: SmallVec<[MachTrap; 16]>,/// Any call site records referring to this code.call_sites: SmallVec<[MachCallSite; 16]>,+ /// Any patchable call site locations.+ patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,/// Any exception-handler records referred to at call sites.exception_handlers: SmallVec<[MachExceptionHandler; 16]>,/// Any source location mappings referring to this code.@@ -340,6 +342,7 @@ impl MachBufferFinalized<Stencil> {relocs: self.relocs,traps: self.traps,call_sites: self.call_sites,+ patchable_call_sites: self.patchable_call_sites,exception_handlers: self.exception_handlers,srclocs: self.srclocs@@ -352,6 +355,7 @@ impl MachBufferFinalized<Stencil> {unwind_info: self.unwind_info,alignment: self.alignment,frame_layout: self.frame_layout,+ nop: self.nop,}}}@@ -374,6 +378,8 @@ pub struct MachBufferFinalized<T: CompilePhase> {pub(crate) traps: SmallVec<[MachTrap; 16]>,/// Any call site records referring to this code.pub(crate) call_sites: SmallVec<[MachCallSite; 16]>,+ /// Any patchable call site locations refering to this code.+ pub(crate) patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,/// Any exception-handler records referred to at call sites.pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,/// Any source location mappings referring to this code.@@ -395,6 +401,12 @@ pub struct MachBufferFinalized<T: CompilePhase> {pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,/// The required alignment of this buffer.pub alignment: u32,+ /// The means by which to NOP out patchable call sites.+ ///+ /// This allows a consumer of a `MachBufferFinalized` to disable+ /// patchable call sites (which are enabled by default) without+ /// specific knowledge of the target ISA.+ pub nop: SmallVec<[u8; 8]>,}const UNKNOWN_LABEL_OFFSET: CodeOffset = 0xffff_ffff;@@ -464,6 +476,7 @@ impl<I: VCodeInst> MachBuffer<I> {relocs: SmallVec::new(),traps: SmallVec::new(),call_sites: SmallVec::new(),+ patchable_call_sites: SmallVec::new(),exception_handlers: SmallVec::new(),srclocs: SmallVec::new(),debug_tags: vec![],@@ -1564,6 +1577,7 @@ impl<I: VCodeInst> MachBuffer<I> {relocs: finalized_relocs,traps: self.traps,call_sites: self.call_sites,+ patchable_call_sites: self.patchable_call_sites,exception_handlers: finalized_exception_handlers,srclocs,debug_tags: self.debug_tags,@@ -1572,6 +1586,7 @@ impl<I: VCodeInst> MachBuffer<I> {unwind_info: self.unwind_info,alignment,frame_layout: self.frame_layout,+ nop: I::gen_nop_unit(),}}@@ -1667,6 +1682,17 @@ impl<I: VCodeInst> MachBuffer<I> {});}+ /// Add a patchable call record at the current offset The actual+ /// call is expected to have been emitted; the VCodeInst trait+ /// specifies how to NOP it out, and we carry that information to+ /// the finalized Machbuffer.+ pub fn add_patchable_call_site(&mut self, len: u32) {+ self.patchable_call_sites.push(MachPatchableCallSite {+ ret_addr: self.cur_offset(),+ len,+ });+ }+/// Add an unwind record at the current offset.pub fn add_unwind(&mut self, unwind: UnwindInst) {self.unwind_info.push((self.cur_offset(), unwind));@@ -1863,6 +1889,22 @@ impl<T: CompilePhase> MachBufferFinalized<T> {pub fn frame_layout(&self) -> Option<&MachBufferFrameLayout> {self.frame_layout.as_ref()}++ /// Get the list of patchable call sites for this code.+ ///+ /// Each location in the buffer contains the bytes for a call+ /// instruction to the specified target. If the call is to be+ /// patched out, the bytes in the region should be replaced with+ /// those given in the `MachBufferFinalized::nop` array, repeated+ /// as many times as necessary. (The length of the patchable+ /// region is guaranteed to be an integer multiple of that NOP+ /// unit size.)+ pub fn patchable_call_sites(&self) -> impl Iterator<Item = &MachPatchableCallSite> + '_ {+ self.patchable_call_sites.iter().map(|call_site| {+ debug_assert!(call_site.len as usize % self.nop.len() == 0);+ call_site+ })+ }}/// An item in the exception-handler list for a callsite, with label@@ -2134,6 +2176,22 @@ pub struct FinalizedMachCallSite<'a> {pub exception_handlers: &'a [FinalizedMachExceptionHandler],}+/// A patchable call site record resulting from a compilation.+#[derive(Clone, Debug, PartialEq)]+#[cfg_attr(+ feature = "enable-serde",+ derive(serde_derive::Serialize, serde_derive::Deserialize)+)]+pub struct MachPatchableCallSite {+ /// The offset of the call's return address (i.e., the address+ /// after the end of the patchable region), *relative to the start+ /// of the buffer*.+ pub ret_addr: CodeOffset,++ /// The length of the region to be patched by NOP bytes.+ pub len: u32,+}+/// A source-location mapping resulting from a compilation.#[derive(PartialEq, Debug, Clone)]#[cfg_attr(diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rsindex 4f9a37a406..aba18a15c0 100644--- a/cranelift/codegen/src/machinst/isle.rs+++ b/cranelift/codegen/src/machinst/isle.rs@@ -620,6 +620,10 @@ macro_rules! isle_lower_prelude_methods {self.lower_ctx.gen_try_call_rets(sig)}+ fn gen_patchable_call_rets(&mut self) -> CallRetList {+ smallvec::smallvec![]+ }+fn try_call_none(&mut self) -> OptionTryCallInfo {None}diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rsindex 032c09a3e7..15d8ed55c6 100644--- a/cranelift/codegen/src/machinst/mod.rs+++ b/cranelift/codegen/src/machinst/mod.rs@@ -177,6 +177,10 @@ pub trait MachInst: Clone + Debug {/// the instruction must have a nonzero size if preferred_size is nonzero.fn gen_nop(preferred_size: usize) -> Self;+ /// The smallest possible NOP, as a unit that can be used to patch+ /// out code.+ fn gen_nop_unit() -> SmallVec<[u8; 8]>;+/// Align a basic block offset (from start of function). By default, no/// alignment occurs.fn align_basic_block(offset: CodeOffset) -> CodeOffset {@@ -446,6 +450,7 @@ impl<T: CompilePhase> CompiledCodeBase<T> {let relocs = self.buffer.relocs();let traps = self.buffer.traps();+ let mut patchables = self.buffer.patchable_call_sites().peekable();// Normalize the block starts to include an initial block of offset 0.let mut block_starts = Vec::new();@@ -495,6 +500,17 @@ impl<T: CompilePhase> CompiledCodeBase<T> {write!(buf, " ; trap: {}", trap.code)?;}+ if let Some(patchable) = patchables.peek()+ && patchable.ret_addr == end as u32+ {+ write!(+ buf,+ " ; patchable call: NOP out last {} bytes",+ patchable.len+ )?;+ patchables.next();+ }+writeln!(buf)?;}}diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isleindex 3c30a8bf34..3fafa3fdc4 100644--- a/cranelift/codegen/src/prelude_lower.isle+++ b/cranelift/codegen/src/prelude_lower.isle@@ -1173,6 +1173,9 @@(decl gen_try_call_rets (Sig) CallRetList)(extern constructor gen_try_call_rets gen_try_call_rets)+(decl gen_patchable_call_rets () CallRetList)+(extern constructor gen_patchable_call_rets gen_patchable_call_rets)+(type OptionTryCallInfo (primitive OptionTryCallInfo))(decl try_call_info (ExceptionTable MachLabelSlice) OptionTryCallInfo)(extern constructor try_call_info try_call_info)diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rsindex 1a64f166ea..34f1f7385e 100644--- a/cranelift/codegen/src/verifier/mod.rs+++ b/cranelift/codegen/src/verifier/mod.rs@@ -587,10 +587,14 @@ impl<'a> Verifier<'a> {self.verify_jump_table(inst, table, errors)?;}Call {- func_ref, ref args, ..+ opcode,+ func_ref,+ ref args,+ ..} => {self.verify_func_ref(inst, func_ref, errors)?;self.verify_value_list(inst, args, errors)?;+ self.verify_callee_abi(inst, func_ref, opcode, errors)?;}CallIndirect {sig_ref, ref args, ..@@ -951,6 +955,38 @@ impl<'a> Verifier<'a> {Ok(())}+ fn verify_callee_abi(+ &self,+ inst: Inst,+ func_ref: FuncRef,+ opcode: Opcode,+ errors: &mut VerifierErrors,+ ) -> VerifierStepResult {+ let callee_sig_ref = self.func.dfg.ext_funcs[func_ref].signature;+ let callee_sig = &self.func.dfg.signatures[callee_sig_ref];+ let callee_abi = callee_sig.call_conv;+ match (opcode, callee_abi) {+ (Opcode::PatchableCall, CallConv::Patchable) => {+ if !self.func.dfg.ext_funcs[func_ref].colocated {+ errors.fatal((+ inst,+ self.context(inst),+ "patchable call to non-colocated function".to_string(),+ ))?;+ }+ }+ (Opcode::PatchableCall, _) => {+ errors.fatal((+ inst,+ self.context(inst),+ "patchable call with non-patchable-ABI callee".to_string(),+ ))?;+ }+ _ => {}+ }+ Ok(())+ }+fn verify_value(&self,loc_inst: Inst,diff --git a/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif b/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clifindex fa69c981de..7f9660801a 100644--- a/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif@@ -163,3 +163,27 @@ block0(v0: i64):; ldp x29, x30, [sp], #0x10; ret+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; block0:+; bl 0 ; patchable+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; bl #8 ; reloc_external Call %f 0 ; patchable call: NOP out last 4 bytes+; ldp x29, x30, [sp], #0x10+; ret+diff --git a/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif b/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clifindex 09a1a333bf..4ac1a3a297 100644--- a/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif@@ -370,3 +370,23 @@ block0(v0: i64):; pop_frame_restore 896,; ret+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; push_frame+; block0:+; patchable_call CallInfo { dest: PulleyCall { name: TestCase(%f), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: Patchable, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None }+; pop_frame+; ret+;+; Disassembled:+; push_frame+; call 0x1 // target = 0x2+; pop_frame+; ret+diff --git a/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif b/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clifindex 2cf2a50bfa..b2b6e8671f 100644--- a/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif@@ -370,3 +370,23 @@ block0(v0: i64):; pop_frame_restore 896,; ret+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; push_frame+; block0:+; patchable_call CallInfo { dest: PulleyCall { name: TestCase(%f), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: Patchable, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None }+; pop_frame+; ret+;+; Disassembled:+; push_frame+; call 0x1 // target = 0x2+; pop_frame+; ret+diff --git a/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif b/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clifindex a00785face..903d683833 100644--- a/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif@@ -424,3 +424,36 @@ block0(v0: i64):; addi sp, sp, 0x10; ret+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; addi sp,sp,-16+; sd ra,8(sp)+; sd fp,0(sp)+; mv fp,sp+; block0:+; call %f ; patchable+; ld ra,8(sp)+; ld fp,0(sp)+; addi sp,sp,16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; addi sp, sp, -0x10+; sd ra, 8(sp)+; sd s0, 0(sp)+; mv s0, sp+; block1: ; offset 0x10+; auipc ra, 0 ; reloc_external RiscvCallPlt %f 0+; jalr ra ; patchable call: NOP out last 8 bytes+; ld ra, 8(sp)+; ld s0, 0(sp)+; addi sp, sp, 0x10+; ret+diff --git a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clifindex 5339d53138..8ecdf5c1f5 100644--- a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif@@ -193,3 +193,27 @@ block0(v0: i64):; lmg %r0, %r15, 0x1a0(%r15); br %r14+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; stmg %r14, %r15, 112(%r15)+; aghi %r15, -160+; block0:+; brasl %r14, %f ; patchable+; lmg %r14, %r15, 272(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r14, %r15, 0x70(%r15)+; aghi %r15, -0xa0+; block1: ; offset 0xa+; brasl %r14, 0xa ; reloc_external PLTRel32Dbl %f 2 ; patchable call: NOP out last 6 bytes+; lmg %r14, %r15, 0x110(%r15)+; br %r14+diff --git a/cranelift/filetests/filetests/isa/x64/patchable-abi.clif b/cranelift/filetests/filetests/isa/x64/patchable-abi.clifindex 24d1eaa974..a197062b49 100644--- a/cranelift/filetests/filetests/isa/x64/patchable-abi.clif+++ b/cranelift/filetests/filetests/isa/x64/patchable-abi.clif@@ -168,3 +168,29 @@ block0(v0: i64):; popq %rbp; retq+function %patchable_call(i64) system_v {+ fn0 = colocated %f(i64) patchable+block0(v0: i64):+ patchable_call fn0(v0)+ return+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; patchable_call TestCase(%f)+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; callq 9 ; reloc_external CallPCRel4 %f -4 ; patchable call: NOP out last 5 bytes+; movq %rbp, %rsp+; popq %rbp+; retq+diff --git a/cranelift/filetests/filetests/verifier/patchable-abi.clif b/cranelift/filetests/filetests/verifier/patchable-abi.clifindex 412f7e75a6..f43a3dad13 100644--- a/cranelift/filetests/filetests/verifier/patchable-abi.clif+++ b/cranelift/filetests/filetests/verifier/patchable-abi.clif@@ -27,3 +27,17 @@ function %patchable_abi_sigref(i32) system_v {block0(v0: i32):return}++function %patchable_call_to_non_patchable_abi() system_v {+ fn0 = %f() system_v+block0:+ patchable_call fn0() ; error: patchable call with non-patchable-ABI callee+ return+}++function %patchable_call_to_non_colocated_patchable_abi() system_v {+ fn0 = %f() patchable+block0:+ patchable_call fn0() ; error: patchable call to non-colocated function+ return+}diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rsindex 8a7fad2323..d69b8db53f 100644--- a/cranelift/interpreter/src/step.rs+++ b/cranelift/interpreter/src/step.rs@@ -1323,6 +1323,7 @@ whereOpcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"),Opcode::SequencePoint => unimplemented!("SequencePoint"),+ Opcode::PatchableCall => unimplemented!("PatchableCall"),})}diff --git a/pulley/src/encode.rs b/pulley/src/encode.rsindex 29ffe000bb..2815d892bc 100644--- a/pulley/src/encode.rs+++ b/pulley/src/encode.rs@@ -332,3 +332,14 @@ macro_rules! impl_extended_encoders {};}for_each_extended_op!(impl_extended_encoders);++#[test]+#[cfg(feature = "std")]+fn nop_is_single_byte() {+ // NOP needs to be a single byte so that it can be used to NOP out+ // an instruction sequence of any length.+ let inst = crate::op::Nop {};+ let mut bytes = vec![];+ inst.encode(&mut bytes);+ assert_eq!(bytes.len(), 1);+}diff --git a/pulley/src/interp.rs b/pulley/src/interp.rsindex f394822aae..3b72697b3d 100644--- a/pulley/src/interp.rs+++ b/pulley/src/interp.rs@@ -1343,6 +1343,10 @@ impl OpVisitor for Interpreter<'_> {&mut self.pc}+ fn nop(&mut self) -> ControlFlow<Done> {+ ControlFlow::Continue(())+ }+fn ret(&mut self) -> ControlFlow<Done> {let lr = self.state.lr;if lr == HOST_RETURN_ADDR {@@ -2827,10 +2831,6 @@ impl OpVisitor for Interpreter<'_> {}impl ExtendedOpVisitor for Interpreter<'_> {- fn nop(&mut self) -> ControlFlow<Done> {- ControlFlow::Continue(())- }-fn trap(&mut self) -> ControlFlow<Done> {self.done_trap::<crate::Trap>()}diff --git a/pulley/src/lib.rs b/pulley/src/lib.rsindex d9a794a783..40d7122206 100644--- a/pulley/src/lib.rs+++ b/pulley/src/lib.rs@@ -86,6 +86,9 @@ extern crate alloc;macro_rules! for_each_op {( $macro:ident ) => {$macro! {+ /// No-operation.+ nop = Nop;+/// Transfer control the address in the `lr` register.ret = Ret;@@ -623,9 +626,6 @@ macro_rules! for_each_extended_op {/// Raise a trap.trap = Trap;- /// Do nothing.- nop = Nop;-/// A special opcode to halt interpreter execution and yield control/// back to the host.///