Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: add patchable call instructions. (#12101)

Chris Fallinc00e9ea2ecb7
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index 9604c6904b..8e1365dd85 100644
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -3918,4 +3918,31 @@ pub(crate) fn define(
)
.other_side_effects(),
);
+
+ ig.push(
+ Inst::new(
+ "patchable_call",
+ r#"
+ A call instruction that can be turned on/off by patching machine code.
+
+ This call is different than an ordinary call in a few ways:
+ - It can only call functions with the `patchable` ABI.
+ - As a result of that, the called function cannot have any
+ return values.
+ - The call emits metadata into the MachBuffer that allows the consumer
+ of the machine code to edit the code to either make the call or not.
+
+ Note that the lack of return values is necessary: because the call
+ may or may not occur, dynamically, we cannot rely on any values
+ actually being defined.
+ "#,
+ &formats.call,
+ )
+ .operands_in(vec![
+ Operand::new("FN", &entities.func_ref)
+ .with_doc("function to call, declared by `function`"),
+ Operand::new("args", &entities.varargs).with_doc("call arguments"),
+ ])
+ .call(),
+ );
}
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 7b42e17286..60507594ba 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -149,7 +149,11 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool {
| Opcode::Fence
| Opcode::Debugtrap
| Opcode::SequencePoint => true,
- Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true,
+ Opcode::Call
+ | Opcode::CallIndirect
+ | Opcode::TryCall
+ | Opcode::TryCallIndirect
+ | Opcode::PatchableCall => true,
op if op.can_trap() => true,
_ => false,
}
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 1c7bc76fdd..44c9c023f3 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -817,6 +817,9 @@
;; An indirect return-call macro instruction.
(ReturnCallInd (info BoxReturnCallIndInfo))
+ ;; A patchable call instruction.
+ (PatchableCall (info BoxCallInfo))
+
;; A pseudo-instruction that captures register arguments in vregs.
(Args
(args VecArgPair))
@@ -4510,6 +4513,11 @@
(rule (return_call_ind_impl info)
(SideEffectNoResult.Inst (MInst.ReturnCallInd info)))
+;; Helper for creating `MInst.PatchableCall` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
;; Helpers for pinned register manipulation.
(decl write_pinned_reg (Reg) SideEffectNoResult)
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 91beb84efb..678ffa5a1b 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2941,7 +2941,8 @@ impl MachInstEmit for Inst {
sink.put4(0xd65f0bff | (op2 << 9)); // reta{key}
}
}
- &Inst::Call { ref info } => {
+ &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {
+ let is_patchable = matches!(self, Inst::PatchableCall { .. });
let user_stack_map = state.take_stack_map();
sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
sink.put4(enc_jump26(0b100101, 0));
@@ -2955,6 +2956,8 @@ impl MachInstEmit for Inst {
Some(state.frame_layout.sp_to_fp()),
try_call.exception_handlers(&state.frame_layout),
);
+ } else if is_patchable {
+ sink.add_patchable_call_site(4);
} else {
sink.add_call_site();
}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 1ba8ec50ff..f8c8b997cc 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -801,7 +801,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
}
Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
Inst::Jump { .. } => {}
- Inst::Call { info, .. } => {
+ Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {
let CallInfo { uses, defs, .. } = &mut **info;
for CallArgPair { vreg, preg } in uses {
collector.reg_fixed_use(vreg, *preg);
@@ -1008,6 +1008,7 @@ impl MachInst for Inst {
match self {
Inst::Call { .. }
| Inst::CallInd { .. }
+ | Inst::PatchableCall { .. }
| Inst::ElfTlsGetAddr { .. }
| Inst::MachOTlsGetAddr { .. } => CallType::Regular,
@@ -1092,7 +1093,7 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::Call { .. } | Inst::CallInd { .. } => true,
+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,
_ => false,
}
}
@@ -1110,6 +1111,10 @@ impl MachInst for Inst {
Inst::Nop4
}
+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+ smallvec![0x1f, 0x20, 0x03, 0xd5]
+ }
+
fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
match ty {
I8 => Ok((&[RegClass::Int], &[I8])),
@@ -2594,6 +2599,9 @@ impl Inst {
.unwrap_or_default();
format!("blr {rn}{try_call}")
}
+ &Inst::PatchableCall { .. } => {
+ format!("bl 0 ; patchable")
+ }
&Inst::ReturnCall { ref info } => {
let mut s = format!(
"return_call {:?} new_stack_arg_size:{}",
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 33c06c4c4a..783c015b5a 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2573,6 +2573,17 @@
(_ Unit (emit_side_effect (call_ind_impl info))))
output))
+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+ (let ((abi Sig (abi_sig sig_ref))
+ (uses CallArgList (gen_call_args abi args))
+ (defs CallRetList (gen_patchable_call_rets))
+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+ (_ Unit (emit_side_effect (patchable_call_impl info))))
+ (output_none)))
+
;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Direct call to an in-range function.
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle
index 901753841c..176346cf13 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle
@@ -51,6 +51,9 @@
;; An indirect call to an unknown callee.
(IndirectCall (info BoxCallIndInfo))
+ ;; A patchable direct call to a known callee.
+ (PatchableCall (info BoxCallInfo))
+
;; A direct return-call macro instruction.
(ReturnCall (info BoxReturnCallInfo))
@@ -749,6 +752,11 @@
(rule (indirect_call_host_impl info)
(SideEffectNoResult.Inst (MInst.IndirectCallHost info)))
+;; Helper for creating `MInst.Call` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
;; Helper for creating `MInst.ReturnCall` instructions.
(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
(rule (return_call_impl info)
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
index c8de063259..70ee02ffdf 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs
@@ -166,7 +166,10 @@ fn pulley_emit<P>(
sink.add_reloc_at_offset(end - size, Reloc::Abs8, &**name, *offset);
}
- Inst::Call { info } => {
+ Inst::Call { info } | Inst::PatchableCall { info } => {
+ let is_patchable = matches!(inst, Inst::PatchableCall { .. });
+ let start = sink.cur_offset();
+
// If arguments happen to already be in the right register for the
// ABI then remove them from this list. Otherwise emit the
// appropriate `Call` instruction depending on how many arguments we
@@ -196,6 +199,8 @@ fn pulley_emit<P>(
Some(state.frame_layout.sp_to_fp()),
try_call.exception_handlers(&state.frame_layout),
);
+ } else if is_patchable {
+ sink.add_patchable_call_site(sink.cur_offset() - start);
} else {
sink.add_call_site();
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
index 606ec93ae3..bc7082ee6e 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs
@@ -148,7 +148,7 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_def(dst);
}
- Inst::Call { info } => {
+ Inst::Call { info } | Inst::PatchableCall { info } => {
let CallInfo {
uses,
defs,
@@ -435,7 +435,8 @@ where
}
| Inst::Call { .. }
| Inst::IndirectCall { .. }
- | Inst::IndirectCallHost { .. } => true,
+ | Inst::IndirectCallHost { .. }
+ | Inst::PatchableCall { .. } => true,
_ => false,
}
}
@@ -498,9 +499,10 @@ where
fn call_type(&self) -> CallType {
match &self.inst {
- Inst::Call { .. } | Inst::IndirectCall { .. } | Inst::IndirectCallHost { .. } => {
- CallType::Regular
- }
+ Inst::Call { .. }
+ | Inst::IndirectCall { .. }
+ | Inst::IndirectCallHost { .. }
+ | Inst::PatchableCall { .. } => CallType::Regular,
Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => CallType::TailCall,
@@ -533,6 +535,16 @@ where
todo!()
}
+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+ let mut bytes = smallvec::smallvec![];
+ let nop = pulley_interpreter::op::Nop {};
+ nop.encode(&mut bytes);
+ // NOP needs to be a 1-byte opcode so it can be used to
+ // overwrite a callsite of any length.
+ assert_eq!(bytes.len(), 1);
+ bytes
+ }
+
fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
match ty {
I8 => Ok((&[RegClass::Int], &[I8])),
@@ -712,6 +724,10 @@ impl Inst {
format!("indirect_call {callee}, {info:?}{try_call}")
}
+ Inst::PatchableCall { info } => {
+ format!("patchable_call {info:?}")
+ }
+
Inst::ReturnCall { info } => {
format!("return_call {info:?}")
}
diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle
index 0373d45157..b5d9a87525 100644
--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle
@@ -171,6 +171,17 @@
(_ Unit (emit_side_effect (indirect_call_impl info))))
output))
+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+ (let ((abi Sig (abi_sig sig_ref))
+ (uses CallArgList (gen_call_args abi args))
+ (defs CallRetList (gen_patchable_call_rets))
+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+ (_ Unit (emit_side_effect (patchable_call_impl info))))
+ (output_none)))
+
;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Direct call to a pulley function.
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 62774b59fe..9a4c29edaf 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -111,11 +111,15 @@
(from_bits u8)
(to_bits u8))
+ ;; A machine direct-call instruction.
(Call (info BoxCallInfo))
- ;; A machine indirect-call instruction.
+ ;; A machine indirect-call instruction.
(CallInd (info BoxCallIndInfo))
+ ;; A patchable machine direct-call instruction.
+ (PatchableCall (info BoxCallInfo))
+
;; A direct return-call macro instruction.
(ReturnCall (info BoxReturnCallInfo))
@@ -3114,6 +3118,11 @@
(rule (call_ind_impl info)
(SideEffectNoResult.Inst (MInst.CallInd info)))
+;; Helper for creating `MInst.PatchableCall` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+ (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
;; Helper for creating `MInst.ReturnCall` instructions.
(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
(rule (return_call_impl info)
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index 77bd801c4f..a4801081a9 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -167,6 +167,7 @@ impl Inst {
| Inst::Extend { .. }
| Inst::Call { .. }
| Inst::CallInd { .. }
+ | Inst::PatchableCall { .. }
| Inst::ReturnCall { .. }
| Inst::ReturnCallInd { .. }
| Inst::Jal { .. }
@@ -1193,9 +1194,12 @@ impl Inst {
.for_each(|i| i.emit(sink, emit_info, state));
}
- &Inst::Call { ref info } => {
+ &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {
+ let is_patchable = matches!(self, Inst::PatchableCall { .. });
+
sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0);
+ let start = sink.cur_offset();
Inst::construct_auipc_and_jalr(Some(writable_link_reg()), writable_link_reg(), 0)
.into_iter()
.for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off));
@@ -1210,6 +1214,9 @@ impl Inst {
Some(state.frame_layout.sp_to_fp()),
try_call.exception_handlers(&state.frame_layout),
);
+ } else if is_patchable {
+ let len = sink.cur_offset() - start;
+ sink.add_patchable_call_site(len);
} else {
sink.add_call_site();
}
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 4a3890b093..1d2d5f6fdb 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -330,7 +330,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_use(rn);
collector.reg_def(rd);
}
- Inst::Call { info, .. } => {
+ Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {
let CallInfo { uses, defs, .. } = &mut **info;
for CallArgPair { vreg, preg } in uses {
collector.reg_fixed_use(vreg, *preg);
@@ -725,7 +725,7 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::Call { .. } | Inst::CallInd { .. } => true,
+ Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,
_ => false,
}
}
@@ -764,9 +764,10 @@ impl MachInst for Inst {
fn call_type(&self) -> CallType {
match self {
- Inst::Call { .. } | Inst::CallInd { .. } | Inst::ElfTlsGetAddr { .. } => {
- CallType::Regular
- }
+ Inst::Call { .. }
+ | Inst::CallInd { .. }
+ | Inst::PatchableCall { .. }
+ | Inst::ElfTlsGetAddr { .. } => CallType::Regular,
Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
@@ -810,6 +811,10 @@ impl MachInst for Inst {
Inst::Nop4
}
+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+ smallvec![0x13, 0x00, 0x00, 0x00]
+ }
+
fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
match ty {
I8 => Ok((&[RegClass::Int], &[I8])),
@@ -1352,6 +1357,9 @@ impl Inst {
.unwrap_or_default();
format!("callind {rd}{try_call}")
}
+ &MInst::PatchableCall { ref info } => {
+ format!("call {} ; patchable", info.dest.display(None))
+ }
&MInst::ReturnCall { ref info } => {
let mut s = format!(
"return_call {:?} new_stack_arg_size:{}",
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 920e5c2119..e1c5f02266 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -2731,6 +2731,15 @@
(_ Unit (emit_side_effect (call_ind_impl info))))
output))
+;; Patchable direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+ (let ((abi Sig (abi_sig sig_ref))
+ (uses CallArgList (gen_call_args abi args))
+ (defs CallRetList (gen_patchable_call_rets))
+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+ (_ Unit (emit_side_effect (patchable_call_impl info))))
+ (output_none)))
+
;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Direct call to an in-range function.
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 46576a42d4..04d6839ddf 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -901,6 +901,11 @@
(link WritableReg)
(info BoxCallInfo))
+ ;; A patchable machine call instruction.
+ (PatchableCall
+ (link WritableReg)
+ (info BoxCallInfo))
+
;; A machine tail call instruction (direct or indirect).
(ReturnCall
(info BoxReturnCallInfo))
@@ -2712,6 +2717,11 @@
(rule (call_impl reg info)
(SideEffectNoResult.Inst (MInst.Call reg info)))
+;; Helper for emitting `MInst.PatchableCall` instructions.
+(decl patchable_call_impl (WritableReg BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl reg info)
+ (SideEffectNoResult.Inst (MInst.PatchableCall reg info)))
+
;; Helper for emitting `MInst.ReturnCall` instructions.
(decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
(rule (return_call_impl info)
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index d67bd5f56c..1e8ac63932 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -3297,7 +3297,10 @@ impl Inst {
assert_eq!(state.nominal_sp_offset, 0);
state.nominal_sp_offset += size;
}
- &Inst::Call { link, ref info } => {
+ &Inst::Call { link, ref info } | &Inst::PatchableCall { link, ref info } => {
+ let is_patchable = matches!(self, Inst::PatchableCall { .. });
+ let start = sink.cur_offset();
+
let enc: &[u8] = match &info.dest {
CallInstDest::Direct { name } => {
let offset = sink.cur_offset() + 2;
@@ -3321,6 +3324,8 @@ impl Inst {
Some(state.frame_layout.sp_to_fp()),
try_call.exception_handlers(&state.frame_layout),
);
+ } else if is_patchable {
+ sink.add_patchable_call_site(sink.cur_offset() - start);
} else {
sink.add_call_site();
}
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 8c16117ec6..d960e6eba0 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -222,6 +222,7 @@ impl Inst {
| Inst::AllocateArgs { .. }
| Inst::Call { .. }
| Inst::ReturnCall { .. }
+ | Inst::PatchableCall { .. }
| Inst::Args { .. }
| Inst::Rets { .. }
| Inst::Ret { .. }
@@ -899,7 +900,7 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Ope
collector.reg_use(rn);
}
Inst::AllocateArgs { .. } => {}
- Inst::Call { link, info, .. } => {
+ Inst::Call { link, info, .. } | Inst::PatchableCall { link, info, .. } => {
let CallInfo {
dest,
uses,
@@ -1125,14 +1126,16 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::Call { .. } => true,
+ Inst::Call { .. } | Inst::PatchableCall { .. } => true,
_ => false,
}
}
fn call_type(&self) -> CallType {
match self {
- Inst::Call { .. } | Inst::ElfTlsGetOffset { .. } => CallType::Regular,
+ Inst::Call { .. } | Inst::PatchableCall { .. } | Inst::ElfTlsGetOffset { .. } => {
+ CallType::Regular
+ }
Inst::ReturnCall { .. } => CallType::TailCall,
@@ -1161,6 +1164,10 @@ impl MachInst for Inst {
}
}
+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+ smallvec::smallvec![0x07, 0x07]
+ }
+
fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
match ty {
types::I8 => Ok((&[RegClass::Int], &[types::I8])),
@@ -3188,7 +3195,8 @@ impl Inst {
format!("slgfi {}, {}", show_reg(stack_reg()), size)
}
}
- &Inst::Call { link, ref info } => {
+ &Inst::Call { link, ref info } | &Inst::PatchableCall { link, ref info } => {
+ let is_patchable = matches!(self, Inst::PatchableCall { .. });
state.nominal_sp_offset = 0;
let link = link.to_reg();
let (opcode, dest) = match &info.dest {
@@ -3219,14 +3227,16 @@ impl Inst {
} else {
"".to_string()
};
+ let patchable = if is_patchable { " ; patchable" } else { "" };
format!(
- "{} {}, {}{}{}{}",
+ "{} {}, {}{}{}{}{}",
opcode,
show_reg(link),
dest,
callee_pop_size,
retval_loads,
- try_call
+ try_call,
+ patchable
)
}
&Inst::ReturnCall { ref info } => {
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index da1b648c69..885634f6a1 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -4000,6 +4000,15 @@
(_ Unit (emit_side_effect (call_impl (writable_link_reg) info))))
output))
+;; Patchable direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+ (let ((abi Sig (abi_sig sig_ref))
+ (_ Unit (abi_emit_call_adjust_stack abi))
+ (uses CallArgList (gen_call_args abi (abi_prepare_args abi args)))
+ (defs CallRetList (gen_patchable_call_rets))
+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+ (_ Unit (emit_side_effect (patchable_call_impl (writable_link_reg) info))))
+ (output_none)))
;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index c5c1c7c891..b6272f43eb 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -323,6 +323,9 @@
;; A pseudoinstruction that serves as a sequence point.
(SequencePoint)
+ ;; A patchable call to a known location.
+ (PatchableCallKnown (info BoxCallInfo))
+
;; An instruction assembled outside of cranelift-codegen.
(External (inst AssemblerInst))))
@@ -1342,6 +1345,10 @@
(rule (return_call_unknown info)
(SideEffectNoResult.Inst (MInst.ReturnCallUnknown info)))
+;; Helper for creating `PatchableCallKnown` instructions.
+(decl patchable_call_known (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_known info)
+ (SideEffectNoResult.Inst (MInst.PatchableCallKnown info)))
;;;; Helpers for emitting stack switches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index e4afbd2a14..251f4a6985 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -408,7 +408,8 @@ pub(crate) fn emit(
Inst::addq_mi(rsp, guard_plus_count).emit(sink, info, state);
}
- Inst::CallKnown { info: call_info } => {
+ Inst::CallKnown { info: call_info } | Inst::PatchableCallKnown { info: call_info } => {
+ let is_patchable = matches!(inst, Inst::PatchableCallKnown { .. });
let stack_map = state.take_stack_map();
asm::inst::callq_d::new(0).emit(sink, info, state);
@@ -430,6 +431,10 @@ pub(crate) fn emit(
Some(state.frame_layout().sp_to_fp()),
try_call.exception_handlers(&state.frame_layout()),
);
+ } else if is_patchable {
+ // Patchable call-site of length 5 bytes. The
+ // MachBuffer knows how to NOP out the callsite.
+ sink.add_patchable_call_site(5);
} else {
sink.add_call_site();
}
@@ -438,6 +443,7 @@ pub(crate) fn emit(
// callee, to ensure that StackAMode values are always computed from
// a consistent SP.
if call_info.callee_pop_size > 0 {
+ assert!(!is_patchable);
let rsp = Writable::from_reg(regs::rsp());
let callee_pop_size = i32::try_from(call_info.callee_pop_size)
.expect("`callee_pop_size` is too large to fit in a 32-bit immediate");
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index c3ee37a958..1d96403d28 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -77,6 +77,7 @@ impl Inst {
| Inst::CallUnknown { .. }
| Inst::ReturnCallKnown { .. }
| Inst::ReturnCallUnknown { .. }
+ | Inst::PatchableCallKnown { .. }
| Inst::CheckedSRemSeq { .. }
| Inst::CheckedSRemSeq8 { .. }
| Inst::CvtFloatToSintSeq { .. }
@@ -644,6 +645,11 @@ impl PrettyPrint for Inst {
s
}
+ Inst::PatchableCallKnown { info } => {
+ let op = ljustify("patchable_call".to_string());
+ format!("{op} {:?}", info.dest)
+ }
+
Inst::Rets { rets } => {
let mut s = "rets".to_string();
for ret in rets {
@@ -959,7 +965,7 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_early_def(tmp);
}
- Inst::CallKnown { info } => {
+ Inst::CallKnown { info } | Inst::PatchableCallKnown { info } => {
// Probestack is special and is only inserted after
// regalloc, so we do not need to represent its ABI to the
// register allocator. Assert that we don't alter that
@@ -1385,6 +1391,10 @@ impl MachInst for Inst {
Inst::nop(std::cmp::min(preferred_size, 9) as u8)
}
+ fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+ smallvec![0x90]
+ }
+
fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
match ty {
types::I8 => Ok((&[RegClass::Int], &[types::I8])),
@@ -1447,7 +1457,9 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
+ Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::PatchableCallKnown { .. } => {
+ true
+ }
_ => false,
}
}
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index ebfd43bbf1..018a6fa272 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -3568,6 +3568,17 @@
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))
(emit_side_effect (call_unknown info))))
+;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+ (let ((abi Sig (abi_sig sig_ref))
+ (uses CallArgList (gen_call_args abi args))
+ (defs CallRetList (gen_patchable_call_rets))
+ (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+ (_ Unit (emit_side_effect (patchable_call_known info))))
+ (output_none)))
+
;; Rules for `stack_switch` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; currently, only the Basic model is supported
diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs
index 09b356d699..48dda69840 100644
--- a/cranelift/codegen/src/isa/x64/pcc.rs
+++ b/cranelift/codegen/src/isa/x64/pcc.rs
@@ -213,6 +213,8 @@ pub(crate) fn check(
Inst::SequencePoint { .. } => Ok(()),
+ Inst::PatchableCallKnown { .. } => Ok(()),
+
Inst::External { .. } => Ok(()), // TODO: unsure what to do about this!
}
}
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index da77d12ae9..c003ee18f7 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -250,6 +250,8 @@ pub struct MachBuffer<I: VCodeInst> {
traps: SmallVec<[MachTrap; 16]>,
/// Any call site records referring to this code.
call_sites: SmallVec<[MachCallSite; 16]>,
+ /// Any patchable call site locations.
+ patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,
/// Any exception-handler records referred to at call sites.
exception_handlers: SmallVec<[MachExceptionHandler; 16]>,
/// Any source location mappings referring to this code.
@@ -340,6 +342,7 @@ impl MachBufferFinalized<Stencil> {
relocs: self.relocs,
traps: self.traps,
call_sites: self.call_sites,
+ patchable_call_sites: self.patchable_call_sites,
exception_handlers: self.exception_handlers,
srclocs: self
.srclocs
@@ -352,6 +355,7 @@ impl MachBufferFinalized<Stencil> {
unwind_info: self.unwind_info,
alignment: self.alignment,
frame_layout: self.frame_layout,
+ nop: self.nop,
}
}
}
@@ -374,6 +378,8 @@ pub struct MachBufferFinalized<T: CompilePhase> {
pub(crate) traps: SmallVec<[MachTrap; 16]>,
/// Any call site records referring to this code.
pub(crate) call_sites: SmallVec<[MachCallSite; 16]>,
+ /// Any patchable call site locations refering to this code.
+ pub(crate) patchable_call_sites: SmallVec<[MachPatchableCallSite; 16]>,
/// Any exception-handler records referred to at call sites.
pub(crate) exception_handlers: SmallVec<[FinalizedMachExceptionHandler; 16]>,
/// Any source location mappings referring to this code.
@@ -395,6 +401,12 @@ pub struct MachBufferFinalized<T: CompilePhase> {
pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
/// The required alignment of this buffer.
pub alignment: u32,
+ /// The means by which to NOP out patchable call sites.
+ ///
+ /// This allows a consumer of a `MachBufferFinalized` to disable
+ /// patchable call sites (which are enabled by default) without
+ /// specific knowledge of the target ISA.
+ pub nop: SmallVec<[u8; 8]>,
}
const UNKNOWN_LABEL_OFFSET: CodeOffset = 0xffff_ffff;
@@ -464,6 +476,7 @@ impl<I: VCodeInst> MachBuffer<I> {
relocs: SmallVec::new(),
traps: SmallVec::new(),
call_sites: SmallVec::new(),
+ patchable_call_sites: SmallVec::new(),
exception_handlers: SmallVec::new(),
srclocs: SmallVec::new(),
debug_tags: vec![],
@@ -1564,6 +1577,7 @@ impl<I: VCodeInst> MachBuffer<I> {
relocs: finalized_relocs,
traps: self.traps,
call_sites: self.call_sites,
+ patchable_call_sites: self.patchable_call_sites,
exception_handlers: finalized_exception_handlers,
srclocs,
debug_tags: self.debug_tags,
@@ -1572,6 +1586,7 @@ impl<I: VCodeInst> MachBuffer<I> {
unwind_info: self.unwind_info,
alignment,
frame_layout: self.frame_layout,
+ nop: I::gen_nop_unit(),
}
}
@@ -1667,6 +1682,17 @@ impl<I: VCodeInst> MachBuffer<I> {
});
}
+ /// Add a patchable call record at the current offset The actual
+ /// call is expected to have been emitted; the VCodeInst trait
+ /// specifies how to NOP it out, and we carry that information to
+ /// the finalized Machbuffer.
+ pub fn add_patchable_call_site(&mut self, len: u32) {
+ self.patchable_call_sites.push(MachPatchableCallSite {
+ ret_addr: self.cur_offset(),
+ len,
+ });
+ }
+
/// Add an unwind record at the current offset.
pub fn add_unwind(&mut self, unwind: UnwindInst) {
self.unwind_info.push((self.cur_offset(), unwind));
@@ -1863,6 +1889,22 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
pub fn frame_layout(&self) -> Option<&MachBufferFrameLayout> {
self.frame_layout.as_ref()
}
+
+ /// Get the list of patchable call sites for this code.
+ ///
+ /// Each location in the buffer contains the bytes for a call
+ /// instruction to the specified target. If the call is to be
+ /// patched out, the bytes in the region should be replaced with
+ /// those given in the `MachBufferFinalized::nop` array, repeated
+ /// as many times as necessary. (The length of the patchable
+ /// region is guaranteed to be an integer multiple of that NOP
+ /// unit size.)
+ pub fn patchable_call_sites(&self) -> impl Iterator<Item = &MachPatchableCallSite> + '_ {
+ self.patchable_call_sites.iter().map(|call_site| {
+ debug_assert!(call_site.len as usize % self.nop.len() == 0);
+ call_site
+ })
+ }
}
/// An item in the exception-handler list for a callsite, with label
@@ -2134,6 +2176,22 @@ pub struct FinalizedMachCallSite<'a> {
pub exception_handlers: &'a [FinalizedMachExceptionHandler],
}
+/// A patchable call site record resulting from a compilation.
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(
+ feature = "enable-serde",
+ derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub struct MachPatchableCallSite {
+ /// The offset of the call's return address (i.e., the address
+ /// after the end of the patchable region), *relative to the start
+ /// of the buffer*.
+ pub ret_addr: CodeOffset,
+
+ /// The length of the region to be patched by NOP bytes.
+ pub len: u32,
+}
+
/// A source-location mapping resulting from a compilation.
#[derive(PartialEq, Debug, Clone)]
#[cfg_attr(
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 4f9a37a406..aba18a15c0 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -620,6 +620,10 @@ macro_rules! isle_lower_prelude_methods {
self.lower_ctx.gen_try_call_rets(sig)
}
+ fn gen_patchable_call_rets(&mut self) -> CallRetList {
+ smallvec::smallvec![]
+ }
+
fn try_call_none(&mut self) -> OptionTryCallInfo {
None
}
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index 032c09a3e7..15d8ed55c6 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -177,6 +177,10 @@ pub trait MachInst: Clone + Debug {
/// the instruction must have a nonzero size if preferred_size is nonzero.
fn gen_nop(preferred_size: usize) -> Self;
+ /// The smallest possible NOP, as a unit that can be used to patch
+ /// out code.
+ fn gen_nop_unit() -> SmallVec<[u8; 8]>;
+
/// Align a basic block offset (from start of function). By default, no
/// alignment occurs.
fn align_basic_block(offset: CodeOffset) -> CodeOffset {
@@ -446,6 +450,7 @@ impl<T: CompilePhase> CompiledCodeBase<T> {
let relocs = self.buffer.relocs();
let traps = self.buffer.traps();
+ let mut patchables = self.buffer.patchable_call_sites().peekable();
// Normalize the block starts to include an initial block of offset 0.
let mut block_starts = Vec::new();
@@ -495,6 +500,17 @@ impl<T: CompilePhase> CompiledCodeBase<T> {
write!(buf, " ; trap: {}", trap.code)?;
}
+ if let Some(patchable) = patchables.peek()
+ && patchable.ret_addr == end as u32
+ {
+ write!(
+ buf,
+ " ; patchable call: NOP out last {} bytes",
+ patchable.len
+ )?;
+ patchables.next();
+ }
+
writeln!(buf)?;
}
}
diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle
index 3c30a8bf34..3fafa3fdc4 100644
--- a/cranelift/codegen/src/prelude_lower.isle
+++ b/cranelift/codegen/src/prelude_lower.isle
@@ -1173,6 +1173,9 @@
(decl gen_try_call_rets (Sig) CallRetList)
(extern constructor gen_try_call_rets gen_try_call_rets)
+(decl gen_patchable_call_rets () CallRetList)
+(extern constructor gen_patchable_call_rets gen_patchable_call_rets)
+
(type OptionTryCallInfo (primitive OptionTryCallInfo))
(decl try_call_info (ExceptionTable MachLabelSlice) OptionTryCallInfo)
(extern constructor try_call_info try_call_info)
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index 1a64f166ea..34f1f7385e 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -587,10 +587,14 @@ impl<'a> Verifier<'a> {
self.verify_jump_table(inst, table, errors)?;
}
Call {
- func_ref, ref args, ..
+ opcode,
+ func_ref,
+ ref args,
+ ..
} => {
self.verify_func_ref(inst, func_ref, errors)?;
self.verify_value_list(inst, args, errors)?;
+ self.verify_callee_abi(inst, func_ref, opcode, errors)?;
}
CallIndirect {
sig_ref, ref args, ..
@@ -951,6 +955,38 @@ impl<'a> Verifier<'a> {
Ok(())
}
+ fn verify_callee_abi(
+ &self,
+ inst: Inst,
+ func_ref: FuncRef,
+ opcode: Opcode,
+ errors: &mut VerifierErrors,
+ ) -> VerifierStepResult {
+ let callee_sig_ref = self.func.dfg.ext_funcs[func_ref].signature;
+ let callee_sig = &self.func.dfg.signatures[callee_sig_ref];
+ let callee_abi = callee_sig.call_conv;
+ match (opcode, callee_abi) {
+ (Opcode::PatchableCall, CallConv::Patchable) => {
+ if !self.func.dfg.ext_funcs[func_ref].colocated {
+ errors.fatal((
+ inst,
+ self.context(inst),
+ "patchable call to non-colocated function".to_string(),
+ ))?;
+ }
+ }
+ (Opcode::PatchableCall, _) => {
+ errors.fatal((
+ inst,
+ self.context(inst),
+ "patchable call with non-patchable-ABI callee".to_string(),
+ ))?;
+ }
+ _ => {}
+ }
+ Ok(())
+ }
+
fn verify_value(
&self,
loc_inst: Inst,
diff --git a/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif b/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif
index fa69c981de..7f9660801a 100644
--- a/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/patchable-abi.clif
@@ -163,3 +163,27 @@ block0(v0: i64):
; ldp x29, x30, [sp], #0x10
; ret
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; block0:
+; bl 0 ; patchable
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; block1: ; offset 0x8
+; bl #8 ; reloc_external Call %f 0 ; patchable call: NOP out last 4 bytes
+; ldp x29, x30, [sp], #0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif b/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif
index 09a1a333bf..4ac1a3a297 100644
--- a/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/pulley32/patchable-abi.clif
@@ -370,3 +370,23 @@ block0(v0: i64):
; pop_frame_restore 896,
; ret
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; push_frame
+; block0:
+; patchable_call CallInfo { dest: PulleyCall { name: TestCase(%f), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: Patchable, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None }
+; pop_frame
+; ret
+;
+; Disassembled:
+; push_frame
+; call 0x1 // target = 0x2
+; pop_frame
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif b/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif
index 2cf2a50bfa..b2b6e8671f 100644
--- a/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/pulley64/patchable-abi.clif
@@ -370,3 +370,23 @@ block0(v0: i64):
; pop_frame_restore 896,
; ret
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; push_frame
+; block0:
+; patchable_call CallInfo { dest: PulleyCall { name: TestCase(%f), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: Patchable, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None }
+; pop_frame
+; ret
+;
+; Disassembled:
+; push_frame
+; call 0x1 // target = 0x2
+; pop_frame
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif b/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif
index a00785face..903d683833 100644
--- a/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/patchable-abi.clif
@@ -424,3 +424,36 @@ block0(v0: i64):
; addi sp, sp, 0x10
; ret
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
+; block0:
+; call %f ; patchable
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; block1: ; offset 0x10
+; auipc ra, 0 ; reloc_external RiscvCallPlt %f 0
+; jalr ra ; patchable call: NOP out last 8 bytes
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
index 5339d53138..8ecdf5c1f5 100644
--- a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
@@ -193,3 +193,27 @@ block0(v0: i64):
; lmg %r0, %r15, 0x1a0(%r15)
; br %r14
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; stmg %r14, %r15, 112(%r15)
+; aghi %r15, -160
+; block0:
+; brasl %r14, %f ; patchable
+; lmg %r14, %r15, 272(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r14, %r15, 0x70(%r15)
+; aghi %r15, -0xa0
+; block1: ; offset 0xa
+; brasl %r14, 0xa ; reloc_external PLTRel32Dbl %f 2 ; patchable call: NOP out last 6 bytes
+; lmg %r14, %r15, 0x110(%r15)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/x64/patchable-abi.clif b/cranelift/filetests/filetests/isa/x64/patchable-abi.clif
index 24d1eaa974..a197062b49 100644
--- a/cranelift/filetests/filetests/isa/x64/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/x64/patchable-abi.clif
@@ -168,3 +168,29 @@ block0(v0: i64):
; popq %rbp
; retq
+function %patchable_call(i64) system_v {
+ fn0 = colocated %f(i64) patchable
+block0(v0: i64):
+ patchable_call fn0(v0)
+ return
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; patchable_call TestCase(%f)
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; callq 9 ; reloc_external CallPCRel4 %f -4 ; patchable call: NOP out last 5 bytes
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/verifier/patchable-abi.clif b/cranelift/filetests/filetests/verifier/patchable-abi.clif
index 412f7e75a6..f43a3dad13 100644
--- a/cranelift/filetests/filetests/verifier/patchable-abi.clif
+++ b/cranelift/filetests/filetests/verifier/patchable-abi.clif
@@ -27,3 +27,17 @@ function %patchable_abi_sigref(i32) system_v {
block0(v0: i32):
return
}
+
+function %patchable_call_to_non_patchable_abi() system_v {
+ fn0 = %f() system_v
+block0:
+ patchable_call fn0() ; error: patchable call with non-patchable-ABI callee
+ return
+}
+
+function %patchable_call_to_non_colocated_patchable_abi() system_v {
+ fn0 = %f() patchable
+block0:
+ patchable_call fn0() ; error: patchable call to non-colocated function
+ return
+}
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 8a7fad2323..d69b8db53f 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -1323,6 +1323,7 @@ where
Opcode::GetExceptionHandlerAddress => unimplemented!("GetExceptionHandlerAddress"),
Opcode::SequencePoint => unimplemented!("SequencePoint"),
+ Opcode::PatchableCall => unimplemented!("PatchableCall"),
})
}
diff --git a/pulley/src/encode.rs b/pulley/src/encode.rs
index 29ffe000bb..2815d892bc 100644
--- a/pulley/src/encode.rs
+++ b/pulley/src/encode.rs
@@ -332,3 +332,14 @@ macro_rules! impl_extended_encoders {
};
}
for_each_extended_op!(impl_extended_encoders);
+
+#[test]
+#[cfg(feature = "std")]
+fn nop_is_single_byte() {
+ // NOP needs to be a single byte so that it can be used to NOP out
+ // an instruction sequence of any length.
+ let inst = crate::op::Nop {};
+ let mut bytes = vec![];
+ inst.encode(&mut bytes);
+ assert_eq!(bytes.len(), 1);
+}
diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs
index f394822aae..3b72697b3d 100644
--- a/pulley/src/interp.rs
+++ b/pulley/src/interp.rs
@@ -1343,6 +1343,10 @@ impl OpVisitor for Interpreter<'_> {
&mut self.pc
}
+ fn nop(&mut self) -> ControlFlow<Done> {
+ ControlFlow::Continue(())
+ }
+
fn ret(&mut self) -> ControlFlow<Done> {
let lr = self.state.lr;
if lr == HOST_RETURN_ADDR {
@@ -2827,10 +2831,6 @@ impl OpVisitor for Interpreter<'_> {
}
impl ExtendedOpVisitor for Interpreter<'_> {
- fn nop(&mut self) -> ControlFlow<Done> {
- ControlFlow::Continue(())
- }
-
fn trap(&mut self) -> ControlFlow<Done> {
self.done_trap::<crate::Trap>()
}
diff --git a/pulley/src/lib.rs b/pulley/src/lib.rs
index d9a794a783..40d7122206 100644
--- a/pulley/src/lib.rs
+++ b/pulley/src/lib.rs
@@ -86,6 +86,9 @@ extern crate alloc;
macro_rules! for_each_op {
( $macro:ident ) => {
$macro! {
+ /// No-operation.
+ nop = Nop;
+
/// Transfer control the address in the `lr` register.
ret = Ret;
@@ -623,9 +626,6 @@ macro_rules! for_each_extended_op {
/// Raise a trap.
trap = Trap;
- /// Do nothing.
- nop = Nop;
-
/// A special opcode to halt interpreter execution and yield control
/// back to the host.
///