A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch: catch exceptions (#14180)
Daniel Macovei0ac37df998ef
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rsindex 24446b8369..787aff2b11 100644--- a/cranelift/codegen/src/lib.rs+++ b/cranelift/codegen/src/lib.rs@@ -71,8 +71,8 @@ pub mod write;pub use crate::entity::packed_option;pub use crate::machinst::buffer::{ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, FinalizedMachReloc,- FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachTextSectionBuilder, MachTrap,- OpenPatchRegion, PatchRegion,+ FinalizedRelocTarget, MachCallSite, MachExceptionHandler, MachSrcLoc, MachTextSectionBuilder,+ MachTrap, OpenPatchRegion, PatchRegion,};pub use crate::machinst::{CallInfo, CompiledCode, Final, FrameLayout, MachBuffer, MachBufferDebugTagList,diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex c480e53b4e..3687e65d63 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -567,9 +567,6 @@ impl WastTest {"misc_testsuite/externref-table-dropped-segment-issue-8281.wast","misc_testsuite/many_table_gets_lead_to_gc.wast","misc_testsuite/no-panic.wast",- // Winch does not implement exception handlers yet.- "misc_testsuite/traps-skip-catch-all.wast",- "spec_testsuite/throw.wast",];if unsupported.iter().any(|part| self.path.ends_with(part)) {diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rsindex 7c5d070de5..ad8fe2c6b9 100644--- a/tests/all/exceptions.rs+++ b/tests/all/exceptions.rs@@ -3,8 +3,7 @@ use std::sync::atomic::{AtomicBool, Ordering::Relaxed};use wasmtime::*;use wasmtime_test_macros::wasmtime_test;-// Winch does not implement catches yet. Re-enable after catch is implemented.-#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))]+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn basic_throw(config: &mut Config) -> Result<()> {let engine = Engine::new(config)?;@@ -40,8 +39,147 @@ fn basic_throw(config: &mut Config) -> Result<()> {Ok(())}-// Winch does not implement catches yet. Re-enable after catch is implemented.-#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))]+#[wasmtime_test(wasm_features(exceptions))]+#[cfg_attr(miri, ignore)]+fn try_table_fallthrough_with_multi_value_results(config: &mut Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param i32))+ (func $callee (result i32) (i32.const 9))+ (func (export "f") (result i32)+ (block $h (result i32)+ (try_table (result i32 i32 i32 i32 i32 i32) (catch $e $h)+ (i32.const 1)+ (i32.const 2)+ (i32.const 3)+ (i32.const 4)+ (i32.const 5)+ (call $callee))+ drop drop drop drop drop)))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let f = instance.get_typed_func::<(), i32>(&mut store, "f")?;+ assert_eq!(f.call(&mut store, ())?, 1);+ Ok(())+}++#[wasmtime_test(wasm_features(exceptions))]+#[cfg_attr(miri, ignore)]+fn try_table_exception_with_multi_value_payload(config: &mut Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param i32 i32 i32 i32 i32 i32))++ (func $throw (result i32 i32 i32 i32 i32 i32)+ (throw $e+ (i32.const 1)+ (i32.const 2)+ (i32.const 3)+ (i32.const 4)+ (i32.const 5)+ (i32.const 6)))++ (func (export "f") (result i32)+ (block $handler (result i32 i32 i32 i32 i32 i32)+ (try_table+ (result i32 i32 i32 i32 i32 i32)+ (catch $e $handler)+ (call $throw)))+ drop drop drop drop drop))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let f = instance.get_typed_func::<(), i32>(&mut store, "f")?;+ assert_eq!(f.call(&mut store, ())?, 1);+ Ok(())+}++#[wasmtime_test(wasm_features(exceptions))]+#[cfg_attr(miri, ignore)]+fn try_table_branch_and_fallthrough_with_multi_value_results(config: &mut Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param i32))++ (func (export "f") (param $branch i32) (result i32)+ (block $handler (result i32)+ (try_table+ (result i32 i32 i32 i32 i32 i32)+ (catch $e $handler)+ (i32.const 1)+ (i32.const 2)+ (i32.const 3)+ (i32.const 4)+ (i32.const 5)+ (i32.const 6)+ (local.get $branch)+ br_if 0)+ drop drop drop drop drop)))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let f = instance.get_typed_func::<i32, i32>(&mut store, "f")?;+ assert_eq!(f.call(&mut store, 0)?, 1);+ assert_eq!(f.call(&mut store, 1)?, 1);+ Ok(())+}++#[wasmtime_test(wasm_features(exceptions))]+#[cfg_attr(miri, ignore)]+fn try_table_unreachable_fallthrough_with_multi_value_results(config: &mut Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param i32))+ (func $callee)++ (func (export "f") (result i32)+ (block $handler (result i32)+ (try_table+ (result i32 i32 i32 i32 i32 i32)+ (catch $e $handler)+ (call $callee)+ (i32.const 1)+ (i32.const 2)+ (i32.const 3)+ (i32.const 4)+ (i32.const 5)+ (i32.const 6)+ br 0)+ drop drop drop drop drop)))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let f = instance.get_typed_func::<(), i32>(&mut store, "f")?;+ assert_eq!(f.call(&mut store, ())?, 1);+ Ok(())+}++#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn dynamic_tags(config: &mut Config) -> Result<()> {let engine = Engine::new(config)?;@@ -101,6 +239,58 @@ fn dynamic_tags(config: &mut Config) -> Result<()> {Ok(())}+#[wasmtime_test(wasm_features(exceptions))]+#[cfg_attr(miri, ignore)]+fn nested_handler_scopes(config: &mut Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $outer)+ (tag $inner)++ (func $throw_outer+ (throw $outer))++ (func $throw_inner+ (throw $inner))++ ;; While both handlers are active, the inner tag does not match and+ ;; lookup continues to the outer handler.+ (func (export "nested") (result i32)+ (block $outer_handler+ (try_table (catch $outer $outer_handler)+ (block $inner_handler+ (try_table (catch $inner $inner_handler)+ (call $throw_outer)))))+ (i32.const 1))++ ;; After the inner try_table ends, its handler is no longer active.+ ;; The outer catch_all handles the throw instead.+ (func (export "after") (result i32)+ (block $stale_inner_handler+ (block $outer_handler+ (try_table (catch_all $outer_handler)+ (try_table (catch $inner $stale_inner_handler)+ (nop))+ (call $throw_inner)))+ (return (i32.const 1)))+ (i32.const 2)))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let nested = instance.get_typed_func::<(), i32>(&mut store, "nested")?;+ let after = instance.get_typed_func::<(), i32>(&mut store, "after")?;+ assert_eq!(nested.call(&mut store, ())?, 1);+ assert_eq!(after.call(&mut store, ())?, 1);++ Ok(())+}+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn exception_escape_to_host(config: &mut Config) -> Result<()> {@@ -193,10 +383,71 @@ fn funcref_exception_payload_escape_to_host(config: &mut Config) -> Result<()> {Ok(())}+#[wasmtime_test(wasm_features(exceptions, reference_types))]+#[cfg_attr(miri, ignore)]+fn caught_funcref_payload(config: &mut Config) -> Result<()> {+ for collector in [+ Collector::Null,+ Collector::Copying,+ Collector::DeferredReferenceCounting,+ ] {+ println!("Using GC collector: {collector:?}");+ config.collector(collector);+ run_caught_funcref_payload(config)?;+ }++ Ok(())+}++fn run_caught_funcref_payload(config: &Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param funcref i32))++ (func $throw (param funcref)+ (throw $e (local.get 0) (i32.const 42)))++ (func (export "catch") (param funcref) (result funcref i32)+ (block $handler (result funcref i32)+ (try_table (result funcref i32) (catch $e $handler)+ (call $throw (local.get 0))+ (ref.null func)+ (i32.const 0)))))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let catch = instance.get_func(&mut store, "catch").unwrap();+ let expected = Func::wrap(&mut store, || 126_i32);+ let mut results = [Val::null_func_ref(), Val::I32(0)];+ catch.call(&mut store, &[Val::FuncRef(Some(expected))], &mut results)?;++ let actual = results[0].unwrap_funcref().unwrap();+ let actual = actual.typed::<(), i32>(&store)?;+ assert_eq!(actual.call(&mut store, ())?, 126);+ assert_eq!(results[1].unwrap_i32(), 42);++ Ok(())+}+#[wasmtime_test(wasm_features(exceptions, reference_types))]#[cfg_attr(miri, ignore)]fn thrown_externref_payload_survives_gc(config: &mut Config) -> Result<()> {- config.collector(Collector::DeferredReferenceCounting);+ for collector in [Collector::Copying, Collector::DeferredReferenceCounting] {+ println!("Using GC collector: {collector:?}");+ config.collector(collector);+ run_thrown_externref_payload_survives_gc(config)?;+ }++ Ok(())+}++fn run_thrown_externref_payload_survives_gc(config: &Config) -> Result<()> {let engine = Engine::new(config)?;let mut store = Store::new(&engine, ());@@ -234,6 +485,66 @@ fn thrown_externref_payload_survives_gc(config: &mut Config) -> Result<()> {Ok(())}+#[wasmtime_test(wasm_features(exceptions, reference_types))]+#[cfg_attr(miri, ignore)]+fn caught_externref_payload_survives_gc(config: &mut Config) -> Result<()> {+ for collector in [+ Collector::Null,+ Collector::Copying,+ Collector::DeferredReferenceCounting,+ ] {+ println!("Using GC collector: {collector:?}");+ config.collector(collector);+ run_caught_externref_payload_survives_gc(config)?;+ }++ Ok(())+}++fn run_caught_externref_payload_survives_gc(config: &Config) -> Result<()> {+ let engine = Engine::new(config)?;+ let mut store = Store::new(&engine, ());++ let module = Module::new(+ &engine,+ r#"+ (module+ (tag $e (param externref i32))++ (func $throw (param externref)+ (throw $e (local.get 0) (i32.const 42)))++ (func (export "catch") (param externref) (result externref i32)+ (block $handler (result externref i32)+ (try_table (result externref i32) (catch $e $handler)+ (call $throw (local.get 0))+ (ref.null extern)+ (i32.const 0)))))+ "#,+ )?;++ let instance = Instance::new(&mut store, &module, &[])?;+ let catch = instance+ .get_typed_func::<Option<Rooted<ExternRef>>, (Option<Rooted<ExternRef>>, i32)>(+ &mut store, "catch",+ )?;+ let dropped = Arc::new(AtomicBool::new(false));++ let caught = {+ let mut scope = RootScope::new(&mut store);+ let payload = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?;+ let (caught, value) = catch.call(&mut scope, Some(payload))?;+ assert_eq!(value, 42);+ caught.unwrap().to_owned_rooted(&mut scope)?+ };++ store.gc(None)?;+ assert!(!dropped.load(Relaxed));+ assert!(caught.data(&store)?.is_some());++ Ok(())+}+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn throw_with_null_collector(config: &mut Config) -> Result<()> {@@ -261,8 +572,7 @@ fn throw_with_null_collector(config: &mut Config) -> Result<()> {Ok(())}-// Winch does not implement catches yet. Re-enable after catch is implemented.-#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))]+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn exception_from_host(config: &mut Config) -> Result<()> {let engine = Engine::new(config)?;@@ -402,8 +712,7 @@ fn thrown_exception_without_throwing(config: &mut Config) -> Result<()> {Ok(())}-// Winch does not implement catches yet. Re-enable after catch is implemented.-#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))]+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn wasm_exceptions_have_backtraces(config: &mut Config) -> Result<()> {let engine = Engine::new(config)?;@@ -429,8 +738,7 @@ fn wasm_exceptions_have_backtraces(config: &mut Config) -> Result<()> {Ok(())}-// Winch does not implement catches yet. Re-enable after catch is implemented.-#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))]+#[wasmtime_test(wasm_features(exceptions))]#[cfg_attr(miri, ignore)]fn store_pending_exnref_is_cloned(config: &mut Config) -> wasmtime::Result<()> {config.collector(Collector::DeferredReferenceCounting);@@ -485,7 +793,7 @@ fn store_pending_exnref_is_cloned(config: &mut Config) -> wasmtime::Result<()> {Ok(())}-// Winch does not implement catches yet. Re-enable after catch is implemented.+// Winch does not implement `catch_ref` yet.#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions, reference_types))]#[cfg_attr(miri, ignore)]fn store_pending_exnref_is_exposed(config: &mut Config) -> wasmtime::Result<()> {diff --git a/tests/disas/winch/aarch64/exceptions/landing-pads.wat b/tests/disas/winch/aarch64/exceptions/landing-pads.watnew file mode 100644index 0000000000..3b28571619--- /dev/null+++ b/tests/disas/winch/aarch64/exceptions/landing-pads.wat@@ -0,0 +1,93 @@+;;! target = "aarch64"+;;! test = "winch"+;;! flags = "-W exceptions -C collector=null"++;; A call with tagged and default handlers records both landing pads.+(module+ (tag $e (param i32))+ (import "m" "may_throw" (func $may_throw (param i32)))++ (func (param i32) (result i32)+ (block $catch_all+ (block $tagged (result i32)+ (try_table (catch $e $tagged) (catch_all $catch_all)+ (call $may_throw (local.get 0)))+ (return (i32.const 0)))+ (return))+ (i32.const -1)))+;; wasm[0]::function[1]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x118+;; 2c: mov x9, x0+;; sub x28, x28, #0x18+;; mov sp, x28+;; stur x0, [x28, #0x10]+;; stur x1, [x28, #8]+;; stur w2, [x28, #4]+;; ldur x4, [x9, #0x48]+;; ldur x3, [x9, #0x38]+;; ldur w16, [x28, #4]+;; sub x28, x28, #4+;; mov sp, x28+;; stur w16, [x28]+;; sub x28, x28, #4+;; mov sp, x28+;; mov x0, x4+;; mov x1, x9+;; ldur w2, [x28, #4]+;; blr x3+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ├─╼ exception handler: tag=0, context at [SP+0x18], handler=0xa8+;; ╰─╼ exception handler: default handler, context at [SP+0x18], handler=0x8c+;; 74: add x28, x28, #4+;; mov sp, x28+;; add x28, x28, #4+;; mov sp, x28+;; ldur x9, [x28, #0x10]+;; b #0xf4+;; 8c: mov x28, x29+;; sub x28, x28, #0x10+;; mov sp, x28+;; sub x28, x28, #0x18+;; mov sp, x28+;; ldur x9, [x28, #0x10]+;; b #0xfc+;; a8: mov x28, x29+;; sub x28, x28, #0x10+;; mov sp, x28+;; sub x28, x28, #0x18+;; mov sp, x28+;; ldur x9, [x28, #0x10]+;; ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; mov x16, x0+;; add x16, x16, #0x18+;; cmp x16, x2, uxtx+;; sub sp, x28, #8+;; b.hi #0x11c+;; e0: mov sp, x28+;; mov x2, x1+;; add x2, x2, x0, uxtx+;; ldur w0, [x2, #0x10]+;; b #0x100+;; f4: mov x0, #0+;; b #0x100+;; fc: mov x0, #0xffffffff+;; add x28, x28, #0x18+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 118: udf #0xc11f+;; 11c: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/exceptions/throw-drc.wat b/tests/disas/winch/aarch64/exceptions/throw-drc.watindex 275008a6fd..178ea237cf 100644--- a/tests/disas/winch/aarch64/exceptions/throw-drc.wat+++ b/tests/disas/winch/aarch64/exceptions/throw-drc.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=drc"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -21,14 +21,16 @@;; movk x17, #0x20;; add x16, x16, x17;; cmp sp, x16-;; b.lo #0x120+;; b.lo #0x160;; 2c: mov x9, x0;; sub x28, x28, #0x10;; mov sp, x28;; stur x0, [x28, #8];; stur x1, [x28];; mov x0, x9-;; bl #0x25c+;; bl #0x2a0+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x108;; 48: ldur x9, [x28, #8];; ldur x1, [x9, #0x28];; ldur w1, [x1, #8]@@ -45,7 +47,9 @@;; ldur w2, [x28, #8];; mov x3, #0x28;; mov x4, #8-;; bl #0x20c+;; bl #0x250+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108;; 8c: add x28, x28, #8;; mov sp, x28;; add x28, x28, #4@@ -71,16 +75,35 @@;; mov sp, x28;; mov x0, x9;; ldur w1, [x28, #0xc]-;; bl #0x28c+;; bl #0x2d0+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108;; f4: add x28, x28, #0xc;; mov sp, x28;; add x28, x28, #4;; mov sp, x28;; ldur x9, [x28, #8]+;; mov x28, x29+;; sub x28, x28, #0x10+;; mov sp, x28+;; sub x28, x28, #0x10+;; mov sp, x28+;; ldur x9, [x28, #8]+;; ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; mov x16, x0+;; add x16, x16, #0x28+;; cmp x16, x2, uxtx+;; b.hi #0x164+;; 13c: mov x2, x1+;; add x2, x2, x0, uxtx+;; ldur w0, [x2, #0x20];; add x28, x28, #0x10;; mov sp, x28;; mov sp, x28;; ldr x28, [sp], #0x10;; ldp x29, x30, [sp], #0x10;; ret-;; 120: udf #0xc11f+;; 160: udf #0xc11f+;; 164: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/exceptions/throw-null.wat b/tests/disas/winch/aarch64/exceptions/throw-null.watindex 5fefe1833b..d73446d07b 100644--- a/tests/disas/winch/aarch64/exceptions/throw-null.wat+++ b/tests/disas/winch/aarch64/exceptions/throw-null.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=null"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -21,23 +21,25 @@;; movk x17, #0x20;; add x16, x16, x17;; cmp sp, x16-;; b.lo #0x190+;; b.lo #0x1d0;; 2c: mov x9, x0;; sub x28, x28, #0x10;; mov sp, x28;; stur x0, [x28, #8];; stur x1, [x28];; mov x0, x9-;; bl #0x2d8+;; bl #0x31c+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x178;; 48: ldur x9, [x28, #8];; ldur x16, [x9, #0x20];; ldur w1, [x16];; adds w1, w1, #7-;; b.hs #0x194+;; b.hs #0x1d4;; 5c: and w1, w1, #0xfffffff8;; mov w2, w1;; adds w2, w2, #0x18-;; b.hs #0x198+;; b.hs #0x1d8;; 6c: sub x28, x28, #4;; mov sp, x28;; stur w0, [x28]@@ -62,7 +64,9 @@;; stur x0, [x28];; mov x0, x9;; ldur x1, [x28]-;; bl #0x284+;; bl #0x2c8+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178;; d0: add x28, x28, #8;; mov sp, x28;; ldur x9, [x28, #0x10]@@ -99,18 +103,37 @@;; mov sp, x28;; mov x0, x9;; ldur w1, [x28, #0xc]-;; bl #0x308+;; bl #0x34c+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178;; 164: add x28, x28, #0xc;; mov sp, x28;; add x28, x28, #4;; mov sp, x28;; ldur x9, [x28, #8]+;; mov x28, x29+;; sub x28, x28, #0x10+;; mov sp, x28+;; sub x28, x28, #0x10+;; mov sp, x28+;; ldur x9, [x28, #8]+;; ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; mov x16, x0+;; add x16, x16, #0x18+;; cmp x16, x2, uxtx+;; b.hi #0x1dc+;; 1ac: mov x2, x1+;; add x2, x2, x0, uxtx+;; ldur w0, [x2, #0x10];; add x28, x28, #0x10;; mov sp, x28;; mov sp, x28;; ldr x28, [sp], #0x10;; ldp x29, x30, [sp], #0x10;; ret-;; 190: udf #0xc11f-;; 194: udf #0xc11f-;; 198: udf #0xc11f+;; 1d0: udf #0xc11f+;; 1d4: udf #0xc11f+;; 1d8: udf #0xc11f+;; 1dc: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/exceptions/throw.wat b/tests/disas/winch/aarch64/exceptions/throw.watindex 7b75c22df4..c0c4ceb155 100644--- a/tests/disas/winch/aarch64/exceptions/throw.wat+++ b/tests/disas/winch/aarch64/exceptions/throw.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=copying"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -21,14 +21,16 @@;; movk x17, #0x20;; add x16, x16, x17;; cmp sp, x16-;; b.lo #0x124+;; b.lo #0x164;; 2c: mov x9, x0;; sub x28, x28, #0x10;; mov sp, x28;; stur x0, [x28, #8];; stur x1, [x28];; mov x0, x9-;; bl #0x260+;; bl #0x2a4+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x10c;; 48: ldur x9, [x28, #8];; ldur x1, [x9, #0x28];; ldur w1, [x1, #8]@@ -46,7 +48,9 @@;; ldur w2, [x28, #8];; mov x3, #0x20;; mov x4, #0x10-;; bl #0x210+;; bl #0x254+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c;; 90: add x28, x28, #8;; mov sp, x28;; add x28, x28, #4@@ -72,16 +76,35 @@;; mov sp, x28;; mov x0, x9;; ldur w1, [x28, #0xc]-;; bl #0x290+;; bl #0x2d4+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c;; f8: add x28, x28, #0xc;; mov sp, x28;; add x28, x28, #4;; mov sp, x28;; ldur x9, [x28, #8]+;; mov x28, x29+;; sub x28, x28, #0x10+;; mov sp, x28+;; sub x28, x28, #0x10+;; mov sp, x28+;; ldur x9, [x28, #8]+;; ldur x1, [x9, #8]+;; ldur x2, [x1, #0x28]+;; ldur x1, [x1, #0x20]+;; mov x16, x0+;; add x16, x16, #0x20+;; cmp x16, x2, uxtx+;; b.hi #0x168+;; 140: mov x2, x1+;; add x2, x2, x0, uxtx+;; ldur w0, [x2, #0x18];; add x28, x28, #0x10;; mov sp, x28;; mov sp, x28;; ldr x28, [sp], #0x10;; ldp x29, x30, [sp], #0x10;; ret-;; 124: udf #0xc11f+;; 164: udf #0xc11f+;; 168: udf #0xc11fdiff --git a/tests/disas/winch/x64/exceptions/landing-pads.wat b/tests/disas/winch/x64/exceptions/landing-pads.watnew file mode 100644index 0000000000..b28f59360c--- /dev/null+++ b/tests/disas/winch/x64/exceptions/landing-pads.wat@@ -0,0 +1,73 @@+;;! target = "x86_64"+;;! test = "winch"+;;! flags = "-W exceptions -C collector=null"++;; A call with tagged and default handlers records both landing pads.+(module+ (tag $e (param i32))+ (import "m" "may_throw" (func $may_throw (param i32)))++ (func (param i32) (result i32)+ (block $catch_all+ (block $tagged (result i32)+ (try_table (catch $e $tagged) (catch_all $catch_all)+ (call $may_throw (local.get 0)))+ (return (i32.const 0)))+ (return))+ (i32.const -1)))+;; wasm[0]::function[1]:+;; pushq %rbp+;; movq %rsp, %rbp+;; movq 8(%rdi), %r11+;; movq 0x18(%r11), %r11+;; addq $0x30, %r11+;; cmpq %rsp, %r11+;; ja 0xdf+;; 1c: movq %rdi, %r14+;; subq $0x20, %rsp+;; movq %rdi, 0x18(%rsp)+;; movq %rsi, 0x10(%rsp)+;; movl %edx, 0xc(%rsp)+;; movq 0x48(%r14), %rcx+;; movq 0x38(%r14), %rax+;; movl 0xc(%rsp), %r11d+;; subq $4, %rsp+;; movl %r11d, (%rsp)+;; subq $0xc, %rsp+;; movq %rcx, %rdi+;; movq %r14, %rsi+;; movl 0xc(%rsp), %edx+;; callq *%rax+;; ├─╼ exception frame offset: SP = FP - 0x30+;; ├─╼ exception handler: tag=0, context at [SP+0x28], handler=0x8b+;; ╰─╼ exception handler: default handler, context at [SP+0x28], handler=0x77+;; addq $0xc, %rsp+;; addq $4, %rsp+;; movq 0x18(%rsp), %r14+;; jmp 0xc7+;; 77: movq %rbp, %rsp+;; subq $0x20, %rsp+;; movq 0x18(%rsp), %r14+;; jmp 0xd1+;; 8b: movq %rbp, %rsp+;; subq $0x20, %rsp+;; movq 0x18(%rsp), %r14+;; movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movq %rax, %r11+;; addq $0x18, %r11+;; cmpq %rdx, %r11+;; ja 0xe1+;; b9: movq %rcx, %rdx+;; addq %rax, %rdx+;; movl 0x10(%rdx), %eax+;; jmp 0xd6+;; c7: movl $0, %eax+;; jmp 0xd6+;; d1: movl $0xffffffff, %eax+;; addq $0x20, %rsp+;; popq %rbp+;; retq+;; df: ud2+;; e1: ud2diff --git a/tests/disas/winch/x64/exceptions/throw-drc.wat b/tests/disas/winch/x64/exceptions/throw-drc.watindex 7dcaab3d93..0aa8836563 100644--- a/tests/disas/winch/x64/exceptions/throw-drc.wat+++ b/tests/disas/winch/x64/exceptions/throw-drc.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=drc"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -17,13 +17,15 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0xf3+;; ja 0x12a;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movq %r14, %rdi-;; callq 0x1f8+;; callq 0x231+;; ├─╼ exception frame offset: SP = FP - 0x10+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea;; movq 8(%rsp), %r14;; movq 0x28(%r14), %rcx;; movl 8(%rcx), %ecx@@ -37,7 +39,9 @@;; movl 8(%rsp), %edx;; movl $0x28, %ecx;; movl $8, %r8d-;; callq 0x1a9+;; callq 0x1e2+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea;; addq $8, %rsp;; addq $4, %rsp;; movq 0xc(%rsp), %r14@@ -56,11 +60,27 @@;; subq $0xc, %rsp;; movq %r14, %rdi;; movl 0xc(%rsp), %esi-;; callq 0x225+;; callq 0x25e+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea;; addq $0xc, %rsp;; addq $4, %rsp;; movq 8(%rsp), %r14+;; movq %rbp, %rsp+;; subq $0x10, %rsp+;; movq 8(%rsp), %r14+;; movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movq %rax, %r11+;; addq $0x28, %r11+;; cmpq %rdx, %r11+;; ja 0x12c+;; 118: movq %rcx, %rdx+;; addq %rax, %rdx+;; movl 0x20(%rdx), %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; f3: ud2+;; 12a: ud2+;; 12c: ud2diff --git a/tests/disas/winch/x64/exceptions/throw-null.wat b/tests/disas/winch/x64/exceptions/throw-null.watindex 2b08339818..7498d39859 100644--- a/tests/disas/winch/x64/exceptions/throw-null.wat+++ b/tests/disas/winch/x64/exceptions/throw-null.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=null"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -17,22 +17,24 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0x14f+;; ja 0x186;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movq %r14, %rdi-;; callq 0x250+;; callq 0x289+;; ├─╼ exception frame offset: SP = FP - 0x10+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x146;; movq 8(%rsp), %r14;; movq 0x20(%r14), %r11;; movl (%r11), %ecx;; addl $7, %ecx-;; jb 0x151+;; jb 0x188;; 4f: andl $0xfffffff8, %ecx;; movl %ecx, %edx;; addl $0x18, %edx-;; jb 0x153+;; jb 0x18a;; 63: subq $4, %rsp;; movl %eax, (%rsp);; subq $4, %rsp@@ -50,7 +52,9 @@;; pushq %rax;; movq %r14, %rdi;; movq (%rsp), %rsi-;; callq 0x209+;; callq 0x242+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146;; addq $8, %rsp;; movq 0x10(%rsp), %r14;; movl (%rsp), %eax@@ -78,13 +82,29 @@;; subq $0xc, %rsp;; movq %r14, %rdi;; movl 0xc(%rsp), %esi-;; callq 0x27d+;; callq 0x2b6+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146;; addq $0xc, %rsp;; addq $4, %rsp;; movq 8(%rsp), %r14+;; movq %rbp, %rsp+;; subq $0x10, %rsp+;; movq 8(%rsp), %r14+;; movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movq %rax, %r11+;; addq $0x18, %r11+;; cmpq %rdx, %r11+;; ja 0x18c+;; 174: movq %rcx, %rdx+;; addq %rax, %rdx+;; movl 0x10(%rdx), %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; 14f: ud2-;; 151: ud2-;; 153: ud2+;; 186: ud2+;; 188: ud2+;; 18a: ud2+;; 18c: ud2diff --git a/tests/disas/winch/x64/exceptions/throw.wat b/tests/disas/winch/x64/exceptions/throw.watindex 9dff95865d..38a0582083 100644--- a/tests/disas/winch/x64/exceptions/throw.wat+++ b/tests/disas/winch/x64/exceptions/throw.wat@@ -2,8 +2,8 @@;;! test = "winch";;! flags = "-W exceptions -C collector=copying"-;; `throw` builds an exception that escapes to the host, while `try_table`-;; still compiles as a plain block.+;; Calls made while the `try_table` handler is active carry exception metadata.+;; Its landing pad loads the exception's payload and branches to `$h`.(module(tag $e (param i32))(func (result i32)@@ -17,13 +17,15 @@;; movq 0x18(%r11), %r11;; addq $0x20, %r11;; cmpq %rsp, %r11-;; ja 0xf3+;; ja 0x12a;; 1c: movq %rdi, %r14;; subq $0x10, %rsp;; movq %rdi, 8(%rsp);; movq %rsi, (%rsp);; movq %r14, %rdi-;; callq 0x1f8+;; callq 0x231+;; ├─╼ exception frame offset: SP = FP - 0x10+;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea;; movq 8(%rsp), %r14;; movq 0x28(%r14), %rcx;; movl 8(%rcx), %ecx@@ -37,7 +39,9 @@;; movl 8(%rsp), %edx;; movl $0x20, %ecx;; movl $0x10, %r8d-;; callq 0x1a9+;; callq 0x1e2+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea;; addq $8, %rsp;; addq $4, %rsp;; movq 0xc(%rsp), %r14@@ -56,11 +60,27 @@;; subq $0xc, %rsp;; movq %r14, %rdi;; movl 0xc(%rsp), %esi-;; callq 0x225+;; callq 0x25e+;; ├─╼ exception frame offset: SP = FP - 0x20+;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea;; addq $0xc, %rsp;; addq $4, %rsp;; movq 8(%rsp), %r14+;; movq %rbp, %rsp+;; subq $0x10, %rsp+;; movq 8(%rsp), %r14+;; movq 8(%r14), %rcx+;; movq 0x28(%rcx), %rdx+;; movq 0x20(%rcx), %rcx+;; movq %rax, %r11+;; addq $0x20, %r11+;; cmpq %rdx, %r11+;; ja 0x12c+;; 118: movq %rcx, %rdx+;; addq %rax, %rdx+;; movl 0x18(%rdx), %eax;; addq $0x10, %rsp;; popq %rbp;; retq-;; f3: ud2+;; 12a: ud2+;; 12c: ud2diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rsindex a45b8d839d..5a0d148446 100644--- a/winch/codegen/src/codegen/call.rs+++ b/winch/codegen/src/codegen/call.rs@@ -83,7 +83,9 @@ impl FnCall {/// 3. Spills the value stack./// 4. Creates the stack space needed for the return area./// 5. Emits the call.- /// 6. Cleans up the stack space.+ /// 6. Records any GC stack map and active exception handlers at the call's+ /// return address.+ /// 7. Cleans up the stack space.pub fn emit<M: MacroAssembler>(env: &mut FuncEnv<M::Ptr>,masm: &mut M,@@ -109,6 +111,13 @@ impl FnCall {if !offsets.is_empty() {masm.emit_stack_map(sp, &offsets)?;}+ if !context.exception_handlers.is_empty() {+ masm.emit_try_call_site(+ sp,+ context.frame.vmctx_slot().offset,+ context.exception_handlers.handlers(),+ )?;+ }Ok(())},)?;diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rsindex f31c336b6e..e575407343 100644--- a/winch/codegen/src/codegen/context.rs+++ b/winch/codegen/src/codegen/context.rs@@ -3,7 +3,9 @@ use crate::{Result,abi::{ABIOperand, ABIResults, RetArea, vmctx},bail,- codegen::{BranchState, CodeGenError, CodeGenPhase, Emission, Prologue},+ codegen::{+ BranchState, CodeGenError, CodeGenPhase, Emission, Prologue, exceptions::HandlerState,+ },ensure, format_err,frame::Frame,isa::reg::RegClass,@@ -45,6 +47,8 @@ pub(crate) struct CodeGenContext<'a, P: CodeGenPhase> {pub reachable: bool,/// A reference to the VMOffsets.pub vmoffsets: &'a VMOffsets<u8>,+ /// The exception handlers currently in scope.+ pub exception_handlers: HandlerState,}impl<'a> CodeGenContext<'a, Emission> {@@ -123,6 +127,7 @@ impl<'a> CodeGenContext<'a, Prologue> {frame,reachable: true,vmoffsets,+ exception_handlers: Default::default(),}}@@ -134,6 +139,7 @@ impl<'a> CodeGenContext<'a, Prologue> {reachable: self.reachable,vmoffsets: self.vmoffsets,frame: self.frame.for_emission(),+ exception_handlers: self.exception_handlers,}}}diff --git a/winch/codegen/src/codegen/control.rs b/winch/codegen/src/codegen/control.rsindex 77d0376ea2..3a6d2d31e8 100644--- a/winch/codegen/src/codegen/control.rs+++ b/winch/codegen/src/codegen/control.rs@@ -6,7 +6,9 @@//! next instruction is a control instruction, we could avoid emitting//! a [`crate::masm::MacroAssembler::cmp_with_set`] and instead emit//! a conditional jump inline when emitting the control flow instruction.-use super::{CodeGenContext, CodeGenError, Emission, OperandSize, Reg, TypedReg};+use super::{+ CodeGenContext, CodeGenError, Emission, OperandSize, Reg, TypedReg, exceptions::TryTableInfo,+};use crate::{CallingConvention, Result,abi::{ABI, ABIOperand, ABIResults, ABISig, RetArea},@@ -244,6 +246,8 @@ pub(crate) enum ControlStackFrame {/// target. By default, this is false, and it's updated when/// emitting a `br` or `br_if`.is_branch_target: bool,+ /// Exception-handling information when this block is a `try_table`.+ try_table_info: Option<TryTableInfo>,},Loop {/// The start of the Loop.@@ -279,18 +283,54 @@ impl ControlStackFrame {sig: BlockSig,masm: &mut M,context: &mut CodeGenContext<Emission>,+ ) -> Result<Self> {+ Self::block_impl(sig, None, masm, context)+ }++ /// Returns a block control frame with exception-handler information.+ pub fn try_table<M: MacroAssembler>(+ sig: BlockSig,+ info: TryTableInfo,+ masm: &mut M,+ context: &mut CodeGenContext<Emission>,+ ) -> Result<Self> {+ Self::block_impl(sig, Some(info), masm, context)+ }++ fn block_impl<M: MacroAssembler>(+ sig: BlockSig,+ try_table_info: Option<TryTableInfo>,+ masm: &mut M,+ context: &mut CodeGenContext<Emission>,) -> Result<Self> {let mut control = Self::Block {sig,is_branch_target: false,exit: masm.get_label()?,stack_state: Default::default(),+ try_table_info,};control.emit(masm, context)?;Ok(control)}+ /// Returns this block's try-table information, if present.+ pub fn try_table_info(&self) -> Option<&TryTableInfo> {+ match self {+ Self::Block { try_table_info, .. } => try_table_info.as_ref(),+ _ => None,+ }+ }++ /// Takes this block's try-table information, if present.+ pub fn take_try_table_info(&mut self) -> Option<TryTableInfo> {+ match self {+ Self::Block { try_table_info, .. } => try_table_info.take(),+ _ => None,+ }+ }+/// Returns [`ControlStackFrame`] for a loop.pub fn r#loop<M: MacroAssembler>(sig: BlockSig,diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rsindex 94de6ae249..abfc88bea6 100644--- a/winch/codegen/src/codegen/drc.rs+++ b/winch/codegen/src/codegen/drc.rs@@ -48,7 +48,7 @@ whereOk(())}- /// Emits a DRC read barrier for a value loaded from `addr`.+ /// Emits a DRC read barrier for the value in `gc_ref`.////// The loaded reference is first pushed and spilled so it is represented/// in a stack map if this barrier calls `force_gc`. Null and i31 references@@ -59,16 +59,8 @@ where/// Finally it forces a collection when the roots list reaches both the/// proportional and absolute thresholds.///- /// Leaves the loaded reference on the value stack and marks- /// `storage_base` available for register reuse before returning.- pub(crate) fn emit_drc_read_barrier(- &mut self,- ty: WasmValType,- storage_base: Reg,- addr: M::Address,- ) -> Result<()> {- let gc_ref = self.context.reg_for_type(ty, self.masm)?;- self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;+ /// Leaves the loaded reference on the value stack.+ pub(crate) fn emit_drc_read_barrier(&mut self, ty: WasmValType, gc_ref: Reg) -> Result<()> {self.context.stack.push(Val::reg(gc_ref, ty));// Spill the loaded result into a stack-map-visible slot before the@@ -121,7 +113,6 @@ whereself.emit_maybe_force_gc(roots_len, heap_data_reg)?;self.masm.bind(skip_barrier)?;- self.context.free_reg(storage_base);Ok(())}diff --git a/winch/codegen/src/codegen/exceptions.rs b/winch/codegen/src/codegen/exceptions.rsindex 9a0aad1297..f62e76a2f8 100644--- a/winch/codegen/src/codegen/exceptions.rs+++ b/winch/codegen/src/codegen/exceptions.rs@@ -1,20 +1,267 @@-use super::{Callee, CodeGen, CodeGenError, Emission, FnCall};+use super::{Callee, CodeGen, CodeGenError, ControlStackFrame, Emission, FnCall};use crate::{- Result, format_err,+ Result,+ codegen::{UnconditionalBranch, control_index},+ ensure, format_err,masm::{IntScratch, MacroAssembler, OperandSize, RegImm},- reg::Reg,- stack::TypedReg,+ reg::{Reg, writable},+ stack::{TypedReg, Val},};-use wasmtime_environ::copying::InlineTraceInfo;+use cranelift_codegen::{MachExceptionHandler, MachLabel, ir::ExceptionTag};+use smallvec::SmallVec;use wasmtime_environ::{Collector, GcStructLayout, GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TagIndex, VMGcKind,- WasmExnType, WasmHeapType, WasmStorageType, WasmValType,+ WasmExnType, WasmHeapType, WasmStorageType, WasmValType, packed_option::ReservedValue,};+use wasmtime_environ::{WasmCompositeInnerType, copying::InlineTraceInfo};++/// The exception handlers that are currently in scope.+#[derive(Default)]+pub(crate) struct HandlerState {+ handlers: Vec<(Option<ExceptionTag>, MachLabel)>,+}++/// A checkpoint that can restore the exception handlers in scope.+#[derive(Clone, Copy, Debug)]+pub(crate) struct HandlerStateCheckpoint(usize);++#[derive(Debug)]+pub(crate) struct CatchInfo {+ pub(crate) tag: Option<TagIndex>,+ pub(crate) target_depth: u32,+ pub(crate) landing_pad: MachLabel,+}++#[derive(Debug)]+pub(crate) struct TryTableInfo {+ pub(crate) checkpoint: HandlerStateCheckpoint,+ pub(crate) catches: Vec<CatchInfo>,+}++impl HandlerState {+ /// Adds an exception handler.+ pub(crate) fn add_handler(&mut self, tag: Option<ExceptionTag>, label: MachLabel) {+ self.handlers.push((tag, label));+ }++ /// Takes a checkpoint of the exception handlers currently in scope.+ pub(crate) fn take_checkpoint(&self) -> HandlerStateCheckpoint {+ HandlerStateCheckpoint(self.handlers.len())+ }++ /// Restores the exception handlers to a previous checkpoint.+ pub(crate) fn restore_checkpoint(&mut self, checkpoint: HandlerStateCheckpoint) {+ assert!(checkpoint.0 <= self.handlers.len());+ self.handlers.truncate(checkpoint.0);+ }++ /// Iterates over exception handlers from the innermost to the outermost.+ pub(crate) fn handlers(&self) -> impl Iterator<Item = MachExceptionHandler> + '_ {+ self.handlers+ .iter()+ .copied()+ .rev()+ .map(|(tag, label)| match tag {+ Some(tag) => MachExceptionHandler::Tag(tag, label),+ None => MachExceptionHandler::Default(label),+ })+ }++ /// Returns whether there are no exception handlers in scope.+ pub(crate) fn is_empty(&self) -> bool {+ self.handlers.is_empty()+ }+}impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>whereM: MacroAssembler,{+ /// Emits the end of a try-table block and its exception landing pads.+ pub(crate) fn emit_try_table_end(+ &mut self,+ mut control: ControlStackFrame,+ info: TryTableInfo,+ ) -> Result<()> {+ let stack_state = *control.stack_state();++ let fallthrough_reachable = self.context.reachable;+ let end_reachable = fallthrough_reachable || control.is_next_sequence_reachable();++ if fallthrough_reachable {+ ensure!(+ control.stack_state().target_len == self.context.stack.len(),+ CodeGenError::control_frame_state_mismatch()+ );++ control.pop_abi_results(&mut self.context, self.masm, |results, _, _| {+ Ok(results.ret_area().copied())+ })?;++ ensure!(+ self.masm.sp_offset()? == stack_state.target_offset,+ CodeGenError::invalid_sp_offset()+ );++ self.masm.jmp(*control.label())?;+ }++ for catch in info.catches {+ self.masm.bind(catch.landing_pad)?;++ self.context.reachable = true;+ let raw_exception_reg = self+ .masm+ .prepare_for_exception_handler(stack_state.base_offset)?;+ self.context.truncate_stack_to(stack_state.base_len)?;+ self.context.load_vmctx(self.masm)?;++ if let Some(tag) = catch.tag {+ self.emit_load_exception_payload_fields(tag, raw_exception_reg)?;+ }+ self.emit_catch_branch(catch.target_depth)?;+ }++ self.context.reachable = end_reachable;++ if end_reachable {+ // Landing pads leave the Masm tracking handler state, so restore+ // the block's end state before binding the join.+ control.ensure_stack_state(self.masm, &mut self.context)?;+ control.bind_end(self.masm, &mut self.context)+ } else {+ Ok(())+ }+ }++ fn emit_catch_branch(&mut self, target_depth: u32) -> Result<()> {+ let index = control_index(target_depth, self.control_frames.len())?;+ let frame = &mut self.control_frames[index];++ self.context+ .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, context, frame| {+ frame.pop_abi_results::<M, _>(context, masm, |results, _, _| {+ Ok(results.ret_area().copied())+ })+ })+ }++ fn emit_load_exception_payload_fields(+ &mut self,+ tag_index: TagIndex,+ raw_exception_reg: Reg,+ ) -> Result<()> {+ let exception_reg = self.context.reg(raw_exception_reg, self.masm)?;+ let interned = self.env.translation.module.tags[tag_index]+ .exception+ .unwrap_module_type_index();++ let exn_ty = match &self.env.types[interned].composite_type.inner {+ WasmCompositeInnerType::Exn(exn_ty) => exn_ty,+ _ => return Err(format_err!(CodeGenError::unsupported_wasm_type())),+ };+ let gc_codegen_config = self.require_gc_codegen_config();+ let layouts = gc_codegen_config.layouts();++ let layout = layouts+ .exn_layout(exn_ty)+ .map_err(|_| format_err!(CodeGenError::unsupported_wasm_type()))?;++ let fields: SmallVec<[(WasmStorageType, u32); 8]> = exn_ty+ .fields+ .iter()+ .zip(layout.fields.iter())+ .map(|(field_ty, field_layout)| (field_ty.element_type, field_layout.offset))+ .collect();++ let (heap_base, heap_bound) = self.emit_load_gc_heap_base_and_bound()?;++ self.emit_gc_ref_bounds_check(exception_reg, heap_bound, i64::from(layout.size))?;++ self.context.free_reg(heap_bound);++ let mut object_addr = self.emit_gc_ref_addr(exception_reg, heap_base)?;+ self.context.free_reg(heap_base);+ self.context.free_reg(exception_reg);+ for (field_ty, field_offset) in fields {+ let ty = match field_ty {+ WasmStorageType::Val(ty @ WasmValType::Ref(r))+ if r.heap_type == WasmHeapType::Func =>+ {+ let func_ref_id = self.context.any_gpr(self.masm)?;+ let addr = self.masm.address_at_reg(object_addr, field_offset)?;+ self.masm+ .load(addr, writable!(func_ref_id), OperandSize::S32)?;++ // The builtin call can clobber allocated registers. Preserve+ // the object address beneath the call's arguments so later+ // payload fields can still be loaded.+ self.context.stack.push(TypedReg::i64(object_addr).into());+ self.context.stack.push(TypedReg::i32(func_ref_id).into());+ self.context.stack.push(Val::i32(+ ModuleInternedTypeIndex::reserved_value()+ .as_bits()+ .cast_signed(),+ ));++ let get = self.env.builtins.get_interned_func_ref::<M::ABI>()?;+ FnCall::emit::<M>(+ &mut self.env,+ self.masm,+ &mut self.context,+ Callee::Builtin(get),+ )?;++ let func_ref = self.context.pop_to_reg(self.masm, None)?;+ object_addr = self.context.pop_to_reg(self.masm, None)?.reg;+ self.context+ .stack+ .push(TypedReg::new(ty, func_ref.reg).into());+ continue;+ }+ WasmStorageType::Val(ty @ WasmValType::Ref(r))+ if r.heap_type == WasmHeapType::Extern =>+ {+ let addr = self.masm.address_at_reg(object_addr, field_offset)?;+ if gc_codegen_config.collector() == Collector::DeferredReferenceCounting {+ // The DRC read barrier can make an out-of-line call.+ // Preserve the object address across the call so later+ // payload fields can still be loaded.+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;+ self.context.stack.push(TypedReg::i64(object_addr).into());+ self.emit_drc_read_barrier(ty, gc_ref)?;+ let payload = self.context.pop_to_reg(self.masm, None)?;+ object_addr = self.context.pop_to_reg(self.masm, None)?.reg;+ self.context.stack.push(payload.into());+ } else {+ let value = self.context.reg_for_type(ty, self.masm)?;+ self.masm.load(addr, writable!(value), ty.try_into()?)?;+ self.context.stack.push(TypedReg::new(ty, value).into());+ }+ continue;+ }+ WasmStorageType::Val(WasmValType::Ref(_)) => {+ return Err(format_err!(CodeGenError::unsupported_wasm_type()));+ }+ WasmStorageType::Val(ty) => ty,+ WasmStorageType::I8 | WasmStorageType::I16 => {+ return Err(format_err!(CodeGenError::unsupported_wasm_type()));+ }+ };++ let value = self.context.reg_for_type(ty, self.masm)?;+ let addr = self.masm.address_at_reg(object_addr, field_offset)?;++ self.masm.load(addr, writable!(value), ty.try_into()?)?;++ self.context.stack.push(TypedReg::new(ty, value).into());+ }++ self.context.free_reg(object_addr);+ Ok(())+ }+/// Allocates an exception and initializes its tag identity.////// Exception tags are identified by the defining instance and the tag's@@ -99,7 +346,7 @@ where////// This method consumes and frees `object_addr`. It returns `gc_ref`, which/// remains owned by the caller.- pub(crate) fn emit_exception_payload_fields(+ pub(crate) fn emit_store_exception_payload_fields(&mut self,exn_ty: &WasmExnType,layout: &GcStructLayout,diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex f668b7206e..f7360cd1af 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -42,6 +42,7 @@ pub use builtin::*;pub(crate) mod bounds;mod drc;mod exceptions;+pub(crate) use exceptions::{CatchInfo, TryTableInfo};mod gc;use gc::GcCodegenConfig;@@ -318,9 +319,17 @@ where/// Pops a control frame from the control frame stack.pub fn pop_control_frame(&mut self) -> Result<ControlStackFrame> {- self.control_frames+ let frame = self+ .control_frames.pop()- .ok_or_else(|| format_err!(CodeGenError::control_frame_expected()))+ .ok_or_else(|| format_err!(CodeGenError::control_frame_expected()))?;+ if let Some(info) = frame.try_table_info() {+ self.context+ .exception_handlers+ .restore_checkpoint(info.checkpoint);+ }++ Ok(frame)}/// Derives a [RelSourceLoc] from a [SourceLoc].@@ -358,6 +367,9 @@ wherepub fn handle_unreachable_end(&mut self) -> Result<()> {let mut frame = self.pop_control_frame()?;+ if let Some(info) = frame.take_try_table_info() {+ return self.emit_try_table_end(frame, info);+ }// We just popped the outermost block.let is_outermost = self.control_frames.len() == 0;@@ -439,7 +451,7 @@ wherefn visit_op_when_unreachable(op: &Operator) -> bool {use Operator::*;match op {- If { .. } | Block { .. } | Loop { .. } | Else | End => true,+ If { .. } | Block { .. } | TryTable { .. } | Loop { .. } | Else | End => true,_ => false,}}diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 0c1328736b..08b89edd56 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -29,14 +29,16 @@ use crate::{stack::{TypedReg, Val},};use cranelift_codegen::{- Final, MachBufferFinalized, MachLabel,+ ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{MemFlagsData, RelSourceLoc, SourceLoc, types},- isa::aarch64,- isa::aarch64::inst::{- self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,- VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,- VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize,+ isa::aarch64::{+ self,+ inst::{+ self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,+ VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,+ VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize,+ },},settings,};@@ -319,6 +321,28 @@ impl Masm for MacroAssembler {Ok(())}+ fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result<Reg> {+ let shadow_sp = regs::shadow_sp();++ self.asm+ .mov_rr(regs::fp(), writable!(shadow_sp), OperandSize::S64);++ let initial_offset =+ Imm12::maybe_from_u64(u64::from(SHADOW_STACK_POINTER_SLOT_SIZE)).unwrap();+ self.asm.sub_ir(+ initial_offset,+ shadow_sp,+ writable!(shadow_sp),+ OperandSize::S64,+ );++ self.move_shadow_sp_to_sp();+ self.sp_offset = 0;+ self.reserve_stack(target_offset.as_u32())?;++ Ok(regs::xreg(0))+ }+fn local_address(&mut self, local: &LocalSlot) -> Result<Address> {let (reg, offset) = local.addressed_from_sp()@@ -1362,6 +1386,30 @@ impl Masm for MacroAssembler {Ok(())}+ fn emit_try_call_site(+ &mut self,+ sp_offset: SPOffset,+ vmctx_slot_offset: u32,+ handlers: impl Iterator<Item = MachExceptionHandler>,+ ) -> Result<()> {+ let frame_offset = sp_offset.as_u32() + u32::from(SHADOW_STACK_POINTER_SLOT_SIZE);+ let vmctx_offset = sp_offset+ .as_u32()+ .checked_sub(vmctx_slot_offset)+ .ok_or_else(CodeGenError::invalid_local_offset)?;++ let handlers = std::iter::once(MachExceptionHandler::Context(+ ExceptionContextLoc::SPOffset(vmctx_offset),+ ))+ .chain(handlers);++ self.asm+ .buffer_mut()+ .add_try_call_site(Some(frame_offset), handlers);++ Ok(())+ }+fn current_code_offset(&self) -> Result<CodeOffset> {Ok(self.asm.buffer().cur_offset())}diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex e7fe439b24..a1c9c456dd 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -34,7 +34,7 @@ use crate::{masm::CalleeKind,};use cranelift_codegen::{- Final, MachBufferFinalized, MachLabel,+ ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{MemFlagsData, RelSourceLoc, SourceLoc},isa::{@@ -223,6 +223,13 @@ impl Masm for MacroAssembler {Ok(())}+ fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result<Reg> {+ self.asm.mov_rr(rbp(), writable!(rsp()), OperandSize::S64);+ self.sp_offset = 0;+ self.reserve_stack(target_offset.as_u32())?;+ Ok(regs::rax())+ }+fn local_address(&mut self, local: &LocalSlot) -> Result<Address> {let (reg, offset) = if local.addressed_from_sp() {let offset = self@@ -1419,6 +1426,29 @@ impl Masm for MacroAssembler {Ok(())}+ fn emit_try_call_site(+ &mut self,+ sp_offset: SPOffset,+ vmctx_slot_offset: u32,+ handlers: impl Iterator<Item = MachExceptionHandler>,+ ) -> Result<()> {+ let frame_offset = sp_offset.as_u32();+ let vmctx_offset = frame_offset+ .checked_sub(vmctx_slot_offset)+ .ok_or_else(CodeGenError::invalid_local_offset)?;++ let handlers = std::iter::once(MachExceptionHandler::Context(+ ExceptionContextLoc::SPOffset(vmctx_offset),+ ))+ .chain(handlers);++ self.asm+ .buffer_mut()+ .add_try_call_site(Some(frame_offset), handlers);++ Ok(())+ }+fn current_code_offset(&self) -> Result<CodeOffset> {Ok(self.asm.buffer().cur_offset())}diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rsindex b64cd66551..f112f54f86 100644--- a/winch/codegen/src/masm.rs+++ b/winch/codegen/src/masm.rs@@ -6,7 +6,7 @@ use crate::isa::{reg::{Reg, RegClass, WritableReg, writable},};use cranelift_codegen::{- Final, MachBufferFinalized, MachLabel,+ Final, MachBufferFinalized, MachExceptionHandler, MachLabel,binemit::CodeOffset,ir::{Endianness, MemFlagsData, RelSourceLoc, SourceLoc, UserExternalNameRef},};@@ -1432,6 +1432,10 @@ pub(crate) trait MacroAssembler {/// when dealing with unreachable code.fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()>;+ /// Prepare to enter an exception handler at the given stack offset and+ /// return the register containing the exception reference.+ fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result<Reg>;+/// Get the address of a local slot.fn local_address(&mut self, local: &LocalSlot) -> Result<Self::Address>;@@ -1471,6 +1475,15 @@ pub(crate) trait MacroAssembler {/// a live GC reference.fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()>;+ /// Record the active exception handlers for the call emitted immediately+ /// before this point.+ fn emit_try_call_site(+ &mut self,+ sp_offset: SPOffset,+ vmctx_slot_offset: u32,+ handlers: impl Iterator<Item = MachExceptionHandler>,+ ) -> Result<()>;+/// Acquire a scratch register and execute the given callback.fn with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R;diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rsindex e8ca63754a..115d6562af 100644--- a/winch/codegen/src/visitor.rs+++ b/winch/codegen/src/visitor.rs@@ -6,8 +6,8 @@use crate::abi::RetArea;use crate::codegen::{- Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, FnCall,- UnconditionalBranch, control_index,+ Callee, CatchInfo, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission,+ FnCall, TryTableInfo, UnconditionalBranch, control_index,};use crate::masm::{AtomicWaitKind, DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind,@@ -20,6 +20,7 @@ use crate::masm::{use crate::reg::{Reg, writable};use crate::stack::{TypedReg, Val};use crate::{Result, bail, format_err};+use cranelift_codegen::ir::ExceptionTag;use regalloc2::RegClass;use smallvec::{SmallVec, smallvec};use wasmparser::{@@ -1511,7 +1512,11 @@ whereself.handle_unreachable_end()} else {let mut control = self.pop_control_frame()?;- control.emit_end(self.masm, &mut self.context)+ if let Some(info) = control.take_try_table_info() {+ self.emit_try_table_end(control, info)+ } else {+ control.emit_end(self.masm, &mut self.context)+ }}}@@ -1848,12 +1853,44 @@ whereOk(())}- // Winch does not implement exception handlers yet, so a `try_table`- // compiles like a `block`. Thrown exceptions escape to the host, and no- // handler metadata is emitted.+ // Record the handlers that apply to calls within this `try_table`. Their+ // landing pads are emitted when the control frame ends.fn visit_try_table(&mut self, try_table: TryTable) -> Self::Output {- self.control_frames.push(ControlStackFrame::block(+ let checkpoint = self.context.exception_handlers.take_checkpoint();+ let mut catches = Vec::with_capacity(try_table.catches.len());++ for catch in try_table.catches.iter().rev() {+ let (tag, target_depth) = match catch {+ wasmparser::Catch::One { tag, label } => (Some(TagIndex::from_u32(*tag)), *label),+ wasmparser::Catch::All { label } => (None, *label),+ wasmparser::Catch::OneRef { .. } | wasmparser::Catch::AllRef { .. } => {+ bail!(CodeGenError::unimplemented_wasm_instruction())+ }+ };++ let landing_pad = self.masm.get_label()?;++ let target = control_index(target_depth, self.control_frames.len())?;+ self.control_frames[target].set_as_target();++ let exception_tag = tag.map(|tag| ExceptionTag::from_u32(tag.as_u32()));+ self.context+ .exception_handlers+ .add_handler(exception_tag, landing_pad);++ catches.push(CatchInfo {+ tag,+ target_depth,+ landing_pad,+ });+ }+ let info = TryTableInfo {+ checkpoint,+ catches,+ };+ self.control_frames.push(ControlStackFrame::try_table(self.env.resolve_block_sig(try_table.ty)?,+ info,self.masm,&mut self.context,)?);@@ -1879,7 +1916,8 @@ wherelet (gc_ref, object_addr) =self.emit_exception_alloc(tag_index, interned, &layout, layouts)?;- let gc_ref = self.emit_exception_payload_fields(exn_ty, &layout, gc_ref, object_addr)?;+ let gc_ref =+ self.emit_store_exception_payload_fields(exn_ty, &layout, gc_ref, object_addr)?;self.context.stack.push(gc_ref.into());self.visit_throw_ref()}@@ -2122,14 +2160,14 @@ wherelet index = GlobalIndex::from_u32(global_index);let (ty, base, offset) = self.emit_get_global_addr(index)?;let addr = self.masm.address_at_reg(base, offset)?;+ let gc_ref = self.context.reg_for_type(ty, self.masm)?;+ self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;+ self.context.free_reg(base);+if self.gc_barrier_needed(&ty) {- self.emit_drc_read_barrier(ty, base, addr)?;+ self.emit_drc_read_barrier(ty, gc_ref)?;} else {- let gc_ref = self.context.reg_for_type(ty, self.masm)?;- self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?;self.context.stack.push(Val::reg(gc_ref, ty));-- self.context.free_reg(base);}Ok(())