A GameCube/Wii emulator and debugger written in Rust.
ppc jit merge
Layle389f73681607
diff --git a/.gitignore b/.gitignoreindex 306bcef..4c87172 100644--- a/.gitignore+++ b/.gitignore@@ -11,3 +11,4 @@ __pycache__//fs/screenshots/textures/+screenshotdb/\ No newline at end of filediff --git a/Cargo.lock b/Cargo.lockindex 27e5197..40a9941 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -181,6 +181,12 @@ version = "1.0.102"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"+[[package]]+name = "arbitrary"+version = "1.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"+[[package]]name = "arboard"version = "3.6.1"@@ -359,6 +365,9 @@ name = "bumpalo"version = "3.20.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"+dependencies = [+ "allocator-api2",+][[package]]name = "bytemuck"@@ -727,7 +736,7 @@ dependencies = ["jni 0.21.1","js-sys","libc",- "mach2",+ "mach2 0.5.0","ndk","ndk-context","num-derive",@@ -754,6 +763,173 @@ dependencies = ["libc",]+[[package]]+name = "cranelift-assembler-x64"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8628cc4ba7f88a9205a7ee42327697abc61195a1e3d92cfae172d6a946e722e"+dependencies = [+ "cranelift-assembler-x64-meta",+]++[[package]]+name = "cranelift-assembler-x64-meta"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d582754487e6c9a065a91c42ccf1bdd8d5977af33468dac5ae9bec0ce88acb3e"+dependencies = [+ "cranelift-srcgen",+]++[[package]]+name = "cranelift-bforest"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fb59c81ace12ee7c33074db7903d4d75d1f40b28cd3e8e6f491de57b29129eb9"+dependencies = [+ "cranelift-entity",+ "wasmtime-internal-core",+]++[[package]]+name = "cranelift-bitset"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f25c06993a681be9cf3140798a3d4ac5bec955e7444416a2fdc87fda8567285d"+dependencies = [+ "wasmtime-internal-core",+]++[[package]]+name = "cranelift-codegen"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27b61f95c5a211918f5d336254a61a488b36a5818de47a868e8c4658dce9cccc"+dependencies = [+ "bumpalo",+ "cranelift-assembler-x64",+ "cranelift-bforest",+ "cranelift-bitset",+ "cranelift-codegen-meta",+ "cranelift-codegen-shared",+ "cranelift-control",+ "cranelift-entity",+ "cranelift-isle",+ "gimli",+ "hashbrown 0.16.1",+ "libm",+ "log",+ "regalloc2",+ "rustc-hash 2.1.2",+ "serde",+ "smallvec",+ "target-lexicon",+ "wasmtime-internal-core",+]++[[package]]+name = "cranelift-codegen-meta"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b85aa822fce72080d041d7c2cf7c3f5c6ecdea7afae68379ba4ef85269c4fa5"+dependencies = [+ "cranelift-assembler-x64-meta",+ "cranelift-codegen-shared",+ "cranelift-srcgen",+ "heck",+]++[[package]]+name = "cranelift-codegen-shared"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "833eb9fc89326cd072cc19e96892f09b5692c0dfe17cd4da2858ba30c2cd85c0"++[[package]]+name = "cranelift-control"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d005320f487e6e8a3edcc7f2fd4f43fcc9946d1013bf206ea649789ac1617fc"+dependencies = [+ "arbitrary",+]++[[package]]+name = "cranelift-entity"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e62ef34c6e720f347a79ece043e8584e242d168911da640bac654a33a6aaaf5"+dependencies = [+ "cranelift-bitset",+ "wasmtime-internal-core",+]++[[package]]+name = "cranelift-frontend"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfa2ad00399dd47e7e7e33cb1dc23b0e39ed9dcd01e8f026fc37af91655031b8"+dependencies = [+ "cranelift-codegen",+ "log",+ "smallvec",+ "target-lexicon",+]++[[package]]+name = "cranelift-isle"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "02c51975ed217b4e8e5a7fd11e9ec83a96104bdff311dddcb505d1d8a9fd7fc6"++[[package]]+name = "cranelift-jit"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4070d2acc5c976887c10c273d38dd2bebebb472fd1ba65fa17b548adf5f56350"+dependencies = [+ "anyhow",+ "cranelift-codegen",+ "cranelift-control",+ "cranelift-entity",+ "cranelift-module",+ "cranelift-native",+ "libc",+ "log",+ "region",+ "target-lexicon",+ "wasmtime-internal-jit-icache-coherence",+ "windows-sys 0.61.2",+]++[[package]]+name = "cranelift-module"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "052a396328dbc7dadc29d1bd27d4aa57d9e9e493ead8ef6e2ab3b75c1bf9644c"+dependencies = [+ "anyhow",+ "cranelift-codegen",+ "cranelift-control",+]++[[package]]+name = "cranelift-native"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9b1889e00da9729d8f8525f3c12998ded86ea709058ff844ebe00b97548de0e"+dependencies = [+ "cranelift-codegen",+ "libc",+ "target-lexicon",+]++[[package]]+name = "cranelift-srcgen"+version = "0.131.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d5a8f82fd5124f009f72167e60139245cd3b56cfd4b53050f22110c48c5f4da1"+[[package]]name = "crc32fast"version = "1.5.0"@@ -1372,6 +1548,11 @@ dependencies = ["bytemuck","chapa","chipi-build",+ "cranelift-codegen",+ "cranelift-frontend",+ "cranelift-jit",+ "cranelift-module",+ "cranelift-native","crc32fast","crossbeam-channel","disasm",@@ -1417,6 +1598,18 @@ dependencies = ["wasip2",]+[[package]]+name = "gimli"+version = "0.33.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c"+dependencies = [+ "fnv",+ "hashbrown 0.16.1",+ "indexmap",+ "stable_deref_trait",+]+[[package]]name = "gl_generator"version = "0.14.0"@@ -1531,6 +1724,12 @@ dependencies = ["foldhash 0.2.0",]+[[package]]+name = "hashbrown"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"+[[package]]name = "heck"version = "0.5.0"@@ -2100,6 +2299,15 @@ dependencies = ["cc",]+[[package]]+name = "mach2"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"+dependencies = [+ "libc",+]+[[package]]name = "mach2"version = "0.5.0"@@ -3080,6 +3288,20 @@ dependencies = ["bitflags 2.11.0",]+[[package]]+name = "regalloc2"+version = "0.15.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186"+dependencies = [+ "allocator-api2",+ "bumpalo",+ "hashbrown 0.17.0",+ "log",+ "rustc-hash 2.1.2",+ "smallvec",+]+[[package]]name = "regex"version = "1.12.3"@@ -3109,6 +3331,18 @@ version = "0.8.10"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"+[[package]]+name = "region"+version = "3.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7"+dependencies = [+ "bitflags 1.3.2",+ "libc",+ "mach2 0.4.3",+ "windows-sys 0.52.0",+]+[[package]]name = "relative-path"version = "1.9.3"@@ -3569,6 +3803,12 @@ version = "0.1.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"+[[package]]+name = "target-lexicon"+version = "0.13.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"+[[package]]name = "term"version = "1.2.1"@@ -4032,6 +4272,28 @@ dependencies = ["unicode-ident",]+[[package]]+name = "wasmtime-internal-core"+version = "44.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f2c7fa6523647262bfb4095dbdf4087accefe525813e783f81a0c682f418ce4"+dependencies = [+ "hashbrown 0.16.1",+ "libm",+]++[[package]]+name = "wasmtime-internal-jit-icache-coherence"+version = "44.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a1859e920871515d324fb9757c3e448d6ed1512ca6ccdff14b6e016505d6ada"+dependencies = [+ "cfg-if",+ "libc",+ "wasmtime-internal-core",+ "windows-sys 0.61.2",+]+[[package]]name = "wayland-backend"version = "0.3.14"diff --git a/crates/gecko/Cargo.toml b/crates/gecko/Cargo.tomlindex b1b64e9..ea790d3 100644--- a/crates/gecko/Cargo.toml+++ b/crates/gecko/Cargo.toml@@ -4,6 +4,7 @@ version = "0.1.0"edition = "2024"[features]+default = ["jit", "idle-skip"]idle-skip = []hooks = []hooks-mut-traps = ["hooks"]@@ -14,6 +15,15 @@ hooks-mut-traps = ["hooks"]efb-writeback = ["dep:crossbeam-channel"]# Enable the WAV file audio sinkaudio-wav-dump = ["dep:hound"]+# Cranelift JIT for Gekko+jit = [+ "dep:cranelift-codegen",+ "dep:cranelift-frontend",+ "dep:cranelift-jit",+ "dep:cranelift-module",+ "dep:cranelift-native",+]+jit-stats = ["jit"][build-dependencies]chipi-build = "0.9.1"@@ -32,3 +42,8 @@ multiversion = "0.8.0"rustc-hash = "2.1.2"crc32fast = "1.5.0"zerocopy = { version = "0.8.48", features = ["derive"] }+cranelift-codegen = { version = "0.131.1", optional = true }+cranelift-frontend = { version = "0.131.1", optional = true }+cranelift-jit = { version = "0.131.1", optional = true }+cranelift-module = { version = "0.131.1", optional = true }+cranelift-native = { version = "0.131.1", optional = true }diff --git a/crates/gecko/build.rs b/crates/gecko/build.rsindex 8df98a1..2eb6c25 100644--- a/crates/gecko/build.rs+++ b/crates/gecko/build.rs@@ -3,4 +3,6 @@ fn main() {chipi_build::generate_bindings("dsp.bindings.chipi").expect("chipi codegen failed (dsp)");chipi_build::generate_bindings("wii_gekko.bindings.chipi").expect("chipi codegen failed (wii gekko)");chipi_build::generate_bindings("wii_dsp.bindings.chipi").expect("chipi codegen failed (wii dsp)");+ chipi_build::generate_bindings("gekko_jit.bindings.chipi").expect("chipi codegen failed (gekko jit)");+ chipi_build::generate_bindings("wii_gekko_jit.bindings.chipi").expect("chipi codegen failed (wii gekko jit)");}diff --git a/crates/gecko/gekko_jit.bindings.chipi b/crates/gecko/gekko_jit.bindings.chipinew file mode 100644index 0000000..17e6c2a--- /dev/null+++ b/crates/gecko/gekko_jit.bindings.chipi@@ -0,0 +1,276 @@+include "../../submodules/chipi-spec/gamecube/gekko.chipi"++target rust {+ dispatch Gekko {+ output "$OUT_DIR/gekko_jit_lut.rs"++ context crate::gekko::jit::translator::JitTranslator+ handlers crate::gekko::jit::handlers+ handler_const crate::system::GC+ strategy fn_ptr_lut++ invalid_handler crate::gekko::jit::handlers::invalid++ instruction_type crate::gekko::instruction::Instruction {+ output "$OUT_DIR/gekko_jit_instr.rs"+ }++ handler branch {+ bx+ bcx+ bclrx+ bcctrx+ }++ handler alu {+ ori+ oris+ addx+ addi+ addis+ xori+ xoris+ andi_dot+ andis_dot+ subfx+ negx+ addcx+ subfcx+ addex+ subfex+ addzex+ subfzex+ addmex+ subfmex+ mullwx+ divwux+ divwx+ mulhwux+ mulhwx+ subfic+ addic+ addic_dot+ mulli+ }++ handler rotate {+ rlwinmx+ rlwimix+ rlwnmx+ }++ handler msr {+ mtmsr+ mfmsr+ rfi+ }++ handler spr {+ mtspr+ mfspr+ }++ handler segment {+ mtsr+ mfsr+ }++ handler store_load {+ stw+ stwu+ sth+ sthu+ lwz+ lwzu+ lbz+ lbzu+ stb+ stbu+ lhz+ lhzu+ lha+ lhau+ lmw+ stmw+ lwzx+ lbzx+ lhzx+ lhax+ stwx+ stbx+ sthx+ lwzux+ lbzux+ lhzux+ lhaux+ stwux+ stbux+ sthux+ lwbrx+ lhbrx+ stwbrx+ sthbrx+ }++ handler store_load_fp {+ lfd+ lfdu+ stfd+ stfdu+ lfs+ lfsu+ stfs+ stfsu+ lfsx+ lfsux+ lfdx+ lfdux+ stfsx+ stfsux+ stfdx+ stfdux+ stfiwx+ }++ handler store_load_psq {+ psq_l+ psq_lu+ psq_st+ psq_stu+ psq_lx+ psq_lux+ psq_stx+ psq_stux+ }++ handler compare {+ cmp+ cmpi+ cmpli+ cmpl+ }++ handler nop {+ isync+ sync+ eieio+ dcbf+ dcbi+ dcbst+ dcbt+ dcbtst+ dcba+ dcbz+ dcbz_l+ icbi+ tlbie+ tlbia+ tlbsync+ tlbld+ tlbli+ }++ handler logical {+ andx+ orx+ xorx+ nandx+ norx+ andcx+ orcx+ eqvx+ slwx+ srwx+ srawx+ srawix+ cntlzwx+ extshx+ extsbx+ }++ handler cr_ops {+ mtcrf+ mfcr+ crxor+ cror+ crand+ creqv+ crnor+ crnand+ crandc+ crorc+ mcrf+ }++ handler fp_ops {+ mtfsfx+ mffsx+ mtfsb0x+ mtfsb1x+ mtfsfix+ mcrfs+ fmrx+ fnegx+ fabsx+ fnabsx+ frspx+ fctiwx+ fctiwzx+ fcmpu+ fcmpo+ faddx+ fsubx+ fmulx+ fdivx+ fmaddx+ fmsubx+ fnmaddx+ fnmsubx+ faddsx+ fsubsx+ fmulsx+ fdivsx+ fmaddsx+ fmsubsx+ fnmaddsx+ fnmsubsx+ fsqrtsx+ fresx+ fsqrtx+ fselx+ frsqrtex+ }++ handler ps_ops {+ ps_cmpu0+ ps_cmpo0+ ps_cmpu1+ ps_cmpo1+ ps_res+ ps_rsqrte+ ps_neg+ ps_mr+ ps_nabs+ ps_abs+ ps_merge00+ ps_merge01+ ps_merge10+ ps_merge11+ ps_sum0+ ps_sum1+ ps_muls0+ ps_muls1+ ps_madds0+ ps_madds1+ ps_div+ ps_sub+ ps_add+ ps_sel+ ps_mul+ ps_msub+ ps_madd+ ps_nmsub+ ps_nmadd+ }+ }+}diff --git a/crates/gecko/src/gekko.rs b/crates/gecko/src/gekko.rsindex 881cd67..fdf31d5 100644--- a/crates/gecko/src/gekko.rs+++ b/crates/gecko/src/gekko.rs@@ -5,6 +5,8 @@ pub mod fpscr;pub mod instruction;pub mod interpreter;pub mod irq;+#[cfg(feature = "jit")]+pub mod jit;pub mod msr;pub mod spr;pub mod sr;@@ -37,12 +39,15 @@ pub struct Gekko {// These are used during instruction execution to track the current// and next PC values. In essence, by writing to `next_pc`, instructions// can change the flow of execution (e.g. for branches and jumps).- pub cia: u32, // Current Instruction Address- pub nia: u32, // Next Instruction Address- pub reserve_addr: Option<u32>, // lwarx/stwcx. reservation address+ pub cia: u32, // Current Instruction Address+ pub nia: u32, // Next Instruction Address+ pub reserve_addr: u32,}impl Gekko {+ /// Sentinel used by `lwarx` / `stwcx.`+ pub const NO_RESERVATION: u32 = 0xFF69_1337;+pub fn new(initial_pc: u32) -> Self {let mut spr = spr::Spr::default();spr.dec = u32::MAX;@@ -60,7 +65,7 @@ impl Gekko {msr: msr::Msr::default(),fpscr: fpscr::Fpscr::default(),sr: [sr::Sr::default(); 16],- reserve_addr: None,+ reserve_addr: Self::NO_RESERVATION,}}diff --git a/crates/gecko/src/gekko/interpreter/store_load.rs b/crates/gecko/src/gekko/interpreter/store_load.rsindex d17d2f7..92def56 100644--- a/crates/gecko/src/gekko/interpreter/store_load.rs+++ b/crates/gecko/src/gekko/interpreter/store_load.rs@@ -191,7 +191,7 @@ pub fn lwarx<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instructio.wrapping_add(ctx.gekko.read_gpr(instr.rb()));let val = ctx.read_u32(addr);ctx.gekko.write_gpr(instr.rd(), val);- ctx.gekko.reserve_addr = Some(addr);+ ctx.gekko.reserve_addr = addr;}#[inline(always)]@@ -201,10 +201,10 @@ pub fn stwcx_dot<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instru.read_gpr_or_zero(instr.ra()).wrapping_add(ctx.gekko.read_gpr(instr.rb()));let so = ctx.gekko.spr.xer.summary_overflow();- let store_performed = ctx.gekko.reserve_addr == Some(addr);+ let store_performed = ctx.gekko.reserve_addr == addr;if store_performed {ctx.write_u32(addr, ctx.gekko.read_gpr(instr.rs()));- ctx.gekko.reserve_addr = None;+ ctx.gekko.reserve_addr = crate::gekko::Gekko::NO_RESERVATION;ctx.gekko.cr.set_cr0(ConditionField::new().with_eq(true).with_so(so));} else {ctx.gekko.cr.set_cr0(ConditionField::new().with_so(so));diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rsnew file mode 100644index 0000000..8a47fe2--- /dev/null+++ b/crates/gecko/src/gekko/jit.rs@@ -0,0 +1,805 @@+pub mod abi;+pub mod block;+pub mod handlers;+pub mod idle;+pub mod insn;+pub mod runtime;+pub mod translator;++#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]+pub mod lut {+ include!(concat!(env!("OUT_DIR"), "/gekko_jit_lut.rs"));+}++#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]+pub mod lut_wii {+ include!(concat!(env!("OUT_DIR"), "/gekko_jit_lut_wii.rs"));+}++use cranelift_codegen::Context;+use cranelift_codegen::ir::{AbiParam, InstBuilder, Signature, types};+use cranelift_codegen::isa::CallConv;+use cranelift_codegen::settings::{self, Configurable};+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};+use cranelift_jit::{JITBuilder, JITModule};+use cranelift_module::{FuncId, Linkage, Module, default_libcall_names};+use rustc_hash::FxHashMap;++use crate::system::{GC, System, SystemId, WII};++pub type BlockEntry = usize;++type TrampolineFn = unsafe extern "C" fn(*mut core::ffi::c_void, usize) -> u32;++#[repr(C)]+#[derive(Clone, Copy)]+pub struct BlockLookupSlot {+ pub pc: u32,+ pub _pad: u32,+ pub entry: usize,+}++pub const BLOCK_LOOKUP_TABLE_SIZE: usize = 8192;+pub const BLOCK_LOOKUP_TABLE_MASK: u32 = (BLOCK_LOOKUP_TABLE_SIZE as u32) - 1;++#[derive(Clone, Copy)]+pub struct ExternFuncs {+ pub cause_invalid_opcode: FuncId,+ pub advance_to_deadline: FuncId,+ pub read_u8: FuncId,+ pub read_u16: FuncId,+ pub read_u32: FuncId,+ pub write_u8: FuncId,+ pub write_u16: FuncId,+ pub write_u32: FuncId,+ pub read_f32: FuncId,+ pub read_f64: FuncId,+ pub write_f32: FuncId,+ pub write_f64: FuncId,+ pub write_msr: FuncId,+ pub read_spr: FuncId,+ pub write_spr: FuncId,+ pub read_sr: FuncId,+ pub write_sr: FuncId,+ pub cause_trap_exception: FuncId,+ pub cause_syscall_interrupt: FuncId,+ pub do_rfi: FuncId,+ pub cause_fp_unavailable: FuncId,+ pub set_reservation: FuncId,+ pub try_clear_reservation: FuncId,+ pub do_lswi: FuncId,+ pub do_stswi: FuncId,+ pub do_lswx: FuncId,+ pub do_stswx: FuncId,+ pub do_psq_load: FuncId,+ pub do_psq_store: FuncId,+ pub read_timebase: FuncId,+}++pub struct JitEngine<const SYSTEM: SystemId> {+ module: JITModule,+ ctx: Context,+ builder_ctx: FunctionBuilderContext,+ cache: FxHashMap<u32, BlockEntry>,+ block_func_ids: FxHashMap<u32, FuncId>,+ pending_chain_slots: FxHashMap<u32, Vec<usize>>,+ #[cfg(feature = "jit-stats")]+ hits: FxHashMap<u32, u64>,+ block_sig: Signature,+ extern_funcs: ExternFuncs,+ trampoline_fn: TrampolineFn,+ block_lookup_table_addr: usize,+ block_seq: u64,+}++impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {+ pub fn new() -> Self {+ let mut flag_builder = settings::builder();+ flag_builder.set("use_colocated_libcalls", "false").unwrap();+ flag_builder.set("is_pic", "false").unwrap();+ flag_builder.set("opt_level", "speed").unwrap();+ flag_builder.set("preserve_frame_pointers", "true").unwrap();+ flag_builder.set("enable_verifier", "false").unwrap();+ flag_builder.set("enable_probestack", "false").unwrap();+ flag_builder.set("unwind_info", "false").unwrap();+ flag_builder+ .set("enable_heap_access_spectre_mitigation", "false")+ .unwrap();+ flag_builder+ .set("enable_table_access_spectre_mitigation", "false")+ .unwrap();+ let isa_builder = cranelift_native::builder().expect("host ISA");+ let isa = isa_builder+ .finish(settings::Flags::new(flag_builder))+ .expect("ISA finish");++ let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());++ struct SymTable {+ cause_invalid_opcode: (&'static str, *const u8),+ advance_to_deadline: (&'static str, *const u8),+ read_u8: (&'static str, *const u8),+ read_u16: (&'static str, *const u8),+ read_u32: (&'static str, *const u8),+ write_u8: (&'static str, *const u8),+ write_u16: (&'static str, *const u8),+ write_u32: (&'static str, *const u8),+ read_f32: (&'static str, *const u8),+ read_f64: (&'static str, *const u8),+ write_f32: (&'static str, *const u8),+ write_f64: (&'static str, *const u8),+ write_msr: (&'static str, *const u8),+ read_spr: (&'static str, *const u8),+ write_spr: (&'static str, *const u8),+ read_sr: (&'static str, *const u8),+ write_sr: (&'static str, *const u8),+ cause_trap_exception: (&'static str, *const u8),+ cause_syscall_interrupt: (&'static str, *const u8),+ do_rfi: (&'static str, *const u8),+ cause_fp_unavailable: (&'static str, *const u8),+ set_reservation: (&'static str, *const u8),+ try_clear_reservation: (&'static str, *const u8),+ do_lswi: (&'static str, *const u8),+ do_stswi: (&'static str, *const u8),+ do_lswx: (&'static str, *const u8),+ do_stswx: (&'static str, *const u8),+ do_psq_load: (&'static str, *const u8),+ do_psq_store: (&'static str, *const u8),+ read_timebase: (&'static str, *const u8),+ }+ let syms: SymTable = match SYSTEM {+ GC => SymTable {+ cause_invalid_opcode: (+ "gecko_jit_cause_invalid_opcode_gc",+ runtime::cause_invalid_opcode_gc as *const u8,+ ),+ advance_to_deadline: (+ "gecko_jit_advance_to_deadline_gc",+ runtime::advance_to_deadline_gc as *const u8,+ ),+ read_u8: ("gecko_jit_read_u8_gc", runtime::read_u8_gc as *const u8),+ read_u16: ("gecko_jit_read_u16_gc", runtime::read_u16_gc as *const u8),+ read_u32: ("gecko_jit_read_u32_gc", runtime::read_u32_gc as *const u8),+ write_u8: ("gecko_jit_write_u8_gc", runtime::write_u8_gc as *const u8),+ write_u16: ("gecko_jit_write_u16_gc", runtime::write_u16_gc as *const u8),+ write_u32: ("gecko_jit_write_u32_gc", runtime::write_u32_gc as *const u8),+ read_f32: ("gecko_jit_read_f32_gc", runtime::read_f32_gc as *const u8),+ read_f64: ("gecko_jit_read_f64_gc", runtime::read_f64_gc as *const u8),+ write_f32: ("gecko_jit_write_f32_gc", runtime::write_f32_gc as *const u8),+ write_f64: ("gecko_jit_write_f64_gc", runtime::write_f64_gc as *const u8),+ write_msr: ("gecko_jit_write_msr_gc", runtime::write_msr_gc as *const u8),+ read_spr: ("gecko_jit_read_spr_gc", runtime::read_spr_gc as *const u8),+ write_spr: ("gecko_jit_write_spr_gc", runtime::write_spr_gc as *const u8),+ read_sr: ("gecko_jit_read_sr_gc", runtime::read_sr_gc as *const u8),+ write_sr: ("gecko_jit_write_sr_gc", runtime::write_sr_gc as *const u8),+ cause_trap_exception: (+ "gecko_jit_cause_trap_exception_gc",+ runtime::cause_trap_exception_gc as *const u8,+ ),+ cause_syscall_interrupt: (+ "gecko_jit_cause_syscall_interrupt_gc",+ runtime::cause_syscall_interrupt_gc as *const u8,+ ),+ do_rfi: ("gecko_jit_do_rfi_gc", runtime::do_rfi_gc as *const u8),+ cause_fp_unavailable: (+ "gecko_jit_cause_fp_unavailable_gc",+ runtime::cause_fp_unavailable_gc as *const u8,+ ),+ set_reservation: ("gecko_jit_set_reservation_gc", runtime::set_reservation_gc as *const u8),+ try_clear_reservation: (+ "gecko_jit_try_clear_reservation_gc",+ runtime::try_clear_reservation_gc as *const u8,+ ),+ do_lswi: ("gecko_jit_do_lswi_gc", runtime::do_lswi_gc as *const u8),+ do_stswi: ("gecko_jit_do_stswi_gc", runtime::do_stswi_gc as *const u8),+ do_lswx: ("gecko_jit_do_lswx_gc", runtime::do_lswx_gc as *const u8),+ do_stswx: ("gecko_jit_do_stswx_gc", runtime::do_stswx_gc as *const u8),+ do_psq_load: ("gecko_jit_do_psq_load_gc", runtime::do_psq_load_gc as *const u8),+ do_psq_store: ("gecko_jit_do_psq_store_gc", runtime::do_psq_store_gc as *const u8),+ read_timebase: ("gecko_jit_read_timebase_gc", runtime::read_timebase_gc as *const u8),+ },+ WII => SymTable {+ cause_invalid_opcode: (+ "gecko_jit_cause_invalid_opcode_wii",+ runtime::cause_invalid_opcode_wii as *const u8,+ ),+ advance_to_deadline: (+ "gecko_jit_advance_to_deadline_wii",+ runtime::advance_to_deadline_wii as *const u8,+ ),+ read_u8: ("gecko_jit_read_u8_wii", runtime::read_u8_wii as *const u8),+ read_u16: ("gecko_jit_read_u16_wii", runtime::read_u16_wii as *const u8),+ read_u32: ("gecko_jit_read_u32_wii", runtime::read_u32_wii as *const u8),+ write_u8: ("gecko_jit_write_u8_wii", runtime::write_u8_wii as *const u8),+ write_u16: ("gecko_jit_write_u16_wii", runtime::write_u16_wii as *const u8),+ write_u32: ("gecko_jit_write_u32_wii", runtime::write_u32_wii as *const u8),+ read_f32: ("gecko_jit_read_f32_wii", runtime::read_f32_wii as *const u8),+ read_f64: ("gecko_jit_read_f64_wii", runtime::read_f64_wii as *const u8),+ write_f32: ("gecko_jit_write_f32_wii", runtime::write_f32_wii as *const u8),+ write_f64: ("gecko_jit_write_f64_wii", runtime::write_f64_wii as *const u8),+ write_msr: ("gecko_jit_write_msr_wii", runtime::write_msr_wii as *const u8),+ read_spr: ("gecko_jit_read_spr_wii", runtime::read_spr_wii as *const u8),+ write_spr: ("gecko_jit_write_spr_wii", runtime::write_spr_wii as *const u8),+ read_sr: ("gecko_jit_read_sr_wii", runtime::read_sr_wii as *const u8),+ write_sr: ("gecko_jit_write_sr_wii", runtime::write_sr_wii as *const u8),+ cause_trap_exception: (+ "gecko_jit_cause_trap_exception_wii",+ runtime::cause_trap_exception_wii as *const u8,+ ),+ cause_syscall_interrupt: (+ "gecko_jit_cause_syscall_interrupt_wii",+ runtime::cause_syscall_interrupt_wii as *const u8,+ ),+ do_rfi: ("gecko_jit_do_rfi_wii", runtime::do_rfi_wii as *const u8),+ cause_fp_unavailable: (+ "gecko_jit_cause_fp_unavailable_wii",+ runtime::cause_fp_unavailable_wii as *const u8,+ ),+ set_reservation: (+ "gecko_jit_set_reservation_wii",+ runtime::set_reservation_wii as *const u8,+ ),+ try_clear_reservation: (+ "gecko_jit_try_clear_reservation_wii",+ runtime::try_clear_reservation_wii as *const u8,+ ),+ do_lswi: ("gecko_jit_do_lswi_wii", runtime::do_lswi_wii as *const u8),+ do_stswi: ("gecko_jit_do_stswi_wii", runtime::do_stswi_wii as *const u8),+ do_lswx: ("gecko_jit_do_lswx_wii", runtime::do_lswx_wii as *const u8),+ do_stswx: ("gecko_jit_do_stswx_wii", runtime::do_stswx_wii as *const u8),+ do_psq_load: ("gecko_jit_do_psq_load_wii", runtime::do_psq_load_wii as *const u8),+ do_psq_store: ("gecko_jit_do_psq_store_wii", runtime::do_psq_store_wii as *const u8),+ read_timebase: ("gecko_jit_read_timebase_wii", runtime::read_timebase_wii as *const u8),+ },+ _ => unreachable!(),+ };++ for &(name, addr) in &[+ syms.cause_invalid_opcode,+ syms.advance_to_deadline,+ syms.read_u8,+ syms.read_u16,+ syms.read_u32,+ syms.write_u8,+ syms.write_u16,+ syms.write_u32,+ syms.read_f32,+ syms.read_f64,+ syms.write_f32,+ syms.write_f64,+ syms.write_msr,+ syms.read_spr,+ syms.write_spr,+ syms.read_sr,+ syms.write_sr,+ syms.cause_trap_exception,+ syms.cause_syscall_interrupt,+ syms.do_rfi,+ syms.cause_fp_unavailable,+ syms.set_reservation,+ syms.try_clear_reservation,+ syms.do_lswi,+ syms.do_stswi,+ syms.do_lswx,+ syms.do_stswx,+ syms.do_psq_load,+ syms.do_psq_store,+ syms.read_timebase,+ ] {+ jit_builder.symbol(name, addr);+ }++ let mut module = JITModule::new(jit_builder);++ let pointer_type = module.target_config().pointer_type();+ let call_conv = module.target_config().default_call_conv;++ let mut block_sig = Signature::new(CallConv::Tail);+ block_sig.params.push(AbiParam::new(pointer_type));+ block_sig.returns.push(AbiParam::new(types::I32));++ let mut fallback_sig = Signature::new(call_conv);+ fallback_sig.params.push(AbiParam::new(pointer_type));+ fallback_sig.params.push(AbiParam::new(types::I32));+ fallback_sig.params.push(AbiParam::new(types::I32));+ fallback_sig.returns.push(AbiParam::new(types::I32));++ let mut deadline_sig = Signature::new(call_conv);+ deadline_sig.params.push(AbiParam::new(pointer_type));++ let mut mem_read_sig = Signature::new(call_conv);+ mem_read_sig.params.push(AbiParam::new(pointer_type));+ mem_read_sig.params.push(AbiParam::new(types::I32));+ mem_read_sig.returns.push(AbiParam::new(types::I32));++ let mut mem_write_sig = Signature::new(call_conv);+ mem_write_sig.params.push(AbiParam::new(pointer_type));+ mem_write_sig.params.push(AbiParam::new(types::I32));+ mem_write_sig.params.push(AbiParam::new(types::I32));++ let mut fp_read_sig = Signature::new(call_conv);+ fp_read_sig.params.push(AbiParam::new(pointer_type));+ fp_read_sig.params.push(AbiParam::new(types::I32));+ fp_read_sig.returns.push(AbiParam::new(types::F64));++ let mut fp_write_sig = Signature::new(call_conv);+ fp_write_sig.params.push(AbiParam::new(pointer_type));+ fp_write_sig.params.push(AbiParam::new(types::I32));+ fp_write_sig.params.push(AbiParam::new(types::F64));++ let mut ctx_u32_sig = Signature::new(call_conv);+ ctx_u32_sig.params.push(AbiParam::new(pointer_type));+ ctx_u32_sig.params.push(AbiParam::new(types::I32));++ let extern_funcs = ExternFuncs {+ cause_invalid_opcode: module+ .declare_function(syms.cause_invalid_opcode.0, Linkage::Import, &fallback_sig)+ .expect("declare cause_invalid_opcode"),+ advance_to_deadline: module+ .declare_function(syms.advance_to_deadline.0, Linkage::Import, &deadline_sig)+ .expect("declare advance_to_deadline"),+ read_u8: module+ .declare_function(syms.read_u8.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_u8"),+ read_u16: module+ .declare_function(syms.read_u16.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_u16"),+ read_u32: module+ .declare_function(syms.read_u32.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_u32"),+ write_u8: module+ .declare_function(syms.write_u8.0, Linkage::Import, &mem_write_sig)+ .expect("declare write_u8"),+ write_u16: module+ .declare_function(syms.write_u16.0, Linkage::Import, &mem_write_sig)+ .expect("declare write_u16"),+ write_u32: module+ .declare_function(syms.write_u32.0, Linkage::Import, &mem_write_sig)+ .expect("declare write_u32"),+ read_f32: module+ .declare_function(syms.read_f32.0, Linkage::Import, &fp_read_sig)+ .expect("declare read_f32"),+ read_f64: module+ .declare_function(syms.read_f64.0, Linkage::Import, &fp_read_sig)+ .expect("declare read_f64"),+ write_f32: module+ .declare_function(syms.write_f32.0, Linkage::Import, &fp_write_sig)+ .expect("declare write_f32"),+ write_f64: module+ .declare_function(syms.write_f64.0, Linkage::Import, &fp_write_sig)+ .expect("declare write_f64"),+ write_msr: module+ .declare_function(syms.write_msr.0, Linkage::Import, &ctx_u32_sig)+ .expect("declare write_msr"),+ read_spr: module+ .declare_function(syms.read_spr.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_spr"),+ write_spr: module+ .declare_function(syms.write_spr.0, Linkage::Import, &mem_write_sig)+ .expect("declare write_spr"),+ read_sr: module+ .declare_function(syms.read_sr.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_sr"),+ write_sr: module+ .declare_function(syms.write_sr.0, Linkage::Import, &mem_write_sig)+ .expect("declare write_sr"),+ cause_trap_exception: module+ .declare_function(syms.cause_trap_exception.0, Linkage::Import, &deadline_sig)+ .expect("declare cause_trap_exception"),+ cause_syscall_interrupt: module+ .declare_function(syms.cause_syscall_interrupt.0, Linkage::Import, &deadline_sig)+ .expect("declare cause_syscall_interrupt"),+ do_rfi: module+ .declare_function(syms.do_rfi.0, Linkage::Import, &deadline_sig)+ .expect("declare do_rfi"),+ cause_fp_unavailable: module+ .declare_function(syms.cause_fp_unavailable.0, Linkage::Import, &ctx_u32_sig)+ .expect("declare cause_fp_unavailable"),+ set_reservation: module+ .declare_function(syms.set_reservation.0, Linkage::Import, &ctx_u32_sig)+ .expect("declare set_reservation"),+ try_clear_reservation: module+ .declare_function(syms.try_clear_reservation.0, Linkage::Import, &mem_read_sig)+ .expect("declare try_clear_reservation"),+ do_lswi: module+ .declare_function(syms.do_lswi.0, Linkage::Import, &{+ let mut sig = Signature::new(call_conv);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+ })+ .expect("declare do_lswi"),+ do_stswi: module+ .declare_function(syms.do_stswi.0, Linkage::Import, &{+ let mut sig = Signature::new(call_conv);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+ })+ .expect("declare do_stswi"),+ do_lswx: module+ .declare_function(syms.do_lswx.0, Linkage::Import, &mem_write_sig)+ .expect("declare do_lswx"),+ do_stswx: module+ .declare_function(syms.do_stswx.0, Linkage::Import, &mem_write_sig)+ .expect("declare do_stswx"),+ do_psq_load: module+ .declare_function(syms.do_psq_load.0, Linkage::Import, &{+ let mut sig = Signature::new(call_conv);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+ })+ .expect("declare do_psq_load"),+ do_psq_store: module+ .declare_function(syms.do_psq_store.0, Linkage::Import, &{+ let mut sig = Signature::new(call_conv);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+ })+ .expect("declare do_psq_store"),+ read_timebase: module+ .declare_function(syms.read_timebase.0, Linkage::Import, &mem_read_sig)+ .expect("declare read_timebase"),+ };++ let trampoline_fn = build_trampoline(&mut module, call_conv, pointer_type, &block_sig);++ let block_lookup_table = vec![+ BlockLookupSlot {+ pc: 0,+ _pad: 0,+ entry: 0+ };+ BLOCK_LOOKUP_TABLE_SIZE+ ]+ .into_boxed_slice();+ let block_lookup_table_addr = Box::leak(block_lookup_table).as_mut_ptr() as usize;++ Self {+ module,+ ctx: Context::new(),+ builder_ctx: FunctionBuilderContext::new(),+ cache: FxHashMap::default(),+ block_func_ids: FxHashMap::default(),+ pending_chain_slots: FxHashMap::default(),+ #[cfg(feature = "jit-stats")]+ hits: FxHashMap::default(),+ block_sig,+ extern_funcs,+ trampoline_fn,+ block_lookup_table_addr,+ block_seq: 0,+ }+ }++ pub fn block_lookup_table_addr(&self) -> usize {+ self.block_lookup_table_addr+ }++ fn register_in_lookup_table(&self, pc: u32, entry: usize) {+ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;+ unsafe {+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;+ (*table.add(idx)).pc = pc;+ (*table.add(idx)).entry = entry;+ }+ }++ pub fn lookup_or_compile(&mut self, sys: &mut System<SYSTEM>, pc: u32) -> BlockEntry {+ unsafe {+ let table = self.block_lookup_table_addr as *const BlockLookupSlot;+ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;+ let slot = &*table.add(idx);+ if slot.pc == pc && slot.entry != 0 {+ #[cfg(feature = "jit-stats")]+ {+ *self.hits.entry(pc).or_insert(0) += 1;+ }+ return slot.entry;+ }+ }++ if let Some(&entry) = self.cache.get(&pc) {+ #[cfg(feature = "jit-stats")]+ {+ *self.hits.entry(pc).or_insert(0) += 1;+ }+ return entry;+ }++ let spec = block::discover::<SYSTEM>(sys, pc);+ let gprs_snapshot = sys.gekko.gprs;++ let entry = self.compile(&spec, &gprs_snapshot);+ self.cache.insert(pc, entry);++ entry+ }++ fn func_id_for(&mut self, pc: u32) -> FuncId {+ if let Some(&id) = self.block_func_ids.get(&pc) {+ return id;+ }++ self.block_seq = self.block_seq.wrapping_add(1);+ let name = format!("gecko_block_{:08x}_{}", pc, self.block_seq);+ let id = self+ .module+ .declare_function(&name, Linkage::Local, &self.block_sig)+ .expect("declare block");+ self.block_func_ids.insert(pc, id);++ id+ }++ pub fn run_block(&mut self, sys: &mut System<SYSTEM>) {+ let pc = sys.gekko.pc;+ let entry = self.lookup_or_compile(sys, pc);+ let ctx_ptr = sys as *mut System<SYSTEM> as *mut core::ffi::c_void;+ let next_pc = unsafe { (self.trampoline_fn)(ctx_ptr, entry) };+ sys.gekko.pc = next_pc;+ }++ #[cfg(feature = "jit-stats")]+ pub fn dump_idle_candidates(&self, sys: &System<SYSTEM>, top_k: usize) {+ const SHAPE_BODY_CAP: usize = 12;++ let mut entries: Vec<(u32, u64, block::BlockSpec)> = self+ .hits+ .iter()+ .filter_map(|(&pc, &hits)| {+ let spec = block::discover::<SYSTEM>(sys, pc);+ if !looks_like_polling_shape(&spec, SHAPE_BODY_CAP) {+ return None;+ }++ let class = idle::classify::<SYSTEM>(&spec, &sys.gekko.gprs);+ if class != idle::IdleClass::None {+ return None;+ }++ Some((pc, hits, spec))+ })+ .collect();+ entries.sort_by(|a, b| b.1.cmp(&a.1));++ if entries.is_empty() {+ tracing::info!("no idle-skip candidates recorded");+ return;+ }++ let total: u64 = entries.iter().map(|e| e.1).sum();+ tracing::info!(+ "idle-skip candidates (top {}, total hits = {})",+ top_k.min(entries.len()),+ total+ );+ for (rank, (pc, hits, spec)) in entries.iter().take(top_k).enumerate() {+ let pct = (*hits as f64) * 100.0 / (total as f64);+ tracing::info!(+ " #{rank} pc={:08X} hits={hits} ({:.1}%) len={} term={:?}",+ pc,+ pct,+ spec.instrs.len(),+ spec.terminator+ );+ }+ }++ #[cfg(feature = "jit-stats")]+ pub fn dump_hot_blocks(&self, sys: &System<SYSTEM>, top_k: usize) {+ let mut entries: Vec<(u32, u64)> = self.hits.iter().map(|(&pc, &n)| (pc, n)).collect();+ entries.sort_by(|a, b| b.1.cmp(&a.1));++ if entries.is_empty() {+ tracing::info!("no dispatcher entry hits recorded");+ return;+ }++ let total: u64 = entries.iter().map(|e| e.1).sum();+ tracing::info!(+ "JIT hot blocks (top {}, total hits = {}) ===",+ top_k.min(entries.len()),+ total+ );++ for (rank, (pc, hits)) in entries.iter().take(top_k).enumerate() {+ let spec = block::discover::<SYSTEM>(sys, *pc);+ let idle_class = idle::classify::<SYSTEM>(&spec, &sys.gekko.gprs);+ let pct = (*hits as f64) * 100.0 / (total as f64);+ tracing::info!(+ " #{rank} pc={:08X} hits={hits} ({:.1}%) len={} idle={:?} term={:?}",+ pc,+ pct,+ spec.instrs.len(),+ idle_class,+ spec.terminator,+ );+ }+ }++ pub fn flush(&mut self) {+ self.cache.clear();+ self.block_func_ids.clear();+ self.pending_chain_slots.clear();++ unsafe {+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;+ for i in 0..BLOCK_LOOKUP_TABLE_SIZE {+ (*table.add(i)).pc = 0;+ (*table.add(i)).entry = 0;+ }+ }+ }++ fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {+ let func_id = self.func_id_for(spec.start_pc);++ self.ctx.clear();+ self.ctx.func.signature = self.block_sig.clone();++ let self_slot: &'static mut usize = Box::leak(Box::new(0usize));+ let self_slot_addr = self_slot as *mut usize as i64;++ let mut pending_for_this_block: Vec<(u32, usize)> = Vec::new();++ let chain = translator::ChainContext {+ self_pc: spec.start_pc,+ self_func_id: func_id,+ self_slot_addr,+ compiled_func_ids: &self.block_func_ids,+ compiled_cache: &self.cache,+ pending: std::cell::RefCell::new(&mut pending_for_this_block),+ block_lookup_table_addr: self.block_lookup_table_addr as i64,+ };++ translator::translate::<SYSTEM>(+ &mut self.ctx,+ &mut self.builder_ctx,+ &mut self.module,+ &self.extern_funcs,+ spec,+ gprs,+ &chain,+ );++ drop(chain);+ for (target_pc, slot_addr) in pending_for_this_block.drain(..) {+ self.pending_chain_slots.entry(target_pc).or_default().push(slot_addr);+ }++ self.module+ .define_function(func_id, &mut self.ctx)+ .expect("define block");++ self.module.finalize_definitions().expect("finalize");+ let entry = self.module.get_finalized_function(func_id) as usize;++ unsafe {+ *(self_slot_addr as *mut usize) = entry;+ }++ if let Some(slots) = self.pending_chain_slots.remove(&spec.start_pc) {+ for slot_addr in slots {+ unsafe {+ *(slot_addr as *mut usize) = entry;+ }+ }+ }++ self.register_in_lookup_table(spec.start_pc, entry);++ entry+ }+}++impl<const SYSTEM: SystemId> Default for JitEngine<SYSTEM> {+ fn default() -> Self {+ Self::new()+ }+}++#[cfg(feature = "jit-stats")]+fn looks_like_polling_shape(spec: &block::BlockSpec, body_cap: usize) -> bool {+ if spec.terminator != block::TermKind::BranchCond {+ return false;+ }++ if spec.instrs.is_empty() || spec.instrs.len() > body_cap {+ return false;+ }++ let last_idx = spec.instrs.len() - 1;+ let raw = spec.instrs[last_idx];+ if (raw >> 26) != 16 {+ return false;+ }++ if raw & 1 != 0 {+ return false;+ }++ let bo = (raw >> 21) & 0x1F;+ if bo & 0b00100 == 0 {+ return false;+ }++ let aa = (raw >> 1) & 1 != 0;+ let bd = (((raw >> 2) & 0x3FFF) as i32) << 18 >> 18;+ let bd_bytes = bd << 2;+ let term_pc = spec.start_pc.wrapping_add((last_idx as u32) * 4);+ let target = if aa {+ bd_bytes as u32+ } else {+ term_pc.wrapping_add_signed(bd_bytes)+ };++ target == spec.start_pc+}++fn build_trampoline(+ module: &mut JITModule,+ host_cc: CallConv,+ pointer_type: cranelift_codegen::ir::Type,+ block_sig: &Signature,+) -> TrampolineFn {+ let mut tramp_sig = Signature::new(host_cc);+ tramp_sig.params.push(AbiParam::new(pointer_type));+ tramp_sig.params.push(AbiParam::new(pointer_type));+ tramp_sig.returns.push(AbiParam::new(types::I32));++ let func_id = module+ .declare_function("gecko_jit_trampoline", Linkage::Local, &tramp_sig)+ .expect("declare trampoline");++ let mut ctx = Context::new();+ ctx.func.signature = tramp_sig.clone();+ let block_sig_ref = ctx.func.import_signature(block_sig.clone());++ {+ let mut bcx_ctx = FunctionBuilderContext::new();+ let mut builder = FunctionBuilder::new(&mut ctx.func, &mut bcx_ctx);+ let entry = builder.create_block();++ builder.append_block_params_for_function_params(entry);+ builder.switch_to_block(entry);+ builder.seal_block(entry);++ let ctx_ptr = builder.block_params(entry)[0];+ let block_ptr = builder.block_params(entry)[1];+ let call = builder.ins().call_indirect(block_sig_ref, block_ptr, &[ctx_ptr]);+ let result = builder.inst_results(call)[0];+ builder.ins().return_(&[result]);+ builder.finalize();+ }++ module.define_function(func_id, &mut ctx).expect("define trampoline");+ module.finalize_definitions().expect("finalize trampoline");++ let ptr = module.get_finalized_function(func_id);+ unsafe { core::mem::transmute::<*const u8, TrampolineFn>(ptr) }+}++#[inline(always)]+pub fn dispatch<const SYSTEM: SystemId>(+ t: &mut translator::JitTranslator,+ instr: crate::gekko::instruction::Instruction,+) {+ if SYSTEM == GC {+ self::lut::dispatch(t, instr);+ } else {+ self::lut_wii::dispatch(t, instr);+ }+}diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rsnew file mode 100644index 0000000..c4f8d81--- /dev/null+++ b/crates/gecko/src/gekko/jit/abi.rs@@ -0,0 +1,238 @@+use core::mem::offset_of;++use crate::gekko::Gekko;+use crate::gekko::condition::ConditionRegister;+use crate::gekko::msr::Msr;+use crate::gekko::spr::{Spr, Xer};+use crate::mmio::Mmio;+use crate::scheduler::Scheduler;+use crate::system::{System, SystemId};++const _: () = {+ if core::mem::size_of::<ConditionRegister>() != 4 {+ panic!("ConditionRegister is no longer 4 bytes; JIT CR access needs to be revisited");+ }+ if core::mem::align_of::<ConditionRegister>() != 4 {+ panic!("ConditionRegister alignment changed; JIT CR access needs to be revisited");+ }+ if core::mem::size_of::<Xer>() != 4 {+ panic!("Xer is no longer 4 bytes; JIT XER access needs to be revisited");+ }+ if core::mem::size_of::<Msr>() != 4 {+ panic!("Msr is no longer 4 bytes; JIT MSR access needs to be revisited");+ }+};++#[inline(always)]+pub const fn gekko_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, gekko)+}++#[inline(always)]+pub const fn scheduler_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, scheduler)+}++#[inline(always)]+pub const fn pc_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, pc)+}++#[inline(always)]+pub const fn cia_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, cia)+}++#[inline(always)]+pub const fn nia_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, nia)+}++#[inline(always)]+pub const fn gpr_base_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, gprs)+}++#[inline(always)]+pub const fn fpr_base_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, fprs)+}++#[inline(always)]+pub const fn ps1_base_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, ps1s)+}++#[inline(always)]+pub const fn cycles_offset<const SYSTEM: SystemId>() -> usize {+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, cycles)+}++#[inline(always)]+pub const fn next_deadline_offset<const SYSTEM: SystemId>() -> usize {+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, next_deadline)+}++#[inline(always)]+pub const fn cr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, cr)+}++#[inline(always)]+pub const fn xer_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, xer)+}++#[inline(always)]+pub const fn fpscr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, fpscr)+}++#[inline(always)]+pub const fn ram_ptr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, ram_ptr)+}++#[inline(always)]+pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)+}++#[inline(always)]+pub const fn gqr_offset<const SYSTEM: SystemId>(i: u8) -> usize {+ let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);+ spr_off+ + match i {+ 0 => offset_of!(Spr, gqr0),+ 1 => offset_of!(Spr, gqr1),+ 2 => offset_of!(Spr, gqr2),+ 3 => offset_of!(Spr, gqr3),+ 4 => offset_of!(Spr, gqr4),+ 5 => offset_of!(Spr, gqr5),+ 6 => offset_of!(Spr, gqr6),+ 7 => offset_of!(Spr, gqr7),+ _ => panic!("GQR index out of range"),+ }+}++#[inline(always)]+pub const fn lr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, lr)+}++#[inline(always)]+pub const fn ctr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, ctr)+}++#[inline(always)]+pub const fn msr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, msr)+}++#[inline(always)]+pub const fn timebase_offset_offset<const SYSTEM: SystemId>() -> usize {+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, timebase_offset)+}++#[inline(always)]+pub const fn reserve_addr_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, reserve_addr)+}++#[inline(always)]+pub const fn sr_base_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, sr)+}++#[inline(always)]+pub fn spr_field_offset<const SYSTEM: SystemId>(spr_num: u16) -> Option<usize> {+ let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);+ let inner = match spr_num {+ 1 => offset_of!(Spr, xer),+ 8 => offset_of!(Spr, lr),+ 9 => offset_of!(Spr, ctr),+ 18 => offset_of!(Spr, dsisr),+ 19 => offset_of!(Spr, dar),+ 25 => offset_of!(Spr, sdr1),+ 26 => offset_of!(Spr, srr0),+ 27 => offset_of!(Spr, srr1),+ 272 => offset_of!(Spr, sprg0),+ 273 => offset_of!(Spr, sprg1),+ 274 => offset_of!(Spr, sprg2),+ 275 => offset_of!(Spr, sprg3),+ 282 => offset_of!(Spr, ear),+ 287 => offset_of!(Spr, pvr),+ 528 => offset_of!(Spr, ibat0u),+ 529 => offset_of!(Spr, ibat0l),+ 530 => offset_of!(Spr, ibat1u),+ 531 => offset_of!(Spr, ibat1l),+ 532 => offset_of!(Spr, ibat2u),+ 533 => offset_of!(Spr, ibat2l),+ 534 => offset_of!(Spr, ibat3u),+ 535 => offset_of!(Spr, ibat3l),+ 536 => offset_of!(Spr, dbat0u),+ 537 => offset_of!(Spr, dbat0l),+ 538 => offset_of!(Spr, dbat1u),+ 539 => offset_of!(Spr, dbat1l),+ 540 => offset_of!(Spr, dbat2u),+ 541 => offset_of!(Spr, dbat2l),+ 542 => offset_of!(Spr, dbat3u),+ 543 => offset_of!(Spr, dbat3l),++ 560 => offset_of!(Spr, ibat4u),+ 561 => offset_of!(Spr, ibat4l),+ 562 => offset_of!(Spr, ibat5u),+ 563 => offset_of!(Spr, ibat5l),+ 564 => offset_of!(Spr, ibat6u),+ 565 => offset_of!(Spr, ibat6l),+ 566 => offset_of!(Spr, ibat7u),+ 567 => offset_of!(Spr, ibat7l),+ 568 => offset_of!(Spr, dbat4u),+ 569 => offset_of!(Spr, dbat4l),+ 570 => offset_of!(Spr, dbat5u),+ 571 => offset_of!(Spr, dbat5l),+ 572 => offset_of!(Spr, dbat6u),+ 573 => offset_of!(Spr, dbat6l),+ 574 => offset_of!(Spr, dbat7u),+ 575 => offset_of!(Spr, dbat7l),+ 912 => offset_of!(Spr, gqr0),+ 913 => offset_of!(Spr, gqr1),+ 914 => offset_of!(Spr, gqr2),+ 915 => offset_of!(Spr, gqr3),+ 916 => offset_of!(Spr, gqr4),+ 917 => offset_of!(Spr, gqr5),+ 918 => offset_of!(Spr, gqr6),+ 919 => offset_of!(Spr, gqr7),+ 920 => offset_of!(Spr, hid2),+ 921 => offset_of!(Spr, wpar),+ 936 => offset_of!(Spr, ummcr0),+ 937 => offset_of!(Spr, upmc1),+ 938 => offset_of!(Spr, upmc2),+ 939 => offset_of!(Spr, usia),+ 940 => offset_of!(Spr, ummcr1),+ 941 => offset_of!(Spr, upmc3),+ 942 => offset_of!(Spr, upmc4),+ 943 => offset_of!(Spr, usda),+ 952 => offset_of!(Spr, mmcr0),+ 953 => offset_of!(Spr, pmc1),+ 954 => offset_of!(Spr, pmc2),+ 955 => offset_of!(Spr, sia),+ 956 => offset_of!(Spr, mmcr1),+ 957 => offset_of!(Spr, pmc3),+ 958 => offset_of!(Spr, pmc4),+ 959 => offset_of!(Spr, sda),+ 1008 => offset_of!(Spr, hid0),+ 1009 => offset_of!(Spr, hid1),+ 1010 => offset_of!(Spr, iabr),+ 1011 => offset_of!(Spr, hid4),+ 1013 => offset_of!(Spr, dabr),+ 1017 => offset_of!(Spr, l2cr),+ 1019 => offset_of!(Spr, ictc),+ 1020 => offset_of!(Spr, thrm1),+ 1021 => offset_of!(Spr, thrm2),+ 1022 => offset_of!(Spr, thrm3),+ _ => return None,+ };+ Some(spr_off + inner)+}diff --git a/crates/gecko/src/gekko/jit/block.rs b/crates/gecko/src/gekko/jit/block.rsnew file mode 100644index 0000000..e2d155d--- /dev/null+++ b/crates/gecko/src/gekko/jit/block.rs@@ -0,0 +1,143 @@+use crate::gekko::instruction::Instruction;+use crate::system::{System, SystemId};++pub const MAX_BLOCK_INSTRS: usize = 64;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TermKind {+ Branch,+ BranchCond,+ BranchToReg,+ SystemCall,+ Rfi,+ Mtmsr,+ Mtspr,+ Isync,+ LengthCap,+}++#[derive(Debug, Clone)]+pub struct BlockSpec {+ pub start_pc: u32,+ pub instrs: Vec<u32>,+ pub pcs: Vec<u32>,+ pub terminator: TermKind,+}++impl BlockSpec {+ #[inline(always)]+ pub fn len(&self) -> usize {+ self.instrs.len()+ }++ #[inline(always)]+ pub fn end_pc(&self) -> u32 {+ self.pcs+ .last()+ .copied()+ .map(|p| p.wrapping_add(4))+ .unwrap_or(self.start_pc)+ }++ #[inline(always)]+ pub fn pc_of(&self, i: usize) -> u32 {+ self.pcs[i]+ }+}++pub fn discover<const SYSTEM: SystemId>(sys: &System<SYSTEM>, start_pc: u32) -> BlockSpec {+ const EXTENSION_MAX_FORWARD_BYTES: u32 = 256;++ let mut instrs: Vec<u32> = Vec::with_capacity(8);+ let mut pcs: Vec<u32> = Vec::with_capacity(8);+ let mut terminator = TermKind::LengthCap;+ let mut pc = start_pc;++ while instrs.len() < MAX_BLOCK_INSTRS {+ let instr = Instruction(sys.mmio.fetch_instruction(pc));+ let cur_pc = pc;++ if let Some(target) = extension_target(instr, cur_pc) {+ if target > cur_pc && target.wrapping_sub(cur_pc) <= EXTENSION_MAX_FORWARD_BYTES && !pcs.contains(&target) {+ pc = target;+ continue;+ }+ }++ instrs.push(instr.0);+ pcs.push(cur_pc);++ if let Some(t) = classify_terminator(instr) {+ terminator = t;+ break;+ }+ pc = pc.wrapping_add(4);+ }++ BlockSpec {+ start_pc,+ instrs,+ pcs,+ terminator,+ }+}++#[inline]+fn extension_target(instr: Instruction, pc: u32) -> Option<u32> {+ if instr.primary_opcode() != 18 {+ return None;+ }+ if instr.lk() {+ return None;+ }+ let target = if instr.aa() {+ instr.li() as u32+ } else {+ pc.wrapping_add_signed(instr.li())+ };+ Some(target)+}++#[inline]+fn mtspr_is_block_safe(spr: u16) -> bool {+ matches!(+ spr,+ 1+ | 8+ | 9+ | 22+ | 26 | 27+ | 272..=275+ | 912..=919+ | 920+ | 1008 | 1009+ )+}++#[inline]+pub fn classify_terminator(instr: Instruction) -> Option<TermKind> {+ match instr.primary_opcode() {+ 16 => Some(TermKind::BranchCond),+ 17 => Some(TermKind::SystemCall),+ 18 => Some(TermKind::Branch),+ 19 => match instr.xo10() {+ 16 | 528 => Some(TermKind::BranchToReg),+ 50 => Some(TermKind::Rfi),+ 150 => Some(TermKind::Isync),+ _ => None,+ },+ 31 => match instr.xo10() {+ 146 => Some(TermKind::Mtmsr),+ 467 => {+ let spr_num = instr.spr_swapped() as u16;+ if mtspr_is_block_safe(spr_num) {+ None+ } else {+ Some(TermKind::Mtspr)+ }+ }+ _ => None,+ },+ _ => None,+ }+}diff --git a/crates/gecko/src/gekko/jit/handlers.rs b/crates/gecko/src/gekko/jit/handlers.rsnew file mode 100644index 0000000..e671aa4--- /dev/null+++ b/crates/gecko/src/gekko/jit/handlers.rs@@ -0,0 +1,1042 @@+use cranelift_codegen::ir::condcodes::IntCC;+use cranelift_codegen::ir::{InstBuilder, MemFlags, types};+use cranelift_frontend::FunctionBuilder;++use crate::gekko::instruction::Instruction;+use crate::gekko::jit::translator::{+ self, AddmeSubfmeKind, AddzeSubfzeKind, CYCLES_PER_INSTR, ImmLogical, JitTranslator, LocalFuncs, LogicalFullOp,+ LogicalOp, MemSize, ShiftKind, TermEmit,+};+use crate::gekko::jit::{abi, lut};+use crate::system::SystemId;++#[inline(always)]+unsafe fn parts<'a>(builder_ptr: usize, local_ptr: usize) -> (&'a mut FunctionBuilder<'a>, &'a LocalFuncs) {+ (unsafe { &mut *(builder_ptr as *mut FunctionBuilder<'a>) }, unsafe {+ &*(local_ptr as *const LocalFuncs)+ })+}++#[inline(always)]+pub fn alu<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_ADDI => translator::emit_addi::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_ADDIS => translator::emit_addis::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_ORI => translator::emit_ori::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_ORIS => translator::emit_oris::<SYSTEM>(builder, t.ctx_ptr, instr),++ lut::OP_XORI => {+ translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::Xor, false);+ }+ lut::OP_XORIS => {+ translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::Xor, true);+ }+ lut::OP_ANDI_DOT => {+ let r = translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::And, false);+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ lut::OP_ANDIS_DOT => {+ let r = translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::And, true);+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }++ lut::OP_MULLI => translator::emit_mulli::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_SUBFIC => translator::emit_subfic::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_ADDIC => translator::emit_addic::<SYSTEM>(builder, t.ctx_ptr, instr, false),+ lut::OP_ADDIC_DOT => translator::emit_addic::<SYSTEM>(builder, t.ctx_ptr, instr, true),++ lut::OP_ADDX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_add_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_add_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_ADDCX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_addc_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_add_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_ADDEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_adde_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_add_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_ADDZEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let r = translator::emit_addze_subfze::<SYSTEM>(builder, t.ctx_ptr, instr, AddzeSubfzeKind::Addze);++ if instr.oe() {+ let zero = builder.ins().iconst(types::I32, 0);+ let ov = translator::compute_add_overflow(builder, a, zero, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_ADDMEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let r = translator::emit_addme_subfme::<SYSTEM>(builder, t.ctx_ptr, instr, AddmeSubfmeKind::Addme);++ if instr.oe() {+ let neg_one = builder.ins().iconst(types::I32, -1i64);+ let ov = translator::compute_add_overflow(builder, a, neg_one, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }++ lut::OP_SUBFX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_subf_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_sub_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_SUBFCX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_subfc_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_sub_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_SUBFEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_subfe_xform::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov = translator::compute_sub_overflow(builder, a, b, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_SUBFZEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let r = translator::emit_addze_subfze::<SYSTEM>(builder, t.ctx_ptr, instr, AddzeSubfzeKind::Subfze);++ if instr.oe() {+ let not_a = builder.ins().bxor_imm(a, !0i64);+ let zero = builder.ins().iconst(types::I32, 0);+ let ov = translator::compute_add_overflow(builder, not_a, zero, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_SUBFMEX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let r = translator::emit_addme_subfme::<SYSTEM>(builder, t.ctx_ptr, instr, AddmeSubfmeKind::Subfme);++ if instr.oe() {+ let not_a = builder.ins().bxor_imm(a, !0i64);+ let neg_one = builder.ins().iconst(types::I32, -1i64);+ let ov = translator::compute_add_overflow(builder, not_a, neg_one, r);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }++ lut::OP_NEGX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let r = translator::emit_neg::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov_b = builder.ins().icmp_imm(IntCC::Equal, a, 0x8000_0000u32 as i64);+ let ov = builder.ins().uextend(types::I32, ov_b);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }++ lut::OP_MULLWX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_mullw::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let a64 = builder.ins().sextend(types::I64, a);+ let b64 = builder.ins().sextend(types::I64, b);+ let full = builder.ins().imul(a64, b64);+ let r_sext = builder.ins().sextend(types::I64, r);+ let neq = builder.ins().icmp(IntCC::NotEqual, full, r_sext);+ let ov = builder.ins().uextend(types::I32, neq);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_MULHWX => {+ let r = translator::emit_mulhw::<SYSTEM>(builder, t.ctx_ptr, instr, true);++ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_MULHWUX => {+ let r = translator::emit_mulhw::<SYSTEM>(builder, t.ctx_ptr, instr, false);++ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }++ lut::OP_DIVWUX => {+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_divwu::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let ov_b = builder.ins().icmp_imm(IntCC::Equal, b, 0);+ let ov = builder.ins().uextend(types::I32, ov_b);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }+ lut::OP_DIVWX => {+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let r = translator::emit_divw::<SYSTEM>(builder, t.ctx_ptr, instr);++ if instr.oe() {+ let b_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);+ let a_min = builder.ins().icmp_imm(IntCC::Equal, a, 0x8000_0000u32 as i64);+ let b_neg1 = builder.ins().icmp_imm(IntCC::Equal, b, -1i64);+ let signed_min_neg1 = builder.ins().band(a_min, b_neg1);+ let any = builder.ins().bor(b_zero, signed_min_neg1);+ let ov = builder.ins().uextend(types::I32, any);+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);+ }+ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }+ }++ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn rotate<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let r = match OP {+ lut::OP_RLWIMIX => translator::emit_rlwimi::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_RLWINMX => translator::emit_rlwinm::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_RLWNMX => translator::emit_rlwnm::<SYSTEM>(builder, t.ctx_ptr, instr),+ _ => return,+ };++ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn logical<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let r = match OP {+ lut::OP_ANDX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::And),+ lut::OP_ORX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::Or),+ lut::OP_XORX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::Xor),++ lut::OP_NORX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Nor),+ lut::OP_NANDX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Nand),+ lut::OP_ANDCX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Andc),+ lut::OP_ORCX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Orc),+ lut::OP_EQVX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Eqv),++ lut::OP_SLWX => translator::emit_shift_xform::<SYSTEM>(builder, t.ctx_ptr, instr, ShiftKind::Slw),+ lut::OP_SRWX => translator::emit_shift_xform::<SYSTEM>(builder, t.ctx_ptr, instr, ShiftKind::Srw),+ lut::OP_SRAWX => translator::emit_sraw::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_SRAWIX => translator::emit_srawi::<SYSTEM>(builder, t.ctx_ptr, instr),++ lut::OP_CNTLZWX => translator::emit_cntlzw::<SYSTEM>(builder, t.ctx_ptr, instr),+ lut::OP_EXTSBX => translator::emit_extend::<SYSTEM>(builder, t.ctx_ptr, instr, types::I8),+ lut::OP_EXTSHX => translator::emit_extend::<SYSTEM>(builder, t.ctx_ptr, instr, types::I16),++ _ => return,+ };++ if instr.rc() {+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn compare<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_CMPI => translator::emit_cmp_imm::<SYSTEM>(builder, t.ctx_ptr, instr, true),+ lut::OP_CMPLI => translator::emit_cmp_imm::<SYSTEM>(builder, t.ctx_ptr, instr, false),+ lut::OP_CMP => translator::emit_cmp_xform::<SYSTEM>(builder, t.ctx_ptr, instr, true),+ lut::OP_CMPL => translator::emit_cmp_xform::<SYSTEM>(builder, t.ctx_ptr, instr, false),+ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn cr_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_MFCR => {+ let cr = translator::cr_load::<SYSTEM>(builder, t.ctx_ptr);+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), cr);+ }+ lut::OP_MTCRF => {+ let s = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());++ let mut mask: u32 = 0;+ for i in 0..8 {+ if (instr.crm() >> (7 - i)) & 1 != 0 {+ mask |= 0xF << ((7 - i) * 4);+ }+ }++ let cr = translator::cr_load::<SYSTEM>(builder, t.ctx_ptr);+ let kept = builder.ins().band_imm(cr, !mask as i64 as u32 as i64);+ let new_bits = builder.ins().band_imm(s, mask as i64);+ let merged = builder.ins().bor(kept, new_bits);+ translator::cr_store::<SYSTEM>(builder, t.ctx_ptr, merged);+ }++ lut::OP_MCRF => translator::emit_mcrf::<SYSTEM>(builder, t.ctx_ptr, instr),++ lut::OP_CRNOR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 33),+ lut::OP_CRANDC => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 129),+ lut::OP_CRXOR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 193),+ lut::OP_CRNAND => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 225),+ lut::OP_CRAND => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 257),+ lut::OP_CREQV => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 289),+ lut::OP_CRORC => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 417),+ lut::OP_CROR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 449),++ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn segment<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_MFSR => {+ let off = (abi::sr_base_offset::<SYSTEM>() + instr.sr() as usize * 4) as i32;+ let val = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, off);+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);+ }+ lut::OP_MTSR => {+ let off = (abi::sr_base_offset::<SYSTEM>() + instr.sr() as usize * 4) as i32;+ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());+ builder.ins().store(MemFlags::trusted(), val, t.ctx_ptr, off);+ }+ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_MFMSR => {+ let msr = builder.ins().load(+ types::I32,+ MemFlags::trusted(),+ t.ctx_ptr,+ abi::msr_offset::<SYSTEM>() as i32,+ );+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), msr);++ t.handled_natively = true;+ }+ lut::OP_MTMSR if t.is_terminator => {+ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());+ builder+ .ins()+ .store(MemFlags::trusted(), val, t.ctx_ptr, abi::msr_offset::<SYSTEM>() as i32);++ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));+ t.handled_natively = true;+ }+ lut::OP_RFI if t.is_terminator => {+ const RFI_MSR_MASK: i64 = 0x87C0_FFFF;++ let msr_off = abi::msr_offset::<SYSTEM>() as i32;+ let srr0_off = abi::spr_field_offset::<SYSTEM>(26).unwrap() as i32;+ let srr1_off = abi::spr_field_offset::<SYSTEM>(27).unwrap() as i32;+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;++ let msr_v = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, msr_off);+ let srr1 = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, srr1_off);+ let msr_keep = builder.ins().band_imm(msr_v, !RFI_MSR_MASK & 0xFFFF_FFFFi64);+ let srr1_take = builder.ins().band_imm(srr1, RFI_MSR_MASK);+ let new_msr = builder.ins().bor(msr_keep, srr1_take);+ let new_msr_clr = builder.ins().band_imm(new_msr, !0x0004_0000i64 & 0xFFFF_FFFFi64);+ builder+ .ins()+ .store(MemFlags::trusted(), new_msr_clr, t.ctx_ptr, msr_off);++ let srr0_raw = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, srr0_off);+ let nia = builder.ins().band_imm(srr0_raw, 0xFFFF_FFFCi64);+ builder.ins().store(MemFlags::trusted(), nia, t.ctx_ptr, nia_off);++ t.last_terminator_nia = Some(nia);+ t.handled_natively = true;+ }+ _ => {}+ }+}++#[inline(always)]+pub fn spr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_MFSPR => {+ if translator::emit_mfspr::<SYSTEM>(builder, t.ctx_ptr, instr, local) {+ t.handled_natively = true;+ }+ }+ lut::OP_MTSPR => {+ if translator::emit_mtspr::<SYSTEM>(builder, t.ctx_ptr, instr, local) {+ if t.is_terminator {+ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));+ }+ t.handled_natively = true;+ }+ }+ _ => {}+ }+}++#[inline(always)]+pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_LWZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false),+ lut::OP_LWZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, true),+ lut::OP_LBZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, false),+ lut::OP_LBZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, true),+ lut::OP_LHZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, false),+ lut::OP_LHZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, true),+ lut::OP_LHA => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),+ lut::OP_LHAU => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),++ lut::OP_STW => {+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)+ }+ lut::OP_STWU => {+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)+ }+ lut::OP_STB => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false),+ lut::OP_STBU => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true),+ lut::OP_STH => {+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)+ }+ lut::OP_STHU => {+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)+ }++ lut::OP_LMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, false),+ lut::OP_STMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, true),++ lut::OP_LWZX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false)+ }+ lut::OP_LWZUX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, true)+ }+ lut::OP_LBZX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, false)+ }+ lut::OP_LBZUX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, true)+ }+ lut::OP_LHZX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, false)+ }+ lut::OP_LHZUX => {+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, true)+ }+ lut::OP_LHAX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),+ lut::OP_LHAUX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),++ lut::OP_STWX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)+ }+ lut::OP_STWUX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)+ }+ lut::OP_STBX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false)+ }+ lut::OP_STBUX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true)+ }+ lut::OP_STHX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)+ }+ lut::OP_STHUX => {+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)+ }++ lut::OP_LWBRX => {+ translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32)+ }+ lut::OP_LHBRX => {+ translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16)+ }+ lut::OP_STWBRX => {+ translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32)+ }+ lut::OP_STHBRX => {+ translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16)+ }++ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ match OP {+ lut::OP_LFS => translator::emit_lfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32, local.read_u32),+ lut::OP_LFSU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),+ lut::OP_LFD => translator::emit_lfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),+ lut::OP_LFDU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),++ lut::OP_STFS => translator::emit_stfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.write_u32),+ lut::OP_STFSU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),+ lut::OP_STFD => translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),+ lut::OP_STFDU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),++ lut::OP_LFSX => translator::emit_lfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32),+ lut::OP_LFSUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),+ lut::OP_LFDX => translator::emit_lfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),+ lut::OP_LFDUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),++ lut::OP_STFSX => translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32),+ lut::OP_STFSUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),+ lut::OP_STFDX => translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),+ lut::OP_STFDUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),++ lut::OP_STFIWX => translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32),++ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn store_load_psq<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ match OP {+ lut::OP_PSQ_L => translator::emit_psq_l::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, false),+ lut::OP_PSQ_LU => translator::emit_psq_l::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, true),+ lut::OP_PSQ_ST => {+ translator::emit_psq_st::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, false)+ }+ lut::OP_PSQ_STU => {+ translator::emit_psq_st::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, true)+ }++ lut::OP_PSQ_LX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, false, false),+ lut::OP_PSQ_LUX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, false, true),+ lut::OP_PSQ_STX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, true, false),+ lut::OP_PSQ_STUX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, true, true),++ _ => return,+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let _ = (instr, SYSTEM);+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_ISYNC if t.is_terminator => {+ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));+ t.handled_natively = true;+ }+ lut::OP_SYNC+ | lut::OP_EIEIO+ | lut::OP_DCBF+ | lut::OP_DCBI+ | lut::OP_DCBST+ | lut::OP_DCBT+ | lut::OP_DCBTST+ | lut::OP_DCBA+ | lut::OP_DCBZ+ | lut::OP_DCBZ_L+ | lut::OP_ICBI+ | lut::OP_TLBIE+ | lut::OP_TLBIA+ | lut::OP_TLBSYNC+ | lut::OP_TLBLD+ | lut::OP_TLBLI => {+ t.handled_natively = true;+ }+ _ => {}+ }+}++#[inline(always)]+pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ if !t.is_terminator {+ return;+ }++ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ match OP {+ lut::OP_BX => {+ if let Some(slot_addr) = t.chain_slot_addr {+ let target = if instr.aa() {+ instr.li() as u32+ } else {+ t.pc.wrapping_add_signed(instr.li())+ };++ if instr.lk() {+ let lr_val = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);+ translator::lr_store::<SYSTEM>(builder, t.ctx_ptr, lr_val);+ }++ translator::emit_cycles_add::<SYSTEM>(builder, t.ctx_ptr, CYCLES_PER_INSTR);++ let target_v = builder.ins().iconst(types::I32, target as i64);+ if instr.lk() {+ translator::emit_call_or_exit::<SYSTEM>(+ builder,+ t.ctx_ptr,+ slot_addr,+ t.block_sig_ref,+ target_v,+ t.exit_block,+ );+ } else {+ translator::emit_chain_or_exit::<SYSTEM>(+ builder,+ t.ctx_ptr,+ slot_addr,+ t.block_sig_ref,+ target_v,+ t.exit_block,+ );+ }++ t.chained = true;+ t.handled_natively = true;+ } else {+ t.last_terminator_nia = Some(translator::emit_b::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc));+ t.handled_natively = true;+ }+ }++ lut::OP_BCX => {+ match translator::emit_bc_with_chain::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ t.pc,+ t.exit_block,+ t.chain_slot_addr,+ t.chain_fall_addr,+ t.block_sig_ref,+ ) {+ TermEmit::HandledNia(nia) => {+ t.last_terminator_nia = Some(nia);+ t.handled_natively = true;+ }+ TermEmit::HandledChained => {+ t.chained = true;+ t.handled_natively = true;+ }+ TermEmit::NotHandled => {}+ }+ }++ lut::OP_BCLRX => {+ match translator::emit_bclr_dispatch::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ t.pc,+ t.exit_block,+ t.block_sig_ref,+ t.block_lookup_table_addr,+ ) {+ TermEmit::HandledNia(nia) => {+ t.last_terminator_nia = Some(nia);+ t.handled_natively = true;+ }+ TermEmit::HandledChained => {+ t.chained = true;+ t.handled_natively = true;+ }+ TermEmit::NotHandled => {}+ }+ }++ lut::OP_BCCTRX => {+ match translator::emit_bcctr_dispatch::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ t.pc,+ t.exit_block,+ t.block_sig_ref,+ t.block_lookup_table_addr,+ ) {+ TermEmit::HandledNia(nia) => {+ t.last_terminator_nia = Some(nia);+ t.handled_natively = true;+ }+ TermEmit::HandledChained => {+ t.chained = true;+ t.handled_natively = true;+ }+ TermEmit::NotHandled => {}+ }+ }++ _ => {}+ }+}++#[inline(always)]+pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ if translator::emit_fp_arith::<OP, SYSTEM>(builder, t.ctx_ptr, instr) {+ if instr.rc() {+ translator::emit_update_cr1_from_fpscr::<SYSTEM>(builder, t.ctx_ptr);+ }+ t.handled_natively = true;+ }+}++#[inline(always)]+pub fn ps_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ if translator::emit_ps_arith::<OP, SYSTEM>(builder, t.ctx_ptr, instr) {+ if instr.rc() {+ translator::emit_update_cr1_from_fpscr::<SYSTEM>(builder, t.ctx_ptr);+ }+ t.handled_natively = true;+ }+}++#[inline(always)]+pub fn sc<const SYSTEM: SystemId>(t: &mut JitTranslator, _instr: Instruction) {+ if !t.is_terminator {+ return;+ }++ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let cia_off = abi::cia_offset::<SYSTEM>() as i32;+ let pc_const = builder.ins().iconst(types::I32, t.pc as i64);+ builder.ins().store(MemFlags::trusted(), pc_const, t.ctx_ptr, cia_off);++ let srr0_value = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);+ t.last_terminator_nia = Some(translator::emit_exception_dispatch::<SYSTEM>(+ builder,+ t.ctx_ptr,+ srr0_value,+ 0,+ 0x0000_0C00,+ ));++ t.handled_natively = true;+}++#[inline(always)]+pub fn mcrxr<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_mcrxr::<SYSTEM>(builder, t.ctx_ptr, instr);++ t.handled_natively = true;+}++#[inline(always)]+pub fn lwarx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_lwarx::<SYSTEM>(builder, t.ctx_ptr, instr, local);++ t.handled_natively = true;+}++#[inline(always)]+pub fn stwcx_dot<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_stwcx_dot::<SYSTEM>(builder, t.ctx_ptr, instr, local);++ t.handled_natively = true;+}++#[inline(always)]+pub fn lswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };+ let ea = if instr.ra() == 0 {+ builder.ins().iconst(types::I32, 0)+ } else {+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra())+ };++ let rd_v = builder.ins().iconst(types::I32, instr.rd() as i64);+ let n_v = builder.ins().iconst(types::I32, n as i64);+ builder.ins().call(local.do_lswi, &[t.ctx_ptr, ea, rd_v, n_v]);++ t.handled_natively = true;+}++#[inline(always)]+pub fn stswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };+ let ea = if instr.ra() == 0 {+ builder.ins().iconst(types::I32, 0)+ } else {+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra())+ };++ let rs_v = builder.ins().iconst(types::I32, instr.rs() as i64);+ let n_v = builder.ins().iconst(types::I32, n as i64);+ builder.ins().call(local.do_stswi, &[t.ctx_ptr, ea, rs_v, n_v]);++ t.handled_natively = true;+}++#[inline(always)]+pub fn lswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);+ let rd_v = builder.ins().iconst(types::I32, instr.rd() as i64);+ builder.ins().call(local.do_lswx, &[t.ctx_ptr, ea, rd_v]);++ t.handled_natively = true;+}++#[inline(always)]+pub fn stswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);+ let rs_v = builder.ins().iconst(types::I32, instr.rs() as i64);+ builder.ins().call(local.do_stswx, &[t.ctx_ptr, ea, rs_v]);++ t.handled_natively = true;+}++#[inline(always)]+pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false);++ t.handled_natively = true;+}++#[inline(always)]+pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false);++ t.handled_natively = true;+}++#[inline(always)]+pub fn mfsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let idx = builder.ins().ushr_imm(b, 28);+ let idx_masked = builder.ins().band_imm(idx, 0xF);+ let byte_off = builder.ins().ishl_imm(idx_masked, 2);+ let byte_off64 = builder.ins().uextend(types::I64, byte_off);++ let base = builder+ .ins()+ .iadd_imm(t.ctx_ptr, abi::sr_base_offset::<SYSTEM>() as i64);+ let addr = builder.ins().iadd(base, byte_off64);++ let val = builder.ins().load(types::I32, MemFlags::trusted(), addr, 0);+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);++ t.handled_natively = true;+}++#[inline(always)]+pub fn mtsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());+ let idx = builder.ins().ushr_imm(b, 28);+ let idx_masked = builder.ins().band_imm(idx, 0xF);+ let byte_off = builder.ins().ishl_imm(idx_masked, 2);+ let byte_off64 = builder.ins().uextend(types::I64, byte_off);++ let base = builder+ .ins()+ .iadd_imm(t.ctx_ptr, abi::sr_base_offset::<SYSTEM>() as i64);+ let addr = builder.ins().iadd(base, byte_off64);++ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());+ builder.ins().store(MemFlags::trusted(), val, addr, 0);++ t.handled_natively = true;+}++#[inline(always)]+pub fn mftb<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let tbr = instr.spr_swapped() as u16;++ if tbr == 268 || tbr == 269 {+ let cycles = builder.ins().load(+ types::I64,+ MemFlags::trusted(),+ t.ctx_ptr,+ abi::cycles_offset::<SYSTEM>() as i32,+ );+ let cycles_div = builder.ins().udiv_imm(cycles, 12);+ let tb_off = builder.ins().load(+ types::I64,+ MemFlags::trusted(),+ t.ctx_ptr,+ abi::timebase_offset_offset::<SYSTEM>() as i32,+ );+ let tb = builder.ins().iadd(cycles_div, tb_off);++ let val = if tbr == 269 {+ let shifted = builder.ins().ushr_imm(tb, 32);+ builder.ins().ireduce(types::I32, shifted)+ } else {+ builder.ins().ireduce(types::I32, tb)+ };++ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);+ } else {+ let tbr_v = builder.ins().iconst(types::I32, tbr as i64);+ let call = builder.ins().call(local.read_timebase, &[t.ctx_ptr, tbr_v]);+ let val = builder.inst_results(call)[0];+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);+ }++ t.handled_natively = true;+}++#[inline(always)]+pub fn tw<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_tw::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ t.handled_natively = true;+}++#[inline(always)]+pub fn twi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ translator::emit_twi::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);++ t.handled_natively = true;+}++#[cold]+#[inline(never)]+pub fn invalid(_t: &mut JitTranslator, _instr: Instruction) {}diff --git a/crates/gecko/src/gekko/jit/idle.rs b/crates/gecko/src/gekko/jit/idle.rsnew file mode 100644index 0000000..b370509--- /dev/null+++ b/crates/gecko/src/gekko/jit/idle.rs@@ -0,0 +1,158 @@+use crate::gekko::jit::block::{BlockSpec, TermKind};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum IdleClass {+ None,+ BranchToSelf,+ PollingLoop,+}++pub fn classify<const SYSTEM: crate::system::SystemId>(spec: &BlockSpec, gprs: &[u32; 32]) -> IdleClass {+ let _ = gprs;++ if let Some(class) = classify_branch_to_self(spec) {+ return class;+ }++ if classify_polling_loop(spec) {+ return IdleClass::PollingLoop;+ }++ IdleClass::None+}++fn classify_branch_to_self(spec: &BlockSpec) -> Option<IdleClass> {+ if spec.terminator != TermKind::Branch || spec.instrs.len() != 1 {+ return None;+ }++ let instr = crate::gekko::instruction::Instruction(spec.instrs[0]);+ if instr.lk() {+ return None;+ }++ let li = instr.li();+ let target = if instr.aa() {+ li as u32+ } else {+ spec.start_pc.wrapping_add_signed(li)+ };++ if target == spec.start_pc {+ Some(IdleClass::BranchToSelf)+ } else {+ None+ }+}++fn classify_polling_loop(spec: &BlockSpec) -> bool {+ const MAX_IDLE_BODY: usize = 6;++ if spec.terminator != TermKind::BranchCond {+ return false;+ }++ let last_idx = match spec.instrs.len().checked_sub(1) {+ Some(i) if i > 0 => i,+ _ => return false,+ };++ if last_idx > MAX_IDLE_BODY {+ return false;+ }++ let term = crate::gekko::instruction::Instruction(spec.instrs[last_idx]);+ let term_pc = spec.start_pc.wrapping_add((last_idx as u32) * 4);+ if !is_idle_loop_terminator(term, term_pc, spec.start_pc) {+ return false;+ }++ validate_idle_loop(&spec.instrs[..last_idx])+}++fn is_idle_loop_terminator(instr: crate::gekko::instruction::Instruction, branch_pc: u32, block_start_pc: u32) -> bool {+ if instr.primary_opcode() != 16 {+ return false;+ }++ if instr.lk() {+ return false;+ }++ if instr.bo() & 0b00100 == 0 {+ return false;+ }++ let target = if instr.aa() {+ instr.bd() as u32+ } else {+ branch_pc.wrapping_add_signed(instr.bd())+ };++ target == block_start_pc+}++fn validate_idle_loop(body: &[u32]) -> bool {+ let mut write_disallowed: u32 = 0;+ let mut written: u32 = 0;++ for &raw in body {+ let (reads, writes) = match gpr_dataflow(crate::gekko::instruction::Instruction(raw)) {+ Some(p) => p,+ None => return false,+ };++ let externals = reads & !written;+ write_disallowed |= externals;+ if writes & write_disallowed != 0 {+ return false;+ }++ written |= writes;+ }++ true+}++fn gpr_dataflow(instr: crate::gekko::instruction::Instruction) -> Option<(u32, u32)> {+ let rd_or_s = instr.rd() as u32;+ let ra = instr.ra() as u32;+ let rb = instr.rb() as u32;+ let bit = |r: u32| 1u32 << r;+ let read_a_or_zero = if ra == 0 { 0 } else { bit(ra) };++ Some(match instr.primary_opcode() {+ 14 | 15 => (read_a_or_zero, bit(rd_or_s)),+ 7 | 8 | 12 | 13 => (bit(ra), bit(rd_or_s)),+ 10 | 11 => (bit(ra), 0),+ 24 | 25 | 26 | 27 | 28 | 29 => (bit(rd_or_s), bit(ra)),+ 20 => (bit(rd_or_s) | bit(ra), bit(ra)),+ 21 => (bit(rd_or_s), bit(ra)),+ 23 => (bit(rd_or_s) | bit(rb), bit(ra)),+ 32 | 34 | 40 | 42 => (read_a_or_zero, bit(rd_or_s)),+ 33 | 35 | 41 | 43 => (bit(ra), bit(rd_or_s) | bit(ra)),+ 31 => return xform_dataflow(instr),+ _ => return None,+ })+}++fn xform_dataflow(instr: crate::gekko::instruction::Instruction) -> Option<(u32, u32)> {+ let rd_or_s = instr.rd() as u32;+ let ra = instr.ra() as u32;+ let rb = instr.rb() as u32;+ let bit = |r: u32| 1u32 << r;+ let read_a_or_zero = if ra == 0 { 0 } else { bit(ra) };++ Some(match instr.xo10() {+ 266 | 40 | 10 | 138 | 202 | 234 | 8 | 136 | 200 | 232 | 104 | 235 | 75 | 11 | 491 | 459 | 778 | 552 | 522+ | 650 | 714 | 746 | 520 | 648 | 712 | 744 | 616 | 747 | 1003 | 971 => (bit(ra) | bit(rb), bit(rd_or_s)),+ 28 | 60 | 124 | 284 | 316 | 412 | 444 | 476 => (bit(rd_or_s) | bit(rb), bit(ra)),+ 26 | 922 | 954 => (bit(rd_or_s), bit(ra)),+ 24 | 536 | 792 => (bit(rd_or_s) | bit(rb), bit(ra)),+ 824 => (bit(rd_or_s), bit(ra)),+ 0 | 32 => (bit(ra) | bit(rb), 0),+ 23 | 87 | 279 | 343 | 534 | 790 => (read_a_or_zero | bit(rb), bit(rd_or_s)),+ 55 | 119 | 311 | 375 => (bit(ra) | bit(rb), bit(rd_or_s) | bit(ra)),+ _ => return None,+ })+}diff --git a/crates/gecko/src/gekko/jit/insn.rs b/crates/gecko/src/gekko/jit/insn.rsnew file mode 100644index 0000000..125a5d6--- /dev/null+++ b/crates/gecko/src/gekko/jit/insn.rs@@ -0,0 +1,24 @@+use crate::gekko::instruction::Instruction;++#[allow(dead_code)]+impl Instruction {+ #[inline(always)]+ pub fn primary_opcode(&self) -> u8 {+ (self.0 >> 26) as u8+ }++ #[inline(always)]+ pub fn xo10(&self) -> u32 {+ (self.0 >> 1) & 0x3FF+ }++ #[inline(always)]+ pub fn xo5(&self) -> u32 {+ (self.0 >> 1) & 0x1F+ }++ #[inline(always)]+ pub fn xo6(&self) -> u32 {+ (self.0 >> 1) & 0x3F+ }+}diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rsnew file mode 100644index 0000000..636026a--- /dev/null+++ b/crates/gecko/src/gekko/jit/runtime.rs@@ -0,0 +1,538 @@+use crate::gekko::msr::Msr;+use crate::gekko::sr::Sr;+use crate::system::{GC, System, SystemId, WII};++#[cold]+#[inline(never)]+fn cause_invalid_opcode<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, raw_instr: u32, pc: u32) -> u32 {+ let sys = unsafe { &mut *sys };+ panic!(+ "unimplemented Gekko opcode {:#010x} at pc={:#010x} lr={:#010x}",+ raw_instr, pc, sys.gekko.spr.lr,+ );+}++pub extern "C" fn cause_invalid_opcode_gc(sys: *mut core::ffi::c_void, raw_instr: u32, pc: u32) -> u32 {+ cause_invalid_opcode::<GC>(sys.cast(), raw_instr, pc)+}++pub extern "C" fn cause_invalid_opcode_wii(sys: *mut core::ffi::c_void, raw_instr: u32, pc: u32) -> u32 {+ cause_invalid_opcode::<WII>(sys.cast(), raw_instr, pc)+}++#[inline(always)]+fn advance_to_deadline<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ let sys = unsafe { &mut *sys };+ let deadline = sys.scheduler.next_deadline();+ if sys.scheduler.cycles < deadline {+ sys.scheduler.cycles = deadline;+ }+}++pub extern "C" fn advance_to_deadline_gc(sys: *mut core::ffi::c_void) {+ advance_to_deadline::<GC>(sys.cast());+}++pub extern "C" fn advance_to_deadline_wii(sys: *mut core::ffi::c_void) {+ advance_to_deadline::<WII>(sys.cast());+}++#[inline(always)]+fn slow_read_u8<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u8 {+ unsafe { (*sys).read_u8(ea) }+}+#[inline(always)]+fn slow_read_u16<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u16 {+ unsafe { (*sys).read_u16(ea) }+}+#[inline(always)]+fn slow_read_u32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u32 {+ unsafe { (*sys).read_u32(ea) }+}+#[inline(always)]+fn slow_write_u8<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u8) {+ unsafe { (*sys).write_u8(ea, v) }+}+#[inline(always)]+fn slow_write_u16<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u16) {+ unsafe { (*sys).write_u16(ea, v) }+}+#[inline(always)]+fn slow_write_u32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u32) {+ unsafe { (*sys).write_u32(ea, v) }+}++pub extern "C" fn read_u8_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u8::<GC>(sys.cast(), ea) as u32+}+pub extern "C" fn read_u16_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u16::<GC>(sys.cast(), ea) as u32+}+pub extern "C" fn read_u32_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u32::<GC>(sys.cast(), ea)+}+pub extern "C" fn write_u8_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u8::<GC>(sys.cast(), ea, v as u8);+}+pub extern "C" fn write_u16_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u16::<GC>(sys.cast(), ea, v as u16);+}+pub extern "C" fn write_u32_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u32::<GC>(sys.cast(), ea, v);+}++pub extern "C" fn read_u8_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u8::<WII>(sys.cast(), ea) as u32+}+pub extern "C" fn read_u16_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u16::<WII>(sys.cast(), ea) as u32+}+pub extern "C" fn read_u32_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {+ slow_read_u32::<WII>(sys.cast(), ea)+}+pub extern "C" fn write_u8_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u8::<WII>(sys.cast(), ea, v as u8);+}+pub extern "C" fn write_u16_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u16::<WII>(sys.cast(), ea, v as u16);+}+pub extern "C" fn write_u32_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {+ slow_write_u32::<WII>(sys.cast(), ea, v);+}++#[inline(always)]+fn slow_read_f32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> f64 {+ unsafe { (*sys).read_f32(ea) }+}+#[inline(always)]+fn slow_write_f32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: f64) {+ unsafe { (*sys).write_f32(ea, v) }+}+#[inline(always)]+fn slow_read_f64<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> f64 {+ unsafe { (*sys).read_f64(ea) }+}+#[inline(always)]+fn slow_write_f64<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: f64) {+ unsafe { (*sys).write_f64(ea, v) }+}++pub extern "C" fn read_f32_gc(sys: *mut core::ffi::c_void, ea: u32) -> f64 {+ slow_read_f32::<GC>(sys.cast(), ea)+}+pub extern "C" fn write_f32_gc(sys: *mut core::ffi::c_void, ea: u32, v: f64) {+ slow_write_f32::<GC>(sys.cast(), ea, v)+}+pub extern "C" fn read_f64_gc(sys: *mut core::ffi::c_void, ea: u32) -> f64 {+ slow_read_f64::<GC>(sys.cast(), ea)+}+pub extern "C" fn write_f64_gc(sys: *mut core::ffi::c_void, ea: u32, v: f64) {+ slow_write_f64::<GC>(sys.cast(), ea, v)+}+pub extern "C" fn read_f32_wii(sys: *mut core::ffi::c_void, ea: u32) -> f64 {+ slow_read_f32::<WII>(sys.cast(), ea)+}+pub extern "C" fn write_f32_wii(sys: *mut core::ffi::c_void, ea: u32, v: f64) {+ slow_write_f32::<WII>(sys.cast(), ea, v)+}+pub extern "C" fn read_f64_wii(sys: *mut core::ffi::c_void, ea: u32) -> f64 {+ slow_read_f64::<WII>(sys.cast(), ea)+}+pub extern "C" fn write_f64_wii(sys: *mut core::ffi::c_void, ea: u32, v: f64) {+ slow_write_f64::<WII>(sys.cast(), ea, v)+}++#[inline(always)]+fn write_msr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, val: u32) {+ unsafe { (*sys).gekko.msr = Msr::from(val) }+}++pub extern "C" fn write_msr_gc(sys: *mut core::ffi::c_void, val: u32) {+ write_msr::<GC>(sys.cast(), val);+}+pub extern "C" fn write_msr_wii(sys: *mut core::ffi::c_void, val: u32) {+ write_msr::<WII>(sys.cast(), val);+}++#[inline(always)]+fn read_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32) -> u32 {+ let sys = unsafe { &mut *sys };+ match num {+ 22 => {+ sys.gekko.spr.dec = sys.gekko.dec.read(sys.scheduler.cycles);+ sys.gekko.spr.dec+ }+ 268 => sys.scheduler.timebase_lower(),+ 269 => sys.scheduler.timebase_upper(),+ _ => sys.gekko.spr.read(num),+ }+}++pub extern "C" fn read_spr_gc(sys: *mut core::ffi::c_void, num: u32) -> u32 {+ read_spr::<GC>(sys.cast(), num)+}+pub extern "C" fn read_spr_wii(sys: *mut core::ffi::c_void, num: u32) -> u32 {+ read_spr::<WII>(sys.cast(), num)+}++#[inline(always)]+fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u32) {+ let sys = unsafe { &mut *sys };+ match num {+ 22 => {+ sys.scheduler.cancel(crate::gekko::dec::underflow_handler::<SYSTEM>);+ sys.gekko.dec.write(sys.scheduler.cycles, val);+ sys.gekko.spr.dec = val;+ sys.scheduler.schedule_in(+ crate::gekko::dec::cycles_until_underflow(val),+ crate::gekko::dec::underflow_handler::<SYSTEM>,+ );+ }+ 284 => sys.scheduler.set_timebase_lower(val),+ 285 => sys.scheduler.set_timebase_upper(val),+ 923 => {+ sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);+ if sys.gekko.spr.dmal.trigger() {+ sys.mmio+ .process_locked_cache_dma(&sys.gekko.spr.dmau, &sys.gekko.spr.dmal);+ }+ }+ _ => sys.gekko.spr.write(num, val),+ }+}++pub extern "C" fn write_spr_gc(sys: *mut core::ffi::c_void, num: u32, val: u32) {+ write_spr::<GC>(sys.cast(), num, val);+}+pub extern "C" fn write_spr_wii(sys: *mut core::ffi::c_void, num: u32, val: u32) {+ write_spr::<WII>(sys.cast(), num, val);+}++#[inline(always)]+fn read_sr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32) -> u32 {+ unsafe { (*sys).gekko.sr[(idx & 0xF) as usize].raw() }+}+pub extern "C" fn read_sr_gc(sys: *mut core::ffi::c_void, idx: u32) -> u32 {+ read_sr::<GC>(sys.cast(), idx)+}+pub extern "C" fn read_sr_wii(sys: *mut core::ffi::c_void, idx: u32) -> u32 {+ read_sr::<WII>(sys.cast(), idx)+}++#[inline(always)]+fn write_sr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32, val: u32) {+ unsafe { (*sys).gekko.sr[(idx & 0xF) as usize] = Sr::from_raw(val) }+}+pub extern "C" fn write_sr_gc(sys: *mut core::ffi::c_void, idx: u32, val: u32) {+ write_sr::<GC>(sys.cast(), idx, val);+}+pub extern "C" fn write_sr_wii(sys: *mut core::ffi::c_void, idx: u32, val: u32) {+ write_sr::<WII>(sys.cast(), idx, val);+}++#[inline(always)]+#[allow(dead_code)]+fn cause_trap_exception<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ unsafe { (*sys).cause_trap_exception() }+}+#[allow(dead_code)]+pub extern "C" fn cause_trap_exception_gc(sys: *mut core::ffi::c_void) {+ cause_trap_exception::<GC>(sys.cast());+}+#[allow(dead_code)]+pub extern "C" fn cause_trap_exception_wii(sys: *mut core::ffi::c_void) {+ cause_trap_exception::<WII>(sys.cast());+}++#[inline(always)]+#[allow(dead_code)]+fn cause_syscall_interrupt<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ unsafe { (*sys).cause_syscall_interrupt() }+}+#[allow(dead_code)]+pub extern "C" fn cause_syscall_interrupt_gc(sys: *mut core::ffi::c_void) {+ cause_syscall_interrupt::<GC>(sys.cast());+}+#[allow(dead_code)]+pub extern "C" fn cause_syscall_interrupt_wii(sys: *mut core::ffi::c_void) {+ cause_syscall_interrupt::<WII>(sys.cast());+}++#[inline(always)]+#[allow(dead_code)]+fn cause_fp_unavailable<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, pc: u32) {+ let sys = unsafe { &mut *sys };+ sys.gekko.cia = pc;+ sys.gekko.nia = pc.wrapping_add(4);+ sys.cause_fp_unavailable();+}+#[allow(dead_code)]+pub extern "C" fn cause_fp_unavailable_gc(sys: *mut core::ffi::c_void, pc: u32) {+ cause_fp_unavailable::<GC>(sys.cast(), pc);+}+#[allow(dead_code)]+pub extern "C" fn cause_fp_unavailable_wii(sys: *mut core::ffi::c_void, pc: u32) {+ cause_fp_unavailable::<WII>(sys.cast(), pc);+}++#[inline(always)]+#[allow(dead_code)]+fn set_reservation<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, addr: u32) {+ unsafe { (*sys).gekko.reserve_addr = addr }+}+#[allow(dead_code)]+pub extern "C" fn set_reservation_gc(sys: *mut core::ffi::c_void, addr: u32) {+ set_reservation::<GC>(sys.cast(), addr);+}+#[allow(dead_code)]+pub extern "C" fn set_reservation_wii(sys: *mut core::ffi::c_void, addr: u32) {+ set_reservation::<WII>(sys.cast(), addr);+}++#[inline(always)]+#[allow(dead_code)]+fn try_clear_reservation<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, addr: u32) -> u32 {+ let sys = unsafe { &mut *sys };+ if sys.gekko.reserve_addr == addr {+ sys.gekko.reserve_addr = crate::gekko::Gekko::NO_RESERVATION;+ 1+ } else {+ 0+ }+}+#[allow(dead_code)]+pub extern "C" fn try_clear_reservation_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ try_clear_reservation::<GC>(sys.cast(), addr)+}+#[allow(dead_code)]+pub extern "C" fn try_clear_reservation_wii(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ try_clear_reservation::<WII>(sys.cast(), addr)+}++#[inline(always)]+fn do_lsw_packed<const SYSTEM: SystemId, const STORE: bool>(+ sys: *mut System<SYSTEM>,+ ea: u32,+ rd_or_rs: u32,+ n_bytes: u32,+) {+ let sys = unsafe { &mut *sys };+ let mut n = n_bytes;+ let mut r = rd_or_rs.wrapping_sub(1) & 31;+ let mut i = 0u32;+ let mut addr = ea;+ while n > 0 {+ if i == 0 {+ r = (r + 1) & 31;+ if !STORE {+ sys.gekko.write_gpr(r as u8, 0);+ }+ }+ if STORE {+ let byte = (sys.gekko.read_gpr(r as u8) >> (24 - i)) as u8;+ sys.write_u8(addr, byte);+ } else {+ let byte = sys.read_u8(addr) as u32;+ let shift = 24 - i;+ let val = sys.gekko.read_gpr(r as u8) | (byte << shift);+ sys.gekko.write_gpr(r as u8, val);+ }+ i += 8;+ if i == 32 {+ i = 0;+ }+ addr = addr.wrapping_add(1);+ n -= 1;+ }+}++pub extern "C" fn do_lswi_gc(sys: *mut core::ffi::c_void, ea: u32, rd: u32, nb: u32) {+ do_lsw_packed::<GC, false>(sys.cast(), ea, rd, nb);+}+pub extern "C" fn do_lswi_wii(sys: *mut core::ffi::c_void, ea: u32, rd: u32, nb: u32) {+ do_lsw_packed::<WII, false>(sys.cast(), ea, rd, nb);+}+pub extern "C" fn do_stswi_gc(sys: *mut core::ffi::c_void, ea: u32, rs: u32, nb: u32) {+ do_lsw_packed::<GC, true>(sys.cast(), ea, rs, nb);+}+pub extern "C" fn do_stswi_wii(sys: *mut core::ffi::c_void, ea: u32, rs: u32, nb: u32) {+ do_lsw_packed::<WII, true>(sys.cast(), ea, rs, nb);+}++pub extern "C" fn do_lswx_gc(sys_v: *mut core::ffi::c_void, ea: u32, rd: u32) {+ let sys = unsafe { &mut *(sys_v as *mut System<GC>) };+ let n = sys.gekko.spr.xer.byte_count() as u32;+ if n == 0 {+ return;+ }+ do_lsw_packed::<GC, false>(sys_v.cast(), ea, rd, n);+}+pub extern "C" fn do_lswx_wii(sys_v: *mut core::ffi::c_void, ea: u32, rd: u32) {+ let sys = unsafe { &mut *(sys_v as *mut System<WII>) };+ let n = sys.gekko.spr.xer.byte_count() as u32;+ if n == 0 {+ return;+ }+ do_lsw_packed::<WII, false>(sys_v.cast(), ea, rd, n);+}+pub extern "C" fn do_stswx_gc(sys_v: *mut core::ffi::c_void, ea: u32, rs: u32) {+ let sys = unsafe { &mut *(sys_v as *mut System<GC>) };+ let n = sys.gekko.spr.xer.byte_count() as u32;+ do_lsw_packed::<GC, true>(sys_v.cast(), ea, rs, n);+}+pub extern "C" fn do_stswx_wii(sys_v: *mut core::ffi::c_void, ea: u32, rs: u32) {+ let sys = unsafe { &mut *(sys_v as *mut System<WII>) };+ let n = sys.gekko.spr.xer.byte_count() as u32;+ do_lsw_packed::<WII, true>(sys_v.cast(), ea, rs, n);+}++pub static DEQUANT_TABLE: [f32; 64] = {+ let mut t = [0.0f32; 64];+ let mut i = 0u32;+ while i < 32 {+ t[i as usize] = 1.0 / (1u64 << i) as f32;+ i += 1;+ }+ while i < 64 {+ t[i as usize] = (1u64 << (64 - i)) as f32;+ i += 1;+ }+ t+};+pub static QUANT_TABLE: [f32; 64] = {+ let mut t = [0.0f32; 64];+ let mut i = 0u32;+ while i < 32 {+ t[i as usize] = (1u64 << i) as f32;+ i += 1;+ }+ while i < 64 {+ t[i as usize] = 1.0 / (1u64 << (64 - i)) as f32;+ i += 1;+ }+ t+};++#[inline(always)]+fn quant_element_size(qtype: u8) -> u32 {+ match qtype {+ 0 => 4,+ 4 | 6 => 1,+ 5 | 7 => 2,+ _ => 4,+ }+}++#[inline(always)]+fn psq_dequant<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, addr: u32, ld_type: u8, ld_scale: u8) -> f64 {+ let scale = DEQUANT_TABLE[ld_scale as usize];+ match ld_type {+ 0 => sys.read_f32(addr),+ 4 => (sys.read_u8(addr) as f32 * scale) as f64,+ 5 => (sys.read_u16(addr) as f32 * scale) as f64,+ 6 => (sys.read_u8(addr) as i8 as f32 * scale) as f64,+ 7 => (sys.read_u16(addr) as i16 as f32 * scale) as f64,+ _ => 0.0,+ }+}++#[inline(always)]+fn psq_quant<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, addr: u32, value: f64, st_type: u8, st_scale: u8) {+ let scale = QUANT_TABLE[st_scale as usize];+ match st_type {+ 0 => sys.write_f32(addr, value),+ 4 => {+ let v = (value as f32 * scale).clamp(u8::MIN as f32, u8::MAX as f32) as u8;+ sys.write_u8(addr, v);+ }+ 5 => {+ let v = (value as f32 * scale).clamp(u16::MIN as f32, u16::MAX as f32) as u16;+ sys.write_u16(addr, v);+ }+ 6 => {+ let v = (value as f32 * scale).clamp(i8::MIN as f32, i8::MAX as f32) as i8;+ sys.write_u8(addr, v as u8);+ }+ 7 => {+ let v = (value as f32 * scale).clamp(i16::MIN as f32, i16::MAX as f32) as i16;+ sys.write_u16(addr, v as u16);+ }+ _ => {}+ }+}++#[inline(always)]+fn psq_load_inner<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, fd: u32, ea: u32, w: u32, gqr_idx: u32) {+ let sys = unsafe { &mut *sys };+ let gqr = sys.gekko.spr.read_gqr((gqr_idx & 0x7) as u8);+ let ld_type = ((gqr >> 16) & 0x7) as u8;+ let ld_scale = ((gqr >> 24) & 0x3F) as u8;+ let ps0 = psq_dequant(sys, ea, ld_type, ld_scale);+ let ps1 = if w != 0 {+ 1.0+ } else {+ let elem = quant_element_size(ld_type);+ psq_dequant(sys, ea.wrapping_add(elem), ld_type, ld_scale)+ };+ sys.gekko.write_fpr(fd as u8, ps0);+ sys.gekko.write_ps1(fd as u8, ps1);+}++#[inline(always)]+fn psq_store_inner<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, fs: u32, ea: u32, w: u32, gqr_idx: u32) {+ let sys = unsafe { &mut *sys };+ let gqr = sys.gekko.spr.read_gqr((gqr_idx & 0x7) as u8);+ let st_type = (gqr & 0x7) as u8;+ let st_scale = ((gqr >> 8) & 0x3F) as u8;+ let ps0 = sys.gekko.read_fpr(fs as u8);+ psq_quant(sys, ea, ps0, st_type, st_scale);+ if w == 0 {+ let ps1 = sys.gekko.read_ps1(fs as u8);+ let elem = quant_element_size(st_type);+ psq_quant(sys, ea.wrapping_add(elem), ps1, st_type, st_scale);+ }+}++pub extern "C" fn do_psq_load_gc(sys: *mut core::ffi::c_void, fd: u32, ea: u32, w: u32, gqr: u32) {+ psq_load_inner::<GC>(sys.cast(), fd, ea, w, gqr);+}+pub extern "C" fn do_psq_load_wii(sys: *mut core::ffi::c_void, fd: u32, ea: u32, w: u32, gqr: u32) {+ psq_load_inner::<WII>(sys.cast(), fd, ea, w, gqr);+}+pub extern "C" fn do_psq_store_gc(sys: *mut core::ffi::c_void, fs: u32, ea: u32, w: u32, gqr: u32) {+ psq_store_inner::<GC>(sys.cast(), fs, ea, w, gqr);+}+pub extern "C" fn do_psq_store_wii(sys: *mut core::ffi::c_void, fs: u32, ea: u32, w: u32, gqr: u32) {+ psq_store_inner::<WII>(sys.cast(), fs, ea, w, gqr);+}++#[inline(always)]+fn read_timebase<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, tbr: u32) -> u32 {+ let sys = unsafe { &*sys };+ match tbr {+ 268 => sys.scheduler.timebase_lower(),+ 269 => sys.scheduler.timebase_upper(),+ _ => 0,+ }+}+pub extern "C" fn read_timebase_gc(sys: *mut core::ffi::c_void, tbr: u32) -> u32 {+ read_timebase::<GC>(sys.cast(), tbr)+}+pub extern "C" fn read_timebase_wii(sys: *mut core::ffi::c_void, tbr: u32) -> u32 {+ read_timebase::<WII>(sys.cast(), tbr)+}++#[inline(always)]+fn do_rfi<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ let sys = unsafe { &mut *sys };+ const RFI_MSR_MASK: u32 = 0x87C0_FFFF;+ let new_msr = (sys.gekko.msr.raw() & !RFI_MSR_MASK) | (sys.gekko.spr.srr1 & RFI_MSR_MASK);+ sys.gekko.msr = Msr::from(new_msr & !0x0004_0000);+ sys.gekko.nia = sys.gekko.spr.srr0.value() << 2;+}+pub extern "C" fn do_rfi_gc(sys: *mut core::ffi::c_void) {+ do_rfi::<GC>(sys.cast());+}+pub extern "C" fn do_rfi_wii(sys: *mut core::ffi::c_void) {+ do_rfi::<WII>(sys.cast());+}diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rsnew file mode 100644index 0000000..4f70572--- /dev/null+++ b/crates/gecko/src/gekko/jit/translator.rs@@ -0,0 +1,4205 @@+use cranelift_codegen::Context;+use cranelift_codegen::ir::condcodes::IntCC;+use cranelift_codegen::ir::{Block, FuncRef, InstBuilder, MemFlags, Value, types};+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};+use cranelift_jit::JITModule;+use cranelift_module::{FuncId, Module};+use rustc_hash::FxHashMap;++use crate::gekko::instruction::Instruction;+use crate::gekko::jit::block::{BlockSpec, TermKind};+use crate::gekko::jit::idle::{self, IdleClass};+use crate::gekko::jit::{ExternFuncs, abi};+use crate::system::SystemId;++pub struct ChainContext<'a> {+ pub self_pc: u32,+ pub self_func_id: FuncId,+ pub self_slot_addr: i64,+ pub compiled_func_ids: &'a FxHashMap<u32, FuncId>,+ pub compiled_cache: &'a FxHashMap<u32, super::BlockEntry>,+ pub pending: std::cell::RefCell<&'a mut Vec<(u32, usize)>>,+ pub block_lookup_table_addr: i64,+}++impl<'a> ChainContext<'a> {+ fn target_for(&self, target_pc: u32) -> Option<i64> {+ if target_pc == self.self_pc {+ return Some(self.self_slot_addr);+ }++ if let Some(&entry) = self.compiled_cache.get(&target_pc) {+ let slot: &'static mut usize = Box::leak(Box::new(entry));+ return Some(slot as *mut usize as i64);+ }++ let slot: &'static mut usize = Box::leak(Box::new(0usize));+ let slot_addr_usize = slot as *mut usize as usize;+ self.pending.borrow_mut().push((target_pc, slot_addr_usize));+ Some(slot_addr_usize as i64)+ }+}++pub(crate) const CYCLES_PER_INSTR: i64 = 2;++#[allow(dead_code)]+pub(crate) struct LocalFuncs {+ pub(crate) cause_invalid_opcode: FuncRef,+ pub(crate) advance_to_deadline: FuncRef,+ pub(crate) read_u8: FuncRef,+ pub(crate) read_u16: FuncRef,+ pub(crate) read_u32: FuncRef,+ pub(crate) write_u8: FuncRef,+ pub(crate) write_u16: FuncRef,+ pub(crate) write_u32: FuncRef,+ pub(crate) read_f32: FuncRef,+ pub(crate) read_f64: FuncRef,+ pub(crate) write_f32: FuncRef,+ pub(crate) write_f64: FuncRef,+ pub(crate) write_msr: FuncRef,+ pub(crate) read_spr: FuncRef,+ pub(crate) write_spr: FuncRef,+ pub(crate) read_sr: FuncRef,+ pub(crate) write_sr: FuncRef,+ pub(crate) cause_trap_exception: FuncRef,+ pub(crate) cause_syscall_interrupt: FuncRef,+ pub(crate) do_rfi: FuncRef,+ pub(crate) cause_fp_unavailable: FuncRef,+ pub(crate) set_reservation: FuncRef,+ pub(crate) try_clear_reservation: FuncRef,+ pub(crate) do_lswi: FuncRef,+ pub(crate) do_stswi: FuncRef,+ pub(crate) do_lswx: FuncRef,+ pub(crate) do_stswx: FuncRef,+ pub(crate) do_psq_load: FuncRef,+ pub(crate) do_psq_store: FuncRef,+ pub(crate) read_timebase: FuncRef,+ pub(crate) fp_guard_emitted: std::cell::Cell<bool>,+}++pub struct JitTranslator {+ pub(crate) builder_ptr: usize,+ pub(crate) local_ptr: usize,+ pub ctx_ptr: Value,+ pub pc: u32,+ pub exit_block: cranelift_codegen::ir::Block,+ pub chain_slot_addr: Option<i64>,+ pub chain_fall_addr: Option<i64>,+ pub block_sig_ref: cranelift_codegen::ir::SigRef,+ pub block_lookup_table_addr: i64,+ pub is_terminator: bool,+ pub chained: bool,+ pub last_terminator_nia: Option<Value>,+ pub handled_natively: bool,+}++pub fn translate<const SYSTEM: SystemId>(+ ctx: &mut Context,+ builder_ctx: &mut FunctionBuilderContext,+ module: &mut JITModule,+ extern_funcs: &ExternFuncs,+ spec: &BlockSpec,+ gprs: &[u32; 32],+ chain: &ChainContext<'_>,+) {+ let local = LocalFuncs {+ cause_invalid_opcode: module.declare_func_in_func(extern_funcs.cause_invalid_opcode, &mut ctx.func),+ advance_to_deadline: module.declare_func_in_func(extern_funcs.advance_to_deadline, &mut ctx.func),+ read_u8: module.declare_func_in_func(extern_funcs.read_u8, &mut ctx.func),+ read_u16: module.declare_func_in_func(extern_funcs.read_u16, &mut ctx.func),+ read_u32: module.declare_func_in_func(extern_funcs.read_u32, &mut ctx.func),+ write_u8: module.declare_func_in_func(extern_funcs.write_u8, &mut ctx.func),+ write_u16: module.declare_func_in_func(extern_funcs.write_u16, &mut ctx.func),+ write_u32: module.declare_func_in_func(extern_funcs.write_u32, &mut ctx.func),+ read_f32: module.declare_func_in_func(extern_funcs.read_f32, &mut ctx.func),+ read_f64: module.declare_func_in_func(extern_funcs.read_f64, &mut ctx.func),+ write_f32: module.declare_func_in_func(extern_funcs.write_f32, &mut ctx.func),+ write_f64: module.declare_func_in_func(extern_funcs.write_f64, &mut ctx.func),+ write_msr: module.declare_func_in_func(extern_funcs.write_msr, &mut ctx.func),+ read_spr: module.declare_func_in_func(extern_funcs.read_spr, &mut ctx.func),+ write_spr: module.declare_func_in_func(extern_funcs.write_spr, &mut ctx.func),+ read_sr: module.declare_func_in_func(extern_funcs.read_sr, &mut ctx.func),+ write_sr: module.declare_func_in_func(extern_funcs.write_sr, &mut ctx.func),+ cause_trap_exception: module.declare_func_in_func(extern_funcs.cause_trap_exception, &mut ctx.func),+ cause_syscall_interrupt: module.declare_func_in_func(extern_funcs.cause_syscall_interrupt, &mut ctx.func),+ do_rfi: module.declare_func_in_func(extern_funcs.do_rfi, &mut ctx.func),+ cause_fp_unavailable: module.declare_func_in_func(extern_funcs.cause_fp_unavailable, &mut ctx.func),+ set_reservation: module.declare_func_in_func(extern_funcs.set_reservation, &mut ctx.func),+ try_clear_reservation: module.declare_func_in_func(extern_funcs.try_clear_reservation, &mut ctx.func),+ do_lswi: module.declare_func_in_func(extern_funcs.do_lswi, &mut ctx.func),+ do_stswi: module.declare_func_in_func(extern_funcs.do_stswi, &mut ctx.func),+ do_lswx: module.declare_func_in_func(extern_funcs.do_lswx, &mut ctx.func),+ do_stswx: module.declare_func_in_func(extern_funcs.do_stswx, &mut ctx.func),+ do_psq_load: module.declare_func_in_func(extern_funcs.do_psq_load, &mut ctx.func),+ do_psq_store: module.declare_func_in_func(extern_funcs.do_psq_store, &mut ctx.func),+ read_timebase: module.declare_func_in_func(extern_funcs.read_timebase, &mut ctx.func),+ fp_guard_emitted: std::cell::Cell::new(false),+ };++ let idle_class = idle::classify::<SYSTEM>(spec, gprs);++ let (chain_slot_addr, chain_fall_addr): (Option<i64>, Option<i64>) = if matches!(idle_class, IdleClass::None) {+ let taken = terminator_static_taken_pc(spec).and_then(|t| chain.target_for(t));+ let fall = terminator_fall_pc(spec).and_then(|t| chain.target_for(t));+ (taken, fall)+ } else {+ (None, None)+ };++ let sig_clone = ctx.func.signature.clone();+ let block_sig_ref = ctx.func.import_signature(sig_clone);++ let mut builder = FunctionBuilder::new(&mut ctx.func, builder_ctx);+ let entry = builder.create_block();+ builder.append_block_params_for_function_params(entry);+ builder.switch_to_block(entry);+ builder.seal_block(entry);++ let ctx_ptr: Value = builder.block_params(entry)[0];++ let exit_block = builder.create_block();+ builder.append_block_param(exit_block, types::I32);++ let mut last_terminator_nia: Option<Value> = None;+ let mut chained = false;++ let mut t = JitTranslator {+ builder_ptr: &mut builder as *mut FunctionBuilder<'_> as *mut () as usize,+ local_ptr: &local as *const LocalFuncs as usize,+ ctx_ptr,+ pc: 0,+ exit_block,+ chain_slot_addr,+ chain_fall_addr,+ block_sig_ref,+ block_lookup_table_addr: chain.block_lookup_table_addr,+ is_terminator: false,+ chained: false,+ last_terminator_nia: None,+ handled_natively: false,+ };++ for (i, &word) in spec.instrs.iter().enumerate() {+ let instr = Instruction(word);+ let pc = spec.pc_of(i);+ let is_terminator = i + 1 == spec.instrs.len() && !matches!(spec.terminator, TermKind::LengthCap);++ t.pc = pc;+ t.is_terminator = is_terminator;+ t.handled_natively = false;+ let chained_before = t.chained;++ super::dispatch::<SYSTEM>(&mut t, instr);++ if t.handled_natively {+ if !t.chained || chained_before {+ emit_cycles_add::<SYSTEM>(&mut builder, ctx_ptr, CYCLES_PER_INSTR);+ }++ if t.chained && !chained_before {+ chained = true;+ }++ if let Some(nia) = t.last_terminator_nia.take() {+ last_terminator_nia = Some(nia);+ }+ } else {+ let pc_const = builder.ins().iconst(types::I32, pc as i64);+ let raw_const = builder.ins().iconst(types::I32, instr.0 as i64);+ let call = builder+ .ins()+ .call(local.cause_invalid_opcode, &[ctx_ptr, raw_const, pc_const]);+ let nia = builder.inst_results(call)[0];++ emit_cycles_add::<SYSTEM>(&mut builder, ctx_ptr, CYCLES_PER_INSTR);++ if is_terminator {+ last_terminator_nia = Some(nia);+ } else {+ let expected = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ let is_seq = builder.ins().icmp(IntCC::Equal, nia, expected);+ let cont = builder.create_block();++ builder.ins().brif(is_seq, cont, &[], exit_block, &[nia.into()]);+ builder.switch_to_block(cont);+ builder.seal_block(cont);+ }+ }+ }+ drop(t);++ if !chained {+ if idle_class != IdleClass::None {+ builder.ins().call(local.advance_to_deadline, &[ctx_ptr]);+ }++ let final_nia = match spec.terminator {+ TermKind::LengthCap => builder.ins().iconst(types::I32, spec.end_pc() as i64),+ _ => last_terminator_nia.unwrap_or_else(|| {+ let nia_off = abi::nia_offset::<SYSTEM>() as i64;+ let nia_addr = builder.ins().iadd_imm(ctx_ptr, nia_off);+ builder.ins().load(types::I32, MemFlags::trusted(), nia_addr, 0)+ }),+ };+ builder.ins().jump(exit_block, &[final_nia.into()]);+ }++ builder.switch_to_block(exit_block);+ builder.seal_block(exit_block);++ let exit_nia = builder.block_params(exit_block)[0];+ builder.ins().return_(&[exit_nia]);++ builder.finalize();+}++pub(crate) fn emit_add_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().iadd(a, b);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_subf_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().isub(b, a);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_neg<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let r = builder.ins().ineg(a);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_mullw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().imul(a, b);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_divwu<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let zero = builder.ins().iconst(types::I32, 0);++ let div_block = builder.create_block();+ let zero_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let is_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);+ builder.ins().brif(is_zero, zero_block, &[], div_block, &[]);++ builder.switch_to_block(div_block);+ builder.seal_block(div_block);+ let q = builder.ins().udiv(a, b);+ builder.ins().jump(merge, &[q.into()]);++ builder.switch_to_block(zero_block);+ builder.seal_block(zero_block);+ builder.ins().jump(merge, &[zero.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let r = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_divw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());++ let div_block = builder.create_block();+ let bad_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let is_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);+ let a_is_min = builder.ins().icmp_imm(IntCC::Equal, a, i32::MIN as i64);+ let b_is_neg1 = builder.ins().icmp_imm(IntCC::Equal, b, -1i64);+ let overflow = builder.ins().band(a_is_min, b_is_neg1);+ let bad = builder.ins().bor(is_zero, overflow);++ builder.ins().brif(bad, bad_block, &[], div_block, &[]);++ builder.switch_to_block(div_block);+ builder.seal_block(div_block);+ let q = builder.ins().sdiv(a, b);+ builder.ins().jump(merge, &[q.into()]);++ builder.switch_to_block(bad_block);+ builder.seal_block(bad_block);+ let neg = builder.ins().icmp_imm(IntCC::SignedLessThan, a, 0);+ let neg_one = builder.ins().iconst(types::I32, 0xFFFF_FFFFu32 as i64);+ let zero = builder.ins().iconst(types::I32, 0);+ let result_bad = builder.ins().select(neg, neg_one, zero);+ builder.ins().jump(merge, &[result_bad.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let r = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_mulhw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ signed: bool,+) -> Value {+ let a32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let (a64, b64) = if signed {+ (+ builder.ins().sextend(types::I64, a32),+ builder.ins().sextend(types::I64, b32),+ )+ } else {+ (+ builder.ins().uextend(types::I64, a32),+ builder.ins().uextend(types::I64, b32),+ )+ };++ let prod = builder.ins().imul(a64, b64);+ let high = builder.ins().ushr_imm(prod, 32);++ let r = builder.ins().ireduce(types::I32, high);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);++ r+}++pub(crate) fn emit_addic<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ rc: bool,+) {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let imm_v = builder.ins().iconst(types::I32, instr.simm() as i64);+ let (sum, carry) = builder.ins().uadd_overflow(a, imm_v);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), sum);++ let carry32 = builder.ins().uextend(types::I32, carry);+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);++ if rc {+ emit_update_cr0::<SYSTEM>(builder, ctx_ptr, sum);+ }+}++pub(crate) fn emit_subfic<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let imm_v = builder.ins().iconst(types::I32, instr.simm() as i64);+ let res = builder.ins().isub(imm_v, a);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), res);++ let carry_b = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, imm_v, a);+ let carry32 = builder.ins().uextend(types::I32, carry_b);+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);+}++pub(crate) fn write_xer_carry<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, carry: Value) {+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);+ let positioned = builder.ins().ishl_imm(carry, 29);+ let merged = builder.ins().bor(cleared, positioned);+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);+}++pub(crate) fn write_xer_ov_so<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, ov: Value) {+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);++ let cleared_ov = builder.ins().band_imm(xer, !0x4000_0000u32 as i64);+ let ov_pos = builder.ins().ishl_imm(ov, 30);+ let with_ov = builder.ins().bor(cleared_ov, ov_pos);++ let so_pos = builder.ins().ishl_imm(ov, 31);+ let merged = builder.ins().bor(with_ov, so_pos);+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);+}++pub(crate) fn compute_add_overflow(builder: &mut FunctionBuilder, a: Value, b: Value, r: Value) -> Value {+ let xor_a = builder.ins().bxor(a, r);+ let xor_b = builder.ins().bxor(b, r);+ let and = builder.ins().band(xor_a, xor_b);+ let shifted = builder.ins().ushr_imm(and, 31);+ builder.ins().band_imm(shifted, 1)+}++pub(crate) fn compute_sub_overflow(builder: &mut FunctionBuilder, a: Value, b: Value, r: Value) -> Value {+ let xor_ab = builder.ins().bxor(b, a);+ let xor_br = builder.ins().bxor(b, r);+ let and = builder.ins().band(xor_ab, xor_br);+ let shifted = builder.ins().ushr_imm(and, 31);+ builder.ins().band_imm(shifted, 1)+}++pub(crate) fn emit_addc_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let (sum, carry) = builder.ins().uadd_overflow(a, b);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), sum);+ let carry32 = builder.ins().uextend(types::I32, carry);+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);+ sum+}++pub(crate) fn emit_subfc_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let res = builder.ins().isub(b, a);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), res);++ let carry_b = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, b, a);+ let carry32 = builder.ins().uextend(types::I32, carry_b);+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);++ res+}++pub(crate) fn emit_twi<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+) {+ let to = (instr.bo() as u32) as u32;+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = builder.ins().iconst(types::I32, instr.simm() as i64);+ emit_trap_check::<SYSTEM>(builder, ctx_ptr, a, b, to, pc, exit_block, local);+}++pub(crate) fn emit_tw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+) {+ let to = (instr.bo() as u32) as u32;+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ emit_trap_check::<SYSTEM>(builder, ctx_ptr, a, b, to, pc, exit_block, local);+}++pub(crate) fn emit_trap_check<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ a: Value,+ b: Value,+ to: u32,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+) {+ let _ = local;+ let mut cond: Option<Value> = None;+ let or_in = |builder: &mut FunctionBuilder, c: Value, cond: &mut Option<Value>| match *cond {+ Some(prev) => *cond = Some(builder.ins().bor(prev, c)),+ None => *cond = Some(c),+ };++ if to & 0x10 != 0 {+ let c = builder.ins().icmp(IntCC::SignedLessThan, a, b);+ or_in(builder, c, &mut cond);+ }++ if to & 0x08 != 0 {+ let c = builder.ins().icmp(IntCC::SignedGreaterThan, a, b);+ or_in(builder, c, &mut cond);+ }++ if to & 0x04 != 0 {+ let c = builder.ins().icmp(IntCC::Equal, a, b);+ or_in(builder, c, &mut cond);+ }++ if to & 0x02 != 0 {+ let c = builder.ins().icmp(IntCC::UnsignedLessThan, a, b);+ or_in(builder, c, &mut cond);+ }++ if to & 0x01 != 0 {+ let c = builder.ins().icmp(IntCC::UnsignedGreaterThan, a, b);+ or_in(builder, c, &mut cond);+ }++ if let Some(c) = cond {+ let trap_block = builder.create_block();+ let cont_block = builder.create_block();+ builder.ins().brif(c, trap_block, &[], cont_block, &[]);++ builder.switch_to_block(trap_block);+ builder.seal_block(trap_block);+ let srr0_value = builder.ins().iconst(types::I32, pc as i64);+ let nia = emit_exception_dispatch::<SYSTEM>(builder, ctx_ptr, srr0_value, 0x0002_0000, 0x0000_0700);+ builder.ins().jump(exit_block, &[nia.into()]);++ builder.switch_to_block(cont_block);+ builder.seal_block(cont_block);+ }+}++pub(crate) fn emit_mcrxr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let top3 = builder.ins().ushr_imm(xer, 29);+ let masked3 = builder.ins().band_imm(top3, 0x7);+ let shifted = builder.ins().ishl_imm(masked3, 1);+ let dst_shift = 28 - 4 * (instr.crfd() as u32);+ let positioned = builder.ins().ishl_imm(shifted, dst_shift as i64);+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));+ let new_cr = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);++ let new_xer = builder.ins().band_imm(xer, !(0xE000_0000u32 as i64));+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), new_xer, ctx_ptr, off);+}++fn read_xer_carry<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let shifted = builder.ins().ushr_imm(xer, 29);+ builder.ins().band_imm(shifted, 1)+}++fn add_with_carry_in<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ a: Value,+ b: Value,+ cin: Value,+) -> Value {+ let (s1, c1) = builder.ins().uadd_overflow(a, b);+ let (s2, c2) = builder.ins().uadd_overflow(s1, cin);+ let c1_32 = builder.ins().uextend(types::I32, c1);+ let c2_32 = builder.ins().uextend(types::I32, c2);+ let carry_out = builder.ins().bor(c1_32, c2_32);+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry_out);+ s2+}++pub(crate) fn emit_adde_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, b, cin);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_subfe_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let not_a = builder.ins().bxor_imm(a, !0i64);+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, not_a, b, cin);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++#[derive(Clone, Copy)]+pub(crate) enum AddzeSubfzeKind {+ Addze,+ Subfze,+}++pub(crate) fn emit_addze_subfze<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ kind: AddzeSubfzeKind,+) -> Value {+ let a_raw = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let a = match kind {+ AddzeSubfzeKind::Addze => a_raw,+ AddzeSubfzeKind::Subfze => builder.ins().bxor_imm(a_raw, !0i64),+ };+ let zero = builder.ins().iconst(types::I32, 0);+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, zero, cin);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++#[derive(Clone, Copy)]+pub(crate) enum AddmeSubfmeKind {+ Addme,+ Subfme,+}++pub(crate) fn emit_addme_subfme<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ kind: AddmeSubfmeKind,+) -> Value {+ let a_raw = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let a = match kind {+ AddmeSubfmeKind::Addme => a_raw,+ AddmeSubfmeKind::Subfme => builder.ins().bxor_imm(a_raw, !0i64),+ };+ let neg_one = builder.ins().iconst(types::I32, -1i64);+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, neg_one, cin);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ r+}++pub(crate) fn emit_mulli<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let r = builder.ins().imul_imm(a, instr.simm() as i64);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+}++pub(crate) fn emit_srawi<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());++ let result = if instr.rb() == 0 {+ s+ } else {+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);+ builder.ins().sshr(s, amt)+ };+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), result);++ let ca: Value = if instr.rb() == 0 {+ builder.ins().iconst(types::I32, 0)+ } else {+ let mask = ((1u32 << instr.rb()) - 1) as i64;+ let lower = builder.ins().band_imm(s, mask);+ let lower_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, lower, 0);+ let neg = builder.ins().icmp_imm(IntCC::SignedLessThan, s, 0);+ let both = builder.ins().band(neg, lower_nonzero);+ builder.ins().uextend(types::I32, both)+ };++ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);+ let positioned = builder.ins().ishl_imm(ca, 29);+ let merged = builder.ins().bor(cleared, positioned);+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);++ result+}++pub(crate) fn emit_sraw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());++ let count = builder.ins().band_imm(b, 0x3F);+ let big = builder.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, count, 32);+ let count_low5 = builder.ins().band_imm(count, 0x1F);++ let thirtyone = builder.ins().iconst(types::I32, 31);+ let eff_shift = builder.ins().select(big, thirtyone, count_low5);+ let result = builder.ins().sshr(s, eff_shift);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), result);++ let s_neg = builder.ins().icmp_imm(IntCC::SignedLessThan, s, 0);+ let s_neg_v = builder.ins().uextend(types::I32, s_neg);++ let one = builder.ins().iconst(types::I32, 1);+ let shifted_one = builder.ins().ishl(one, count_low5);+ let mask = builder.ins().iadd_imm(shifted_one, -1);+ let lower = builder.ins().band(s, mask);+ let lower_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, lower, 0);+ let lower_nonzero_v = builder.ins().uextend(types::I32, lower_nonzero);+ let small_ca = builder.ins().band(s_neg_v, lower_nonzero_v);++ let ca = builder.ins().select(big, s_neg_v, small_ca);++ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);+ let positioned = builder.ins().ishl_imm(ca, 29);+ let merged = builder.ins().bor(cleared, positioned);+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);++ result+}++pub(crate) fn emit_shift_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ kind: ShiftKind,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let count = builder.ins().band_imm(b, 0x3F);+ let count_low5 = builder.ins().band_imm(count, 0x1F);+ let shifted = match kind {+ ShiftKind::Slw => builder.ins().ishl(s, count_low5),+ ShiftKind::Srw => builder.ins().ushr(s, count_low5),+ };+ let high_bit = builder.ins().band_imm(count, 0x20);+ let out_of_range = builder.ins().icmp_imm(IntCC::NotEqual, high_bit, 0);+ let zero = builder.ins().iconst(types::I32, 0);+ let r = builder.ins().select(out_of_range, zero, shifted);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), r);+ r+}++pub(crate) fn emit_mfspr<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+) -> bool {+ let spr_num = instr.spr_swapped() as u16;+ if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {+ let val = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+ } else {+ let num_v = builder.ins().iconst(types::I32, spr_num as i64);+ let call = builder.ins().call(local.read_spr, &[ctx_ptr, num_v]);+ let val = builder.inst_results(call)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+ }+ true+}++pub(crate) fn emit_mtspr<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+) -> bool {+ let spr_num = instr.spr_swapped() as u16;+ let val = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+ } else {+ let num_v = builder.ins().iconst(types::I32, spr_num as i64);+ builder.ins().call(local.write_spr, &[ctx_ptr, num_v, val]);+ }+ true+}++fn rlw_mask(mb: u8, me: u8) -> u32 {+ debug_assert!(mb < 32 && me < 32);++ let begin = 31u8 - mb;+ let end = 31u8 - me;+ if begin >= end {+ let width = (begin - end + 1) as u32;++ if width >= 32 {+ 0xFFFF_FFFF+ } else {+ (((1u64 << width) - 1) as u32) << end+ }+ } else {+ let lo_part = ((1u64 << (begin + 1)) - 1) as u32;+ let hi_part = !(((1u64 << end) - 1) as u32);+ lo_part | hi_part+ }+}++pub(crate) fn emit_rlwnm<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let mask = rlw_mask(instr.mb(), instr.me());+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let amt = builder.ins().band_imm(b, 0x1F);+ let rotated = builder.ins().rotl(s, amt);+ let masked = builder.ins().band_imm(rotated, mask as i64);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), masked);+ masked+}++pub(crate) fn emit_rlwimi<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let mask = rlw_mask(instr.mb(), instr.me());+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());++ let rotated = if instr.rb() == 0 {+ s+ } else {+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);+ builder.ins().rotl(s, amt)+ };++ let inserted_bits = builder.ins().band_imm(rotated, mask as i64);+ let kept_bits = builder.ins().band_imm(a, !mask as i64);+ let val = builder.ins().bor(kept_bits, inserted_bits);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);++ val+}++pub(crate) fn emit_rlwinm<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let mask = rlw_mask(instr.mb(), instr.me());+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let rotated = if instr.rb() == 0 {+ s+ } else {+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);+ builder.ins().rotl(s, amt)+ };++ let masked = builder.ins().band_imm(rotated, mask as i64);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), masked);++ masked+}++fn cr_bit_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, bi: u8) -> Value {+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let shifted = builder.ins().ushr_imm(cr, (31 - bi) as i64);+ builder.ins().band_imm(shifted, 1)+}++pub(crate) fn ctr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let off = abi::ctr_offset::<SYSTEM>() as i32;+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn ctr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {+ let off = abi::ctr_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++pub(crate) fn lr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {+ let off = abi::lr_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++pub(crate) fn emit_b<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+) -> Value {+ let target: u32 = if instr.aa() {+ instr.li() as u32+ } else {+ pc.wrapping_add_signed(instr.li())+ };+ if instr.lk() {+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);+ }+ builder.ins().iconst(types::I32, target as i64)+}++pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ taken_slot: Option<i64>,+ fall_slot: Option<i64>,+ block_sig_ref: cranelift_codegen::ir::SigRef,+) -> TermEmit {+ if taken_slot.is_none() && fall_slot.is_none() {+ return TermEmit::HandledNia(emit_bc::<SYSTEM>(builder, ctx_ptr, instr, pc));+ }++ let bo = (instr.bo() as u32) as u8;++ let taken_target: u32 = if instr.aa() {+ instr.bd() as u32+ } else {+ pc.wrapping_add_signed(instr.bd())+ };+ let fall_target: u32 = pc.wrapping_add(4);++ let decrement_ctr = (bo & 0x04) == 0;+ let mut ctr_after: Option<Value> = None;+ if decrement_ctr {+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);+ ctr_after = Some(ctr_minus_1);+ }++ let ctr_ok: Value = if decrement_ctr {+ let ctr = ctr_after.unwrap();+ let cc = if (bo & 0x02) != 0 {+ IntCC::Equal+ } else {+ IntCC::NotEqual+ };+ let r = builder.ins().icmp_imm(cc, ctr, 0);+ builder.ins().uextend(types::I32, r)+ } else {+ builder.ins().iconst(types::I32, 1)+ };++ let cond_ok: Value = if (bo & 0x10) != 0 {+ builder.ins().iconst(types::I32, 1)+ } else {+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);+ builder.ins().uextend(types::I32, r)+ };++ let take = builder.ins().band(ctr_ok, cond_ok);+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);++ let taken_block = builder.create_block();+ let fall_block = builder.create_block();+ builder.ins().brif(take_b, taken_block, &[], fall_block, &[]);++ builder.switch_to_block(taken_block);+ builder.seal_block(taken_block);+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);+ if instr.lk() {+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);+ }+ let taken_v = builder.ins().iconst(types::I32, taken_target as i64);+ if let Some(slot_addr) = taken_slot {+ emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, taken_v, exit_block);+ } else {+ builder.ins().jump(exit_block, &[taken_v.into()]);+ }++ builder.switch_to_block(fall_block);+ builder.seal_block(fall_block);+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);++ let fall_v = builder.ins().iconst(types::I32, fall_target as i64);+ if let Some(slot_addr) = fall_slot {+ emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, fall_v, exit_block);+ } else {+ builder.ins().jump(exit_block, &[fall_v.into()]);+ }++ TermEmit::HandledChained+}++pub(crate) fn emit_bc<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+) -> Value {+ let bo = (instr.bo() as u32) as u8;++ let target: u32 = if instr.aa() {+ instr.bd() as u32+ } else {+ pc.wrapping_add_signed(instr.bd())+ };+ let target_v = builder.ins().iconst(types::I32, target as i64);+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);++ let decrement_ctr = (bo & 0x04) == 0;+ let mut ctr_after: Option<Value> = None;+ if decrement_ctr {+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);+ ctr_after = Some(ctr_minus_1);+ }++ let ctr_ok: Value = if decrement_ctr {+ let ctr = ctr_after.unwrap();+ if (bo & 0x02) != 0 {+ let r = builder.ins().icmp_imm(IntCC::Equal, ctr, 0);+ builder.ins().uextend(types::I32, r)+ } else {+ let r = builder.ins().icmp_imm(IntCC::NotEqual, ctr, 0);+ builder.ins().uextend(types::I32, r)+ }+ } else {+ builder.ins().iconst(types::I32, 1)+ };++ let cond_ok: Value = if (bo & 0x10) != 0 {+ builder.ins().iconst(types::I32, 1)+ } else {+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);+ builder.ins().uextend(types::I32, r)+ };++ let take = builder.ins().band(ctr_ok, cond_ok);+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);++ if instr.lk() {+ let lk_block = builder.create_block();+ let after_lk = builder.create_block();+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);+ builder.switch_to_block(lk_block);+ builder.seal_block(lk_block);++ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);++ builder.ins().jump(after_lk, &[]);+ builder.switch_to_block(after_lk);+ builder.seal_block(after_lk);+ }++ builder.ins().select(take_b, target_v, fall_v)+}++pub(crate) fn lr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let off = abi::lr_offset::<SYSTEM>() as i32;+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn emit_bclr_dispatch<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ lookup_table_addr: i64,+) -> TermEmit {+ let bo = (instr.bo() as u32) as u8;++ let unconditional = (bo & 0x14) == 0x14;+ if unconditional && !instr.lk() {+ emit_blr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)+ } else {+ TermEmit::HandledNia(emit_bclr::<SYSTEM>(builder, ctx_ptr, instr, pc))+ }+}++pub(crate) fn emit_bcctr_dispatch<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ lookup_table_addr: i64,+) -> TermEmit {+ let bo = (instr.bo() as u32) as u8;+ let unconditional = (bo & 0x14) == 0x14;+ if unconditional && !instr.lk() {+ emit_bctr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)+ } else {+ TermEmit::HandledNia(emit_bcctr::<SYSTEM>(builder, ctx_ptr, instr, pc))+ }+}++pub(crate) fn emit_blr_with_ic<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ _pc: u32,+ exit_block: Block,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ lookup_table_addr: i64,+) -> TermEmit {+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);++ let lr = lr_load::<SYSTEM>(builder, ctx_ptr);+ let target_pc = builder.ins().band_imm(lr, !3i64);+ emit_inline_cache_dispatch::<SYSTEM>(+ builder,+ ctx_ptr,+ target_pc,+ exit_block,+ block_sig_ref,+ lookup_table_addr,+ );+ TermEmit::HandledChained+}++pub(crate) fn emit_bctr_with_ic<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ _pc: u32,+ exit_block: Block,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ lookup_table_addr: i64,+) -> TermEmit {+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);+ let target_pc = builder.ins().band_imm(ctr, !3i64);+ emit_inline_cache_dispatch::<SYSTEM>(+ builder,+ ctx_ptr,+ target_pc,+ exit_block,+ block_sig_ref,+ lookup_table_addr,+ );+ TermEmit::HandledChained+}++pub(crate) fn emit_inline_cache_dispatch<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ target_pc: Value,+ exit_block: Block,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ lookup_table_addr: i64,+) {+ let mask_v = builder.ins().iconst(types::I32, super::BLOCK_LOOKUP_TABLE_MASK as i64);+ let idx_word = builder.ins().ushr_imm(target_pc, 2);+ let idx = builder.ins().band(idx_word, mask_v);+ let idx64 = builder.ins().uextend(types::I64, idx);+ let entry_size = std::mem::size_of::<super::BlockLookupSlot>() as i64;+ let off64 = builder.ins().imul_imm(idx64, entry_size);+ let table_base = builder.ins().iconst(types::I64, lookup_table_addr);+ let slot_addr = builder.ins().iadd(table_base, off64);++ let slot_pc = builder.ins().load(types::I32, MemFlags::trusted(), slot_addr, 0);+ let slot_entry = builder.ins().load(types::I64, MemFlags::trusted(), slot_addr, 8);+ let pc_eq = builder.ins().icmp(IntCC::Equal, slot_pc, target_pc);++ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);+ let in_budget = builder.ins().icmp(IntCC::UnsignedLessThan, cycles, deadline);++ let pc_eq_i8 = pc_eq;+ let budget_i8 = in_budget;+ let nonzero_i8 = builder.ins().icmp_imm(IntCC::NotEqual, slot_entry, 0);+ let pc_and_budget = builder.ins().band(pc_eq_i8, budget_i8);+ let ok = builder.ins().band(pc_and_budget, nonzero_i8);++ let hit_block = builder.create_block();+ let miss_block = builder.create_block();+ builder.ins().brif(ok, hit_block, &[], miss_block, &[]);++ builder.switch_to_block(miss_block);+ builder.seal_block(miss_block);+ builder.ins().jump(exit_block, &[target_pc.into()]);++ builder.switch_to_block(hit_block);+ builder.seal_block(hit_block);+ builder+ .ins()+ .return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr]);+}++pub(crate) fn emit_bclr<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+) -> Value {+ let bo = (instr.bo() as u32) as u8;++ let lr = lr_load::<SYSTEM>(builder, ctx_ptr);+ let target_v = builder.ins().band_imm(lr, !3i64);+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);++ let decrement_ctr = (bo & 0x04) == 0;+ let mut ctr_after: Option<Value> = None;+ if decrement_ctr {+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);+ ctr_after = Some(ctr_minus_1);+ }++ let ctr_ok: Value = if decrement_ctr {+ let ctr = ctr_after.unwrap();+ let cc = if (bo & 0x02) != 0 {+ IntCC::Equal+ } else {+ IntCC::NotEqual+ };+ let r = builder.ins().icmp_imm(cc, ctr, 0);+ builder.ins().uextend(types::I32, r)+ } else {+ builder.ins().iconst(types::I32, 1)+ };++ let cond_ok: Value = if (bo & 0x10) != 0 {+ builder.ins().iconst(types::I32, 1)+ } else {+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);+ builder.ins().uextend(types::I32, r)+ };++ let take = builder.ins().band(ctr_ok, cond_ok);+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);++ if instr.lk() {+ let lk_block = builder.create_block();+ let after_lk = builder.create_block();+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);+ builder.switch_to_block(lk_block);+ builder.seal_block(lk_block);+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);+ builder.ins().jump(after_lk, &[]);+ builder.switch_to_block(after_lk);+ builder.seal_block(after_lk);+ }++ builder.ins().select(take_b, target_v, fall_v)+}++pub(crate) fn emit_bcctr<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+) -> Value {+ let bo = (instr.bo() as u32) as u8;++ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);+ let target_v = builder.ins().band_imm(ctr, !3i64);+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);++ let unconditional = (bo & 0x10) != 0;+ let take_b = if unconditional {+ builder.ins().iconst(types::I8, 1)+ } else {+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };+ builder.ins().icmp_imm(IntCC::Equal, bit, want)+ };++ if instr.lk() {+ let lk_block = builder.create_block();+ let after_lk = builder.create_block();+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);+ builder.switch_to_block(lk_block);+ builder.seal_block(lk_block);+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);+ builder.ins().jump(after_lk, &[]);+ builder.switch_to_block(after_lk);+ builder.seal_block(after_lk);+ }++ builder.ins().select(take_b, target_v, fall_v)+}++pub(crate) fn emit_cycles_add<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, n: i64) {+ let off = abi::cycles_offset::<SYSTEM>() as i64;+ let addr = builder.ins().iadd_imm(ctx_ptr, off);+ let now = builder.ins().load(types::I64, MemFlags::trusted(), addr, 0);+ let next = builder.ins().iadd_imm(now, n);+ builder.ins().store(MemFlags::trusted(), next, addr, 0);+}++pub(crate) fn emit_mcrf<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let src_shift = 28 - 4 * (instr.crfs() as u32);+ let dst_shift = 28 - 4 * (instr.crfd() as u32);+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let src_field = builder.ins().ushr_imm(cr, src_shift as i64);+ let src_nibble = builder.ins().band_imm(src_field, 0xF);+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));+ let positioned = builder.ins().ishl_imm(src_nibble, dst_shift as i64);+ let new_cr = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);+}++pub(crate) fn emit_cr_bit_op<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ xo: u32,+) {+ let bit_a = 31 - (instr.crba() as u32);+ let bit_b = 31 - (instr.crbb() as u32);+ let bit_d = 31 - (instr.crbd() as u32);++ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let a_shifted = builder.ins().ushr_imm(cr, bit_a as i64);+ let a = builder.ins().band_imm(a_shifted, 1);+ let b_shifted = builder.ins().ushr_imm(cr, bit_b as i64);+ let b = builder.ins().band_imm(b_shifted, 1);++ let result = match xo {+ 33 => {+ let or_v = builder.ins().bor(a, b);+ let inv = builder.ins().bxor_imm(or_v, 1);+ inv+ }+ 129 => {+ let not_b = builder.ins().bxor_imm(b, 1);+ builder.ins().band(a, not_b)+ }+ 193 => builder.ins().bxor(a, b),+ 225 => {+ let and_v = builder.ins().band(a, b);+ builder.ins().bxor_imm(and_v, 1)+ }+ 257 => builder.ins().band(a, b),+ 289 => {+ let xor_v = builder.ins().bxor(a, b);+ builder.ins().bxor_imm(xor_v, 1)+ }+ 417 => {+ let not_b = builder.ins().bxor_imm(b, 1);+ builder.ins().bor(a, not_b)+ }+ 449 => builder.ins().bor(a, b),+ _ => unreachable!("unexpected CR-bit XO {xo}"),+ };++ let cleared = builder.ins().band_imm(cr, !(1i64 << bit_d));+ let positioned = builder.ins().ishl_imm(result, bit_d as i64);+ let new_cr = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);+}++#[derive(Clone, Copy)]+pub(crate) enum ShiftKind {+ Slw,+ Srw,+}++#[derive(Clone, Copy)]+pub(crate) enum LogicalOp {+ And,+ Or,+ Xor,+}++fn terminator_fall_pc(spec: &BlockSpec) -> Option<u32> {+ if !matches!(spec.terminator, TermKind::BranchCond) {+ return None;+ }+ let last_idx = spec.instrs.len().checked_sub(1)?;+ let pc = spec.pc_of(last_idx);+ Some(pc.wrapping_add(4))+}++fn terminator_static_taken_pc(spec: &BlockSpec) -> Option<u32> {+ if spec.instrs.is_empty() {+ return None;+ }+ let last_idx = spec.instrs.len() - 1;+ let pc = spec.pc_of(last_idx);+ let instr = Instruction(spec.instrs[last_idx]);+ match spec.terminator {+ TermKind::Branch => Some(if instr.aa() {+ instr.li() as u32+ } else {+ pc.wrapping_add_signed(instr.li())+ }),+ TermKind::BranchCond => Some(if instr.aa() {+ instr.bd() as u32+ } else {+ pc.wrapping_add_signed(instr.bd())+ }),+ _ => None,+ }+}++pub(crate) enum TermEmit {+ #[allow(dead_code)]+ NotHandled,++ HandledNia(Value),++ HandledChained,+}++pub(crate) fn emit_chain_or_exit<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ slot_addr: i64,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ nia_if_exit: Value,+ exit_block: Block,+) {+ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);+ let exhausted = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cycles, deadline);++ let chain_block = builder.create_block();+ builder+ .ins()+ .brif(exhausted, exit_block, &[nia_if_exit.into()], chain_block, &[]);++ builder.switch_to_block(chain_block);+ builder.seal_block(chain_block);++ let slot_const = builder.ins().iconst(types::I64, slot_addr);+ let target_ptr = builder.ins().load(types::I64, MemFlags::trusted(), slot_const, 0);+ let is_null = builder.ins().icmp_imm(IntCC::Equal, target_ptr, 0);+ let dispatch_block = builder.create_block();+ let tailcall_block = builder.create_block();+ builder.ins().brif(is_null, dispatch_block, &[], tailcall_block, &[]);++ builder.switch_to_block(dispatch_block);+ builder.seal_block(dispatch_block);+ builder.ins().jump(exit_block, &[nia_if_exit.into()]);++ builder.switch_to_block(tailcall_block);+ builder.seal_block(tailcall_block);+ builder+ .ins()+ .return_call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);+}++pub(crate) fn emit_call_or_exit<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ slot_addr: i64,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ nia_if_exit: Value,+ exit_block: Block,+) {+ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);+ let exhausted = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cycles, deadline);++ let chain_block = builder.create_block();+ builder+ .ins()+ .brif(exhausted, exit_block, &[nia_if_exit.into()], chain_block, &[]);++ builder.switch_to_block(chain_block);+ builder.seal_block(chain_block);+ let slot_const = builder.ins().iconst(types::I64, slot_addr);+ let target_ptr = builder.ins().load(types::I64, MemFlags::trusted(), slot_const, 0);+ let is_null = builder.ins().icmp_imm(IntCC::Equal, target_ptr, 0);++ let dispatch_block = builder.create_block();+ let call_block = builder.create_block();+ builder.ins().brif(is_null, dispatch_block, &[], call_block, &[]);++ builder.switch_to_block(dispatch_block);+ builder.seal_block(dispatch_block);+ builder.ins().jump(exit_block, &[nia_if_exit.into()]);++ builder.switch_to_block(call_block);+ builder.seal_block(call_block);++ let call = builder.ins().call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);+ let returned_nia = builder.inst_results(call)[0];++ builder.ins().jump(exit_block, &[returned_nia.into()]);+}++fn gpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {+ debug_assert!(i < 32);+ (abi::gpr_base_offset::<SYSTEM>() + 4 * i as usize) as i32+}++pub(crate) fn gpr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {+ let off = gpr_offset::<SYSTEM>(i);+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn gpr_load_or_zero<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {+ if i == 0 {+ builder.ins().iconst(types::I32, 0)+ } else {+ gpr_load::<SYSTEM>(builder, ctx_ptr, i)+ }+}++pub(crate) fn gpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {+ let off = gpr_offset::<SYSTEM>(i);+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++fn fpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {+ debug_assert!(i < 32);+ (abi::fpr_base_offset::<SYSTEM>() + 8 * i as usize) as i32+}++pub(crate) fn fpr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {+ let off = fpr_offset::<SYSTEM>(i);+ builder.ins().load(types::F64, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn fpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {+ let off = fpr_offset::<SYSTEM>(i);+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++fn ps1_offset<const SYSTEM: SystemId>(i: u8) -> i32 {+ debug_assert!(i < 32);+ (abi::ps1_base_offset::<SYSTEM>() + 8 * i as usize) as i32+}++pub(crate) fn ps1_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {+ let off = ps1_offset::<SYSTEM>(i);+ builder.ins().load(types::F64, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn ps1_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {+ let off = ps1_offset::<SYSTEM>(i);+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++pub(crate) fn round_to_single(builder: &mut FunctionBuilder, val: Value) -> Value {+ let f32v = builder.ins().fdemote(types::F32, val);+ builder.ins().fpromote(types::F64, f32v)+}++pub(crate) fn f32_reciprocal(builder: &mut FunctionBuilder, b_f64: Value) -> Value {+ let b32 = builder.ins().fdemote(types::F32, b_f64);+ let one32 = builder.ins().f32const(1.0f32);+ let q32 = builder.ins().fdiv(one32, b32);+ builder.ins().fpromote(types::F64, q32)+}++pub(crate) fn f32_rsqrte(builder: &mut FunctionBuilder, b_f64: Value) -> Value {+ let b32 = builder.ins().fdemote(types::F32, b_f64);+ let s32 = builder.ins().sqrt(b32);+ let one32 = builder.ins().f32const(1.0f32);+ let q32 = builder.ins().fdiv(one32, s32);+ builder.ins().fpromote(types::F64, q32)+}++pub(crate) fn emit_recompute_fpscr_summary<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) {+ const VX_MASK: u32 = 0x01F8_0700;+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let fpscr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);++ let vx_in = builder.ins().band_imm(fpscr, VX_MASK as i64);+ let vx_b = builder.ins().icmp_imm(IntCC::NotEqual, vx_in, 0);+ let vx_v = builder.ins().uextend(types::I32, vx_b);+ let vx_pos = builder.ins().ishl_imm(vx_v, 29);+ let cleared_vx = builder.ins().band_imm(fpscr, !(1i64 << 29));+ let with_vx = builder.ins().bor(cleared_vx, vx_pos);++ let upper = builder.ins().ushr_imm(with_vx, 22);+ let upper_mask = builder.ins().band_imm(upper, 0xF8);+ let lower_mask = builder.ins().band_imm(with_vx, 0xF8);+ let pairs = builder.ins().band(upper_mask, lower_mask);+ let fex_b = builder.ins().icmp_imm(IntCC::NotEqual, pairs, 0);+ let fex_v = builder.ins().uextend(types::I32, fex_b);+ let fex_pos = builder.ins().ishl_imm(fex_v, 30);+ let cleared_fex = builder.ins().band_imm(with_vx, !(1i64 << 30));+ let new_fpscr = builder.ins().bor(cleared_fex, fex_pos);++ builder.ins().store(MemFlags::trusted(), new_fpscr, ctx_ptr, off);+}++pub(crate) fn fpr_store_paired<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ i: u8,+ val: Value,+) {+ fpr_store::<SYSTEM>(builder, ctx_ptr, i, val);+ ps1_store::<SYSTEM>(builder, ctx_ptr, i, val);+}++pub(crate) fn cr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let off = abi::cr_offset::<SYSTEM>() as i32;+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)+}++pub(crate) fn cr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {+ let off = abi::cr_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);+}++pub(crate) fn emit_update_cr1_from_fpscr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) {+ let fpscr = builder.ins().load(+ types::I32,+ MemFlags::trusted(),+ ctx_ptr,+ abi::fpscr_offset::<SYSTEM>() as i32,+ );++ let top4 = builder.ins().ushr_imm(fpscr, 28);+ let positioned = builder.ins().ishl_imm(top4, 24);+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << 24));+ let new_cr = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);+}++pub(crate) fn xer_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let off = abi::xer_offset::<SYSTEM>() as i32;+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)+}++fn xer_so_bit<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let shifted = builder.ins().ushr_imm(xer, 31);+ builder.ins().band_imm(shifted, 1)+}++fn cr_set_field<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, crfd: u8, nibble: Value) {+ debug_assert!(crfd < 8);+ let shift = 28 - 4 * crfd as i64;+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let mask = !(0xFu32 << shift) as i64 as u64 as i64;+ let cleared = builder.ins().band_imm(cr, mask);+ let nibble_only = builder.ins().band_imm(nibble, 0xF);+ let positioned = builder.ins().ishl_imm(nibble_only, shift);+ let merged = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, merged);+}++fn build_cr_nibble(builder: &mut FunctionBuilder, lt: Value, gt: Value, eq: Value, so: Value) -> Value {+ let lt8 = builder.ins().ishl_imm(lt, 3);+ let gt4 = builder.ins().ishl_imm(gt, 2);+ let eq2 = builder.ins().ishl_imm(eq, 1);+ let a = builder.ins().bor(lt8, gt4);+ let b = builder.ins().bor(eq2, so);+ builder.ins().bor(a, b)+}++fn cmp_to_i32(builder: &mut FunctionBuilder, cc: IntCC, a: Value, b: Value) -> Value {+ let r = builder.ins().icmp(cc, a, b);+ builder.ins().uextend(types::I32, r)+}++pub(crate) fn emit_update_cr0<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {+ let zero = builder.ins().iconst(types::I32, 0);+ let lt = cmp_to_i32(builder, IntCC::SignedLessThan, val, zero);+ let gt = cmp_to_i32(builder, IntCC::SignedGreaterThan, val, zero);+ let eq = cmp_to_i32(builder, IntCC::Equal, val, zero);+ let so = xer_so_bit::<SYSTEM>(builder, ctx_ptr);+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);+ cr_set_field::<SYSTEM>(builder, ctx_ptr, 0, nibble);+}++#[derive(Clone, Copy)]+pub(crate) enum MemSize {+ U8,+ U16,+ U32,+}++impl MemSize {+ fn ir_type(self) -> cranelift_codegen::ir::Type {+ match self {+ MemSize::U8 => types::I8,+ MemSize::U16 => types::I16,+ MemSize::U32 => types::I32,+ }+ }+}++pub(crate) fn emit_d_form_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ builder.ins().iadd_imm(base, instr.simm() as i64)+}++pub(crate) fn emit_x_form_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ builder.ins().iadd(base, b)+}++pub(crate) fn emit_fastmem_lookup<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+) -> (Value, Value) {+ let lut_base = {+ let off = abi::fastmem_lut_ptr_offset::<SYSTEM>() as i32;+ builder+ .ins()+ .load(types::I64, MemFlags::trusted().with_readonly(), ctx_ptr, off)+ };++ let idx_u32 = builder.ins().ushr_imm(ea, crate::mmio::FASTMEM_PAGE_SHIFT as i64);+ let idx = builder.ins().uextend(types::I64, idx_u32);+ let entry_off = builder.ins().ishl_imm(idx, 3);+ let entry_addr = builder.ins().iadd(lut_base, entry_off);+ let host_ptr = builder.ins().load(types::I64, MemFlags::trusted(), entry_addr, 0);++ let page_off_u32 = builder.ins().band_imm(ea, crate::mmio::FASTMEM_PAGE_MASK as i64);+ let page_off = builder.ins().uextend(types::I64, page_off_u32);++ (host_ptr, page_off)+}++pub(crate) fn emit_load<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rd(), instr.ra(), size, slow, update);+}++pub(crate) fn emit_lha_d_form<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_val = builder.ins().load(types::I16, MemFlags::new(), addr, 0);+ let swapped = builder.ins().bswap(raw_val);++ let val32 = builder.ins().sextend(types::I32, swapped);+ builder.ins().jump(merge, &[val32.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);+ let raw_u16_in_u32 = builder.inst_results(call)[0];++ let trunc = builder.ins().ireduce(types::I16, raw_u16_in_u32);+ let sext = builder.ins().sextend(types::I32, trunc);+ builder.ins().jump(merge, &[sext.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+ }+}++pub(crate) fn emit_lfx_update<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ single: bool,+ local: &LocalFuncs,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ if single {+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f32);+ } else {+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f64);+ }+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+}++pub(crate) fn emit_stfx_update<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ single: bool,+ local: &LocalFuncs,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ if single {+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);+ } else {+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);+ }+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+}++pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_u32: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let f = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let u64v = builder.ins().bitcast(types::I64, MemFlags::new(), f);+ let u32v = builder.ins().ireduce(types::I32, u64v);++ emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32);+}++pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ val: Value,+ slow: FuncRef,+) {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let bswapped = builder.ins().bswap(val);+ builder.ins().store(MemFlags::new(), bswapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow, &[ctx_ptr, ea, val]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+ store: bool,+ update: bool,+) {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let ea = builder.ins().iadd(base, b);++ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_22_24(), store);+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ if store {+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);+ if !instr.w_21_21() {+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rd());+ let ea1 = builder.ins().iadd_imm(ea, 4);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);+ }+ } else {+ let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);+ let ps1 = if instr.w_21_21() {+ builder.ins().f64const(1.0)+ } else {+ let ea1 = builder.ins().iadd_imm(ea, 4);+ emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)+ };+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);+ ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);+ }+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ if store {+ emit_psq_store_quantized::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rd(),+ instr.w_21_21(),+ instr.i_22_24(),+ merge,+ local,+ );+ } else {+ emit_psq_load_quantized::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rd(),+ instr.w_21_21(),+ instr.i_22_24(),+ merge,+ local,+ );+ }++ builder.switch_to_block(merge);+ builder.seal_block(merge);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+ }+}++pub(crate) fn emit_lwarx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_load_at_ea::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rd(),+ instr.ra(),+ MemSize::U32,+ local.read_u32,+ false,+ );+ let reserve_off = abi::reserve_addr_offset::<SYSTEM>() as i32;+ builder.ins().store(MemFlags::trusted(), ea, ctx_ptr, reserve_off);+}++pub(crate) fn emit_stwcx_dot<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let reserve_off = abi::reserve_addr_offset::<SYSTEM>() as i32;+ let reserved = builder+ .ins()+ .load(types::I32, MemFlags::trusted(), ctx_ptr, reserve_off);+ let matched_b = builder.ins().icmp(IntCC::Equal, reserved, ea);+ let matched = builder.ins().uextend(types::I32, matched_b);++ let store_block = builder.create_block();+ let cr_block = builder.create_block();+ builder.append_block_param(cr_block, types::I32);+ builder+ .ins()+ .brif(matched_b, store_block, &[], cr_block, &[matched.into()]);++ builder.switch_to_block(store_block);+ builder.seal_block(store_block);++ let no_reservation = builder+ .ins()+ .iconst(types::I32, crate::gekko::Gekko::NO_RESERVATION as i64);+ builder+ .ins()+ .store(MemFlags::trusted(), no_reservation, ctx_ptr, reserve_off);+ emit_store_at_ea::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rs(),+ 0,+ MemSize::U32,+ local.write_u32,+ false,+ );+ builder.ins().jump(cr_block, &[matched.into()]);++ builder.switch_to_block(cr_block);+ builder.seal_block(cr_block);+ let eq_flag = builder.block_params(cr_block)[0];++ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);+ let so = builder.ins().ushr_imm(xer, 31);+ let so_clean = builder.ins().band_imm(so, 1);+ let eq_shifted = builder.ins().ishl_imm(eq_flag, 1);+ let nibble = builder.ins().bor(eq_shifted, so_clean);+ let positioned = builder.ins().ishl_imm(nibble, 28);+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << 28));+ let new_cr = builder.ins().bor(cleared, positioned);+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);+}++pub(crate) fn emit_lmw_stmw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ local: &LocalFuncs,+ store: bool,+) {+ let ea_base = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ for r in instr.rd()..32 {+ let off = ((r - instr.rd()) as i32) * 4;+ let ea = if off == 0 {+ ea_base+ } else {+ builder.ins().iadd_imm(ea_base, off as i64)+ };+ if store {+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.write_u32, false);+ } else {+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.read_u32, false);+ }+ }+}++pub(crate) fn emit_lha_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_val = builder.ins().load(types::I16, MemFlags::new(), addr, 0);+ let swapped = builder.ins().bswap(raw_val);+ let val32 = builder.ins().sextend(types::I32, swapped);+ builder.ins().jump(merge, &[val32.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);+ let raw_u16_in_u32 = builder.inst_results(call)[0];+ let trunc = builder.ins().ireduce(types::I16, raw_u16_in_u32);+ let sext = builder.ins().sextend(types::I32, trunc);+ builder.ins().jump(merge, &[sext.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+ }+}++pub(crate) fn emit_load_xform_brx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_val = builder.ins().load(size.ir_type(), MemFlags::new(), addr, 0);+ let val32 = match size {+ MemSize::U32 => raw_val,+ MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, raw_val),+ };+ builder.ins().jump(merge, &[val32.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);+ let raw_be_in_u32 = builder.inst_results(call)[0];++ let swapped = match size {+ MemSize::U32 => builder.ins().bswap(raw_be_in_u32),+ MemSize::U16 => {+ let trunc = builder.ins().ireduce(types::I16, raw_be_in_u32);+ let sw = builder.ins().bswap(trunc);+ builder.ins().uextend(types::I32, sw)+ }+ MemSize::U8 => raw_be_in_u32,+ };+ builder.ins().jump(merge, &[swapped.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+}++pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];++ let truncated = match size {+ MemSize::U32 => val32,+ MemSize::U16 => builder.ins().ireduce(types::I16, val32),+ MemSize::U8 => builder.ins().ireduce(types::I8, val32),+ };+ builder.ins().store(MemFlags::new(), truncated, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);++ let to_write = match size {+ MemSize::U32 => builder.ins().bswap(val32),+ MemSize::U16 => {+ let trunc = builder.ins().ireduce(types::I16, val32);+ let sw = builder.ins().bswap(trunc);+ builder.ins().uextend(types::I32, sw)+ }+ MemSize::U8 => val32,+ };+ builder.ins().call(slow, &[ctx_ptr, ea, to_write]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_exception_dispatch<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ srr0_value: Value,+ srr1_extra_bits: u32,+ vector_offset: u32,+) -> Value {+ const MSR_CLEAR_MASK: i64 = 0x4_EF37;++ const MSR_PRESERVE_MASK: i64 = 0x87C0_FFFF;++ let msr_off = abi::msr_offset::<SYSTEM>() as i32;+ let srr0_off = abi::spr_field_offset::<SYSTEM>(26).unwrap() as i32;+ let srr1_off = abi::spr_field_offset::<SYSTEM>(27).unwrap() as i32;+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;++ let msr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, msr_off);++ builder.ins().store(MemFlags::trusted(), srr0_value, ctx_ptr, srr0_off);++ let preserved = builder.ins().band_imm(msr, MSR_PRESERVE_MASK);+ let srr1 = if srr1_extra_bits != 0 {+ builder.ins().bor_imm(preserved, srr1_extra_bits as i64)+ } else {+ preserved+ };+ builder.ins().store(MemFlags::trusted(), srr1, ctx_ptr, srr1_off);++ let ip_bit = builder.ins().band_imm(msr, 0x40);+ let want_high = builder.ins().icmp_imm(IntCC::NotEqual, ip_bit, 0);+ let high_base = builder.ins().iconst(types::I32, 0xFFF0_0000i64);+ let zero = builder.ins().iconst(types::I32, 0);+ let base = builder.ins().select(want_high, high_base, zero);+ let nia = builder.ins().bor_imm(base, vector_offset as i64);+ builder.ins().store(MemFlags::trusted(), nia, ctx_ptr, nia_off);++ let msr_cleared = builder.ins().band_imm(msr, !MSR_CLEAR_MASK & 0xFFFF_FFFFi64);+ let ile_shifted = builder.ins().ushr_imm(msr, 16);+ let ile_bit = builder.ins().band_imm(ile_shifted, 1);+ let new_msr = builder.ins().bor(msr_cleared, ile_bit);+ builder.ins().store(MemFlags::trusted(), new_msr, ctx_ptr, msr_off);++ nia+}++pub(crate) fn emit_msr_fp_guard<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ _instr: Instruction,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+) {+ if local.fp_guard_emitted.get() {+ return;+ }+ local.fp_guard_emitted.set(true);++ let msr_off = abi::msr_offset::<SYSTEM>() as i32;+ let msr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, msr_off);+ let fp_bit = builder.ins().band_imm(msr, 0x2000);+ let fp_disabled = builder.ins().icmp_imm(IntCC::Equal, fp_bit, 0);+ let trap_block = builder.create_block();+ let cont_block = builder.create_block();+ builder.ins().brif(fp_disabled, trap_block, &[], cont_block, &[]);++ builder.switch_to_block(trap_block);+ builder.seal_block(trap_block);++ let _ = local;+ let cia_off = abi::cia_offset::<SYSTEM>() as i32;+ let pc_const = builder.ins().iconst(types::I32, pc as i64);+ builder.ins().store(MemFlags::trusted(), pc_const, ctx_ptr, cia_off);+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;+ let pc_plus4 = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);+ builder.ins().store(MemFlags::trusted(), pc_plus4, ctx_ptr, nia_off);+ let nia = emit_exception_dispatch::<SYSTEM>(builder, ctx_ptr, pc_const, 0, 0x0000_0800);+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);+ builder.ins().jump(exit_block, &[nia.into()]);++ builder.switch_to_block(cont_block);+ builder.seal_block(cont_block);+}++pub(crate) fn emit_lf_d_form_update<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ single: bool,+ local: &LocalFuncs,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ if single {+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f32);+ } else {+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f64);+ }+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+}++pub(crate) fn emit_stf_d_form_update<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ single: bool,+ local: &LocalFuncs,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ if single {+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);+ } else {+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);+ }+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+}++pub(crate) fn emit_lfs<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f32: FuncRef,+ _slow_u32: FuncRef,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F64);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);+ let be_u32 = builder.ins().bswap(raw_u32);+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);+ let f64v = builder.ins().fpromote(types::F64, f32v);+ builder.ins().jump(merge, &[f64v.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);+ let f64v_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[f64v_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];++ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+}++pub(crate) fn emit_lfd<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f64: FuncRef,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F64);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_u64 = builder.ins().load(types::I64, MemFlags::new(), addr, 0);+ let be_u64 = builder.ins().bswap(raw_u64);+ let f64v = builder.ins().bitcast(types::F64, MemFlags::new(), be_u64);+ builder.ins().jump(merge, &[f64v.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow_f64, &[ctx_ptr, ea]);+ let f64v_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[f64v_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+}++pub(crate) fn emit_stfs<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f32: FuncRef,+ _slow_u32: FuncRef,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let f32v = builder.ins().fdemote(types::F32, f64v);+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(MemFlags::new(), swapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_stfd<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f64: FuncRef,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(MemFlags::new(), swapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow_f64, &[ctx_ptr, ea, f64v]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_lfsx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f32: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, slow_f32);+}++pub(crate) fn emit_lfdx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f64: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, slow_f64);+}++pub(crate) fn emit_stfsx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f32: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32);+}++pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ slow_f64: FuncRef,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64);+}++pub(crate) fn emit_lfs_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ rd: u8,+ ea: Value,+ slow_f32: FuncRef,+) {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F64);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);+ let be_u32 = builder.ins().bswap(raw_u32);+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);+ let f64v = builder.ins().fpromote(types::F64, f32v);+ builder.ins().jump(merge, &[f64v.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);+ let f64v_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[f64v_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, rd, val);+}++pub(crate) fn emit_lfd_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ rd: u8,+ ea: Value,+ slow_f64: FuncRef,+) {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F64);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_u64 = builder.ins().load(types::I64, MemFlags::new(), addr, 0);+ let be_u64 = builder.ins().bswap(raw_u64);+ let f64v = builder.ins().bitcast(types::F64, MemFlags::new(), be_u64);+ builder.ins().jump(merge, &[f64v.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow_f64, &[ctx_ptr, ea]);+ let f64v_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[f64v_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ fpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);+}++pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ rs: u8,+ ea: Value,+ slow_f32: FuncRef,+) {+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let f32v = builder.ins().fdemote(types::F32, f64v);+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(MemFlags::new(), swapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ rs: u8,+ ea: Value,+ slow_f64: FuncRef,+) {+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(MemFlags::new(), swapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow_f64, &[ctx_ptr, ea, f64v]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+ update: bool,+) {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);++ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), false);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);+ let ps1 = if instr.w_16_16() {+ builder.ins().f64const(1.0)+ } else {+ let ea1 = builder.ins().iadd_imm(ea, 4);+ emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)+ };+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);+ ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let _ = pc;+ let _ = exit_block;+ emit_psq_load_quantized::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rd(),+ instr.w_16_16(),+ instr.i_17_19(),+ merge,+ local,+ );++ builder.switch_to_block(merge);+ builder.seal_block(merge);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+ }+}++pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ pc: u32,+ exit_block: Block,+ local: &LocalFuncs,+ update: bool,+) {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);++ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), true);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);+ if !instr.w_16_16() {+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let ea1 = builder.ins().iadd_imm(ea, 4);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);+ }+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let _ = pc;+ let _ = exit_block;+ emit_psq_store_quantized::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rs(),+ instr.w_16_16(),+ instr.i_17_19(),+ merge,+ local,+ );++ builder.switch_to_block(merge);+ builder.seal_block(merge);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);+ }+}++pub(crate) fn emit_gqr_type_zero_check<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ gqr_idx: u8,+ store: bool,+) -> Value {+ let off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;+ let gqr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let mask = if store { 0x7 } else { 0x70000 };+ let bits = builder.ins().band_imm(gqr, mask);+ builder.ins().icmp_imm(IntCC::Equal, bits, 0)+}++pub(crate) fn emit_psq_dequant_element(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ type_v: Value,+ scale_f32: Value,+ read_u8_fn: FuncRef,+ read_u16_fn: FuncRef,+) -> Value {+ let half_bit = builder.ins().band_imm(type_v, 1);+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);+ let signed_bit = builder.ins().band_imm(type_v, 2);+ let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);++ let byte_block = builder.create_block();+ let half_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F32);++ builder.ins().brif(is_half, half_block, &[], byte_block, &[]);++ builder.switch_to_block(byte_block);+ builder.seal_block(byte_block);+ let call = builder.ins().call(read_u8_fn, &[ctx_ptr, ea]);+ let raw_byte = builder.inst_results(call)[0];+ let trunc = builder.ins().ireduce(types::I8, raw_byte);+ let sext = builder.ins().sextend(types::I32, trunc);+ let val_i32 = builder.ins().select(is_signed, sext, raw_byte);+ let val_f32 = builder.ins().fcvt_from_sint(types::F32, val_i32);+ let scaled = builder.ins().fmul(val_f32, scale_f32);+ builder.ins().jump(merge, &[scaled.into()]);++ builder.switch_to_block(half_block);+ builder.seal_block(half_block);+ let call = builder.ins().call(read_u16_fn, &[ctx_ptr, ea]);+ let raw_half = builder.inst_results(call)[0];+ let trunc = builder.ins().ireduce(types::I16, raw_half);+ let sext = builder.ins().sextend(types::I32, trunc);+ let val_i32 = builder.ins().select(is_signed, sext, raw_half);+ let val_f32 = builder.ins().fcvt_from_sint(types::F32, val_i32);+ let scaled = builder.ins().fmul(val_f32, scale_f32);+ builder.ins().jump(merge, &[scaled.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let scaled_f32 = builder.block_params(merge)[0];+ builder.ins().fpromote(types::F64, scaled_f32)+}++pub(crate) fn emit_psq_quant_element(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ value_f64: Value,+ type_v: Value,+ scale_f32: Value,+ write_u8_fn: FuncRef,+ write_u16_fn: FuncRef,+) {+ let val_f32 = builder.ins().fdemote(types::F32, value_f64);+ let scaled_f32 = builder.ins().fmul(val_f32, scale_f32);++ let half_bit = builder.ins().band_imm(type_v, 1);+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);+ let signed_bit = builder.ins().band_imm(type_v, 2);+ let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);++ let zero_f = builder.ins().f32const(0.0_f32);+ let u8_max = builder.ins().f32const(255.0_f32);+ let u16_max = builder.ins().f32const(65535.0_f32);+ let i8_min = builder.ins().f32const(-128.0_f32);+ let i8_max = builder.ins().f32const(127.0_f32);+ let i16_min = builder.ins().f32const(-32768.0_f32);+ let i16_max = builder.ins().f32const(32767.0_f32);++ let signed_min = builder.ins().select(is_half, i16_min, i8_min);+ let min_f32 = builder.ins().select(is_signed, signed_min, zero_f);+ let signed_max = builder.ins().select(is_half, i16_max, i8_max);+ let unsigned_max = builder.ins().select(is_half, u16_max, u8_max);+ let max_f32 = builder.ins().select(is_signed, signed_max, unsigned_max);++ let lo_clamped = builder.ins().fmax(scaled_f32, min_f32);+ let clamped = builder.ins().fmin(lo_clamped, max_f32);++ let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, clamped);++ let mask_byte = builder.ins().iconst(types::I32, 0xFFi64);+ let mask_half = builder.ins().iconst(types::I32, 0xFFFFi64);+ let mask = builder.ins().select(is_half, mask_half, mask_byte);+ let val_to_write = builder.ins().band(val_i32, mask);++ let byte_block = builder.create_block();+ let half_block = builder.create_block();+ let merge = builder.create_block();+ builder.ins().brif(is_half, half_block, &[], byte_block, &[]);++ builder.switch_to_block(byte_block);+ builder.seal_block(byte_block);+ builder.ins().call(write_u8_fn, &[ctx_ptr, ea, val_to_write]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(half_block);+ builder.seal_block(half_block);+ builder.ins().call(write_u16_fn, &[ctx_ptr, ea, val_to_write]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_psq_elem_size(builder: &mut FunctionBuilder, type_v: Value) -> Value {+ let half_bit = builder.ins().band_imm(type_v, 1);+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);+ let two = builder.ins().iconst(types::I32, 2);+ let one = builder.ins().iconst(types::I32, 1);+ builder.ins().select(is_half, two, one)+}++pub(crate) fn emit_psq_load_quantized<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ fd: u8,+ w: bool,+ gqr_idx: u8,+ merge: Block,+ local: &LocalFuncs,+) {+ let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;+ let gqr_v = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, gqr_off);+ let type_shifted = builder.ins().ushr_imm(gqr_v, 16);+ let type_v = builder.ins().band_imm(type_shifted, 0x7);++ let q_bit = builder.ins().band_imm(type_v, 0x4);+ let is_quant = builder.ins().icmp_imm(IntCC::NotEqual, q_bit, 0);+ let inline_block = builder.create_block();+ let tramp_block = builder.create_block();+ builder.ins().brif(is_quant, inline_block, &[], tramp_block, &[]);++ builder.switch_to_block(inline_block);+ builder.seal_block(inline_block);+ let scale_idx_shifted = builder.ins().ushr_imm(gqr_v, 24);+ let scale_idx = builder.ins().band_imm(scale_idx_shifted, 0x3F);+ let scale_idx_64 = builder.ins().uextend(types::I64, scale_idx);+ let scale_offset = builder.ins().ishl_imm(scale_idx_64, 2);+ let table_addr = builder+ .ins()+ .iconst(types::I64, crate::gekko::jit::runtime::DEQUANT_TABLE.as_ptr() as i64);+ let scale_addr = builder.ins().iadd(table_addr, scale_offset);+ let scale_f32 = builder.ins().load(types::F32, MemFlags::trusted(), scale_addr, 0);++ let ps0 = emit_psq_dequant_element(builder, ctx_ptr, ea, type_v, scale_f32, local.read_u8, local.read_u16);+ let ps1 = if w {+ builder.ins().f64const(1.0)+ } else {+ let elem_size = emit_psq_elem_size(builder, type_v);+ let ea1 = builder.ins().iadd(ea, elem_size);+ emit_psq_dequant_element(builder, ctx_ptr, ea1, type_v, scale_f32, local.read_u8, local.read_u16)+ };+ fpr_store::<SYSTEM>(builder, ctx_ptr, fd, ps0);+ ps1_store::<SYSTEM>(builder, ctx_ptr, fd, ps1);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(tramp_block);+ builder.seal_block(tramp_block);+ let fd_v = builder.ins().iconst(types::I32, fd as i64);+ let w_v = builder.ins().iconst(types::I32, if w { 1 } else { 0 });+ let gqr_const = builder.ins().iconst(types::I32, gqr_idx as i64);+ builder+ .ins()+ .call(local.do_psq_load, &[ctx_ptr, fd_v, ea, w_v, gqr_const]);+ builder.ins().jump(merge, &[]);+}++pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ fs: u8,+ w: bool,+ gqr_idx: u8,+ merge: Block,+ local: &LocalFuncs,+) {+ let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;+ let gqr_v = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, gqr_off);+ let type_v = builder.ins().band_imm(gqr_v, 0x7);++ let q_bit = builder.ins().band_imm(type_v, 0x4);+ let is_quant = builder.ins().icmp_imm(IntCC::NotEqual, q_bit, 0);+ let inline_block = builder.create_block();+ let tramp_block = builder.create_block();+ builder.ins().brif(is_quant, inline_block, &[], tramp_block, &[]);++ builder.switch_to_block(inline_block);+ builder.seal_block(inline_block);+ let scale_idx_shifted = builder.ins().ushr_imm(gqr_v, 8);+ let scale_idx = builder.ins().band_imm(scale_idx_shifted, 0x3F);+ let scale_idx_64 = builder.ins().uextend(types::I64, scale_idx);+ let scale_offset = builder.ins().ishl_imm(scale_idx_64, 2);+ let table_addr = builder+ .ins()+ .iconst(types::I64, crate::gekko::jit::runtime::QUANT_TABLE.as_ptr() as i64);+ let scale_addr = builder.ins().iadd(table_addr, scale_offset);+ let scale_f32 = builder.ins().load(types::F32, MemFlags::trusted(), scale_addr, 0);++ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, fs);+ emit_psq_quant_element(+ builder,+ ctx_ptr,+ ea,+ ps0,+ type_v,+ scale_f32,+ local.write_u8,+ local.write_u16,+ );+ if !w {+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, fs);+ let elem_size = emit_psq_elem_size(builder, type_v);+ let ea1 = builder.ins().iadd(ea, elem_size);+ emit_psq_quant_element(+ builder,+ ctx_ptr,+ ea1,+ ps1,+ type_v,+ scale_f32,+ local.write_u8,+ local.write_u16,+ );+ }+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(tramp_block);+ builder.seal_block(tramp_block);+ let fs_v = builder.ins().iconst(types::I32, fs as i64);+ let w_v = builder.ins().iconst(types::I32, if w { 1 } else { 0 });+ let gqr_const = builder.ins().iconst(types::I32, gqr_idx as i64);+ builder+ .ins()+ .call(local.do_psq_store, &[ctx_ptr, fs_v, ea, w_v, gqr_const]);+ builder.ins().jump(merge, &[]);+}++pub(crate) fn emit_read_f32<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ slow_f32: FuncRef,+) -> Value {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::F64);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);+ let be_u32 = builder.ins().bswap(raw_u32);+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);+ let f64v = builder.ins().fpromote(types::F64, f32v);+ builder.ins().jump(merge, &[f64v.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);+ let f64v_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[f64v_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ builder.block_params(merge)[0]+}++pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ f64v: Value,+ slow_f32: FuncRef,+) {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let f32v = builder.ins().fdemote(types::F32, f64v);+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(MemFlags::new(), swapped, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+}++pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> bool {+ use crate::gekko::jit::lut;++ match OP {+ lut::OP_PS_ADD => {+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s0 = builder.ins().fadd(a0, b0);+ let r0 = round_to_single(builder, s0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s1 = builder.ins().fadd(a1, b1);+ let r1 = round_to_single(builder, s1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_SUB => {+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s0 = builder.ins().fsub(a0, b0);+ let r0 = round_to_single(builder, s0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s1 = builder.ins().fsub(a1, b1);+ let r1 = round_to_single(builder, s1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MUL => {+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let m0 = builder.ins().fmul(a0, c0);+ let r0 = round_to_single(builder, m0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let m1 = builder.ins().fmul(a1, c1);+ let r1 = round_to_single(builder, m1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_DIV => {+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let q0 = builder.ins().fdiv(a0, b0);+ let r0 = round_to_single(builder, q0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let q1 = builder.ins().fdiv(a1, b1);+ let r1 = round_to_single(builder, q1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MADD => {+ let r0 = ps_fma::<SYSTEM>(+ builder,+ ctx_ptr,+ instr.ra(),+ instr.fc(),+ instr.rb(),+ FmaSign::Pos,+ false,+ );+ let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MSUB => {+ let r0 = ps_fma::<SYSTEM>(+ builder,+ ctx_ptr,+ instr.ra(),+ instr.fc(),+ instr.rb(),+ FmaSign::Neg,+ false,+ );+ let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_NMADD => {+ let v0 = ps_fma::<SYSTEM>(+ builder,+ ctx_ptr,+ instr.ra(),+ instr.fc(),+ instr.rb(),+ FmaSign::Pos,+ false,+ );+ let r0 = builder.ins().fneg(v0);+ let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);+ let r1 = builder.ins().fneg(v1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_NMSUB => {+ let v0 = ps_fma::<SYSTEM>(+ builder,+ ctx_ptr,+ instr.ra(),+ instr.fc(),+ instr.rb(),+ FmaSign::Neg,+ false,+ );+ let r0 = builder.ins().fneg(v0);+ let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);+ let r1 = builder.ins().fneg(v1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MULS0 => {+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let m0 = builder.ins().fmul(a0, c0);+ let r0 = round_to_single(builder, m0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let m1 = builder.ins().fmul(a1, c0);+ let r1 = round_to_single(builder, m1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MULS1 => {+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let m0 = builder.ins().fmul(a0, c1);+ let r0 = round_to_single(builder, m0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let m1 = builder.ins().fmul(a1, c1);+ let r1 = round_to_single(builder, m1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MADDS0 => {+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let m0 = builder.ins().fmul(a0, c0);+ let s0 = builder.ins().fadd(m0, b0);+ let r0 = round_to_single(builder, s0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let m1 = builder.ins().fmul(a1, c0);+ let s1 = builder.ins().fadd(m1, b1);+ let r1 = round_to_single(builder, s1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MADDS1 => {+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let m0 = builder.ins().fmul(a0, c1);+ let s0 = builder.ins().fadd(m0, b0);+ let r0 = round_to_single(builder, s0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let m1 = builder.ins().fmul(a1, c1);+ let s1 = builder.ins().fadd(m1, b1);+ let r1 = round_to_single(builder, s1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_SUM0 => {+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s = builder.ins().fadd(a, b);+ let r0 = round_to_single(builder, s);+ let r1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_SUM1 => {+ let r0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s = builder.ins().fadd(a, b);+ let r1 = round_to_single(builder, s);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_SEL => {+ use cranelift_codegen::ir::condcodes::FloatCC;+ let zero = builder.ins().f64const(0.0);+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let cond0 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a0, zero);+ let r0 = builder.ins().select(cond0, c0, b0);+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let cond1 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a1, zero);+ let r1 = builder.ins().select(cond1, c1, b1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_RES => {+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r0 = f32_reciprocal(builder, b0);+ let r1 = f32_reciprocal(builder, b1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_RSQRTE => {+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r0 = f32_rsqrte(builder, b0);+ let r1 = f32_rsqrte(builder, b1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);+ return true;+ }+ lut::OP_PS_MR => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);+ true+ }+ lut::OP_PS_NEG => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let n0 = builder.ins().fneg(p0);+ let n1 = builder.ins().fneg(p1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);+ true+ }+ lut::OP_PS_ABS => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let a0 = builder.ins().fabs(p0);+ let a1 = builder.ins().fabs(p1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), a0, a1);+ true+ }+ lut::OP_PS_NABS => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let a0 = builder.ins().fabs(p0);+ let a1 = builder.ins().fabs(p1);+ let n0 = builder.ins().fneg(a0);+ let n1 = builder.ins().fneg(a1);+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);+ true+ }+ lut::OP_PS_MERGE00 => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let p1 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);+ true+ }+ lut::OP_PS_MERGE01 => {+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);+ true+ }+ lut::OP_PS_MERGE10 => {+ let p0 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let p1 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);+ true+ }+ lut::OP_PS_MERGE11 => {+ let p0 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);+ true+ }+ lut::OP_PS_CMPU0 | lut::OP_PS_CMPO0 => {+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);+ true+ }+ lut::OP_PS_CMPU1 | lut::OP_PS_CMPO1 => {+ let a = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);+ true+ }+ _ => false,+ }+}++#[derive(Clone, Copy)]+pub(crate) enum FmaSign {+ Pos,+ Neg,+}++pub(crate) fn ps_fma<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ fra: u8,+ frc: u8,+ frb: u8,+ sign: FmaSign,+ pick_ps1: bool,+) -> Value {+ let a = if pick_ps1 {+ ps1_load::<SYSTEM>(builder, ctx_ptr, fra)+ } else {+ fpr_load::<SYSTEM>(builder, ctx_ptr, fra)+ };+ let c = if pick_ps1 {+ ps1_load::<SYSTEM>(builder, ctx_ptr, frc)+ } else {+ fpr_load::<SYSTEM>(builder, ctx_ptr, frc)+ };+ let b = if pick_ps1 {+ ps1_load::<SYSTEM>(builder, ctx_ptr, frb)+ } else {+ fpr_load::<SYSTEM>(builder, ctx_ptr, frb)+ };+ let m = builder.ins().fmul(a, c);+ let r = match sign {+ FmaSign::Pos => builder.ins().fadd(m, b),+ FmaSign::Neg => builder.ins().fsub(m, b),+ };+ round_to_single(builder, r)+}++pub(crate) fn ps_write<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ rd: u8,+ ps0: Value,+ ps1: Value,+) {+ fpr_store::<SYSTEM>(builder, ctx_ptr, rd, ps0);+ ps1_store::<SYSTEM>(builder, ctx_ptr, rd, ps1);+}++pub(crate) fn emit_fp_compare<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ crfd: u8,+ a: Value,+ b: Value,+) {+ use cranelift_codegen::ir::condcodes::FloatCC;++ let a_nan = builder.ins().fcmp(FloatCC::NotEqual, a, a);+ let b_nan = builder.ins().fcmp(FloatCC::NotEqual, b, b);+ let so_b = builder.ins().bor(a_nan, b_nan);+ let so = builder.ins().uextend(types::I32, so_b);++ let lt_b = builder.ins().fcmp(FloatCC::LessThan, a, b);+ let gt_b = builder.ins().fcmp(FloatCC::GreaterThan, a, b);+ let eq_b = builder.ins().fcmp(FloatCC::Equal, a, b);+ let lt = builder.ins().uextend(types::I32, lt_b);+ let gt = builder.ins().uextend(types::I32, gt_b);+ let eq = builder.ins().uextend(types::I32, eq_b);+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);+ cr_set_field::<SYSTEM>(builder, ctx_ptr, crfd, nibble);+}++pub(crate) fn emit_fp_arith<const OP: u32, const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> bool {+ use crate::gekko::jit::lut;++ let single = matches!(+ OP,+ lut::OP_FADDSX+ | lut::OP_FSUBSX+ | lut::OP_FMULSX+ | lut::OP_FDIVSX+ | lut::OP_FMADDSX+ | lut::OP_FMSUBSX+ | lut::OP_FNMADDSX+ | lut::OP_FNMSUBSX+ | lut::OP_FSQRTSX+ | lut::OP_FRESX+ );++ match OP {+ lut::OP_FADDSX | lut::OP_FADDX => {+ emit_fp_binop::<SYSTEM>(+ builder,+ ctx_ptr,+ FpBinOp::Add,+ instr.rd(),+ instr.ra(),+ instr.rb(),+ single,+ );+ return true;+ }+ lut::OP_FSUBSX | lut::OP_FSUBX => {+ emit_fp_binop::<SYSTEM>(+ builder,+ ctx_ptr,+ FpBinOp::Sub,+ instr.rd(),+ instr.ra(),+ instr.rb(),+ single,+ );+ return true;+ }+ lut::OP_FMULSX | lut::OP_FMULX => {+ emit_fp_binop::<SYSTEM>(+ builder,+ ctx_ptr,+ FpBinOp::Mul,+ instr.rd(),+ instr.ra(),+ instr.fc(),+ single,+ );+ return true;+ }+ lut::OP_FDIVSX | lut::OP_FDIVX => {+ emit_fp_binop::<SYSTEM>(+ builder,+ ctx_ptr,+ FpBinOp::Div,+ instr.rd(),+ instr.ra(),+ instr.rb(),+ single,+ );+ return true;+ }+ lut::OP_FMADDSX+ | lut::OP_FMADDX+ | lut::OP_FMSUBSX+ | lut::OP_FMSUBX+ | lut::OP_FNMADDSX+ | lut::OP_FNMADDX+ | lut::OP_FNMSUBSX+ | lut::OP_FNMSUBX => {+ let is_add = matches!(+ OP,+ lut::OP_FMADDSX | lut::OP_FMADDX | lut::OP_FNMADDSX | lut::OP_FNMADDX+ );+ let neg = matches!(+ OP,+ lut::OP_FNMADDSX | lut::OP_FNMADDX | lut::OP_FNMSUBSX | lut::OP_FNMSUBX+ );+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let m = builder.ins().fmul(a, c);+ let core = if is_add {+ builder.ins().fadd(m, b)+ } else {+ builder.ins().fsub(m, b)+ };+ let signed = if neg { builder.ins().fneg(core) } else { core };+ let stored = if single {+ round_to_single(builder, signed)+ } else {+ signed+ };+ if single {+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);+ } else {+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);+ }+ return true;+ }+ lut::OP_FSQRTSX | lut::OP_FSQRTX => {+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().sqrt(b);+ let stored = if single { round_to_single(builder, r) } else { r };+ if single {+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);+ } else {+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);+ }+ return true;+ }+ lut::OP_FRESX => {+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = f32_reciprocal(builder, b);+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FRSQRTEX => {+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let s = builder.ins().sqrt(b);+ let one = builder.ins().f64const(1.0);+ let r = builder.ins().fdiv(one, s);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FSELX => {+ use cranelift_codegen::ir::condcodes::FloatCC;+ let zero = builder.ins().f64const(0.0);+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let cond = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a, zero);+ let r = builder.ins().select(cond, c, b);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FMRX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);+ return true;+ }+ lut::OP_FNEGX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().fneg(v);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FABSX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = builder.ins().fabs(v);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FNABSX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let a = builder.ins().fabs(v);+ let r = builder.ins().fneg(a);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FRSPX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let r = round_to_single(builder, v);+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);+ return true;+ }+ lut::OP_FCTIWZX | lut::OP_FCTIWX => {+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let i = builder.ins().fcvt_to_sint_sat(types::I32, v);+ let i64v = builder.ins().uextend(types::I64, i);+ let f = builder.ins().bitcast(types::F64, MemFlags::new(), i64v);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), f);+ return true;+ }+ lut::OP_MTFSB1X => {+ const FPSCR_ANY_X: u32 = 0x1FF8_0700;+ let bit_pos = 31 - (instr.crbd() as u32);+ let bit_value = 1u32 << bit_pos;+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let mut new_v = builder.ins().bor_imm(cur, bit_value as i64);+ if (bit_value & FPSCR_ANY_X) != 0 {+ let was_set = builder.ins().band_imm(cur, bit_value as i64);+ let was_unset = builder.ins().icmp_imm(IntCC::Equal, was_set, 0);+ let was_unset_v = builder.ins().uextend(types::I32, was_unset);+ let fx_bit = builder.ins().ishl_imm(was_unset_v, 31);+ new_v = builder.ins().bor(new_v, fx_bit);+ }+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);+ if instr.rc() {+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);+ }+ true+ }+ lut::OP_MTFSB0X => {+ let bit_pos = 31 - (instr.crbd() as u32);+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let new_v = builder.ins().band_imm(cur, !(1i64 << bit_pos));+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);+ if instr.rc() {+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);+ }+ true+ }+ lut::OP_MTFSFIX => {+ let dst_shift = 28 - 4 * (instr.crfd() as u32);+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let cleared = builder.ins().band_imm(cur, !(0xFi64 << dst_shift));+ let new_v = builder.ins().bor_imm(cleared, (instr.imm() as i64) << dst_shift);+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);+ if instr.rc() {+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);+ }+ true+ }+ lut::OP_MFFSX => {+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let cur_64 = builder.ins().uextend(types::I64, cur);+ let with_nan_hi = builder.ins().bor_imm(cur_64, 0xFFF8_0000_0000_0000u64 as i64);+ let f = builder.ins().bitcast(types::F64, MemFlags::new(), with_nan_hi);+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), f);+ if instr.rc() {+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);+ }+ true+ }+ lut::OP_MTFSFX => {+ let mut mask: u32 = 0;+ for i in 0..8 {+ if (instr.fm() as u32) & (1 << (7 - i)) != 0 {+ mask |= 0xFu32 << (28 - 4 * i);+ }+ }+ let off = abi::fpscr_offset::<SYSTEM>() as i32;+ let frb_v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ let frb_bits = builder.ins().bitcast(types::I64, MemFlags::new(), frb_v);+ let frb_low = builder.ins().ireduce(types::I32, frb_bits);+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);+ let cleared = builder.ins().band_imm(cur, !(mask as i64) & 0xFFFF_FFFF);+ let from_src = builder.ins().band_imm(frb_low, mask as i64);+ let new_v = builder.ins().bor(cleared, from_src);+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);+ if instr.rc() {+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);+ }+ true+ }+ lut::OP_FCMPU | lut::OP_FCMPO => {+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);+ true+ }+ lut::OP_MCRFS => {+ let src_shift = (7 - (instr.crfs() as u32)) * 4;+ let dst_shift = 28 - 4 * (instr.crfd() as u32);++ const FX_ANY_X: u32 = 0x9FF8_0700;+ let fpscr_off = abi::fpscr_offset::<SYSTEM>() as i32;+ let fpscr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, fpscr_off);++ let nibble = builder.ins().ushr_imm(fpscr, src_shift as i64);+ let nibble = builder.ins().band_imm(nibble, 0xF);++ let cr_off = abi::cr_offset::<SYSTEM>() as i32;+ let cr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, cr_off);+ let cr_cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));+ let positioned = builder.ins().ishl_imm(nibble, dst_shift as i64);+ let cr_new = builder.ins().bor(cr_cleared, positioned);+ builder.ins().store(MemFlags::trusted(), cr_new, ctx_ptr, cr_off);++ let clear_mask = (0xFu32 << src_shift) & FX_ANY_X;+ if clear_mask != 0 {+ let fpscr_new = builder.ins().band_imm(fpscr, !(clear_mask as i64) & 0xFFFF_FFFF);+ builder.ins().store(MemFlags::trusted(), fpscr_new, ctx_ptr, fpscr_off);+ }+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);+ true+ }+ _ => false,+ }+}++#[derive(Clone, Copy)]+pub(crate) enum FpBinOp {+ Add,+ Sub,+ Mul,+ Div,+}++pub(crate) fn emit_fp_binop<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ op: FpBinOp,+ frd: u8,+ fra: u8,+ frb_or_c: u8,+ single: bool,+) {+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, fra);+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, frb_or_c);++ let res = match op {+ FpBinOp::Add => builder.ins().fadd(a, b),+ FpBinOp::Sub => builder.ins().fsub(a, b),+ FpBinOp::Mul => builder.ins().fmul(a, b),+ FpBinOp::Div => builder.ins().fdiv(a, b),+ };+ let stored = if single {+ let demoted = builder.ins().fdemote(types::F32, res);+ builder.ins().fpromote(types::F64, demoted)+ } else {+ res+ };+ if single {+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, frd, stored);+ } else {+ fpr_store::<SYSTEM>(builder, ctx_ptr, frd, stored);+ }+}++pub(crate) fn emit_load_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rd(), instr.ra(), size, slow, update);+}++pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ rd: u8,+ ra: u8,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();+ builder.append_block_param(merge, types::I32);++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let flags = MemFlags::new();+ let raw_val = builder.ins().load(size.ir_type(), flags, addr, 0);+ let val_swapped = match size {+ MemSize::U8 => raw_val,+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw_val),+ };+ let val32 = match size {+ MemSize::U32 => val_swapped,+ MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, val_swapped),+ };+ builder.ins().jump(merge, &[val32.into()]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);+ let val32_slow = builder.inst_results(call)[0];+ builder.ins().jump(merge, &[val32_slow.into()]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);+ let val = builder.block_params(merge)[0];+ gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);+ }+}++pub(crate) fn emit_store<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);+}++pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);+}++pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ rs: u8,+ ra: u8,+ size: MemSize,+ slow: FuncRef,+ update: bool,+) {+ let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge = builder.create_block();++ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ builder+ .ins()+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);+ builder.append_block_param(fast_block, types::I64);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let addr = builder.block_params(fast_block)[0];+ let flags = MemFlags::new();+ let truncated = match size {+ MemSize::U32 => val32,+ MemSize::U16 => builder.ins().ireduce(types::I16, val32),+ MemSize::U8 => builder.ins().ireduce(types::I8, val32),+ };+ let val_to_store = match size {+ MemSize::U8 => truncated,+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),+ };+ builder.ins().store(flags, val_to_store, addr, 0);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(slow, &[ctx_ptr, ea, val32]);+ builder.ins().jump(merge, &[]);++ builder.switch_to_block(merge);+ builder.seal_block(merge);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);+ }+}++pub(crate) fn emit_addi<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let val = builder.ins().iadd_imm(base, instr.simm() as i64);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+}++pub(crate) fn emit_addis<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let val = builder.ins().iadd_imm(base, (instr.simm() as i64) << 16);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);+}++pub(crate) fn emit_ori<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let uimm = (instr.uimm() as u32) as i64;+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let val = builder.ins().bor_imm(src, uimm);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);+}++#[derive(Clone, Copy)]+pub(crate) enum ImmLogical {+ And,+ Xor,+}++pub(crate) fn emit_imm_logical<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ op: ImmLogical,+ shift_high: bool,+) -> Value {+ let uimm = (instr.uimm() as u32) as i64;+ let imm = if shift_high { uimm << 16 } else { uimm };+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let val = match op {+ ImmLogical::And => builder.ins().band_imm(src, imm),+ ImmLogical::Xor => builder.ins().bxor_imm(src, imm),+ };+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);+ val+}++pub(crate) fn emit_oris<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {+ let uimm = (instr.uimm() as u32) as i64;+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let val = builder.ins().bor_imm(src, uimm << 16);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);+}++pub(crate) fn emit_cmp_imm<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ signed: bool,+) {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let imm_val: i64 = if signed {+ instr.simm() as i64+ } else {+ (instr.uimm() as u32) as i64+ };+ let imm = builder.ins().iconst(types::I32, imm_val);+ emit_cmp_common::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, imm, signed);+}++pub(crate) fn emit_cmp_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ signed: bool,+) {+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());+ emit_cmp_common::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b, signed);+}++pub(crate) fn emit_cmp_common<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ crfd: u8,+ a: Value,+ b: Value,+ signed: bool,+) {+ let (lt_cc, gt_cc) = if signed {+ (IntCC::SignedLessThan, IntCC::SignedGreaterThan)+ } else {+ (IntCC::UnsignedLessThan, IntCC::UnsignedGreaterThan)+ };+ let lt = cmp_to_i32(builder, lt_cc, a, b);+ let gt = cmp_to_i32(builder, gt_cc, a, b);+ let eq = cmp_to_i32(builder, IntCC::Equal, a, b);+ let so = xer_so_bit::<SYSTEM>(builder, ctx_ptr);+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);+ cr_set_field::<SYSTEM>(builder, ctx_ptr, crfd, nibble);+}++pub(crate) fn emit_cntlzw<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let r = builder.ins().clz(s);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), r);+ r+}++pub(crate) fn emit_extend<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ from_ty: cranelift_codegen::ir::Type,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let truncated = builder.ins().ireduce(from_ty, s);+ let extended = builder.ins().sextend(types::I32, truncated);+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), extended);+ extended+}++#[derive(Clone, Copy)]+pub(crate) enum LogicalFullOp {+ Nor,+ Nand,+ Andc,+ Orc,+ Eqv,+}++pub(crate) fn emit_logical_xform_full<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ op: LogicalFullOp,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());++ let val = match op {+ LogicalFullOp::Nor => {+ let or = builder.ins().bor(s, b);+ builder.ins().bnot(or)+ }+ LogicalFullOp::Nand => {+ let and = builder.ins().band(s, b);+ builder.ins().bnot(and)+ }+ LogicalFullOp::Eqv => {+ let xor = builder.ins().bxor(s, b);+ builder.ins().bnot(xor)+ }+ LogicalFullOp::Andc => {+ let nb = builder.ins().bnot(b);+ builder.ins().band(s, nb)+ }+ LogicalFullOp::Orc => {+ let nb = builder.ins().bnot(b);+ builder.ins().bor(s, nb)+ }+ };++ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);+ val+}++pub(crate) fn emit_logical_xform<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ instr: Instruction,+ op: LogicalOp,+) -> Value {+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());++ let val = match op {+ LogicalOp::And => builder.ins().band(s, b),+ LogicalOp::Or => builder.ins().bor(s, b),+ LogicalOp::Xor => builder.ins().bxor(s, b),+ };++ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);+ val+}diff --git a/crates/gecko/src/idle.rs b/crates/gecko/src/idle.rsdeleted file mode 100644index 45e2b8f..0000000--- a/crates/gecko/src/idle.rs+++ /dev/null@@ -1,214 +0,0 @@-/// Maximum backward branch distance (in bytes) considered a potential polling loop.-pub const IDLE_LOOP_MAX_SIZE: u32 = 10 * 4;-pub const IDLE_LOOP_MAX_INSTRS: usize = (IDLE_LOOP_MAX_SIZE / 4 + 1) as usize;-/// Number of consecutive loop iterations before we attempt to skip.-const IDLE_LOOP_THRESHOLD: u32 = 10;--const MMIO_VIRT_START: u32 = 0xCC00_0000;-const MMIO_VIRT_END: u32 = 0xCC00_FFFF;--pub enum IdleCheck {- /// Nothing special, keep executing normally.- Continue,- /// Branch-to-self detected, always safe to skip to the next event.- Skip,- /// A backward-branch loop hit the threshold, caller must validate the loop- /// body and call [`IdleDetector::set_validated`] with the result.- Validate { start: u32, end: u32 },-}--pub struct IdleDetector {- loop_pc: Option<u32>,- loop_end: u32,- loop_count: u32,- validated: LoopStatus,-}--#[derive(Clone, Copy, PartialEq, Eq)]-enum LoopStatus {- /// Not yet checked.- Unknown,- /// Safe polling loop, skip allowed.- Safe,- /// Has side-effects, never skip.- Unsafe,-}--impl IdleDetector {- pub fn new() -> Self {- Self {- loop_pc: None,- loop_end: 0,- loop_count: 0,- validated: LoopStatus::Unknown,- }- }-- #[inline(always)]- pub fn check(&mut self, cia: u32, nia: u32) -> IdleCheck {- // Branching to current is always a safe idle loop- if nia == cia {- return IdleCheck::Skip;- }-- // Short backward branch, potential polling loop- if nia < cia && cia.wrapping_sub(nia) <= IDLE_LOOP_MAX_SIZE {- return self.track_backward_branch(nia, cia);- }-- self.try_reset(cia);- IdleCheck::Continue- }-- /// Record that the current loop was validated (or not) as a safe polling- /// loop. The result is cached until the loop changes.- pub fn set_validated(&mut self, safe: bool) {- self.validated = if safe { LoopStatus::Safe } else { LoopStatus::Unsafe };- }-- fn track_backward_branch(&mut self, target: u32, branch_pc: u32) -> IdleCheck {- if self.loop_pc == Some(target) {- self.loop_count += 1;- if self.loop_count >= IDLE_LOOP_THRESHOLD {- self.loop_count = 0;- return match self.validated {- LoopStatus::Safe => IdleCheck::Skip,- LoopStatus::Unsafe => IdleCheck::Continue,- LoopStatus::Unknown => IdleCheck::Validate {- start: target,- end: branch_pc,- },- };- }- } else {- // New loop- self.loop_pc = Some(target);- self.loop_end = branch_pc;- self.loop_count = 1;- self.validated = LoopStatus::Unknown;- }- IdleCheck::Continue- }-- fn try_reset(&mut self, cia: u32) {- if let Some(start) = self.loop_pc {- if cia < start || cia > self.loop_end {- self.loop_pc = None;- self.loop_count = 0;- self.validated = LoopStatus::Unknown;- }- }- }-}--/// Analyse a small backward-branch loop and decide whether it is a pure-/// MMIO-polling loop that is safe to skip.-///-/// Requirements:-/// * Every instruction is side effect free (no stores, no calls, no-/// supervisor stuff).-/// * At least one load targets the hardware register address range-/// (`0xCC00_0000 ..= 0xCC00_FFFF`).-pub fn validate_polling_loop(instrs: &[u32], gprs: &[u32; 32]) -> bool {- let mut has_mmio_load = false;-- for &raw in instrs {- let opcode = raw >> 26;- match opcode {- // D-form integer loads- // lwz, lwzu, lbz, lbzu, lhz, lhzu, lha, lhau- 32 | 33 | 34 | 35 | 40 | 41 | 42 | 43 => {- has_mmio_load |= d_form_targets_mmio(raw, gprs);- }- // D-form FP loads (lfs, lfsu, lfd, lfdu)- 48 | 49 | 50 | 51 => {}- // Load multiple word- 46 => {}- // ALU immediate / compares- // mulli, subfic, cmpli, cmpi, addic, addic., addi, addis- 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 => {}- // ori, oris, xori, xoris, andi., andis.- 24 | 25 | 26 | 27 | 28 | 29 => {}- // rlwimi, rlwinm, rlwnm- 20 | 21 | 23 => {}- // FP arith (single / double)- 59 | 63 => {}- // Branches (without link)- 16 | 18 => {- if raw & 1 != 0 {- return false; // bl / bcl = function call- }- }- // Extended integer ops (opcode 31)- 31 => {- let xo = (raw >> 1) & 0x3FF;- if is_unsafe_xo_31(xo) {- return false;- }- if is_indexed_load_xo(xo) {- has_mmio_load |= x_form_targets_mmio(raw, gprs);- }- }- // Rest- _ => return false,- }- }-- has_mmio_load-}--/// Check the effective address of a D-form load (rA + sign-ext d).-fn d_form_targets_mmio(raw: u32, gprs: &[u32; 32]) -> bool {- let ra = ((raw >> 16) & 0x1F) as usize;- let d = (raw & 0xFFFF) as i16 as i32;- let base = if ra == 0 { 0u32 } else { gprs[ra] };- let ea = (base as i32).wrapping_add(d) as u32;- (MMIO_VIRT_START..=MMIO_VIRT_END).contains(&ea)-}--/// Check the effective address of an X-form indexed load (rA + rB).-fn x_form_targets_mmio(raw: u32, gprs: &[u32; 32]) -> bool {- let ra = ((raw >> 16) & 0x1F) as usize;- let rb = ((raw >> 11) & 0x1F) as usize;- let base = if ra == 0 { 0u32 } else { gprs[ra] };- let ea = base.wrapping_add(gprs[rb]);- (MMIO_VIRT_START..=MMIO_VIRT_END).contains(&ea)-}--/// Opcode-31 extended ops that have side-effects (stores, supervisor writes,-/// I/O writes, cache-invalidate).-fn is_unsafe_xo_31(xo: u32) -> bool {- matches!(- xo,- // Integer stores- 150 | 151 | 183 | 215 | 247 | 407 | 439 | 661 | 662 | 725 | 918- // FP stores- | 663 | 695 | 727 | 759 | 983- // dcbz- | 1014- // Supervisor / SPR writes- | 146 // mtmsr- | 210 // mtsr- | 242 // mtsrin- | 467 // mtspr- // I/O write- | 438 // ecowx- )-}--/// Opcode-31 X-form indexed loads.-fn is_indexed_load_xo(xo: u32) -> bool {- matches!(- xo,- 23 // lwzx- | 55 // lwzux- | 87 // lbzx- | 119 // lbzux- | 279 // lhzx- | 311 // lhzux- | 343 // lhax- | 375 // lhaux- | 534 // lwbrx- | 790 // lhbrx- )-}diff --git a/crates/gecko/src/lib.rs b/crates/gecko/src/lib.rsindex 064a158..6c98a21 100644--- a/crates/gecko/src/lib.rs+++ b/crates/gecko/src/lib.rs@@ -6,7 +6,6 @@ pub mod gamecube;pub mod gekko;pub mod hollywood;pub mod host;-pub mod idle;pub mod input;pub mod ipl;pub mod mmio;diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rsindex 22b3339..37fafbc 100644--- a/crates/gecko/src/mmio.rs+++ b/crates/gecko/src/mmio.rs@@ -7,6 +7,11 @@ pub mod traits;use crate::system::{SystemId, WII};use constants::*;+pub const FASTMEM_LUT_PAGES: usize = 1 << 15;+pub const FASTMEM_PAGE_BYTES: usize = 1 << 17;+pub const FASTMEM_PAGE_MASK: u32 = (FASTMEM_PAGE_BYTES as u32) - 1;+pub const FASTMEM_PAGE_SHIFT: u32 = 17;+pub struct Mmio<const SYSTEM: SystemId> {pub ram: Vec<u8>,pub efb: Vec<u8>,@@ -14,6 +19,9 @@ pub struct Mmio<const SYSTEM: SystemId> {pub ipl: Vec<u8>,pub lcache: Vec<u8>,pub mem2: Vec<u8>,+ pub ram_ptr: usize,+ pub fastmem_lut: Vec<usize>,+ pub fastmem_lut_ptr: usize,}/// Read-only view over MEM1 plus (on Wii) MEM2, addressed by physical@@ -75,13 +83,55 @@ impl<'a> RamViewMut<'a> {impl<const SYSTEM: SystemId> Mmio<SYSTEM> {pub fn new() -> Self {+ let ram = vec![0u8; RAM_SIZE];+ let ram_ptr = ram.as_ptr() as usize;+ let mem2 = if SYSTEM == WII { vec![0; MEM2_SIZE] } else { Vec::new() };++ let mut fastmem_lut = vec![0usize; FASTMEM_LUT_PAGES];++ // Map MEM1 (24 MiB) at all three aliases:+ // * physical 0x0000_0000..0x017F_FFFF+ // * cached 0x8000_0000..0x817F_FFFF+ // * uncached 0xC000_0000..0xC17F_FFFF+ let mem1_pages = (RAM_SIZE / FASTMEM_PAGE_BYTES) as u32;+ let bases = [0x0000_0000u32, 0x8000_0000, 0xC000_0000];+ for base in bases {+ let lut_base = (base >> FASTMEM_PAGE_SHIFT) as usize;+ for i in 0..mem1_pages {+ let host_addr = ram_ptr + (i as usize) * FASTMEM_PAGE_BYTES;+ fastmem_lut[lut_base + i as usize] = host_addr;+ }+ }++ // Wii: also map MEM2 (64 MiB) at its three aliases:+ // * physical 0x1000_0000..0x13FF_FFFF+ // * cached 0x9000_0000..0x93FF_FFFF+ // * uncached 0xD000_0000..0xD3FF_FFFF+ if SYSTEM == WII {+ let mem2_pages = (MEM2_SIZE / FASTMEM_PAGE_BYTES) as u32;+ let mem2_ptr = mem2.as_ptr() as usize;+ let bases = [0x1000_0000u32, 0x9000_0000, 0xD000_0000];+ for base in bases {+ let lut_base = (base >> FASTMEM_PAGE_SHIFT) as usize;+ for i in 0..mem2_pages {+ let host_addr = mem2_ptr + (i as usize) * FASTMEM_PAGE_BYTES;+ fastmem_lut[lut_base + i as usize] = host_addr;+ }+ }+ }++ let fastmem_lut_ptr = fastmem_lut.as_ptr() as usize;+Mmio {- ram: vec![0; RAM_SIZE],+ ram,efb: vec![0; EFB_SIZE],hwr: vec![0; HW_REG_SIZE],ipl: vec![0; 0],lcache: vec![0; LCACHE_SIZE],- mem2: if SYSTEM == WII { vec![0; MEM2_SIZE] } else { Vec::new() },+ mem2,+ ram_ptr,+ fastmem_lut,+ fastmem_lut_ptr,}}diff --git a/crates/gecko/src/scheduler.rs b/crates/gecko/src/scheduler.rsindex b4b1132..bf3a770 100644--- a/crates/gecko/src/scheduler.rs+++ b/crates/gecko/src/scheduler.rs@@ -41,8 +41,8 @@ struct ScheduledEvent<const SYSTEM: SystemId> {pub struct Scheduler<const SYSTEM: SystemId> {pub cycles: u64,- next_deadline: u64,- timebase_offset: i64,+ pub(crate) next_deadline: u64,+ pub(crate) timebase_offset: i64,events: VecDeque<ScheduledEvent<SYSTEM>>,}diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rsindex aac6bf7..98ecd0b 100644--- a/crates/gecko/src/system.rs+++ b/crates/gecko/src/system.rs@@ -16,8 +16,6 @@ use crate::hollywood::Hollywood;#[cfg(feature = "hooks")]use crate::hooks::{HookFilters, HookFlags, HookState, Host};use crate::host::{EmptyRenderSink, RenderSink};-#[cfg(feature = "idle-skip")]-use crate::idle::{IDLE_LOOP_MAX_INSTRS, IdleCheck, IdleDetector};use crate::mmio::Mmio;use crate::scheduler::Scheduler;use crate::starlet::Starlet;@@ -55,15 +53,15 @@ pub struct System<const SYSTEM: SystemId> {/// AID DMA pushes 8-frame stereo s16 blocks here.pub audio_sink: Box<dyn AudioSink>,- #[cfg(feature = "idle-skip")]- pub(crate) idle: IdleDetector,-#[cfg(feature = "hooks")]pub hook_host: Option<Box<dyn Host<SYSTEM> + Send>>,#[cfg(feature = "hooks")]pub hook_flags: HookFlags,#[cfg(feature = "hooks")]pub hook_filters: HookFilters,++ #[cfg(feature = "jit")]+ pub jit: Option<Box<crate::gekko::jit::JitEngine<SYSTEM>>>,}impl<const SYSTEM: SystemId> System<SYSTEM> {@@ -91,15 +89,15 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {render_sink: Box::new(EmptyRenderSink),audio_sink: Box::new(EmptyAudioSink),- #[cfg(feature = "idle-skip")]- idle: IdleDetector::new(),-#[cfg(feature = "hooks")]hook_host: None,#[cfg(feature = "hooks")]hook_flags: HookFlags::empty(),#[cfg(feature = "hooks")]hook_filters: HookFilters::default(),++ #[cfg(feature = "jit")]+ jit: None,}}@@ -153,21 +151,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}}- #[cfg(feature = "idle-skip")]- match self.idle.check(self.gekko.cia, self.gekko.nia) {- IdleCheck::Skip => {- self.scheduler.cycles = self.scheduler.next_deadline();- }- IdleCheck::Validate { start, end } => {- let safe = self.is_polling_loop(start, end);- self.idle.set_validated(safe);- if safe {- self.scheduler.cycles = self.scheduler.next_deadline();- }- }- IdleCheck::Continue => {}- }-self.gekko.pc = self.gekko.nia;}@@ -195,26 +178,50 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.prepare_frame();while !self.vsync_pending {self.scheduler.refresh_deadline();- // Execute a slice of CPU instructions until the next event deadline- while self.scheduler.cycles < self.scheduler.next_deadline() {- self.step_cpu();+ #[cfg(feature = "jit")]+ {+ self.run_until_deadline_jit();+ }+ #[cfg(not(feature = "jit"))]+ {+ while self.scheduler.cycles < self.scheduler.next_deadline() {+ self.step_cpu();+ }}// Drain all events that are now dueself.drain_events();}}- /// Read the instructions in `[start, end]` and check whether the loop is a- /// side effect free MMIO polling loop that can safely be skipped.- #[cfg(feature = "idle-skip")]- #[inline(always)]- fn is_polling_loop(&self, start: u32, end: u32) -> bool {- let count = ((end - start) / 4 + 1) as usize;- let mut buf = [0u32; IDLE_LOOP_MAX_INSTRS];- for i in 0..count.min(buf.len()) {- buf[i] = self.mmio.fetch_instruction(start + (i as u32) * 4);+ /// JIT inner loop: runs compiled blocks back-to-back until+ /// `scheduler.cycles >= next_deadline`. Interrupts are checked at block+ /// boundaries.+ #[cfg(feature = "jit")]+ fn run_until_deadline_jit(&mut self) {+ let mut jit = match self.jit.take() {+ Some(jit) => jit,+ None => Box::new(crate::gekko::jit::JitEngine::<SYSTEM>::new()),+ };++ while self.scheduler.cycles < self.scheduler.next_deadline() {+ if self.gekko.msr.external_interrupt_enable() {+ if self.pi.interrupt_pending() {+ self.cause_external_interrupt();+ self.scheduler.cycles += 2;+ continue;+ }++ if self.gekko.dec.interrupt_pending() {+ self.cause_decrementer_interrupt();+ self.scheduler.cycles += 2;+ continue;+ }+ }++ jit.run_block(self);}- crate::idle::validate_polling_loop(&buf[..count.min(buf.len())], &self.gekko.gprs)++ self.jit = Some(jit);}pub fn frame_size(&self) -> (usize, usize) {diff --git a/crates/gecko/wii_gekko_jit.bindings.chipi b/crates/gecko/wii_gekko_jit.bindings.chipinew file mode 100644index 0000000..eedbd87--- /dev/null+++ b/crates/gecko/wii_gekko_jit.bindings.chipi@@ -0,0 +1,276 @@+include "../../submodules/chipi-spec/gamecube/gekko.chipi"++target rust {+ dispatch Gekko {+ output "$OUT_DIR/gekko_jit_lut_wii.rs"++ context crate::gekko::jit::translator::JitTranslator+ handlers crate::gekko::jit::handlers+ handler_const crate::system::WII+ strategy fn_ptr_lut++ invalid_handler crate::gekko::jit::handlers::invalid++ instruction_type crate::gekko::instruction::Instruction {+ output "$OUT_DIR/gekko_jit_instr_wii.rs"+ }++ handler branch {+ bx+ bcx+ bclrx+ bcctrx+ }++ handler alu {+ ori+ oris+ addx+ addi+ addis+ xori+ xoris+ andi_dot+ andis_dot+ subfx+ negx+ addcx+ subfcx+ addex+ subfex+ addzex+ subfzex+ addmex+ subfmex+ mullwx+ divwux+ divwx+ mulhwux+ mulhwx+ subfic+ addic+ addic_dot+ mulli+ }++ handler rotate {+ rlwinmx+ rlwimix+ rlwnmx+ }++ handler msr {+ mtmsr+ mfmsr+ rfi+ }++ handler spr {+ mtspr+ mfspr+ }++ handler segment {+ mtsr+ mfsr+ }++ handler store_load {+ stw+ stwu+ sth+ sthu+ lwz+ lwzu+ lbz+ lbzu+ stb+ stbu+ lhz+ lhzu+ lha+ lhau+ lmw+ stmw+ lwzx+ lbzx+ lhzx+ lhax+ stwx+ stbx+ sthx+ lwzux+ lbzux+ lhzux+ lhaux+ stwux+ stbux+ sthux+ lwbrx+ lhbrx+ stwbrx+ sthbrx+ }++ handler store_load_fp {+ lfd+ lfdu+ stfd+ stfdu+ lfs+ lfsu+ stfs+ stfsu+ lfsx+ lfsux+ lfdx+ lfdux+ stfsx+ stfsux+ stfdx+ stfdux+ stfiwx+ }++ handler store_load_psq {+ psq_l+ psq_lu+ psq_st+ psq_stu+ psq_lx+ psq_lux+ psq_stx+ psq_stux+ }++ handler compare {+ cmp+ cmpi+ cmpli+ cmpl+ }++ handler nop {+ isync+ sync+ eieio+ dcbf+ dcbi+ dcbst+ dcbt+ dcbtst+ dcba+ dcbz+ dcbz_l+ icbi+ tlbie+ tlbia+ tlbsync+ tlbld+ tlbli+ }++ handler logical {+ andx+ orx+ xorx+ nandx+ norx+ andcx+ orcx+ eqvx+ slwx+ srwx+ srawx+ srawix+ cntlzwx+ extshx+ extsbx+ }++ handler cr_ops {+ mtcrf+ mfcr+ crxor+ cror+ crand+ creqv+ crnor+ crnand+ crandc+ crorc+ mcrf+ }++ handler fp_ops {+ mtfsfx+ mffsx+ mtfsb0x+ mtfsb1x+ mtfsfix+ mcrfs+ fmrx+ fnegx+ fabsx+ fnabsx+ frspx+ fctiwx+ fctiwzx+ fcmpu+ fcmpo+ faddx+ fsubx+ fmulx+ fdivx+ fmaddx+ fmsubx+ fnmaddx+ fnmsubx+ faddsx+ fsubsx+ fmulsx+ fdivsx+ fmaddsx+ fmsubsx+ fnmaddsx+ fnmsubsx+ fsqrtsx+ fresx+ fsqrtx+ fselx+ frsqrtex+ }++ handler ps_ops {+ ps_cmpu0+ ps_cmpo0+ ps_cmpu1+ ps_cmpo1+ ps_res+ ps_rsqrte+ ps_neg+ ps_mr+ ps_nabs+ ps_abs+ ps_merge00+ ps_merge01+ ps_merge10+ ps_merge11+ ps_sum0+ ps_sum1+ ps_muls0+ ps_muls1+ ps_madds0+ ps_madds1+ ps_div+ ps_sub+ ps_add+ ps_sel+ ps_mul+ ps_msub+ ps_madd+ ps_nmsub+ ps_nmadd+ }+ }+}diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.tomlindex beda2ca..db20d7b 100644--- a/crates/web/Cargo.toml+++ b/crates/web/Cargo.toml@@ -10,7 +10,7 @@ crate-type = ["cdylib"]debug = ["dep:dbglib", "dep:egui-phosphor"][dependencies]-gecko = { path = "../gecko" }+gecko = { path = "../gecko", default-features = false }image = { path = "../image", default-features = false }backend-wgpu = { path = "../backend-wgpu" }dbglib = { path = "../dbglib", optional = true }