Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

ppc jit merge

Layle389f73681607
diff --git a/.gitignore b/.gitignore
index 306bcef..4c87172 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ __pycache__/
/fs/
screenshots/
textures/
+screenshotdb/
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 27e5197..40a9941 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -181,6 +181,12 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+[[package]]
+name = "arbitrary"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
+
[[package]]
name = "arboard"
version = "3.6.1"
@@ -359,6 +365,9 @@ name = "bumpalo"
version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+dependencies = [
+ "allocator-api2",
+]
[[package]]
name = "bytemuck"
@@ -727,7 +736,7 @@ dependencies = [
"jni 0.21.1",
"js-sys",
"libc",
- "mach2",
+ "mach2 0.5.0",
"ndk",
"ndk-context",
"num-derive",
@@ -754,6 +763,173 @@ dependencies = [
"libc",
]
+[[package]]
+name = "cranelift-assembler-x64"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8628cc4ba7f88a9205a7ee42327697abc61195a1e3d92cfae172d6a946e722e"
+dependencies = [
+ "cranelift-assembler-x64-meta",
+]
+
+[[package]]
+name = "cranelift-assembler-x64-meta"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d582754487e6c9a065a91c42ccf1bdd8d5977af33468dac5ae9bec0ce88acb3e"
+dependencies = [
+ "cranelift-srcgen",
+]
+
+[[package]]
+name = "cranelift-bforest"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb59c81ace12ee7c33074db7903d4d75d1f40b28cd3e8e6f491de57b29129eb9"
+dependencies = [
+ "cranelift-entity",
+ "wasmtime-internal-core",
+]
+
+[[package]]
+name = "cranelift-bitset"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f25c06993a681be9cf3140798a3d4ac5bec955e7444416a2fdc87fda8567285d"
+dependencies = [
+ "wasmtime-internal-core",
+]
+
+[[package]]
+name = "cranelift-codegen"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b61f95c5a211918f5d336254a61a488b36a5818de47a868e8c4658dce9cccc"
+dependencies = [
+ "bumpalo",
+ "cranelift-assembler-x64",
+ "cranelift-bforest",
+ "cranelift-bitset",
+ "cranelift-codegen-meta",
+ "cranelift-codegen-shared",
+ "cranelift-control",
+ "cranelift-entity",
+ "cranelift-isle",
+ "gimli",
+ "hashbrown 0.16.1",
+ "libm",
+ "log",
+ "regalloc2",
+ "rustc-hash 2.1.2",
+ "serde",
+ "smallvec",
+ "target-lexicon",
+ "wasmtime-internal-core",
+]
+
+[[package]]
+name = "cranelift-codegen-meta"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b85aa822fce72080d041d7c2cf7c3f5c6ecdea7afae68379ba4ef85269c4fa5"
+dependencies = [
+ "cranelift-assembler-x64-meta",
+ "cranelift-codegen-shared",
+ "cranelift-srcgen",
+ "heck",
+]
+
+[[package]]
+name = "cranelift-codegen-shared"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "833eb9fc89326cd072cc19e96892f09b5692c0dfe17cd4da2858ba30c2cd85c0"
+
+[[package]]
+name = "cranelift-control"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d005320f487e6e8a3edcc7f2fd4f43fcc9946d1013bf206ea649789ac1617fc"
+dependencies = [
+ "arbitrary",
+]
+
+[[package]]
+name = "cranelift-entity"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e62ef34c6e720f347a79ece043e8584e242d168911da640bac654a33a6aaaf5"
+dependencies = [
+ "cranelift-bitset",
+ "wasmtime-internal-core",
+]
+
+[[package]]
+name = "cranelift-frontend"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfa2ad00399dd47e7e7e33cb1dc23b0e39ed9dcd01e8f026fc37af91655031b8"
+dependencies = [
+ "cranelift-codegen",
+ "log",
+ "smallvec",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-isle"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02c51975ed217b4e8e5a7fd11e9ec83a96104bdff311dddcb505d1d8a9fd7fc6"
+
+[[package]]
+name = "cranelift-jit"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4070d2acc5c976887c10c273d38dd2bebebb472fd1ba65fa17b548adf5f56350"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-control",
+ "cranelift-entity",
+ "cranelift-module",
+ "cranelift-native",
+ "libc",
+ "log",
+ "region",
+ "target-lexicon",
+ "wasmtime-internal-jit-icache-coherence",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "cranelift-module"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "052a396328dbc7dadc29d1bd27d4aa57d9e9e493ead8ef6e2ab3b75c1bf9644c"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-control",
+]
+
+[[package]]
+name = "cranelift-native"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9b1889e00da9729d8f8525f3c12998ded86ea709058ff844ebe00b97548de0e"
+dependencies = [
+ "cranelift-codegen",
+ "libc",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-srcgen"
+version = "0.131.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5a8f82fd5124f009f72167e60139245cd3b56cfd4b53050f22110c48c5f4da1"
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -1372,6 +1548,11 @@ dependencies = [
"bytemuck",
"chapa",
"chipi-build",
+ "cranelift-codegen",
+ "cranelift-frontend",
+ "cranelift-jit",
+ "cranelift-module",
+ "cranelift-native",
"crc32fast",
"crossbeam-channel",
"disasm",
@@ -1417,6 +1598,18 @@ dependencies = [
"wasip2",
]
+[[package]]
+name = "gimli"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c"
+dependencies = [
+ "fnv",
+ "hashbrown 0.16.1",
+ "indexmap",
+ "stable_deref_trait",
+]
+
[[package]]
name = "gl_generator"
version = "0.14.0"
@@ -1531,6 +1724,12 @@ dependencies = [
"foldhash 0.2.0",
]
+[[package]]
+name = "hashbrown"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+
[[package]]
name = "heck"
version = "0.5.0"
@@ -2100,6 +2299,15 @@ dependencies = [
"cc",
]
+[[package]]
+name = "mach2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "mach2"
version = "0.5.0"
@@ -3080,6 +3288,20 @@ dependencies = [
"bitflags 2.11.0",
]
+[[package]]
+name = "regalloc2"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186"
+dependencies = [
+ "allocator-api2",
+ "bumpalo",
+ "hashbrown 0.17.0",
+ "log",
+ "rustc-hash 2.1.2",
+ "smallvec",
+]
+
[[package]]
name = "regex"
version = "1.12.3"
@@ -3109,6 +3331,18 @@ version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+[[package]]
+name = "region"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "mach2 0.4.3",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "relative-path"
version = "1.9.3"
@@ -3569,6 +3803,12 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
[[package]]
name = "term"
version = "1.2.1"
@@ -4032,6 +4272,28 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "wasmtime-internal-core"
+version = "44.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f2c7fa6523647262bfb4095dbdf4087accefe525813e783f81a0c682f418ce4"
+dependencies = [
+ "hashbrown 0.16.1",
+ "libm",
+]
+
+[[package]]
+name = "wasmtime-internal-jit-icache-coherence"
+version = "44.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a1859e920871515d324fb9757c3e448d6ed1512ca6ccdff14b6e016505d6ada"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasmtime-internal-core",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "wayland-backend"
version = "0.3.14"
diff --git a/crates/gecko/Cargo.toml b/crates/gecko/Cargo.toml
index b1b64e9..ea790d3 100644
--- a/crates/gecko/Cargo.toml
+++ b/crates/gecko/Cargo.toml
@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2024"
[features]
+default = ["jit", "idle-skip"]
idle-skip = []
hooks = []
hooks-mut-traps = ["hooks"]
@@ -14,6 +15,15 @@ hooks-mut-traps = ["hooks"]
efb-writeback = ["dep:crossbeam-channel"]
# Enable the WAV file audio sink
audio-wav-dump = ["dep:hound"]
+# Cranelift JIT for Gekko
+jit = [
+ "dep:cranelift-codegen",
+ "dep:cranelift-frontend",
+ "dep:cranelift-jit",
+ "dep:cranelift-module",
+ "dep:cranelift-native",
+]
+jit-stats = ["jit"]
[build-dependencies]
chipi-build = "0.9.1"
@@ -32,3 +42,8 @@ multiversion = "0.8.0"
rustc-hash = "2.1.2"
crc32fast = "1.5.0"
zerocopy = { version = "0.8.48", features = ["derive"] }
+cranelift-codegen = { version = "0.131.1", optional = true }
+cranelift-frontend = { version = "0.131.1", optional = true }
+cranelift-jit = { version = "0.131.1", optional = true }
+cranelift-module = { version = "0.131.1", optional = true }
+cranelift-native = { version = "0.131.1", optional = true }
diff --git a/crates/gecko/build.rs b/crates/gecko/build.rs
index 8df98a1..2eb6c25 100644
--- a/crates/gecko/build.rs
+++ b/crates/gecko/build.rs
@@ -3,4 +3,6 @@ fn main() {
chipi_build::generate_bindings("dsp.bindings.chipi").expect("chipi codegen failed (dsp)");
chipi_build::generate_bindings("wii_gekko.bindings.chipi").expect("chipi codegen failed (wii gekko)");
chipi_build::generate_bindings("wii_dsp.bindings.chipi").expect("chipi codegen failed (wii dsp)");
+ chipi_build::generate_bindings("gekko_jit.bindings.chipi").expect("chipi codegen failed (gekko jit)");
+ chipi_build::generate_bindings("wii_gekko_jit.bindings.chipi").expect("chipi codegen failed (wii gekko jit)");
}
diff --git a/crates/gecko/gekko_jit.bindings.chipi b/crates/gecko/gekko_jit.bindings.chipi
new file mode 100644
index 0000000..17e6c2a
--- /dev/null
+++ b/crates/gecko/gekko_jit.bindings.chipi
@@ -0,0 +1,276 @@
+include "../../submodules/chipi-spec/gamecube/gekko.chipi"
+
+target rust {
+ dispatch Gekko {
+ output "$OUT_DIR/gekko_jit_lut.rs"
+
+ context crate::gekko::jit::translator::JitTranslator
+ handlers crate::gekko::jit::handlers
+ handler_const crate::system::GC
+ strategy fn_ptr_lut
+
+ invalid_handler crate::gekko::jit::handlers::invalid
+
+ instruction_type crate::gekko::instruction::Instruction {
+ output "$OUT_DIR/gekko_jit_instr.rs"
+ }
+
+ handler branch {
+ bx
+ bcx
+ bclrx
+ bcctrx
+ }
+
+ handler alu {
+ ori
+ oris
+ addx
+ addi
+ addis
+ xori
+ xoris
+ andi_dot
+ andis_dot
+ subfx
+ negx
+ addcx
+ subfcx
+ addex
+ subfex
+ addzex
+ subfzex
+ addmex
+ subfmex
+ mullwx
+ divwux
+ divwx
+ mulhwux
+ mulhwx
+ subfic
+ addic
+ addic_dot
+ mulli
+ }
+
+ handler rotate {
+ rlwinmx
+ rlwimix
+ rlwnmx
+ }
+
+ handler msr {
+ mtmsr
+ mfmsr
+ rfi
+ }
+
+ handler spr {
+ mtspr
+ mfspr
+ }
+
+ handler segment {
+ mtsr
+ mfsr
+ }
+
+ handler store_load {
+ stw
+ stwu
+ sth
+ sthu
+ lwz
+ lwzu
+ lbz
+ lbzu
+ stb
+ stbu
+ lhz
+ lhzu
+ lha
+ lhau
+ lmw
+ stmw
+ lwzx
+ lbzx
+ lhzx
+ lhax
+ stwx
+ stbx
+ sthx
+ lwzux
+ lbzux
+ lhzux
+ lhaux
+ stwux
+ stbux
+ sthux
+ lwbrx
+ lhbrx
+ stwbrx
+ sthbrx
+ }
+
+ handler store_load_fp {
+ lfd
+ lfdu
+ stfd
+ stfdu
+ lfs
+ lfsu
+ stfs
+ stfsu
+ lfsx
+ lfsux
+ lfdx
+ lfdux
+ stfsx
+ stfsux
+ stfdx
+ stfdux
+ stfiwx
+ }
+
+ handler store_load_psq {
+ psq_l
+ psq_lu
+ psq_st
+ psq_stu
+ psq_lx
+ psq_lux
+ psq_stx
+ psq_stux
+ }
+
+ handler compare {
+ cmp
+ cmpi
+ cmpli
+ cmpl
+ }
+
+ handler nop {
+ isync
+ sync
+ eieio
+ dcbf
+ dcbi
+ dcbst
+ dcbt
+ dcbtst
+ dcba
+ dcbz
+ dcbz_l
+ icbi
+ tlbie
+ tlbia
+ tlbsync
+ tlbld
+ tlbli
+ }
+
+ handler logical {
+ andx
+ orx
+ xorx
+ nandx
+ norx
+ andcx
+ orcx
+ eqvx
+ slwx
+ srwx
+ srawx
+ srawix
+ cntlzwx
+ extshx
+ extsbx
+ }
+
+ handler cr_ops {
+ mtcrf
+ mfcr
+ crxor
+ cror
+ crand
+ creqv
+ crnor
+ crnand
+ crandc
+ crorc
+ mcrf
+ }
+
+ handler fp_ops {
+ mtfsfx
+ mffsx
+ mtfsb0x
+ mtfsb1x
+ mtfsfix
+ mcrfs
+ fmrx
+ fnegx
+ fabsx
+ fnabsx
+ frspx
+ fctiwx
+ fctiwzx
+ fcmpu
+ fcmpo
+ faddx
+ fsubx
+ fmulx
+ fdivx
+ fmaddx
+ fmsubx
+ fnmaddx
+ fnmsubx
+ faddsx
+ fsubsx
+ fmulsx
+ fdivsx
+ fmaddsx
+ fmsubsx
+ fnmaddsx
+ fnmsubsx
+ fsqrtsx
+ fresx
+ fsqrtx
+ fselx
+ frsqrtex
+ }
+
+ handler ps_ops {
+ ps_cmpu0
+ ps_cmpo0
+ ps_cmpu1
+ ps_cmpo1
+ ps_res
+ ps_rsqrte
+ ps_neg
+ ps_mr
+ ps_nabs
+ ps_abs
+ ps_merge00
+ ps_merge01
+ ps_merge10
+ ps_merge11
+ ps_sum0
+ ps_sum1
+ ps_muls0
+ ps_muls1
+ ps_madds0
+ ps_madds1
+ ps_div
+ ps_sub
+ ps_add
+ ps_sel
+ ps_mul
+ ps_msub
+ ps_madd
+ ps_nmsub
+ ps_nmadd
+ }
+ }
+}
diff --git a/crates/gecko/src/gekko.rs b/crates/gecko/src/gekko.rs
index 881cd67..fdf31d5 100644
--- a/crates/gecko/src/gekko.rs
+++ b/crates/gecko/src/gekko.rs
@@ -5,6 +5,8 @@ pub mod fpscr;
pub mod instruction;
pub mod interpreter;
pub mod irq;
+#[cfg(feature = "jit")]
+pub mod jit;
pub mod msr;
pub mod spr;
pub mod sr;
@@ -37,12 +39,15 @@ pub struct Gekko {
// These are used during instruction execution to track the current
// and next PC values. In essence, by writing to `next_pc`, instructions
// can change the flow of execution (e.g. for branches and jumps).
- pub cia: u32, // Current Instruction Address
- pub nia: u32, // Next Instruction Address
- pub reserve_addr: Option<u32>, // lwarx/stwcx. reservation address
+ pub cia: u32, // Current Instruction Address
+ pub nia: u32, // Next Instruction Address
+ pub reserve_addr: u32,
}
impl Gekko {
+ /// Sentinel used by `lwarx` / `stwcx.`
+ pub const NO_RESERVATION: u32 = 0xFF69_1337;
+
pub fn new(initial_pc: u32) -> Self {
let mut spr = spr::Spr::default();
spr.dec = u32::MAX;
@@ -60,7 +65,7 @@ impl Gekko {
msr: msr::Msr::default(),
fpscr: fpscr::Fpscr::default(),
sr: [sr::Sr::default(); 16],
- reserve_addr: None,
+ reserve_addr: Self::NO_RESERVATION,
}
}
diff --git a/crates/gecko/src/gekko/interpreter/store_load.rs b/crates/gecko/src/gekko/interpreter/store_load.rs
index d17d2f7..92def56 100644
--- a/crates/gecko/src/gekko/interpreter/store_load.rs
+++ b/crates/gecko/src/gekko/interpreter/store_load.rs
@@ -191,7 +191,7 @@ pub fn lwarx<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instructio
.wrapping_add(ctx.gekko.read_gpr(instr.rb()));
let val = ctx.read_u32(addr);
ctx.gekko.write_gpr(instr.rd(), val);
- ctx.gekko.reserve_addr = Some(addr);
+ ctx.gekko.reserve_addr = addr;
}
#[inline(always)]
@@ -201,10 +201,10 @@ pub fn stwcx_dot<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instru
.read_gpr_or_zero(instr.ra())
.wrapping_add(ctx.gekko.read_gpr(instr.rb()));
let so = ctx.gekko.spr.xer.summary_overflow();
- let store_performed = ctx.gekko.reserve_addr == Some(addr);
+ let store_performed = ctx.gekko.reserve_addr == addr;
if store_performed {
ctx.write_u32(addr, ctx.gekko.read_gpr(instr.rs()));
- ctx.gekko.reserve_addr = None;
+ ctx.gekko.reserve_addr = crate::gekko::Gekko::NO_RESERVATION;
ctx.gekko.cr.set_cr0(ConditionField::new().with_eq(true).with_so(so));
} else {
ctx.gekko.cr.set_cr0(ConditionField::new().with_so(so));
diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rs
new file mode 100644
index 0000000..8a47fe2
--- /dev/null
+++ b/crates/gecko/src/gekko/jit.rs
@@ -0,0 +1,805 @@
+pub mod abi;
+pub mod block;
+pub mod handlers;
+pub mod idle;
+pub mod insn;
+pub mod runtime;
+pub mod translator;
+
+#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]
+pub mod lut {
+ include!(concat!(env!("OUT_DIR"), "/gekko_jit_lut.rs"));
+}
+
+#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]
+pub mod lut_wii {
+ include!(concat!(env!("OUT_DIR"), "/gekko_jit_lut_wii.rs"));
+}
+
+use cranelift_codegen::Context;
+use cranelift_codegen::ir::{AbiParam, InstBuilder, Signature, types};
+use cranelift_codegen::isa::CallConv;
+use cranelift_codegen::settings::{self, Configurable};
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
+use cranelift_jit::{JITBuilder, JITModule};
+use cranelift_module::{FuncId, Linkage, Module, default_libcall_names};
+use rustc_hash::FxHashMap;
+
+use crate::system::{GC, System, SystemId, WII};
+
+pub type BlockEntry = usize;
+
+type TrampolineFn = unsafe extern "C" fn(*mut core::ffi::c_void, usize) -> u32;
+
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub struct BlockLookupSlot {
+ pub pc: u32,
+ pub _pad: u32,
+ pub entry: usize,
+}
+
+pub const BLOCK_LOOKUP_TABLE_SIZE: usize = 8192;
+pub const BLOCK_LOOKUP_TABLE_MASK: u32 = (BLOCK_LOOKUP_TABLE_SIZE as u32) - 1;
+
+#[derive(Clone, Copy)]
+pub struct ExternFuncs {
+ pub cause_invalid_opcode: FuncId,
+ pub advance_to_deadline: FuncId,
+ pub read_u8: FuncId,
+ pub read_u16: FuncId,
+ pub read_u32: FuncId,
+ pub write_u8: FuncId,
+ pub write_u16: FuncId,
+ pub write_u32: FuncId,
+ pub read_f32: FuncId,
+ pub read_f64: FuncId,
+ pub write_f32: FuncId,
+ pub write_f64: FuncId,
+ pub write_msr: FuncId,
+ pub read_spr: FuncId,
+ pub write_spr: FuncId,
+ pub read_sr: FuncId,
+ pub write_sr: FuncId,
+ pub cause_trap_exception: FuncId,
+ pub cause_syscall_interrupt: FuncId,
+ pub do_rfi: FuncId,
+ pub cause_fp_unavailable: FuncId,
+ pub set_reservation: FuncId,
+ pub try_clear_reservation: FuncId,
+ pub do_lswi: FuncId,
+ pub do_stswi: FuncId,
+ pub do_lswx: FuncId,
+ pub do_stswx: FuncId,
+ pub do_psq_load: FuncId,
+ pub do_psq_store: FuncId,
+ pub read_timebase: FuncId,
+}
+
+pub struct JitEngine<const SYSTEM: SystemId> {
+ module: JITModule,
+ ctx: Context,
+ builder_ctx: FunctionBuilderContext,
+ cache: FxHashMap<u32, BlockEntry>,
+ block_func_ids: FxHashMap<u32, FuncId>,
+ pending_chain_slots: FxHashMap<u32, Vec<usize>>,
+ #[cfg(feature = "jit-stats")]
+ hits: FxHashMap<u32, u64>,
+ block_sig: Signature,
+ extern_funcs: ExternFuncs,
+ trampoline_fn: TrampolineFn,
+ block_lookup_table_addr: usize,
+ block_seq: u64,
+}
+
+impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
+ pub fn new() -> Self {
+ let mut flag_builder = settings::builder();
+ flag_builder.set("use_colocated_libcalls", "false").unwrap();
+ flag_builder.set("is_pic", "false").unwrap();
+ flag_builder.set("opt_level", "speed").unwrap();
+ flag_builder.set("preserve_frame_pointers", "true").unwrap();
+ flag_builder.set("enable_verifier", "false").unwrap();
+ flag_builder.set("enable_probestack", "false").unwrap();
+ flag_builder.set("unwind_info", "false").unwrap();
+ flag_builder
+ .set("enable_heap_access_spectre_mitigation", "false")
+ .unwrap();
+ flag_builder
+ .set("enable_table_access_spectre_mitigation", "false")
+ .unwrap();
+ let isa_builder = cranelift_native::builder().expect("host ISA");
+ let isa = isa_builder
+ .finish(settings::Flags::new(flag_builder))
+ .expect("ISA finish");
+
+ let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());
+
+ struct SymTable {
+ cause_invalid_opcode: (&'static str, *const u8),
+ advance_to_deadline: (&'static str, *const u8),
+ read_u8: (&'static str, *const u8),
+ read_u16: (&'static str, *const u8),
+ read_u32: (&'static str, *const u8),
+ write_u8: (&'static str, *const u8),
+ write_u16: (&'static str, *const u8),
+ write_u32: (&'static str, *const u8),
+ read_f32: (&'static str, *const u8),
+ read_f64: (&'static str, *const u8),
+ write_f32: (&'static str, *const u8),
+ write_f64: (&'static str, *const u8),
+ write_msr: (&'static str, *const u8),
+ read_spr: (&'static str, *const u8),
+ write_spr: (&'static str, *const u8),
+ read_sr: (&'static str, *const u8),
+ write_sr: (&'static str, *const u8),
+ cause_trap_exception: (&'static str, *const u8),
+ cause_syscall_interrupt: (&'static str, *const u8),
+ do_rfi: (&'static str, *const u8),
+ cause_fp_unavailable: (&'static str, *const u8),
+ set_reservation: (&'static str, *const u8),
+ try_clear_reservation: (&'static str, *const u8),
+ do_lswi: (&'static str, *const u8),
+ do_stswi: (&'static str, *const u8),
+ do_lswx: (&'static str, *const u8),
+ do_stswx: (&'static str, *const u8),
+ do_psq_load: (&'static str, *const u8),
+ do_psq_store: (&'static str, *const u8),
+ read_timebase: (&'static str, *const u8),
+ }
+ let syms: SymTable = match SYSTEM {
+ GC => SymTable {
+ cause_invalid_opcode: (
+ "gecko_jit_cause_invalid_opcode_gc",
+ runtime::cause_invalid_opcode_gc as *const u8,
+ ),
+ advance_to_deadline: (
+ "gecko_jit_advance_to_deadline_gc",
+ runtime::advance_to_deadline_gc as *const u8,
+ ),
+ read_u8: ("gecko_jit_read_u8_gc", runtime::read_u8_gc as *const u8),
+ read_u16: ("gecko_jit_read_u16_gc", runtime::read_u16_gc as *const u8),
+ read_u32: ("gecko_jit_read_u32_gc", runtime::read_u32_gc as *const u8),
+ write_u8: ("gecko_jit_write_u8_gc", runtime::write_u8_gc as *const u8),
+ write_u16: ("gecko_jit_write_u16_gc", runtime::write_u16_gc as *const u8),
+ write_u32: ("gecko_jit_write_u32_gc", runtime::write_u32_gc as *const u8),
+ read_f32: ("gecko_jit_read_f32_gc", runtime::read_f32_gc as *const u8),
+ read_f64: ("gecko_jit_read_f64_gc", runtime::read_f64_gc as *const u8),
+ write_f32: ("gecko_jit_write_f32_gc", runtime::write_f32_gc as *const u8),
+ write_f64: ("gecko_jit_write_f64_gc", runtime::write_f64_gc as *const u8),
+ write_msr: ("gecko_jit_write_msr_gc", runtime::write_msr_gc as *const u8),
+ read_spr: ("gecko_jit_read_spr_gc", runtime::read_spr_gc as *const u8),
+ write_spr: ("gecko_jit_write_spr_gc", runtime::write_spr_gc as *const u8),
+ read_sr: ("gecko_jit_read_sr_gc", runtime::read_sr_gc as *const u8),
+ write_sr: ("gecko_jit_write_sr_gc", runtime::write_sr_gc as *const u8),
+ cause_trap_exception: (
+ "gecko_jit_cause_trap_exception_gc",
+ runtime::cause_trap_exception_gc as *const u8,
+ ),
+ cause_syscall_interrupt: (
+ "gecko_jit_cause_syscall_interrupt_gc",
+ runtime::cause_syscall_interrupt_gc as *const u8,
+ ),
+ do_rfi: ("gecko_jit_do_rfi_gc", runtime::do_rfi_gc as *const u8),
+ cause_fp_unavailable: (
+ "gecko_jit_cause_fp_unavailable_gc",
+ runtime::cause_fp_unavailable_gc as *const u8,
+ ),
+ set_reservation: ("gecko_jit_set_reservation_gc", runtime::set_reservation_gc as *const u8),
+ try_clear_reservation: (
+ "gecko_jit_try_clear_reservation_gc",
+ runtime::try_clear_reservation_gc as *const u8,
+ ),
+ do_lswi: ("gecko_jit_do_lswi_gc", runtime::do_lswi_gc as *const u8),
+ do_stswi: ("gecko_jit_do_stswi_gc", runtime::do_stswi_gc as *const u8),
+ do_lswx: ("gecko_jit_do_lswx_gc", runtime::do_lswx_gc as *const u8),
+ do_stswx: ("gecko_jit_do_stswx_gc", runtime::do_stswx_gc as *const u8),
+ do_psq_load: ("gecko_jit_do_psq_load_gc", runtime::do_psq_load_gc as *const u8),
+ do_psq_store: ("gecko_jit_do_psq_store_gc", runtime::do_psq_store_gc as *const u8),
+ read_timebase: ("gecko_jit_read_timebase_gc", runtime::read_timebase_gc as *const u8),
+ },
+ WII => SymTable {
+ cause_invalid_opcode: (
+ "gecko_jit_cause_invalid_opcode_wii",
+ runtime::cause_invalid_opcode_wii as *const u8,
+ ),
+ advance_to_deadline: (
+ "gecko_jit_advance_to_deadline_wii",
+ runtime::advance_to_deadline_wii as *const u8,
+ ),
+ read_u8: ("gecko_jit_read_u8_wii", runtime::read_u8_wii as *const u8),
+ read_u16: ("gecko_jit_read_u16_wii", runtime::read_u16_wii as *const u8),
+ read_u32: ("gecko_jit_read_u32_wii", runtime::read_u32_wii as *const u8),
+ write_u8: ("gecko_jit_write_u8_wii", runtime::write_u8_wii as *const u8),
+ write_u16: ("gecko_jit_write_u16_wii", runtime::write_u16_wii as *const u8),
+ write_u32: ("gecko_jit_write_u32_wii", runtime::write_u32_wii as *const u8),
+ read_f32: ("gecko_jit_read_f32_wii", runtime::read_f32_wii as *const u8),
+ read_f64: ("gecko_jit_read_f64_wii", runtime::read_f64_wii as *const u8),
+ write_f32: ("gecko_jit_write_f32_wii", runtime::write_f32_wii as *const u8),
+ write_f64: ("gecko_jit_write_f64_wii", runtime::write_f64_wii as *const u8),
+ write_msr: ("gecko_jit_write_msr_wii", runtime::write_msr_wii as *const u8),
+ read_spr: ("gecko_jit_read_spr_wii", runtime::read_spr_wii as *const u8),
+ write_spr: ("gecko_jit_write_spr_wii", runtime::write_spr_wii as *const u8),
+ read_sr: ("gecko_jit_read_sr_wii", runtime::read_sr_wii as *const u8),
+ write_sr: ("gecko_jit_write_sr_wii", runtime::write_sr_wii as *const u8),
+ cause_trap_exception: (
+ "gecko_jit_cause_trap_exception_wii",
+ runtime::cause_trap_exception_wii as *const u8,
+ ),
+ cause_syscall_interrupt: (
+ "gecko_jit_cause_syscall_interrupt_wii",
+ runtime::cause_syscall_interrupt_wii as *const u8,
+ ),
+ do_rfi: ("gecko_jit_do_rfi_wii", runtime::do_rfi_wii as *const u8),
+ cause_fp_unavailable: (
+ "gecko_jit_cause_fp_unavailable_wii",
+ runtime::cause_fp_unavailable_wii as *const u8,
+ ),
+ set_reservation: (
+ "gecko_jit_set_reservation_wii",
+ runtime::set_reservation_wii as *const u8,
+ ),
+ try_clear_reservation: (
+ "gecko_jit_try_clear_reservation_wii",
+ runtime::try_clear_reservation_wii as *const u8,
+ ),
+ do_lswi: ("gecko_jit_do_lswi_wii", runtime::do_lswi_wii as *const u8),
+ do_stswi: ("gecko_jit_do_stswi_wii", runtime::do_stswi_wii as *const u8),
+ do_lswx: ("gecko_jit_do_lswx_wii", runtime::do_lswx_wii as *const u8),
+ do_stswx: ("gecko_jit_do_stswx_wii", runtime::do_stswx_wii as *const u8),
+ do_psq_load: ("gecko_jit_do_psq_load_wii", runtime::do_psq_load_wii as *const u8),
+ do_psq_store: ("gecko_jit_do_psq_store_wii", runtime::do_psq_store_wii as *const u8),
+ read_timebase: ("gecko_jit_read_timebase_wii", runtime::read_timebase_wii as *const u8),
+ },
+ _ => unreachable!(),
+ };
+
+ for &(name, addr) in &[
+ syms.cause_invalid_opcode,
+ syms.advance_to_deadline,
+ syms.read_u8,
+ syms.read_u16,
+ syms.read_u32,
+ syms.write_u8,
+ syms.write_u16,
+ syms.write_u32,
+ syms.read_f32,
+ syms.read_f64,
+ syms.write_f32,
+ syms.write_f64,
+ syms.write_msr,
+ syms.read_spr,
+ syms.write_spr,
+ syms.read_sr,
+ syms.write_sr,
+ syms.cause_trap_exception,
+ syms.cause_syscall_interrupt,
+ syms.do_rfi,
+ syms.cause_fp_unavailable,
+ syms.set_reservation,
+ syms.try_clear_reservation,
+ syms.do_lswi,
+ syms.do_stswi,
+ syms.do_lswx,
+ syms.do_stswx,
+ syms.do_psq_load,
+ syms.do_psq_store,
+ syms.read_timebase,
+ ] {
+ jit_builder.symbol(name, addr);
+ }
+
+ let mut module = JITModule::new(jit_builder);
+
+ let pointer_type = module.target_config().pointer_type();
+ let call_conv = module.target_config().default_call_conv;
+
+ let mut block_sig = Signature::new(CallConv::Tail);
+ block_sig.params.push(AbiParam::new(pointer_type));
+ block_sig.returns.push(AbiParam::new(types::I32));
+
+ let mut fallback_sig = Signature::new(call_conv);
+ fallback_sig.params.push(AbiParam::new(pointer_type));
+ fallback_sig.params.push(AbiParam::new(types::I32));
+ fallback_sig.params.push(AbiParam::new(types::I32));
+ fallback_sig.returns.push(AbiParam::new(types::I32));
+
+ let mut deadline_sig = Signature::new(call_conv);
+ deadline_sig.params.push(AbiParam::new(pointer_type));
+
+ let mut mem_read_sig = Signature::new(call_conv);
+ mem_read_sig.params.push(AbiParam::new(pointer_type));
+ mem_read_sig.params.push(AbiParam::new(types::I32));
+ mem_read_sig.returns.push(AbiParam::new(types::I32));
+
+ let mut mem_write_sig = Signature::new(call_conv);
+ mem_write_sig.params.push(AbiParam::new(pointer_type));
+ mem_write_sig.params.push(AbiParam::new(types::I32));
+ mem_write_sig.params.push(AbiParam::new(types::I32));
+
+ let mut fp_read_sig = Signature::new(call_conv);
+ fp_read_sig.params.push(AbiParam::new(pointer_type));
+ fp_read_sig.params.push(AbiParam::new(types::I32));
+ fp_read_sig.returns.push(AbiParam::new(types::F64));
+
+ let mut fp_write_sig = Signature::new(call_conv);
+ fp_write_sig.params.push(AbiParam::new(pointer_type));
+ fp_write_sig.params.push(AbiParam::new(types::I32));
+ fp_write_sig.params.push(AbiParam::new(types::F64));
+
+ let mut ctx_u32_sig = Signature::new(call_conv);
+ ctx_u32_sig.params.push(AbiParam::new(pointer_type));
+ ctx_u32_sig.params.push(AbiParam::new(types::I32));
+
+ let extern_funcs = ExternFuncs {
+ cause_invalid_opcode: module
+ .declare_function(syms.cause_invalid_opcode.0, Linkage::Import, &fallback_sig)
+ .expect("declare cause_invalid_opcode"),
+ advance_to_deadline: module
+ .declare_function(syms.advance_to_deadline.0, Linkage::Import, &deadline_sig)
+ .expect("declare advance_to_deadline"),
+ read_u8: module
+ .declare_function(syms.read_u8.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_u8"),
+ read_u16: module
+ .declare_function(syms.read_u16.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_u16"),
+ read_u32: module
+ .declare_function(syms.read_u32.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_u32"),
+ write_u8: module
+ .declare_function(syms.write_u8.0, Linkage::Import, &mem_write_sig)
+ .expect("declare write_u8"),
+ write_u16: module
+ .declare_function(syms.write_u16.0, Linkage::Import, &mem_write_sig)
+ .expect("declare write_u16"),
+ write_u32: module
+ .declare_function(syms.write_u32.0, Linkage::Import, &mem_write_sig)
+ .expect("declare write_u32"),
+ read_f32: module
+ .declare_function(syms.read_f32.0, Linkage::Import, &fp_read_sig)
+ .expect("declare read_f32"),
+ read_f64: module
+ .declare_function(syms.read_f64.0, Linkage::Import, &fp_read_sig)
+ .expect("declare read_f64"),
+ write_f32: module
+ .declare_function(syms.write_f32.0, Linkage::Import, &fp_write_sig)
+ .expect("declare write_f32"),
+ write_f64: module
+ .declare_function(syms.write_f64.0, Linkage::Import, &fp_write_sig)
+ .expect("declare write_f64"),
+ write_msr: module
+ .declare_function(syms.write_msr.0, Linkage::Import, &ctx_u32_sig)
+ .expect("declare write_msr"),
+ read_spr: module
+ .declare_function(syms.read_spr.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_spr"),
+ write_spr: module
+ .declare_function(syms.write_spr.0, Linkage::Import, &mem_write_sig)
+ .expect("declare write_spr"),
+ read_sr: module
+ .declare_function(syms.read_sr.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_sr"),
+ write_sr: module
+ .declare_function(syms.write_sr.0, Linkage::Import, &mem_write_sig)
+ .expect("declare write_sr"),
+ cause_trap_exception: module
+ .declare_function(syms.cause_trap_exception.0, Linkage::Import, &deadline_sig)
+ .expect("declare cause_trap_exception"),
+ cause_syscall_interrupt: module
+ .declare_function(syms.cause_syscall_interrupt.0, Linkage::Import, &deadline_sig)
+ .expect("declare cause_syscall_interrupt"),
+ do_rfi: module
+ .declare_function(syms.do_rfi.0, Linkage::Import, &deadline_sig)
+ .expect("declare do_rfi"),
+ cause_fp_unavailable: module
+ .declare_function(syms.cause_fp_unavailable.0, Linkage::Import, &ctx_u32_sig)
+ .expect("declare cause_fp_unavailable"),
+ set_reservation: module
+ .declare_function(syms.set_reservation.0, Linkage::Import, &ctx_u32_sig)
+ .expect("declare set_reservation"),
+ try_clear_reservation: module
+ .declare_function(syms.try_clear_reservation.0, Linkage::Import, &mem_read_sig)
+ .expect("declare try_clear_reservation"),
+ do_lswi: module
+ .declare_function(syms.do_lswi.0, Linkage::Import, &{
+ let mut sig = Signature::new(call_conv);
+ sig.params.push(AbiParam::new(pointer_type));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig
+ })
+ .expect("declare do_lswi"),
+ do_stswi: module
+ .declare_function(syms.do_stswi.0, Linkage::Import, &{
+ let mut sig = Signature::new(call_conv);
+ sig.params.push(AbiParam::new(pointer_type));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig
+ })
+ .expect("declare do_stswi"),
+ do_lswx: module
+ .declare_function(syms.do_lswx.0, Linkage::Import, &mem_write_sig)
+ .expect("declare do_lswx"),
+ do_stswx: module
+ .declare_function(syms.do_stswx.0, Linkage::Import, &mem_write_sig)
+ .expect("declare do_stswx"),
+ do_psq_load: module
+ .declare_function(syms.do_psq_load.0, Linkage::Import, &{
+ let mut sig = Signature::new(call_conv);
+ sig.params.push(AbiParam::new(pointer_type));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig
+ })
+ .expect("declare do_psq_load"),
+ do_psq_store: module
+ .declare_function(syms.do_psq_store.0, Linkage::Import, &{
+ let mut sig = Signature::new(call_conv);
+ sig.params.push(AbiParam::new(pointer_type));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig.params.push(AbiParam::new(types::I32));
+ sig
+ })
+ .expect("declare do_psq_store"),
+ read_timebase: module
+ .declare_function(syms.read_timebase.0, Linkage::Import, &mem_read_sig)
+ .expect("declare read_timebase"),
+ };
+
+ let trampoline_fn = build_trampoline(&mut module, call_conv, pointer_type, &block_sig);
+
+ let block_lookup_table = vec![
+ BlockLookupSlot {
+ pc: 0,
+ _pad: 0,
+ entry: 0
+ };
+ BLOCK_LOOKUP_TABLE_SIZE
+ ]
+ .into_boxed_slice();
+ let block_lookup_table_addr = Box::leak(block_lookup_table).as_mut_ptr() as usize;
+
+ Self {
+ module,
+ ctx: Context::new(),
+ builder_ctx: FunctionBuilderContext::new(),
+ cache: FxHashMap::default(),
+ block_func_ids: FxHashMap::default(),
+ pending_chain_slots: FxHashMap::default(),
+ #[cfg(feature = "jit-stats")]
+ hits: FxHashMap::default(),
+ block_sig,
+ extern_funcs,
+ trampoline_fn,
+ block_lookup_table_addr,
+ block_seq: 0,
+ }
+ }
+
+ pub fn block_lookup_table_addr(&self) -> usize {
+ self.block_lookup_table_addr
+ }
+
+ fn register_in_lookup_table(&self, pc: u32, entry: usize) {
+ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;
+ unsafe {
+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;
+ (*table.add(idx)).pc = pc;
+ (*table.add(idx)).entry = entry;
+ }
+ }
+
+ pub fn lookup_or_compile(&mut self, sys: &mut System<SYSTEM>, pc: u32) -> BlockEntry {
+ unsafe {
+ let table = self.block_lookup_table_addr as *const BlockLookupSlot;
+ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;
+ let slot = &*table.add(idx);
+ if slot.pc == pc && slot.entry != 0 {
+ #[cfg(feature = "jit-stats")]
+ {
+ *self.hits.entry(pc).or_insert(0) += 1;
+ }
+ return slot.entry;
+ }
+ }
+
+ if let Some(&entry) = self.cache.get(&pc) {
+ #[cfg(feature = "jit-stats")]
+ {
+ *self.hits.entry(pc).or_insert(0) += 1;
+ }
+ return entry;
+ }
+
+ let spec = block::discover::<SYSTEM>(sys, pc);
+ let gprs_snapshot = sys.gekko.gprs;
+
+ let entry = self.compile(&spec, &gprs_snapshot);
+ self.cache.insert(pc, entry);
+
+ entry
+ }
+
+ fn func_id_for(&mut self, pc: u32) -> FuncId {
+ if let Some(&id) = self.block_func_ids.get(&pc) {
+ return id;
+ }
+
+ self.block_seq = self.block_seq.wrapping_add(1);
+ let name = format!("gecko_block_{:08x}_{}", pc, self.block_seq);
+ let id = self
+ .module
+ .declare_function(&name, Linkage::Local, &self.block_sig)
+ .expect("declare block");
+ self.block_func_ids.insert(pc, id);
+
+ id
+ }
+
+ pub fn run_block(&mut self, sys: &mut System<SYSTEM>) {
+ let pc = sys.gekko.pc;
+ let entry = self.lookup_or_compile(sys, pc);
+ let ctx_ptr = sys as *mut System<SYSTEM> as *mut core::ffi::c_void;
+ let next_pc = unsafe { (self.trampoline_fn)(ctx_ptr, entry) };
+ sys.gekko.pc = next_pc;
+ }
+
+ #[cfg(feature = "jit-stats")]
+ pub fn dump_idle_candidates(&self, sys: &System<SYSTEM>, top_k: usize) {
+ const SHAPE_BODY_CAP: usize = 12;
+
+ let mut entries: Vec<(u32, u64, block::BlockSpec)> = self
+ .hits
+ .iter()
+ .filter_map(|(&pc, &hits)| {
+ let spec = block::discover::<SYSTEM>(sys, pc);
+ if !looks_like_polling_shape(&spec, SHAPE_BODY_CAP) {
+ return None;
+ }
+
+ let class = idle::classify::<SYSTEM>(&spec, &sys.gekko.gprs);
+ if class != idle::IdleClass::None {
+ return None;
+ }
+
+ Some((pc, hits, spec))
+ })
+ .collect();
+ entries.sort_by(|a, b| b.1.cmp(&a.1));
+
+ if entries.is_empty() {
+ tracing::info!("no idle-skip candidates recorded");
+ return;
+ }
+
+ let total: u64 = entries.iter().map(|e| e.1).sum();
+ tracing::info!(
+ "idle-skip candidates (top {}, total hits = {})",
+ top_k.min(entries.len()),
+ total
+ );
+ for (rank, (pc, hits, spec)) in entries.iter().take(top_k).enumerate() {
+ let pct = (*hits as f64) * 100.0 / (total as f64);
+ tracing::info!(
+ " #{rank} pc={:08X} hits={hits} ({:.1}%) len={} term={:?}",
+ pc,
+ pct,
+ spec.instrs.len(),
+ spec.terminator
+ );
+ }
+ }
+
+ #[cfg(feature = "jit-stats")]
+ pub fn dump_hot_blocks(&self, sys: &System<SYSTEM>, top_k: usize) {
+ let mut entries: Vec<(u32, u64)> = self.hits.iter().map(|(&pc, &n)| (pc, n)).collect();
+ entries.sort_by(|a, b| b.1.cmp(&a.1));
+
+ if entries.is_empty() {
+ tracing::info!("no dispatcher entry hits recorded");
+ return;
+ }
+
+ let total: u64 = entries.iter().map(|e| e.1).sum();
+ tracing::info!(
+ "JIT hot blocks (top {}, total hits = {}) ===",
+ top_k.min(entries.len()),
+ total
+ );
+
+ for (rank, (pc, hits)) in entries.iter().take(top_k).enumerate() {
+ let spec = block::discover::<SYSTEM>(sys, *pc);
+ let idle_class = idle::classify::<SYSTEM>(&spec, &sys.gekko.gprs);
+ let pct = (*hits as f64) * 100.0 / (total as f64);
+ tracing::info!(
+ " #{rank} pc={:08X} hits={hits} ({:.1}%) len={} idle={:?} term={:?}",
+ pc,
+ pct,
+ spec.instrs.len(),
+ idle_class,
+ spec.terminator,
+ );
+ }
+ }
+
+ pub fn flush(&mut self) {
+ self.cache.clear();
+ self.block_func_ids.clear();
+ self.pending_chain_slots.clear();
+
+ unsafe {
+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;
+ for i in 0..BLOCK_LOOKUP_TABLE_SIZE {
+ (*table.add(i)).pc = 0;
+ (*table.add(i)).entry = 0;
+ }
+ }
+ }
+
+ fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {
+ let func_id = self.func_id_for(spec.start_pc);
+
+ self.ctx.clear();
+ self.ctx.func.signature = self.block_sig.clone();
+
+ let self_slot: &'static mut usize = Box::leak(Box::new(0usize));
+ let self_slot_addr = self_slot as *mut usize as i64;
+
+ let mut pending_for_this_block: Vec<(u32, usize)> = Vec::new();
+
+ let chain = translator::ChainContext {
+ self_pc: spec.start_pc,
+ self_func_id: func_id,
+ self_slot_addr,
+ compiled_func_ids: &self.block_func_ids,
+ compiled_cache: &self.cache,
+ pending: std::cell::RefCell::new(&mut pending_for_this_block),
+ block_lookup_table_addr: self.block_lookup_table_addr as i64,
+ };
+
+ translator::translate::<SYSTEM>(
+ &mut self.ctx,
+ &mut self.builder_ctx,
+ &mut self.module,
+ &self.extern_funcs,
+ spec,
+ gprs,
+ &chain,
+ );
+
+ drop(chain);
+ for (target_pc, slot_addr) in pending_for_this_block.drain(..) {
+ self.pending_chain_slots.entry(target_pc).or_default().push(slot_addr);
+ }
+
+ self.module
+ .define_function(func_id, &mut self.ctx)
+ .expect("define block");
+
+ self.module.finalize_definitions().expect("finalize");
+ let entry = self.module.get_finalized_function(func_id) as usize;
+
+ unsafe {
+ *(self_slot_addr as *mut usize) = entry;
+ }
+
+ if let Some(slots) = self.pending_chain_slots.remove(&spec.start_pc) {
+ for slot_addr in slots {
+ unsafe {
+ *(slot_addr as *mut usize) = entry;
+ }
+ }
+ }
+
+ self.register_in_lookup_table(spec.start_pc, entry);
+
+ entry
+ }
+}
+
+impl<const SYSTEM: SystemId> Default for JitEngine<SYSTEM> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[cfg(feature = "jit-stats")]
+fn looks_like_polling_shape(spec: &block::BlockSpec, body_cap: usize) -> bool {
+ if spec.terminator != block::TermKind::BranchCond {
+ return false;
+ }
+
+ if spec.instrs.is_empty() || spec.instrs.len() > body_cap {
+ return false;
+ }
+
+ let last_idx = spec.instrs.len() - 1;
+ let raw = spec.instrs[last_idx];
+ if (raw >> 26) != 16 {
+ return false;
+ }
+
+ if raw & 1 != 0 {
+ return false;
+ }
+
+ let bo = (raw >> 21) & 0x1F;
+ if bo & 0b00100 == 0 {
+ return false;
+ }
+
+ let aa = (raw >> 1) & 1 != 0;
+ let bd = (((raw >> 2) & 0x3FFF) as i32) << 18 >> 18;
+ let bd_bytes = bd << 2;
+ let term_pc = spec.start_pc.wrapping_add((last_idx as u32) * 4);
+ let target = if aa {
+ bd_bytes as u32
+ } else {
+ term_pc.wrapping_add_signed(bd_bytes)
+ };
+
+ target == spec.start_pc
+}
+
+fn build_trampoline(
+ module: &mut JITModule,
+ host_cc: CallConv,
+ pointer_type: cranelift_codegen::ir::Type,
+ block_sig: &Signature,
+) -> TrampolineFn {
+ let mut tramp_sig = Signature::new(host_cc);
+ tramp_sig.params.push(AbiParam::new(pointer_type));
+ tramp_sig.params.push(AbiParam::new(pointer_type));
+ tramp_sig.returns.push(AbiParam::new(types::I32));
+
+ let func_id = module
+ .declare_function("gecko_jit_trampoline", Linkage::Local, &tramp_sig)
+ .expect("declare trampoline");
+
+ let mut ctx = Context::new();
+ ctx.func.signature = tramp_sig.clone();
+ let block_sig_ref = ctx.func.import_signature(block_sig.clone());
+
+ {
+ let mut bcx_ctx = FunctionBuilderContext::new();
+ let mut builder = FunctionBuilder::new(&mut ctx.func, &mut bcx_ctx);
+ let entry = builder.create_block();
+
+ builder.append_block_params_for_function_params(entry);
+ builder.switch_to_block(entry);
+ builder.seal_block(entry);
+
+ let ctx_ptr = builder.block_params(entry)[0];
+ let block_ptr = builder.block_params(entry)[1];
+ let call = builder.ins().call_indirect(block_sig_ref, block_ptr, &[ctx_ptr]);
+ let result = builder.inst_results(call)[0];
+ builder.ins().return_(&[result]);
+ builder.finalize();
+ }
+
+ module.define_function(func_id, &mut ctx).expect("define trampoline");
+ module.finalize_definitions().expect("finalize trampoline");
+
+ let ptr = module.get_finalized_function(func_id);
+ unsafe { core::mem::transmute::<*const u8, TrampolineFn>(ptr) }
+}
+
+#[inline(always)]
+pub fn dispatch<const SYSTEM: SystemId>(
+ t: &mut translator::JitTranslator,
+ instr: crate::gekko::instruction::Instruction,
+) {
+ if SYSTEM == GC {
+ self::lut::dispatch(t, instr);
+ } else {
+ self::lut_wii::dispatch(t, instr);
+ }
+}
diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rs
new file mode 100644
index 0000000..c4f8d81
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/abi.rs
@@ -0,0 +1,238 @@
+use core::mem::offset_of;
+
+use crate::gekko::Gekko;
+use crate::gekko::condition::ConditionRegister;
+use crate::gekko::msr::Msr;
+use crate::gekko::spr::{Spr, Xer};
+use crate::mmio::Mmio;
+use crate::scheduler::Scheduler;
+use crate::system::{System, SystemId};
+
+const _: () = {
+ if core::mem::size_of::<ConditionRegister>() != 4 {
+ panic!("ConditionRegister is no longer 4 bytes; JIT CR access needs to be revisited");
+ }
+ if core::mem::align_of::<ConditionRegister>() != 4 {
+ panic!("ConditionRegister alignment changed; JIT CR access needs to be revisited");
+ }
+ if core::mem::size_of::<Xer>() != 4 {
+ panic!("Xer is no longer 4 bytes; JIT XER access needs to be revisited");
+ }
+ if core::mem::size_of::<Msr>() != 4 {
+ panic!("Msr is no longer 4 bytes; JIT MSR access needs to be revisited");
+ }
+};
+
+#[inline(always)]
+pub const fn gekko_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, gekko)
+}
+
+#[inline(always)]
+pub const fn scheduler_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, scheduler)
+}
+
+#[inline(always)]
+pub const fn pc_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, pc)
+}
+
+#[inline(always)]
+pub const fn cia_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, cia)
+}
+
+#[inline(always)]
+pub const fn nia_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, nia)
+}
+
+#[inline(always)]
+pub const fn gpr_base_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, gprs)
+}
+
+#[inline(always)]
+pub const fn fpr_base_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, fprs)
+}
+
+#[inline(always)]
+pub const fn ps1_base_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, ps1s)
+}
+
+#[inline(always)]
+pub const fn cycles_offset<const SYSTEM: SystemId>() -> usize {
+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, cycles)
+}
+
+#[inline(always)]
+pub const fn next_deadline_offset<const SYSTEM: SystemId>() -> usize {
+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, next_deadline)
+}
+
+#[inline(always)]
+pub const fn cr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, cr)
+}
+
+#[inline(always)]
+pub const fn xer_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, xer)
+}
+
+#[inline(always)]
+pub const fn fpscr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, fpscr)
+}
+
+#[inline(always)]
+pub const fn ram_ptr_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, ram_ptr)
+}
+
+#[inline(always)]
+pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)
+}
+
+#[inline(always)]
+pub const fn gqr_offset<const SYSTEM: SystemId>(i: u8) -> usize {
+ let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);
+ spr_off
+ + match i {
+ 0 => offset_of!(Spr, gqr0),
+ 1 => offset_of!(Spr, gqr1),
+ 2 => offset_of!(Spr, gqr2),
+ 3 => offset_of!(Spr, gqr3),
+ 4 => offset_of!(Spr, gqr4),
+ 5 => offset_of!(Spr, gqr5),
+ 6 => offset_of!(Spr, gqr6),
+ 7 => offset_of!(Spr, gqr7),
+ _ => panic!("GQR index out of range"),
+ }
+}
+
+#[inline(always)]
+pub const fn lr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, lr)
+}
+
+#[inline(always)]
+pub const fn ctr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr) + offset_of!(Spr, ctr)
+}
+
+#[inline(always)]
+pub const fn msr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, msr)
+}
+
+#[inline(always)]
+pub const fn timebase_offset_offset<const SYSTEM: SystemId>() -> usize {
+ scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, timebase_offset)
+}
+
+#[inline(always)]
+pub const fn reserve_addr_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, reserve_addr)
+}
+
+#[inline(always)]
+pub const fn sr_base_offset<const SYSTEM: SystemId>() -> usize {
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, sr)
+}
+
+#[inline(always)]
+pub fn spr_field_offset<const SYSTEM: SystemId>(spr_num: u16) -> Option<usize> {
+ let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);
+ let inner = match spr_num {
+ 1 => offset_of!(Spr, xer),
+ 8 => offset_of!(Spr, lr),
+ 9 => offset_of!(Spr, ctr),
+ 18 => offset_of!(Spr, dsisr),
+ 19 => offset_of!(Spr, dar),
+ 25 => offset_of!(Spr, sdr1),
+ 26 => offset_of!(Spr, srr0),
+ 27 => offset_of!(Spr, srr1),
+ 272 => offset_of!(Spr, sprg0),
+ 273 => offset_of!(Spr, sprg1),
+ 274 => offset_of!(Spr, sprg2),
+ 275 => offset_of!(Spr, sprg3),
+ 282 => offset_of!(Spr, ear),
+ 287 => offset_of!(Spr, pvr),
+ 528 => offset_of!(Spr, ibat0u),
+ 529 => offset_of!(Spr, ibat0l),
+ 530 => offset_of!(Spr, ibat1u),
+ 531 => offset_of!(Spr, ibat1l),
+ 532 => offset_of!(Spr, ibat2u),
+ 533 => offset_of!(Spr, ibat2l),
+ 534 => offset_of!(Spr, ibat3u),
+ 535 => offset_of!(Spr, ibat3l),
+ 536 => offset_of!(Spr, dbat0u),
+ 537 => offset_of!(Spr, dbat0l),
+ 538 => offset_of!(Spr, dbat1u),
+ 539 => offset_of!(Spr, dbat1l),
+ 540 => offset_of!(Spr, dbat2u),
+ 541 => offset_of!(Spr, dbat2l),
+ 542 => offset_of!(Spr, dbat3u),
+ 543 => offset_of!(Spr, dbat3l),
+
+ 560 => offset_of!(Spr, ibat4u),
+ 561 => offset_of!(Spr, ibat4l),
+ 562 => offset_of!(Spr, ibat5u),
+ 563 => offset_of!(Spr, ibat5l),
+ 564 => offset_of!(Spr, ibat6u),
+ 565 => offset_of!(Spr, ibat6l),
+ 566 => offset_of!(Spr, ibat7u),
+ 567 => offset_of!(Spr, ibat7l),
+ 568 => offset_of!(Spr, dbat4u),
+ 569 => offset_of!(Spr, dbat4l),
+ 570 => offset_of!(Spr, dbat5u),
+ 571 => offset_of!(Spr, dbat5l),
+ 572 => offset_of!(Spr, dbat6u),
+ 573 => offset_of!(Spr, dbat6l),
+ 574 => offset_of!(Spr, dbat7u),
+ 575 => offset_of!(Spr, dbat7l),
+ 912 => offset_of!(Spr, gqr0),
+ 913 => offset_of!(Spr, gqr1),
+ 914 => offset_of!(Spr, gqr2),
+ 915 => offset_of!(Spr, gqr3),
+ 916 => offset_of!(Spr, gqr4),
+ 917 => offset_of!(Spr, gqr5),
+ 918 => offset_of!(Spr, gqr6),
+ 919 => offset_of!(Spr, gqr7),
+ 920 => offset_of!(Spr, hid2),
+ 921 => offset_of!(Spr, wpar),
+ 936 => offset_of!(Spr, ummcr0),
+ 937 => offset_of!(Spr, upmc1),
+ 938 => offset_of!(Spr, upmc2),
+ 939 => offset_of!(Spr, usia),
+ 940 => offset_of!(Spr, ummcr1),
+ 941 => offset_of!(Spr, upmc3),
+ 942 => offset_of!(Spr, upmc4),
+ 943 => offset_of!(Spr, usda),
+ 952 => offset_of!(Spr, mmcr0),
+ 953 => offset_of!(Spr, pmc1),
+ 954 => offset_of!(Spr, pmc2),
+ 955 => offset_of!(Spr, sia),
+ 956 => offset_of!(Spr, mmcr1),
+ 957 => offset_of!(Spr, pmc3),
+ 958 => offset_of!(Spr, pmc4),
+ 959 => offset_of!(Spr, sda),
+ 1008 => offset_of!(Spr, hid0),
+ 1009 => offset_of!(Spr, hid1),
+ 1010 => offset_of!(Spr, iabr),
+ 1011 => offset_of!(Spr, hid4),
+ 1013 => offset_of!(Spr, dabr),
+ 1017 => offset_of!(Spr, l2cr),
+ 1019 => offset_of!(Spr, ictc),
+ 1020 => offset_of!(Spr, thrm1),
+ 1021 => offset_of!(Spr, thrm2),
+ 1022 => offset_of!(Spr, thrm3),
+ _ => return None,
+ };
+ Some(spr_off + inner)
+}
diff --git a/crates/gecko/src/gekko/jit/block.rs b/crates/gecko/src/gekko/jit/block.rs
new file mode 100644
index 0000000..e2d155d
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/block.rs
@@ -0,0 +1,143 @@
+use crate::gekko::instruction::Instruction;
+use crate::system::{System, SystemId};
+
+pub const MAX_BLOCK_INSTRS: usize = 64;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TermKind {
+ Branch,
+ BranchCond,
+ BranchToReg,
+ SystemCall,
+ Rfi,
+ Mtmsr,
+ Mtspr,
+ Isync,
+ LengthCap,
+}
+
+#[derive(Debug, Clone)]
+pub struct BlockSpec {
+ pub start_pc: u32,
+ pub instrs: Vec<u32>,
+ pub pcs: Vec<u32>,
+ pub terminator: TermKind,
+}
+
+impl BlockSpec {
+ #[inline(always)]
+ pub fn len(&self) -> usize {
+ self.instrs.len()
+ }
+
+ #[inline(always)]
+ pub fn end_pc(&self) -> u32 {
+ self.pcs
+ .last()
+ .copied()
+ .map(|p| p.wrapping_add(4))
+ .unwrap_or(self.start_pc)
+ }
+
+ #[inline(always)]
+ pub fn pc_of(&self, i: usize) -> u32 {
+ self.pcs[i]
+ }
+}
+
+pub fn discover<const SYSTEM: SystemId>(sys: &System<SYSTEM>, start_pc: u32) -> BlockSpec {
+ const EXTENSION_MAX_FORWARD_BYTES: u32 = 256;
+
+ let mut instrs: Vec<u32> = Vec::with_capacity(8);
+ let mut pcs: Vec<u32> = Vec::with_capacity(8);
+ let mut terminator = TermKind::LengthCap;
+ let mut pc = start_pc;
+
+ while instrs.len() < MAX_BLOCK_INSTRS {
+ let instr = Instruction(sys.mmio.fetch_instruction(pc));
+ let cur_pc = pc;
+
+ if let Some(target) = extension_target(instr, cur_pc) {
+ if target > cur_pc && target.wrapping_sub(cur_pc) <= EXTENSION_MAX_FORWARD_BYTES && !pcs.contains(&target) {
+ pc = target;
+ continue;
+ }
+ }
+
+ instrs.push(instr.0);
+ pcs.push(cur_pc);
+
+ if let Some(t) = classify_terminator(instr) {
+ terminator = t;
+ break;
+ }
+ pc = pc.wrapping_add(4);
+ }
+
+ BlockSpec {
+ start_pc,
+ instrs,
+ pcs,
+ terminator,
+ }
+}
+
+#[inline]
+fn extension_target(instr: Instruction, pc: u32) -> Option<u32> {
+ if instr.primary_opcode() != 18 {
+ return None;
+ }
+ if instr.lk() {
+ return None;
+ }
+ let target = if instr.aa() {
+ instr.li() as u32
+ } else {
+ pc.wrapping_add_signed(instr.li())
+ };
+ Some(target)
+}
+
+#[inline]
+fn mtspr_is_block_safe(spr: u16) -> bool {
+ matches!(
+ spr,
+ 1
+ | 8
+ | 9
+ | 22
+ | 26 | 27
+ | 272..=275
+ | 912..=919
+ | 920
+ | 1008 | 1009
+ )
+}
+
+#[inline]
+pub fn classify_terminator(instr: Instruction) -> Option<TermKind> {
+ match instr.primary_opcode() {
+ 16 => Some(TermKind::BranchCond),
+ 17 => Some(TermKind::SystemCall),
+ 18 => Some(TermKind::Branch),
+ 19 => match instr.xo10() {
+ 16 | 528 => Some(TermKind::BranchToReg),
+ 50 => Some(TermKind::Rfi),
+ 150 => Some(TermKind::Isync),
+ _ => None,
+ },
+ 31 => match instr.xo10() {
+ 146 => Some(TermKind::Mtmsr),
+ 467 => {
+ let spr_num = instr.spr_swapped() as u16;
+ if mtspr_is_block_safe(spr_num) {
+ None
+ } else {
+ Some(TermKind::Mtspr)
+ }
+ }
+ _ => None,
+ },
+ _ => None,
+ }
+}
diff --git a/crates/gecko/src/gekko/jit/handlers.rs b/crates/gecko/src/gekko/jit/handlers.rs
new file mode 100644
index 0000000..e671aa4
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/handlers.rs
@@ -0,0 +1,1042 @@
+use cranelift_codegen::ir::condcodes::IntCC;
+use cranelift_codegen::ir::{InstBuilder, MemFlags, types};
+use cranelift_frontend::FunctionBuilder;
+
+use crate::gekko::instruction::Instruction;
+use crate::gekko::jit::translator::{
+ self, AddmeSubfmeKind, AddzeSubfzeKind, CYCLES_PER_INSTR, ImmLogical, JitTranslator, LocalFuncs, LogicalFullOp,
+ LogicalOp, MemSize, ShiftKind, TermEmit,
+};
+use crate::gekko::jit::{abi, lut};
+use crate::system::SystemId;
+
+#[inline(always)]
+unsafe fn parts<'a>(builder_ptr: usize, local_ptr: usize) -> (&'a mut FunctionBuilder<'a>, &'a LocalFuncs) {
+ (unsafe { &mut *(builder_ptr as *mut FunctionBuilder<'a>) }, unsafe {
+ &*(local_ptr as *const LocalFuncs)
+ })
+}
+
+#[inline(always)]
+pub fn alu<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_ADDI => translator::emit_addi::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_ADDIS => translator::emit_addis::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_ORI => translator::emit_ori::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_ORIS => translator::emit_oris::<SYSTEM>(builder, t.ctx_ptr, instr),
+
+ lut::OP_XORI => {
+ translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::Xor, false);
+ }
+ lut::OP_XORIS => {
+ translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::Xor, true);
+ }
+ lut::OP_ANDI_DOT => {
+ let r = translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::And, false);
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ lut::OP_ANDIS_DOT => {
+ let r = translator::emit_imm_logical::<SYSTEM>(builder, t.ctx_ptr, instr, ImmLogical::And, true);
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+
+ lut::OP_MULLI => translator::emit_mulli::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_SUBFIC => translator::emit_subfic::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_ADDIC => translator::emit_addic::<SYSTEM>(builder, t.ctx_ptr, instr, false),
+ lut::OP_ADDIC_DOT => translator::emit_addic::<SYSTEM>(builder, t.ctx_ptr, instr, true),
+
+ lut::OP_ADDX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_add_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_add_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_ADDCX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_addc_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_add_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_ADDEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_adde_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_add_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_ADDZEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let r = translator::emit_addze_subfze::<SYSTEM>(builder, t.ctx_ptr, instr, AddzeSubfzeKind::Addze);
+
+ if instr.oe() {
+ let zero = builder.ins().iconst(types::I32, 0);
+ let ov = translator::compute_add_overflow(builder, a, zero, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_ADDMEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let r = translator::emit_addme_subfme::<SYSTEM>(builder, t.ctx_ptr, instr, AddmeSubfmeKind::Addme);
+
+ if instr.oe() {
+ let neg_one = builder.ins().iconst(types::I32, -1i64);
+ let ov = translator::compute_add_overflow(builder, a, neg_one, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+
+ lut::OP_SUBFX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_subf_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_sub_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_SUBFCX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_subfc_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_sub_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_SUBFEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_subfe_xform::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov = translator::compute_sub_overflow(builder, a, b, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_SUBFZEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let r = translator::emit_addze_subfze::<SYSTEM>(builder, t.ctx_ptr, instr, AddzeSubfzeKind::Subfze);
+
+ if instr.oe() {
+ let not_a = builder.ins().bxor_imm(a, !0i64);
+ let zero = builder.ins().iconst(types::I32, 0);
+ let ov = translator::compute_add_overflow(builder, not_a, zero, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_SUBFMEX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let r = translator::emit_addme_subfme::<SYSTEM>(builder, t.ctx_ptr, instr, AddmeSubfmeKind::Subfme);
+
+ if instr.oe() {
+ let not_a = builder.ins().bxor_imm(a, !0i64);
+ let neg_one = builder.ins().iconst(types::I32, -1i64);
+ let ov = translator::compute_add_overflow(builder, not_a, neg_one, r);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+
+ lut::OP_NEGX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let r = translator::emit_neg::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov_b = builder.ins().icmp_imm(IntCC::Equal, a, 0x8000_0000u32 as i64);
+ let ov = builder.ins().uextend(types::I32, ov_b);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+
+ lut::OP_MULLWX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_mullw::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let a64 = builder.ins().sextend(types::I64, a);
+ let b64 = builder.ins().sextend(types::I64, b);
+ let full = builder.ins().imul(a64, b64);
+ let r_sext = builder.ins().sextend(types::I64, r);
+ let neq = builder.ins().icmp(IntCC::NotEqual, full, r_sext);
+ let ov = builder.ins().uextend(types::I32, neq);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_MULHWX => {
+ let r = translator::emit_mulhw::<SYSTEM>(builder, t.ctx_ptr, instr, true);
+
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_MULHWUX => {
+ let r = translator::emit_mulhw::<SYSTEM>(builder, t.ctx_ptr, instr, false);
+
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+
+ lut::OP_DIVWUX => {
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_divwu::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let ov_b = builder.ins().icmp_imm(IntCC::Equal, b, 0);
+ let ov = builder.ins().uextend(types::I32, ov_b);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+ lut::OP_DIVWX => {
+ let a = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra());
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let r = translator::emit_divw::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ if instr.oe() {
+ let b_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);
+ let a_min = builder.ins().icmp_imm(IntCC::Equal, a, 0x8000_0000u32 as i64);
+ let b_neg1 = builder.ins().icmp_imm(IntCC::Equal, b, -1i64);
+ let signed_min_neg1 = builder.ins().band(a_min, b_neg1);
+ let any = builder.ins().bor(b_zero, signed_min_neg1);
+ let ov = builder.ins().uextend(types::I32, any);
+ translator::write_xer_ov_so::<SYSTEM>(builder, t.ctx_ptr, ov);
+ }
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+ }
+
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn rotate<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let r = match OP {
+ lut::OP_RLWIMIX => translator::emit_rlwimi::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_RLWINMX => translator::emit_rlwinm::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_RLWNMX => translator::emit_rlwnm::<SYSTEM>(builder, t.ctx_ptr, instr),
+ _ => return,
+ };
+
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn logical<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let r = match OP {
+ lut::OP_ANDX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::And),
+ lut::OP_ORX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::Or),
+ lut::OP_XORX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::Xor),
+
+ lut::OP_NORX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Nor),
+ lut::OP_NANDX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Nand),
+ lut::OP_ANDCX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Andc),
+ lut::OP_ORCX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Orc),
+ lut::OP_EQVX => translator::emit_logical_xform_full::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalFullOp::Eqv),
+
+ lut::OP_SLWX => translator::emit_shift_xform::<SYSTEM>(builder, t.ctx_ptr, instr, ShiftKind::Slw),
+ lut::OP_SRWX => translator::emit_shift_xform::<SYSTEM>(builder, t.ctx_ptr, instr, ShiftKind::Srw),
+ lut::OP_SRAWX => translator::emit_sraw::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_SRAWIX => translator::emit_srawi::<SYSTEM>(builder, t.ctx_ptr, instr),
+
+ lut::OP_CNTLZWX => translator::emit_cntlzw::<SYSTEM>(builder, t.ctx_ptr, instr),
+ lut::OP_EXTSBX => translator::emit_extend::<SYSTEM>(builder, t.ctx_ptr, instr, types::I8),
+ lut::OP_EXTSHX => translator::emit_extend::<SYSTEM>(builder, t.ctx_ptr, instr, types::I16),
+
+ _ => return,
+ };
+
+ if instr.rc() {
+ translator::emit_update_cr0::<SYSTEM>(builder, t.ctx_ptr, r);
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn compare<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_CMPI => translator::emit_cmp_imm::<SYSTEM>(builder, t.ctx_ptr, instr, true),
+ lut::OP_CMPLI => translator::emit_cmp_imm::<SYSTEM>(builder, t.ctx_ptr, instr, false),
+ lut::OP_CMP => translator::emit_cmp_xform::<SYSTEM>(builder, t.ctx_ptr, instr, true),
+ lut::OP_CMPL => translator::emit_cmp_xform::<SYSTEM>(builder, t.ctx_ptr, instr, false),
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn cr_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_MFCR => {
+ let cr = translator::cr_load::<SYSTEM>(builder, t.ctx_ptr);
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), cr);
+ }
+ lut::OP_MTCRF => {
+ let s = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());
+
+ let mut mask: u32 = 0;
+ for i in 0..8 {
+ if (instr.crm() >> (7 - i)) & 1 != 0 {
+ mask |= 0xF << ((7 - i) * 4);
+ }
+ }
+
+ let cr = translator::cr_load::<SYSTEM>(builder, t.ctx_ptr);
+ let kept = builder.ins().band_imm(cr, !mask as i64 as u32 as i64);
+ let new_bits = builder.ins().band_imm(s, mask as i64);
+ let merged = builder.ins().bor(kept, new_bits);
+ translator::cr_store::<SYSTEM>(builder, t.ctx_ptr, merged);
+ }
+
+ lut::OP_MCRF => translator::emit_mcrf::<SYSTEM>(builder, t.ctx_ptr, instr),
+
+ lut::OP_CRNOR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 33),
+ lut::OP_CRANDC => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 129),
+ lut::OP_CRXOR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 193),
+ lut::OP_CRNAND => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 225),
+ lut::OP_CRAND => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 257),
+ lut::OP_CREQV => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 289),
+ lut::OP_CRORC => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 417),
+ lut::OP_CROR => translator::emit_cr_bit_op::<SYSTEM>(builder, t.ctx_ptr, instr, 449),
+
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn segment<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_MFSR => {
+ let off = (abi::sr_base_offset::<SYSTEM>() + instr.sr() as usize * 4) as i32;
+ let val = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, off);
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);
+ }
+ lut::OP_MTSR => {
+ let off = (abi::sr_base_offset::<SYSTEM>() + instr.sr() as usize * 4) as i32;
+ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());
+ builder.ins().store(MemFlags::trusted(), val, t.ctx_ptr, off);
+ }
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_MFMSR => {
+ let msr = builder.ins().load(
+ types::I32,
+ MemFlags::trusted(),
+ t.ctx_ptr,
+ abi::msr_offset::<SYSTEM>() as i32,
+ );
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), msr);
+
+ t.handled_natively = true;
+ }
+ lut::OP_MTMSR if t.is_terminator => {
+ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());
+ builder
+ .ins()
+ .store(MemFlags::trusted(), val, t.ctx_ptr, abi::msr_offset::<SYSTEM>() as i32);
+
+ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));
+ t.handled_natively = true;
+ }
+ lut::OP_RFI if t.is_terminator => {
+ const RFI_MSR_MASK: i64 = 0x87C0_FFFF;
+
+ let msr_off = abi::msr_offset::<SYSTEM>() as i32;
+ let srr0_off = abi::spr_field_offset::<SYSTEM>(26).unwrap() as i32;
+ let srr1_off = abi::spr_field_offset::<SYSTEM>(27).unwrap() as i32;
+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;
+
+ let msr_v = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, msr_off);
+ let srr1 = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, srr1_off);
+ let msr_keep = builder.ins().band_imm(msr_v, !RFI_MSR_MASK & 0xFFFF_FFFFi64);
+ let srr1_take = builder.ins().band_imm(srr1, RFI_MSR_MASK);
+ let new_msr = builder.ins().bor(msr_keep, srr1_take);
+ let new_msr_clr = builder.ins().band_imm(new_msr, !0x0004_0000i64 & 0xFFFF_FFFFi64);
+ builder
+ .ins()
+ .store(MemFlags::trusted(), new_msr_clr, t.ctx_ptr, msr_off);
+
+ let srr0_raw = builder.ins().load(types::I32, MemFlags::trusted(), t.ctx_ptr, srr0_off);
+ let nia = builder.ins().band_imm(srr0_raw, 0xFFFF_FFFCi64);
+ builder.ins().store(MemFlags::trusted(), nia, t.ctx_ptr, nia_off);
+
+ t.last_terminator_nia = Some(nia);
+ t.handled_natively = true;
+ }
+ _ => {}
+ }
+}
+
+#[inline(always)]
+pub fn spr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_MFSPR => {
+ if translator::emit_mfspr::<SYSTEM>(builder, t.ctx_ptr, instr, local) {
+ t.handled_natively = true;
+ }
+ }
+ lut::OP_MTSPR => {
+ if translator::emit_mtspr::<SYSTEM>(builder, t.ctx_ptr, instr, local) {
+ if t.is_terminator {
+ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));
+ }
+ t.handled_natively = true;
+ }
+ }
+ _ => {}
+ }
+}
+
+#[inline(always)]
+pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_LWZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false),
+ lut::OP_LWZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, true),
+ lut::OP_LBZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, false),
+ lut::OP_LBZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, true),
+ lut::OP_LHZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, false),
+ lut::OP_LHZU => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, true),
+ lut::OP_LHA => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),
+ lut::OP_LHAU => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),
+
+ lut::OP_STW => {
+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)
+ }
+ lut::OP_STWU => {
+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)
+ }
+ lut::OP_STB => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false),
+ lut::OP_STBU => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true),
+ lut::OP_STH => {
+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)
+ }
+ lut::OP_STHU => {
+ translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)
+ }
+
+ lut::OP_LMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, false),
+ lut::OP_STMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, true),
+
+ lut::OP_LWZX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false)
+ }
+ lut::OP_LWZUX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, true)
+ }
+ lut::OP_LBZX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, false)
+ }
+ lut::OP_LBZUX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.read_u8, true)
+ }
+ lut::OP_LHZX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, false)
+ }
+ lut::OP_LHZUX => {
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16, true)
+ }
+ lut::OP_LHAX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),
+ lut::OP_LHAUX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),
+
+ lut::OP_STWX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)
+ }
+ lut::OP_STWUX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)
+ }
+ lut::OP_STBX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false)
+ }
+ lut::OP_STBUX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true)
+ }
+ lut::OP_STHX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)
+ }
+ lut::OP_STHUX => {
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)
+ }
+
+ lut::OP_LWBRX => {
+ translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32)
+ }
+ lut::OP_LHBRX => {
+ translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16)
+ }
+ lut::OP_STWBRX => {
+ translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32)
+ }
+ lut::OP_STHBRX => {
+ translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16)
+ }
+
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ match OP {
+ lut::OP_LFS => translator::emit_lfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32, local.read_u32),
+ lut::OP_LFSU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
+ lut::OP_LFD => translator::emit_lfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),
+ lut::OP_LFDU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
+
+ lut::OP_STFS => translator::emit_stfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.write_u32),
+ lut::OP_STFSU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
+ lut::OP_STFD => translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),
+ lut::OP_STFDU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
+
+ lut::OP_LFSX => translator::emit_lfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32),
+ lut::OP_LFSUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
+ lut::OP_LFDX => translator::emit_lfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),
+ lut::OP_LFDUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
+
+ lut::OP_STFSX => translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32),
+ lut::OP_STFSUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
+ lut::OP_STFDX => translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),
+ lut::OP_STFDUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
+
+ lut::OP_STFIWX => translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32),
+
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn store_load_psq<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ match OP {
+ lut::OP_PSQ_L => translator::emit_psq_l::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, false),
+ lut::OP_PSQ_LU => translator::emit_psq_l::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, true),
+ lut::OP_PSQ_ST => {
+ translator::emit_psq_st::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, false)
+ }
+ lut::OP_PSQ_STU => {
+ translator::emit_psq_st::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, true)
+ }
+
+ lut::OP_PSQ_LX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, false, false),
+ lut::OP_PSQ_LUX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, false, true),
+ lut::OP_PSQ_STX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, true, false),
+ lut::OP_PSQ_STUX => translator::emit_psq_x::<SYSTEM>(builder, t.ctx_ptr, instr, local, true, true),
+
+ _ => return,
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let _ = (instr, SYSTEM);
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_ISYNC if t.is_terminator => {
+ t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));
+ t.handled_natively = true;
+ }
+ lut::OP_SYNC
+ | lut::OP_EIEIO
+ | lut::OP_DCBF
+ | lut::OP_DCBI
+ | lut::OP_DCBST
+ | lut::OP_DCBT
+ | lut::OP_DCBTST
+ | lut::OP_DCBA
+ | lut::OP_DCBZ
+ | lut::OP_DCBZ_L
+ | lut::OP_ICBI
+ | lut::OP_TLBIE
+ | lut::OP_TLBIA
+ | lut::OP_TLBSYNC
+ | lut::OP_TLBLD
+ | lut::OP_TLBLI => {
+ t.handled_natively = true;
+ }
+ _ => {}
+ }
+}
+
+#[inline(always)]
+pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ if !t.is_terminator {
+ return;
+ }
+
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ match OP {
+ lut::OP_BX => {
+ if let Some(slot_addr) = t.chain_slot_addr {
+ let target = if instr.aa() {
+ instr.li() as u32
+ } else {
+ t.pc.wrapping_add_signed(instr.li())
+ };
+
+ if instr.lk() {
+ let lr_val = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);
+ translator::lr_store::<SYSTEM>(builder, t.ctx_ptr, lr_val);
+ }
+
+ translator::emit_cycles_add::<SYSTEM>(builder, t.ctx_ptr, CYCLES_PER_INSTR);
+
+ let target_v = builder.ins().iconst(types::I32, target as i64);
+ if instr.lk() {
+ translator::emit_call_or_exit::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ slot_addr,
+ t.block_sig_ref,
+ target_v,
+ t.exit_block,
+ );
+ } else {
+ translator::emit_chain_or_exit::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ slot_addr,
+ t.block_sig_ref,
+ target_v,
+ t.exit_block,
+ );
+ }
+
+ t.chained = true;
+ t.handled_natively = true;
+ } else {
+ t.last_terminator_nia = Some(translator::emit_b::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc));
+ t.handled_natively = true;
+ }
+ }
+
+ lut::OP_BCX => {
+ match translator::emit_bc_with_chain::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ t.pc,
+ t.exit_block,
+ t.chain_slot_addr,
+ t.chain_fall_addr,
+ t.block_sig_ref,
+ ) {
+ TermEmit::HandledNia(nia) => {
+ t.last_terminator_nia = Some(nia);
+ t.handled_natively = true;
+ }
+ TermEmit::HandledChained => {
+ t.chained = true;
+ t.handled_natively = true;
+ }
+ TermEmit::NotHandled => {}
+ }
+ }
+
+ lut::OP_BCLRX => {
+ match translator::emit_bclr_dispatch::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ t.pc,
+ t.exit_block,
+ t.block_sig_ref,
+ t.block_lookup_table_addr,
+ ) {
+ TermEmit::HandledNia(nia) => {
+ t.last_terminator_nia = Some(nia);
+ t.handled_natively = true;
+ }
+ TermEmit::HandledChained => {
+ t.chained = true;
+ t.handled_natively = true;
+ }
+ TermEmit::NotHandled => {}
+ }
+ }
+
+ lut::OP_BCCTRX => {
+ match translator::emit_bcctr_dispatch::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ t.pc,
+ t.exit_block,
+ t.block_sig_ref,
+ t.block_lookup_table_addr,
+ ) {
+ TermEmit::HandledNia(nia) => {
+ t.last_terminator_nia = Some(nia);
+ t.handled_natively = true;
+ }
+ TermEmit::HandledChained => {
+ t.chained = true;
+ t.handled_natively = true;
+ }
+ TermEmit::NotHandled => {}
+ }
+ }
+
+ _ => {}
+ }
+}
+
+#[inline(always)]
+pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ if translator::emit_fp_arith::<OP, SYSTEM>(builder, t.ctx_ptr, instr) {
+ if instr.rc() {
+ translator::emit_update_cr1_from_fpscr::<SYSTEM>(builder, t.ctx_ptr);
+ }
+ t.handled_natively = true;
+ }
+}
+
+#[inline(always)]
+pub fn ps_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ if translator::emit_ps_arith::<OP, SYSTEM>(builder, t.ctx_ptr, instr) {
+ if instr.rc() {
+ translator::emit_update_cr1_from_fpscr::<SYSTEM>(builder, t.ctx_ptr);
+ }
+ t.handled_natively = true;
+ }
+}
+
+#[inline(always)]
+pub fn sc<const SYSTEM: SystemId>(t: &mut JitTranslator, _instr: Instruction) {
+ if !t.is_terminator {
+ return;
+ }
+
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let cia_off = abi::cia_offset::<SYSTEM>() as i32;
+ let pc_const = builder.ins().iconst(types::I32, t.pc as i64);
+ builder.ins().store(MemFlags::trusted(), pc_const, t.ctx_ptr, cia_off);
+
+ let srr0_value = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);
+ t.last_terminator_nia = Some(translator::emit_exception_dispatch::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ srr0_value,
+ 0,
+ 0x0000_0C00,
+ ));
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn mcrxr<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_mcrxr::<SYSTEM>(builder, t.ctx_ptr, instr);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn lwarx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_lwarx::<SYSTEM>(builder, t.ctx_ptr, instr, local);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn stwcx_dot<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_stwcx_dot::<SYSTEM>(builder, t.ctx_ptr, instr, local);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn lswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };
+ let ea = if instr.ra() == 0 {
+ builder.ins().iconst(types::I32, 0)
+ } else {
+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra())
+ };
+
+ let rd_v = builder.ins().iconst(types::I32, instr.rd() as i64);
+ let n_v = builder.ins().iconst(types::I32, n as i64);
+ builder.ins().call(local.do_lswi, &[t.ctx_ptr, ea, rd_v, n_v]);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn stswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };
+ let ea = if instr.ra() == 0 {
+ builder.ins().iconst(types::I32, 0)
+ } else {
+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.ra())
+ };
+
+ let rs_v = builder.ins().iconst(types::I32, instr.rs() as i64);
+ let n_v = builder.ins().iconst(types::I32, n as i64);
+ builder.ins().call(local.do_stswi, &[t.ctx_ptr, ea, rs_v, n_v]);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn lswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);
+ let rd_v = builder.ins().iconst(types::I32, instr.rd() as i64);
+ builder.ins().call(local.do_lswx, &[t.ctx_ptr, ea, rd_v]);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn stswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);
+ let rs_v = builder.ins().iconst(types::I32, instr.rs() as i64);
+ builder.ins().call(local.do_stswx, &[t.ctx_ptr, ea, rs_v]);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn mfsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let idx = builder.ins().ushr_imm(b, 28);
+ let idx_masked = builder.ins().band_imm(idx, 0xF);
+ let byte_off = builder.ins().ishl_imm(idx_masked, 2);
+ let byte_off64 = builder.ins().uextend(types::I64, byte_off);
+
+ let base = builder
+ .ins()
+ .iadd_imm(t.ctx_ptr, abi::sr_base_offset::<SYSTEM>() as i64);
+ let addr = builder.ins().iadd(base, byte_off64);
+
+ let val = builder.ins().load(types::I32, MemFlags::trusted(), addr, 0);
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn mtsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());
+ let idx = builder.ins().ushr_imm(b, 28);
+ let idx_masked = builder.ins().band_imm(idx, 0xF);
+ let byte_off = builder.ins().ishl_imm(idx_masked, 2);
+ let byte_off64 = builder.ins().uextend(types::I64, byte_off);
+
+ let base = builder
+ .ins()
+ .iadd_imm(t.ctx_ptr, abi::sr_base_offset::<SYSTEM>() as i64);
+ let addr = builder.ins().iadd(base, byte_off64);
+
+ let val = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rs());
+ builder.ins().store(MemFlags::trusted(), val, addr, 0);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn mftb<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let tbr = instr.spr_swapped() as u16;
+
+ if tbr == 268 || tbr == 269 {
+ let cycles = builder.ins().load(
+ types::I64,
+ MemFlags::trusted(),
+ t.ctx_ptr,
+ abi::cycles_offset::<SYSTEM>() as i32,
+ );
+ let cycles_div = builder.ins().udiv_imm(cycles, 12);
+ let tb_off = builder.ins().load(
+ types::I64,
+ MemFlags::trusted(),
+ t.ctx_ptr,
+ abi::timebase_offset_offset::<SYSTEM>() as i32,
+ );
+ let tb = builder.ins().iadd(cycles_div, tb_off);
+
+ let val = if tbr == 269 {
+ let shifted = builder.ins().ushr_imm(tb, 32);
+ builder.ins().ireduce(types::I32, shifted)
+ } else {
+ builder.ins().ireduce(types::I32, tb)
+ };
+
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);
+ } else {
+ let tbr_v = builder.ins().iconst(types::I32, tbr as i64);
+ let call = builder.ins().call(local.read_timebase, &[t.ctx_ptr, tbr_v]);
+ let val = builder.inst_results(call)[0];
+ translator::gpr_store::<SYSTEM>(builder, t.ctx_ptr, instr.rd(), val);
+ }
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn tw<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_tw::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ t.handled_natively = true;
+}
+
+#[inline(always)]
+pub fn twi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ translator::emit_twi::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);
+
+ t.handled_natively = true;
+}
+
+#[cold]
+#[inline(never)]
+pub fn invalid(_t: &mut JitTranslator, _instr: Instruction) {}
diff --git a/crates/gecko/src/gekko/jit/idle.rs b/crates/gecko/src/gekko/jit/idle.rs
new file mode 100644
index 0000000..b370509
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/idle.rs
@@ -0,0 +1,158 @@
+use crate::gekko::jit::block::{BlockSpec, TermKind};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum IdleClass {
+ None,
+ BranchToSelf,
+ PollingLoop,
+}
+
+pub fn classify<const SYSTEM: crate::system::SystemId>(spec: &BlockSpec, gprs: &[u32; 32]) -> IdleClass {
+ let _ = gprs;
+
+ if let Some(class) = classify_branch_to_self(spec) {
+ return class;
+ }
+
+ if classify_polling_loop(spec) {
+ return IdleClass::PollingLoop;
+ }
+
+ IdleClass::None
+}
+
+fn classify_branch_to_self(spec: &BlockSpec) -> Option<IdleClass> {
+ if spec.terminator != TermKind::Branch || spec.instrs.len() != 1 {
+ return None;
+ }
+
+ let instr = crate::gekko::instruction::Instruction(spec.instrs[0]);
+ if instr.lk() {
+ return None;
+ }
+
+ let li = instr.li();
+ let target = if instr.aa() {
+ li as u32
+ } else {
+ spec.start_pc.wrapping_add_signed(li)
+ };
+
+ if target == spec.start_pc {
+ Some(IdleClass::BranchToSelf)
+ } else {
+ None
+ }
+}
+
+fn classify_polling_loop(spec: &BlockSpec) -> bool {
+ const MAX_IDLE_BODY: usize = 6;
+
+ if spec.terminator != TermKind::BranchCond {
+ return false;
+ }
+
+ let last_idx = match spec.instrs.len().checked_sub(1) {
+ Some(i) if i > 0 => i,
+ _ => return false,
+ };
+
+ if last_idx > MAX_IDLE_BODY {
+ return false;
+ }
+
+ let term = crate::gekko::instruction::Instruction(spec.instrs[last_idx]);
+ let term_pc = spec.start_pc.wrapping_add((last_idx as u32) * 4);
+ if !is_idle_loop_terminator(term, term_pc, spec.start_pc) {
+ return false;
+ }
+
+ validate_idle_loop(&spec.instrs[..last_idx])
+}
+
+fn is_idle_loop_terminator(instr: crate::gekko::instruction::Instruction, branch_pc: u32, block_start_pc: u32) -> bool {
+ if instr.primary_opcode() != 16 {
+ return false;
+ }
+
+ if instr.lk() {
+ return false;
+ }
+
+ if instr.bo() & 0b00100 == 0 {
+ return false;
+ }
+
+ let target = if instr.aa() {
+ instr.bd() as u32
+ } else {
+ branch_pc.wrapping_add_signed(instr.bd())
+ };
+
+ target == block_start_pc
+}
+
+fn validate_idle_loop(body: &[u32]) -> bool {
+ let mut write_disallowed: u32 = 0;
+ let mut written: u32 = 0;
+
+ for &raw in body {
+ let (reads, writes) = match gpr_dataflow(crate::gekko::instruction::Instruction(raw)) {
+ Some(p) => p,
+ None => return false,
+ };
+
+ let externals = reads & !written;
+ write_disallowed |= externals;
+ if writes & write_disallowed != 0 {
+ return false;
+ }
+
+ written |= writes;
+ }
+
+ true
+}
+
+fn gpr_dataflow(instr: crate::gekko::instruction::Instruction) -> Option<(u32, u32)> {
+ let rd_or_s = instr.rd() as u32;
+ let ra = instr.ra() as u32;
+ let rb = instr.rb() as u32;
+ let bit = |r: u32| 1u32 << r;
+ let read_a_or_zero = if ra == 0 { 0 } else { bit(ra) };
+
+ Some(match instr.primary_opcode() {
+ 14 | 15 => (read_a_or_zero, bit(rd_or_s)),
+ 7 | 8 | 12 | 13 => (bit(ra), bit(rd_or_s)),
+ 10 | 11 => (bit(ra), 0),
+ 24 | 25 | 26 | 27 | 28 | 29 => (bit(rd_or_s), bit(ra)),
+ 20 => (bit(rd_or_s) | bit(ra), bit(ra)),
+ 21 => (bit(rd_or_s), bit(ra)),
+ 23 => (bit(rd_or_s) | bit(rb), bit(ra)),
+ 32 | 34 | 40 | 42 => (read_a_or_zero, bit(rd_or_s)),
+ 33 | 35 | 41 | 43 => (bit(ra), bit(rd_or_s) | bit(ra)),
+ 31 => return xform_dataflow(instr),
+ _ => return None,
+ })
+}
+
+fn xform_dataflow(instr: crate::gekko::instruction::Instruction) -> Option<(u32, u32)> {
+ let rd_or_s = instr.rd() as u32;
+ let ra = instr.ra() as u32;
+ let rb = instr.rb() as u32;
+ let bit = |r: u32| 1u32 << r;
+ let read_a_or_zero = if ra == 0 { 0 } else { bit(ra) };
+
+ Some(match instr.xo10() {
+ 266 | 40 | 10 | 138 | 202 | 234 | 8 | 136 | 200 | 232 | 104 | 235 | 75 | 11 | 491 | 459 | 778 | 552 | 522
+ | 650 | 714 | 746 | 520 | 648 | 712 | 744 | 616 | 747 | 1003 | 971 => (bit(ra) | bit(rb), bit(rd_or_s)),
+ 28 | 60 | 124 | 284 | 316 | 412 | 444 | 476 => (bit(rd_or_s) | bit(rb), bit(ra)),
+ 26 | 922 | 954 => (bit(rd_or_s), bit(ra)),
+ 24 | 536 | 792 => (bit(rd_or_s) | bit(rb), bit(ra)),
+ 824 => (bit(rd_or_s), bit(ra)),
+ 0 | 32 => (bit(ra) | bit(rb), 0),
+ 23 | 87 | 279 | 343 | 534 | 790 => (read_a_or_zero | bit(rb), bit(rd_or_s)),
+ 55 | 119 | 311 | 375 => (bit(ra) | bit(rb), bit(rd_or_s) | bit(ra)),
+ _ => return None,
+ })
+}
diff --git a/crates/gecko/src/gekko/jit/insn.rs b/crates/gecko/src/gekko/jit/insn.rs
new file mode 100644
index 0000000..125a5d6
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/insn.rs
@@ -0,0 +1,24 @@
+use crate::gekko::instruction::Instruction;
+
+#[allow(dead_code)]
+impl Instruction {
+ #[inline(always)]
+ pub fn primary_opcode(&self) -> u8 {
+ (self.0 >> 26) as u8
+ }
+
+ #[inline(always)]
+ pub fn xo10(&self) -> u32 {
+ (self.0 >> 1) & 0x3FF
+ }
+
+ #[inline(always)]
+ pub fn xo5(&self) -> u32 {
+ (self.0 >> 1) & 0x1F
+ }
+
+ #[inline(always)]
+ pub fn xo6(&self) -> u32 {
+ (self.0 >> 1) & 0x3F
+ }
+}
diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rs
new file mode 100644
index 0000000..636026a
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/runtime.rs
@@ -0,0 +1,538 @@
+use crate::gekko::msr::Msr;
+use crate::gekko::sr::Sr;
+use crate::system::{GC, System, SystemId, WII};
+
+#[cold]
+#[inline(never)]
+fn cause_invalid_opcode<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, raw_instr: u32, pc: u32) -> u32 {
+ let sys = unsafe { &mut *sys };
+ panic!(
+ "unimplemented Gekko opcode {:#010x} at pc={:#010x} lr={:#010x}",
+ raw_instr, pc, sys.gekko.spr.lr,
+ );
+}
+
+pub extern "C" fn cause_invalid_opcode_gc(sys: *mut core::ffi::c_void, raw_instr: u32, pc: u32) -> u32 {
+ cause_invalid_opcode::<GC>(sys.cast(), raw_instr, pc)
+}
+
+pub extern "C" fn cause_invalid_opcode_wii(sys: *mut core::ffi::c_void, raw_instr: u32, pc: u32) -> u32 {
+ cause_invalid_opcode::<WII>(sys.cast(), raw_instr, pc)
+}
+
+#[inline(always)]
+fn advance_to_deadline<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {
+ let sys = unsafe { &mut *sys };
+ let deadline = sys.scheduler.next_deadline();
+ if sys.scheduler.cycles < deadline {
+ sys.scheduler.cycles = deadline;
+ }
+}
+
+pub extern "C" fn advance_to_deadline_gc(sys: *mut core::ffi::c_void) {
+ advance_to_deadline::<GC>(sys.cast());
+}
+
+pub extern "C" fn advance_to_deadline_wii(sys: *mut core::ffi::c_void) {
+ advance_to_deadline::<WII>(sys.cast());
+}
+
+#[inline(always)]
+fn slow_read_u8<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u8 {
+ unsafe { (*sys).read_u8(ea) }
+}
+#[inline(always)]
+fn slow_read_u16<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u16 {
+ unsafe { (*sys).read_u16(ea) }
+}
+#[inline(always)]
+fn slow_read_u32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> u32 {
+ unsafe { (*sys).read_u32(ea) }
+}
+#[inline(always)]
+fn slow_write_u8<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u8) {
+ unsafe { (*sys).write_u8(ea, v) }
+}
+#[inline(always)]
+fn slow_write_u16<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u16) {
+ unsafe { (*sys).write_u16(ea, v) }
+}
+#[inline(always)]
+fn slow_write_u32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: u32) {
+ unsafe { (*sys).write_u32(ea, v) }
+}
+
+pub extern "C" fn read_u8_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u8::<GC>(sys.cast(), ea) as u32
+}
+pub extern "C" fn read_u16_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u16::<GC>(sys.cast(), ea) as u32
+}
+pub extern "C" fn read_u32_gc(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u32::<GC>(sys.cast(), ea)
+}
+pub extern "C" fn write_u8_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u8::<GC>(sys.cast(), ea, v as u8);
+}
+pub extern "C" fn write_u16_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u16::<GC>(sys.cast(), ea, v as u16);
+}
+pub extern "C" fn write_u32_gc(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u32::<GC>(sys.cast(), ea, v);
+}
+
+pub extern "C" fn read_u8_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u8::<WII>(sys.cast(), ea) as u32
+}
+pub extern "C" fn read_u16_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u16::<WII>(sys.cast(), ea) as u32
+}
+pub extern "C" fn read_u32_wii(sys: *mut core::ffi::c_void, ea: u32) -> u32 {
+ slow_read_u32::<WII>(sys.cast(), ea)
+}
+pub extern "C" fn write_u8_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u8::<WII>(sys.cast(), ea, v as u8);
+}
+pub extern "C" fn write_u16_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u16::<WII>(sys.cast(), ea, v as u16);
+}
+pub extern "C" fn write_u32_wii(sys: *mut core::ffi::c_void, ea: u32, v: u32) {
+ slow_write_u32::<WII>(sys.cast(), ea, v);
+}
+
+#[inline(always)]
+fn slow_read_f32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> f64 {
+ unsafe { (*sys).read_f32(ea) }
+}
+#[inline(always)]
+fn slow_write_f32<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: f64) {
+ unsafe { (*sys).write_f32(ea, v) }
+}
+#[inline(always)]
+fn slow_read_f64<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) -> f64 {
+ unsafe { (*sys).read_f64(ea) }
+}
+#[inline(always)]
+fn slow_write_f64<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, v: f64) {
+ unsafe { (*sys).write_f64(ea, v) }
+}
+
+pub extern "C" fn read_f32_gc(sys: *mut core::ffi::c_void, ea: u32) -> f64 {
+ slow_read_f32::<GC>(sys.cast(), ea)
+}
+pub extern "C" fn write_f32_gc(sys: *mut core::ffi::c_void, ea: u32, v: f64) {
+ slow_write_f32::<GC>(sys.cast(), ea, v)
+}
+pub extern "C" fn read_f64_gc(sys: *mut core::ffi::c_void, ea: u32) -> f64 {
+ slow_read_f64::<GC>(sys.cast(), ea)
+}
+pub extern "C" fn write_f64_gc(sys: *mut core::ffi::c_void, ea: u32, v: f64) {
+ slow_write_f64::<GC>(sys.cast(), ea, v)
+}
+pub extern "C" fn read_f32_wii(sys: *mut core::ffi::c_void, ea: u32) -> f64 {
+ slow_read_f32::<WII>(sys.cast(), ea)
+}
+pub extern "C" fn write_f32_wii(sys: *mut core::ffi::c_void, ea: u32, v: f64) {
+ slow_write_f32::<WII>(sys.cast(), ea, v)
+}
+pub extern "C" fn read_f64_wii(sys: *mut core::ffi::c_void, ea: u32) -> f64 {
+ slow_read_f64::<WII>(sys.cast(), ea)
+}
+pub extern "C" fn write_f64_wii(sys: *mut core::ffi::c_void, ea: u32, v: f64) {
+ slow_write_f64::<WII>(sys.cast(), ea, v)
+}
+
+#[inline(always)]
+fn write_msr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, val: u32) {
+ unsafe { (*sys).gekko.msr = Msr::from(val) }
+}
+
+pub extern "C" fn write_msr_gc(sys: *mut core::ffi::c_void, val: u32) {
+ write_msr::<GC>(sys.cast(), val);
+}
+pub extern "C" fn write_msr_wii(sys: *mut core::ffi::c_void, val: u32) {
+ write_msr::<WII>(sys.cast(), val);
+}
+
+#[inline(always)]
+fn read_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32) -> u32 {
+ let sys = unsafe { &mut *sys };
+ match num {
+ 22 => {
+ sys.gekko.spr.dec = sys.gekko.dec.read(sys.scheduler.cycles);
+ sys.gekko.spr.dec
+ }
+ 268 => sys.scheduler.timebase_lower(),
+ 269 => sys.scheduler.timebase_upper(),
+ _ => sys.gekko.spr.read(num),
+ }
+}
+
+pub extern "C" fn read_spr_gc(sys: *mut core::ffi::c_void, num: u32) -> u32 {
+ read_spr::<GC>(sys.cast(), num)
+}
+pub extern "C" fn read_spr_wii(sys: *mut core::ffi::c_void, num: u32) -> u32 {
+ read_spr::<WII>(sys.cast(), num)
+}
+
+#[inline(always)]
+fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u32) {
+ let sys = unsafe { &mut *sys };
+ match num {
+ 22 => {
+ sys.scheduler.cancel(crate::gekko::dec::underflow_handler::<SYSTEM>);
+ sys.gekko.dec.write(sys.scheduler.cycles, val);
+ sys.gekko.spr.dec = val;
+ sys.scheduler.schedule_in(
+ crate::gekko::dec::cycles_until_underflow(val),
+ crate::gekko::dec::underflow_handler::<SYSTEM>,
+ );
+ }
+ 284 => sys.scheduler.set_timebase_lower(val),
+ 285 => sys.scheduler.set_timebase_upper(val),
+ 923 => {
+ sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
+ if sys.gekko.spr.dmal.trigger() {
+ sys.mmio
+ .process_locked_cache_dma(&sys.gekko.spr.dmau, &sys.gekko.spr.dmal);
+ }
+ }
+ _ => sys.gekko.spr.write(num, val),
+ }
+}
+
+pub extern "C" fn write_spr_gc(sys: *mut core::ffi::c_void, num: u32, val: u32) {
+ write_spr::<GC>(sys.cast(), num, val);
+}
+pub extern "C" fn write_spr_wii(sys: *mut core::ffi::c_void, num: u32, val: u32) {
+ write_spr::<WII>(sys.cast(), num, val);
+}
+
+#[inline(always)]
+fn read_sr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32) -> u32 {
+ unsafe { (*sys).gekko.sr[(idx & 0xF) as usize].raw() }
+}
+pub extern "C" fn read_sr_gc(sys: *mut core::ffi::c_void, idx: u32) -> u32 {
+ read_sr::<GC>(sys.cast(), idx)
+}
+pub extern "C" fn read_sr_wii(sys: *mut core::ffi::c_void, idx: u32) -> u32 {
+ read_sr::<WII>(sys.cast(), idx)
+}
+
+#[inline(always)]
+fn write_sr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32, val: u32) {
+ unsafe { (*sys).gekko.sr[(idx & 0xF) as usize] = Sr::from_raw(val) }
+}
+pub extern "C" fn write_sr_gc(sys: *mut core::ffi::c_void, idx: u32, val: u32) {
+ write_sr::<GC>(sys.cast(), idx, val);
+}
+pub extern "C" fn write_sr_wii(sys: *mut core::ffi::c_void, idx: u32, val: u32) {
+ write_sr::<WII>(sys.cast(), idx, val);
+}
+
+#[inline(always)]
+#[allow(dead_code)]
+fn cause_trap_exception<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {
+ unsafe { (*sys).cause_trap_exception() }
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_trap_exception_gc(sys: *mut core::ffi::c_void) {
+ cause_trap_exception::<GC>(sys.cast());
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_trap_exception_wii(sys: *mut core::ffi::c_void) {
+ cause_trap_exception::<WII>(sys.cast());
+}
+
+#[inline(always)]
+#[allow(dead_code)]
+fn cause_syscall_interrupt<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {
+ unsafe { (*sys).cause_syscall_interrupt() }
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_syscall_interrupt_gc(sys: *mut core::ffi::c_void) {
+ cause_syscall_interrupt::<GC>(sys.cast());
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_syscall_interrupt_wii(sys: *mut core::ffi::c_void) {
+ cause_syscall_interrupt::<WII>(sys.cast());
+}
+
+#[inline(always)]
+#[allow(dead_code)]
+fn cause_fp_unavailable<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, pc: u32) {
+ let sys = unsafe { &mut *sys };
+ sys.gekko.cia = pc;
+ sys.gekko.nia = pc.wrapping_add(4);
+ sys.cause_fp_unavailable();
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_fp_unavailable_gc(sys: *mut core::ffi::c_void, pc: u32) {
+ cause_fp_unavailable::<GC>(sys.cast(), pc);
+}
+#[allow(dead_code)]
+pub extern "C" fn cause_fp_unavailable_wii(sys: *mut core::ffi::c_void, pc: u32) {
+ cause_fp_unavailable::<WII>(sys.cast(), pc);
+}
+
+#[inline(always)]
+#[allow(dead_code)]
+fn set_reservation<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, addr: u32) {
+ unsafe { (*sys).gekko.reserve_addr = addr }
+}
+#[allow(dead_code)]
+pub extern "C" fn set_reservation_gc(sys: *mut core::ffi::c_void, addr: u32) {
+ set_reservation::<GC>(sys.cast(), addr);
+}
+#[allow(dead_code)]
+pub extern "C" fn set_reservation_wii(sys: *mut core::ffi::c_void, addr: u32) {
+ set_reservation::<WII>(sys.cast(), addr);
+}
+
+#[inline(always)]
+#[allow(dead_code)]
+fn try_clear_reservation<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, addr: u32) -> u32 {
+ let sys = unsafe { &mut *sys };
+ if sys.gekko.reserve_addr == addr {
+ sys.gekko.reserve_addr = crate::gekko::Gekko::NO_RESERVATION;
+ 1
+ } else {
+ 0
+ }
+}
+#[allow(dead_code)]
+pub extern "C" fn try_clear_reservation_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {
+ try_clear_reservation::<GC>(sys.cast(), addr)
+}
+#[allow(dead_code)]
+pub extern "C" fn try_clear_reservation_wii(sys: *mut core::ffi::c_void, addr: u32) -> u32 {
+ try_clear_reservation::<WII>(sys.cast(), addr)
+}
+
+#[inline(always)]
+fn do_lsw_packed<const SYSTEM: SystemId, const STORE: bool>(
+ sys: *mut System<SYSTEM>,
+ ea: u32,
+ rd_or_rs: u32,
+ n_bytes: u32,
+) {
+ let sys = unsafe { &mut *sys };
+ let mut n = n_bytes;
+ let mut r = rd_or_rs.wrapping_sub(1) & 31;
+ let mut i = 0u32;
+ let mut addr = ea;
+ while n > 0 {
+ if i == 0 {
+ r = (r + 1) & 31;
+ if !STORE {
+ sys.gekko.write_gpr(r as u8, 0);
+ }
+ }
+ if STORE {
+ let byte = (sys.gekko.read_gpr(r as u8) >> (24 - i)) as u8;
+ sys.write_u8(addr, byte);
+ } else {
+ let byte = sys.read_u8(addr) as u32;
+ let shift = 24 - i;
+ let val = sys.gekko.read_gpr(r as u8) | (byte << shift);
+ sys.gekko.write_gpr(r as u8, val);
+ }
+ i += 8;
+ if i == 32 {
+ i = 0;
+ }
+ addr = addr.wrapping_add(1);
+ n -= 1;
+ }
+}
+
+pub extern "C" fn do_lswi_gc(sys: *mut core::ffi::c_void, ea: u32, rd: u32, nb: u32) {
+ do_lsw_packed::<GC, false>(sys.cast(), ea, rd, nb);
+}
+pub extern "C" fn do_lswi_wii(sys: *mut core::ffi::c_void, ea: u32, rd: u32, nb: u32) {
+ do_lsw_packed::<WII, false>(sys.cast(), ea, rd, nb);
+}
+pub extern "C" fn do_stswi_gc(sys: *mut core::ffi::c_void, ea: u32, rs: u32, nb: u32) {
+ do_lsw_packed::<GC, true>(sys.cast(), ea, rs, nb);
+}
+pub extern "C" fn do_stswi_wii(sys: *mut core::ffi::c_void, ea: u32, rs: u32, nb: u32) {
+ do_lsw_packed::<WII, true>(sys.cast(), ea, rs, nb);
+}
+
+pub extern "C" fn do_lswx_gc(sys_v: *mut core::ffi::c_void, ea: u32, rd: u32) {
+ let sys = unsafe { &mut *(sys_v as *mut System<GC>) };
+ let n = sys.gekko.spr.xer.byte_count() as u32;
+ if n == 0 {
+ return;
+ }
+ do_lsw_packed::<GC, false>(sys_v.cast(), ea, rd, n);
+}
+pub extern "C" fn do_lswx_wii(sys_v: *mut core::ffi::c_void, ea: u32, rd: u32) {
+ let sys = unsafe { &mut *(sys_v as *mut System<WII>) };
+ let n = sys.gekko.spr.xer.byte_count() as u32;
+ if n == 0 {
+ return;
+ }
+ do_lsw_packed::<WII, false>(sys_v.cast(), ea, rd, n);
+}
+pub extern "C" fn do_stswx_gc(sys_v: *mut core::ffi::c_void, ea: u32, rs: u32) {
+ let sys = unsafe { &mut *(sys_v as *mut System<GC>) };
+ let n = sys.gekko.spr.xer.byte_count() as u32;
+ do_lsw_packed::<GC, true>(sys_v.cast(), ea, rs, n);
+}
+pub extern "C" fn do_stswx_wii(sys_v: *mut core::ffi::c_void, ea: u32, rs: u32) {
+ let sys = unsafe { &mut *(sys_v as *mut System<WII>) };
+ let n = sys.gekko.spr.xer.byte_count() as u32;
+ do_lsw_packed::<WII, true>(sys_v.cast(), ea, rs, n);
+}
+
+pub static DEQUANT_TABLE: [f32; 64] = {
+ let mut t = [0.0f32; 64];
+ let mut i = 0u32;
+ while i < 32 {
+ t[i as usize] = 1.0 / (1u64 << i) as f32;
+ i += 1;
+ }
+ while i < 64 {
+ t[i as usize] = (1u64 << (64 - i)) as f32;
+ i += 1;
+ }
+ t
+};
+pub static QUANT_TABLE: [f32; 64] = {
+ let mut t = [0.0f32; 64];
+ let mut i = 0u32;
+ while i < 32 {
+ t[i as usize] = (1u64 << i) as f32;
+ i += 1;
+ }
+ while i < 64 {
+ t[i as usize] = 1.0 / (1u64 << (64 - i)) as f32;
+ i += 1;
+ }
+ t
+};
+
+#[inline(always)]
+fn quant_element_size(qtype: u8) -> u32 {
+ match qtype {
+ 0 => 4,
+ 4 | 6 => 1,
+ 5 | 7 => 2,
+ _ => 4,
+ }
+}
+
+#[inline(always)]
+fn psq_dequant<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, addr: u32, ld_type: u8, ld_scale: u8) -> f64 {
+ let scale = DEQUANT_TABLE[ld_scale as usize];
+ match ld_type {
+ 0 => sys.read_f32(addr),
+ 4 => (sys.read_u8(addr) as f32 * scale) as f64,
+ 5 => (sys.read_u16(addr) as f32 * scale) as f64,
+ 6 => (sys.read_u8(addr) as i8 as f32 * scale) as f64,
+ 7 => (sys.read_u16(addr) as i16 as f32 * scale) as f64,
+ _ => 0.0,
+ }
+}
+
+#[inline(always)]
+fn psq_quant<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, addr: u32, value: f64, st_type: u8, st_scale: u8) {
+ let scale = QUANT_TABLE[st_scale as usize];
+ match st_type {
+ 0 => sys.write_f32(addr, value),
+ 4 => {
+ let v = (value as f32 * scale).clamp(u8::MIN as f32, u8::MAX as f32) as u8;
+ sys.write_u8(addr, v);
+ }
+ 5 => {
+ let v = (value as f32 * scale).clamp(u16::MIN as f32, u16::MAX as f32) as u16;
+ sys.write_u16(addr, v);
+ }
+ 6 => {
+ let v = (value as f32 * scale).clamp(i8::MIN as f32, i8::MAX as f32) as i8;
+ sys.write_u8(addr, v as u8);
+ }
+ 7 => {
+ let v = (value as f32 * scale).clamp(i16::MIN as f32, i16::MAX as f32) as i16;
+ sys.write_u16(addr, v as u16);
+ }
+ _ => {}
+ }
+}
+
+#[inline(always)]
+fn psq_load_inner<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, fd: u32, ea: u32, w: u32, gqr_idx: u32) {
+ let sys = unsafe { &mut *sys };
+ let gqr = sys.gekko.spr.read_gqr((gqr_idx & 0x7) as u8);
+ let ld_type = ((gqr >> 16) & 0x7) as u8;
+ let ld_scale = ((gqr >> 24) & 0x3F) as u8;
+ let ps0 = psq_dequant(sys, ea, ld_type, ld_scale);
+ let ps1 = if w != 0 {
+ 1.0
+ } else {
+ let elem = quant_element_size(ld_type);
+ psq_dequant(sys, ea.wrapping_add(elem), ld_type, ld_scale)
+ };
+ sys.gekko.write_fpr(fd as u8, ps0);
+ sys.gekko.write_ps1(fd as u8, ps1);
+}
+
+#[inline(always)]
+fn psq_store_inner<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, fs: u32, ea: u32, w: u32, gqr_idx: u32) {
+ let sys = unsafe { &mut *sys };
+ let gqr = sys.gekko.spr.read_gqr((gqr_idx & 0x7) as u8);
+ let st_type = (gqr & 0x7) as u8;
+ let st_scale = ((gqr >> 8) & 0x3F) as u8;
+ let ps0 = sys.gekko.read_fpr(fs as u8);
+ psq_quant(sys, ea, ps0, st_type, st_scale);
+ if w == 0 {
+ let ps1 = sys.gekko.read_ps1(fs as u8);
+ let elem = quant_element_size(st_type);
+ psq_quant(sys, ea.wrapping_add(elem), ps1, st_type, st_scale);
+ }
+}
+
+pub extern "C" fn do_psq_load_gc(sys: *mut core::ffi::c_void, fd: u32, ea: u32, w: u32, gqr: u32) {
+ psq_load_inner::<GC>(sys.cast(), fd, ea, w, gqr);
+}
+pub extern "C" fn do_psq_load_wii(sys: *mut core::ffi::c_void, fd: u32, ea: u32, w: u32, gqr: u32) {
+ psq_load_inner::<WII>(sys.cast(), fd, ea, w, gqr);
+}
+pub extern "C" fn do_psq_store_gc(sys: *mut core::ffi::c_void, fs: u32, ea: u32, w: u32, gqr: u32) {
+ psq_store_inner::<GC>(sys.cast(), fs, ea, w, gqr);
+}
+pub extern "C" fn do_psq_store_wii(sys: *mut core::ffi::c_void, fs: u32, ea: u32, w: u32, gqr: u32) {
+ psq_store_inner::<WII>(sys.cast(), fs, ea, w, gqr);
+}
+
+#[inline(always)]
+fn read_timebase<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, tbr: u32) -> u32 {
+ let sys = unsafe { &*sys };
+ match tbr {
+ 268 => sys.scheduler.timebase_lower(),
+ 269 => sys.scheduler.timebase_upper(),
+ _ => 0,
+ }
+}
+pub extern "C" fn read_timebase_gc(sys: *mut core::ffi::c_void, tbr: u32) -> u32 {
+ read_timebase::<GC>(sys.cast(), tbr)
+}
+pub extern "C" fn read_timebase_wii(sys: *mut core::ffi::c_void, tbr: u32) -> u32 {
+ read_timebase::<WII>(sys.cast(), tbr)
+}
+
+#[inline(always)]
+fn do_rfi<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {
+ let sys = unsafe { &mut *sys };
+ const RFI_MSR_MASK: u32 = 0x87C0_FFFF;
+ let new_msr = (sys.gekko.msr.raw() & !RFI_MSR_MASK) | (sys.gekko.spr.srr1 & RFI_MSR_MASK);
+ sys.gekko.msr = Msr::from(new_msr & !0x0004_0000);
+ sys.gekko.nia = sys.gekko.spr.srr0.value() << 2;
+}
+pub extern "C" fn do_rfi_gc(sys: *mut core::ffi::c_void) {
+ do_rfi::<GC>(sys.cast());
+}
+pub extern "C" fn do_rfi_wii(sys: *mut core::ffi::c_void) {
+ do_rfi::<WII>(sys.cast());
+}
diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rs
new file mode 100644
index 0000000..4f70572
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/translator.rs
@@ -0,0 +1,4205 @@
+use cranelift_codegen::Context;
+use cranelift_codegen::ir::condcodes::IntCC;
+use cranelift_codegen::ir::{Block, FuncRef, InstBuilder, MemFlags, Value, types};
+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
+use cranelift_jit::JITModule;
+use cranelift_module::{FuncId, Module};
+use rustc_hash::FxHashMap;
+
+use crate::gekko::instruction::Instruction;
+use crate::gekko::jit::block::{BlockSpec, TermKind};
+use crate::gekko::jit::idle::{self, IdleClass};
+use crate::gekko::jit::{ExternFuncs, abi};
+use crate::system::SystemId;
+
+pub struct ChainContext<'a> {
+ pub self_pc: u32,
+ pub self_func_id: FuncId,
+ pub self_slot_addr: i64,
+ pub compiled_func_ids: &'a FxHashMap<u32, FuncId>,
+ pub compiled_cache: &'a FxHashMap<u32, super::BlockEntry>,
+ pub pending: std::cell::RefCell<&'a mut Vec<(u32, usize)>>,
+ pub block_lookup_table_addr: i64,
+}
+
+impl<'a> ChainContext<'a> {
+ fn target_for(&self, target_pc: u32) -> Option<i64> {
+ if target_pc == self.self_pc {
+ return Some(self.self_slot_addr);
+ }
+
+ if let Some(&entry) = self.compiled_cache.get(&target_pc) {
+ let slot: &'static mut usize = Box::leak(Box::new(entry));
+ return Some(slot as *mut usize as i64);
+ }
+
+ let slot: &'static mut usize = Box::leak(Box::new(0usize));
+ let slot_addr_usize = slot as *mut usize as usize;
+ self.pending.borrow_mut().push((target_pc, slot_addr_usize));
+ Some(slot_addr_usize as i64)
+ }
+}
+
+pub(crate) const CYCLES_PER_INSTR: i64 = 2;
+
+#[allow(dead_code)]
+pub(crate) struct LocalFuncs {
+ pub(crate) cause_invalid_opcode: FuncRef,
+ pub(crate) advance_to_deadline: FuncRef,
+ pub(crate) read_u8: FuncRef,
+ pub(crate) read_u16: FuncRef,
+ pub(crate) read_u32: FuncRef,
+ pub(crate) write_u8: FuncRef,
+ pub(crate) write_u16: FuncRef,
+ pub(crate) write_u32: FuncRef,
+ pub(crate) read_f32: FuncRef,
+ pub(crate) read_f64: FuncRef,
+ pub(crate) write_f32: FuncRef,
+ pub(crate) write_f64: FuncRef,
+ pub(crate) write_msr: FuncRef,
+ pub(crate) read_spr: FuncRef,
+ pub(crate) write_spr: FuncRef,
+ pub(crate) read_sr: FuncRef,
+ pub(crate) write_sr: FuncRef,
+ pub(crate) cause_trap_exception: FuncRef,
+ pub(crate) cause_syscall_interrupt: FuncRef,
+ pub(crate) do_rfi: FuncRef,
+ pub(crate) cause_fp_unavailable: FuncRef,
+ pub(crate) set_reservation: FuncRef,
+ pub(crate) try_clear_reservation: FuncRef,
+ pub(crate) do_lswi: FuncRef,
+ pub(crate) do_stswi: FuncRef,
+ pub(crate) do_lswx: FuncRef,
+ pub(crate) do_stswx: FuncRef,
+ pub(crate) do_psq_load: FuncRef,
+ pub(crate) do_psq_store: FuncRef,
+ pub(crate) read_timebase: FuncRef,
+ pub(crate) fp_guard_emitted: std::cell::Cell<bool>,
+}
+
+pub struct JitTranslator {
+ pub(crate) builder_ptr: usize,
+ pub(crate) local_ptr: usize,
+ pub ctx_ptr: Value,
+ pub pc: u32,
+ pub exit_block: cranelift_codegen::ir::Block,
+ pub chain_slot_addr: Option<i64>,
+ pub chain_fall_addr: Option<i64>,
+ pub block_sig_ref: cranelift_codegen::ir::SigRef,
+ pub block_lookup_table_addr: i64,
+ pub is_terminator: bool,
+ pub chained: bool,
+ pub last_terminator_nia: Option<Value>,
+ pub handled_natively: bool,
+}
+
+pub fn translate<const SYSTEM: SystemId>(
+ ctx: &mut Context,
+ builder_ctx: &mut FunctionBuilderContext,
+ module: &mut JITModule,
+ extern_funcs: &ExternFuncs,
+ spec: &BlockSpec,
+ gprs: &[u32; 32],
+ chain: &ChainContext<'_>,
+) {
+ let local = LocalFuncs {
+ cause_invalid_opcode: module.declare_func_in_func(extern_funcs.cause_invalid_opcode, &mut ctx.func),
+ advance_to_deadline: module.declare_func_in_func(extern_funcs.advance_to_deadline, &mut ctx.func),
+ read_u8: module.declare_func_in_func(extern_funcs.read_u8, &mut ctx.func),
+ read_u16: module.declare_func_in_func(extern_funcs.read_u16, &mut ctx.func),
+ read_u32: module.declare_func_in_func(extern_funcs.read_u32, &mut ctx.func),
+ write_u8: module.declare_func_in_func(extern_funcs.write_u8, &mut ctx.func),
+ write_u16: module.declare_func_in_func(extern_funcs.write_u16, &mut ctx.func),
+ write_u32: module.declare_func_in_func(extern_funcs.write_u32, &mut ctx.func),
+ read_f32: module.declare_func_in_func(extern_funcs.read_f32, &mut ctx.func),
+ read_f64: module.declare_func_in_func(extern_funcs.read_f64, &mut ctx.func),
+ write_f32: module.declare_func_in_func(extern_funcs.write_f32, &mut ctx.func),
+ write_f64: module.declare_func_in_func(extern_funcs.write_f64, &mut ctx.func),
+ write_msr: module.declare_func_in_func(extern_funcs.write_msr, &mut ctx.func),
+ read_spr: module.declare_func_in_func(extern_funcs.read_spr, &mut ctx.func),
+ write_spr: module.declare_func_in_func(extern_funcs.write_spr, &mut ctx.func),
+ read_sr: module.declare_func_in_func(extern_funcs.read_sr, &mut ctx.func),
+ write_sr: module.declare_func_in_func(extern_funcs.write_sr, &mut ctx.func),
+ cause_trap_exception: module.declare_func_in_func(extern_funcs.cause_trap_exception, &mut ctx.func),
+ cause_syscall_interrupt: module.declare_func_in_func(extern_funcs.cause_syscall_interrupt, &mut ctx.func),
+ do_rfi: module.declare_func_in_func(extern_funcs.do_rfi, &mut ctx.func),
+ cause_fp_unavailable: module.declare_func_in_func(extern_funcs.cause_fp_unavailable, &mut ctx.func),
+ set_reservation: module.declare_func_in_func(extern_funcs.set_reservation, &mut ctx.func),
+ try_clear_reservation: module.declare_func_in_func(extern_funcs.try_clear_reservation, &mut ctx.func),
+ do_lswi: module.declare_func_in_func(extern_funcs.do_lswi, &mut ctx.func),
+ do_stswi: module.declare_func_in_func(extern_funcs.do_stswi, &mut ctx.func),
+ do_lswx: module.declare_func_in_func(extern_funcs.do_lswx, &mut ctx.func),
+ do_stswx: module.declare_func_in_func(extern_funcs.do_stswx, &mut ctx.func),
+ do_psq_load: module.declare_func_in_func(extern_funcs.do_psq_load, &mut ctx.func),
+ do_psq_store: module.declare_func_in_func(extern_funcs.do_psq_store, &mut ctx.func),
+ read_timebase: module.declare_func_in_func(extern_funcs.read_timebase, &mut ctx.func),
+ fp_guard_emitted: std::cell::Cell::new(false),
+ };
+
+ let idle_class = idle::classify::<SYSTEM>(spec, gprs);
+
+ let (chain_slot_addr, chain_fall_addr): (Option<i64>, Option<i64>) = if matches!(idle_class, IdleClass::None) {
+ let taken = terminator_static_taken_pc(spec).and_then(|t| chain.target_for(t));
+ let fall = terminator_fall_pc(spec).and_then(|t| chain.target_for(t));
+ (taken, fall)
+ } else {
+ (None, None)
+ };
+
+ let sig_clone = ctx.func.signature.clone();
+ let block_sig_ref = ctx.func.import_signature(sig_clone);
+
+ let mut builder = FunctionBuilder::new(&mut ctx.func, builder_ctx);
+ let entry = builder.create_block();
+ builder.append_block_params_for_function_params(entry);
+ builder.switch_to_block(entry);
+ builder.seal_block(entry);
+
+ let ctx_ptr: Value = builder.block_params(entry)[0];
+
+ let exit_block = builder.create_block();
+ builder.append_block_param(exit_block, types::I32);
+
+ let mut last_terminator_nia: Option<Value> = None;
+ let mut chained = false;
+
+ let mut t = JitTranslator {
+ builder_ptr: &mut builder as *mut FunctionBuilder<'_> as *mut () as usize,
+ local_ptr: &local as *const LocalFuncs as usize,
+ ctx_ptr,
+ pc: 0,
+ exit_block,
+ chain_slot_addr,
+ chain_fall_addr,
+ block_sig_ref,
+ block_lookup_table_addr: chain.block_lookup_table_addr,
+ is_terminator: false,
+ chained: false,
+ last_terminator_nia: None,
+ handled_natively: false,
+ };
+
+ for (i, &word) in spec.instrs.iter().enumerate() {
+ let instr = Instruction(word);
+ let pc = spec.pc_of(i);
+ let is_terminator = i + 1 == spec.instrs.len() && !matches!(spec.terminator, TermKind::LengthCap);
+
+ t.pc = pc;
+ t.is_terminator = is_terminator;
+ t.handled_natively = false;
+ let chained_before = t.chained;
+
+ super::dispatch::<SYSTEM>(&mut t, instr);
+
+ if t.handled_natively {
+ if !t.chained || chained_before {
+ emit_cycles_add::<SYSTEM>(&mut builder, ctx_ptr, CYCLES_PER_INSTR);
+ }
+
+ if t.chained && !chained_before {
+ chained = true;
+ }
+
+ if let Some(nia) = t.last_terminator_nia.take() {
+ last_terminator_nia = Some(nia);
+ }
+ } else {
+ let pc_const = builder.ins().iconst(types::I32, pc as i64);
+ let raw_const = builder.ins().iconst(types::I32, instr.0 as i64);
+ let call = builder
+ .ins()
+ .call(local.cause_invalid_opcode, &[ctx_ptr, raw_const, pc_const]);
+ let nia = builder.inst_results(call)[0];
+
+ emit_cycles_add::<SYSTEM>(&mut builder, ctx_ptr, CYCLES_PER_INSTR);
+
+ if is_terminator {
+ last_terminator_nia = Some(nia);
+ } else {
+ let expected = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ let is_seq = builder.ins().icmp(IntCC::Equal, nia, expected);
+ let cont = builder.create_block();
+
+ builder.ins().brif(is_seq, cont, &[], exit_block, &[nia.into()]);
+ builder.switch_to_block(cont);
+ builder.seal_block(cont);
+ }
+ }
+ }
+ drop(t);
+
+ if !chained {
+ if idle_class != IdleClass::None {
+ builder.ins().call(local.advance_to_deadline, &[ctx_ptr]);
+ }
+
+ let final_nia = match spec.terminator {
+ TermKind::LengthCap => builder.ins().iconst(types::I32, spec.end_pc() as i64),
+ _ => last_terminator_nia.unwrap_or_else(|| {
+ let nia_off = abi::nia_offset::<SYSTEM>() as i64;
+ let nia_addr = builder.ins().iadd_imm(ctx_ptr, nia_off);
+ builder.ins().load(types::I32, MemFlags::trusted(), nia_addr, 0)
+ }),
+ };
+ builder.ins().jump(exit_block, &[final_nia.into()]);
+ }
+
+ builder.switch_to_block(exit_block);
+ builder.seal_block(exit_block);
+
+ let exit_nia = builder.block_params(exit_block)[0];
+ builder.ins().return_(&[exit_nia]);
+
+ builder.finalize();
+}
+
+pub(crate) fn emit_add_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().iadd(a, b);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_subf_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().isub(b, a);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_neg<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let r = builder.ins().ineg(a);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_mullw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().imul(a, b);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_divwu<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let zero = builder.ins().iconst(types::I32, 0);
+
+ let div_block = builder.create_block();
+ let zero_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let is_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);
+ builder.ins().brif(is_zero, zero_block, &[], div_block, &[]);
+
+ builder.switch_to_block(div_block);
+ builder.seal_block(div_block);
+ let q = builder.ins().udiv(a, b);
+ builder.ins().jump(merge, &[q.into()]);
+
+ builder.switch_to_block(zero_block);
+ builder.seal_block(zero_block);
+ builder.ins().jump(merge, &[zero.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let r = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_divw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+
+ let div_block = builder.create_block();
+ let bad_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let is_zero = builder.ins().icmp_imm(IntCC::Equal, b, 0);
+ let a_is_min = builder.ins().icmp_imm(IntCC::Equal, a, i32::MIN as i64);
+ let b_is_neg1 = builder.ins().icmp_imm(IntCC::Equal, b, -1i64);
+ let overflow = builder.ins().band(a_is_min, b_is_neg1);
+ let bad = builder.ins().bor(is_zero, overflow);
+
+ builder.ins().brif(bad, bad_block, &[], div_block, &[]);
+
+ builder.switch_to_block(div_block);
+ builder.seal_block(div_block);
+ let q = builder.ins().sdiv(a, b);
+ builder.ins().jump(merge, &[q.into()]);
+
+ builder.switch_to_block(bad_block);
+ builder.seal_block(bad_block);
+ let neg = builder.ins().icmp_imm(IntCC::SignedLessThan, a, 0);
+ let neg_one = builder.ins().iconst(types::I32, 0xFFFF_FFFFu32 as i64);
+ let zero = builder.ins().iconst(types::I32, 0);
+ let result_bad = builder.ins().select(neg, neg_one, zero);
+ builder.ins().jump(merge, &[result_bad.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let r = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_mulhw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ signed: bool,
+) -> Value {
+ let a32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let (a64, b64) = if signed {
+ (
+ builder.ins().sextend(types::I64, a32),
+ builder.ins().sextend(types::I64, b32),
+ )
+ } else {
+ (
+ builder.ins().uextend(types::I64, a32),
+ builder.ins().uextend(types::I64, b32),
+ )
+ };
+
+ let prod = builder.ins().imul(a64, b64);
+ let high = builder.ins().ushr_imm(prod, 32);
+
+ let r = builder.ins().ireduce(types::I32, high);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+
+ r
+}
+
+pub(crate) fn emit_addic<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ rc: bool,
+) {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let imm_v = builder.ins().iconst(types::I32, instr.simm() as i64);
+ let (sum, carry) = builder.ins().uadd_overflow(a, imm_v);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), sum);
+
+ let carry32 = builder.ins().uextend(types::I32, carry);
+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);
+
+ if rc {
+ emit_update_cr0::<SYSTEM>(builder, ctx_ptr, sum);
+ }
+}
+
+pub(crate) fn emit_subfic<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let imm_v = builder.ins().iconst(types::I32, instr.simm() as i64);
+ let res = builder.ins().isub(imm_v, a);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), res);
+
+ let carry_b = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, imm_v, a);
+ let carry32 = builder.ins().uextend(types::I32, carry_b);
+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);
+}
+
+pub(crate) fn write_xer_carry<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, carry: Value) {
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);
+ let positioned = builder.ins().ishl_imm(carry, 29);
+ let merged = builder.ins().bor(cleared, positioned);
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);
+}
+
+pub(crate) fn write_xer_ov_so<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, ov: Value) {
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+
+ let cleared_ov = builder.ins().band_imm(xer, !0x4000_0000u32 as i64);
+ let ov_pos = builder.ins().ishl_imm(ov, 30);
+ let with_ov = builder.ins().bor(cleared_ov, ov_pos);
+
+ let so_pos = builder.ins().ishl_imm(ov, 31);
+ let merged = builder.ins().bor(with_ov, so_pos);
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);
+}
+
+pub(crate) fn compute_add_overflow(builder: &mut FunctionBuilder, a: Value, b: Value, r: Value) -> Value {
+ let xor_a = builder.ins().bxor(a, r);
+ let xor_b = builder.ins().bxor(b, r);
+ let and = builder.ins().band(xor_a, xor_b);
+ let shifted = builder.ins().ushr_imm(and, 31);
+ builder.ins().band_imm(shifted, 1)
+}
+
+pub(crate) fn compute_sub_overflow(builder: &mut FunctionBuilder, a: Value, b: Value, r: Value) -> Value {
+ let xor_ab = builder.ins().bxor(b, a);
+ let xor_br = builder.ins().bxor(b, r);
+ let and = builder.ins().band(xor_ab, xor_br);
+ let shifted = builder.ins().ushr_imm(and, 31);
+ builder.ins().band_imm(shifted, 1)
+}
+
+pub(crate) fn emit_addc_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let (sum, carry) = builder.ins().uadd_overflow(a, b);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), sum);
+ let carry32 = builder.ins().uextend(types::I32, carry);
+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);
+ sum
+}
+
+pub(crate) fn emit_subfc_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let res = builder.ins().isub(b, a);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), res);
+
+ let carry_b = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, b, a);
+ let carry32 = builder.ins().uextend(types::I32, carry_b);
+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry32);
+
+ res
+}
+
+pub(crate) fn emit_twi<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+) {
+ let to = (instr.bo() as u32) as u32;
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = builder.ins().iconst(types::I32, instr.simm() as i64);
+ emit_trap_check::<SYSTEM>(builder, ctx_ptr, a, b, to, pc, exit_block, local);
+}
+
+pub(crate) fn emit_tw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+) {
+ let to = (instr.bo() as u32) as u32;
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ emit_trap_check::<SYSTEM>(builder, ctx_ptr, a, b, to, pc, exit_block, local);
+}
+
+pub(crate) fn emit_trap_check<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ a: Value,
+ b: Value,
+ to: u32,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+) {
+ let _ = local;
+ let mut cond: Option<Value> = None;
+ let or_in = |builder: &mut FunctionBuilder, c: Value, cond: &mut Option<Value>| match *cond {
+ Some(prev) => *cond = Some(builder.ins().bor(prev, c)),
+ None => *cond = Some(c),
+ };
+
+ if to & 0x10 != 0 {
+ let c = builder.ins().icmp(IntCC::SignedLessThan, a, b);
+ or_in(builder, c, &mut cond);
+ }
+
+ if to & 0x08 != 0 {
+ let c = builder.ins().icmp(IntCC::SignedGreaterThan, a, b);
+ or_in(builder, c, &mut cond);
+ }
+
+ if to & 0x04 != 0 {
+ let c = builder.ins().icmp(IntCC::Equal, a, b);
+ or_in(builder, c, &mut cond);
+ }
+
+ if to & 0x02 != 0 {
+ let c = builder.ins().icmp(IntCC::UnsignedLessThan, a, b);
+ or_in(builder, c, &mut cond);
+ }
+
+ if to & 0x01 != 0 {
+ let c = builder.ins().icmp(IntCC::UnsignedGreaterThan, a, b);
+ or_in(builder, c, &mut cond);
+ }
+
+ if let Some(c) = cond {
+ let trap_block = builder.create_block();
+ let cont_block = builder.create_block();
+ builder.ins().brif(c, trap_block, &[], cont_block, &[]);
+
+ builder.switch_to_block(trap_block);
+ builder.seal_block(trap_block);
+ let srr0_value = builder.ins().iconst(types::I32, pc as i64);
+ let nia = emit_exception_dispatch::<SYSTEM>(builder, ctx_ptr, srr0_value, 0x0002_0000, 0x0000_0700);
+ builder.ins().jump(exit_block, &[nia.into()]);
+
+ builder.switch_to_block(cont_block);
+ builder.seal_block(cont_block);
+ }
+}
+
+pub(crate) fn emit_mcrxr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let top3 = builder.ins().ushr_imm(xer, 29);
+ let masked3 = builder.ins().band_imm(top3, 0x7);
+ let shifted = builder.ins().ishl_imm(masked3, 1);
+ let dst_shift = 28 - 4 * (instr.crfd() as u32);
+ let positioned = builder.ins().ishl_imm(shifted, dst_shift as i64);
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));
+ let new_cr = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);
+
+ let new_xer = builder.ins().band_imm(xer, !(0xE000_0000u32 as i64));
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), new_xer, ctx_ptr, off);
+}
+
+fn read_xer_carry<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let shifted = builder.ins().ushr_imm(xer, 29);
+ builder.ins().band_imm(shifted, 1)
+}
+
+fn add_with_carry_in<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ a: Value,
+ b: Value,
+ cin: Value,
+) -> Value {
+ let (s1, c1) = builder.ins().uadd_overflow(a, b);
+ let (s2, c2) = builder.ins().uadd_overflow(s1, cin);
+ let c1_32 = builder.ins().uextend(types::I32, c1);
+ let c2_32 = builder.ins().uextend(types::I32, c2);
+ let carry_out = builder.ins().bor(c1_32, c2_32);
+ write_xer_carry::<SYSTEM>(builder, ctx_ptr, carry_out);
+ s2
+}
+
+pub(crate) fn emit_adde_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);
+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, b, cin);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_subfe_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let not_a = builder.ins().bxor_imm(a, !0i64);
+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);
+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, not_a, b, cin);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum AddzeSubfzeKind {
+ Addze,
+ Subfze,
+}
+
+pub(crate) fn emit_addze_subfze<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ kind: AddzeSubfzeKind,
+) -> Value {
+ let a_raw = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let a = match kind {
+ AddzeSubfzeKind::Addze => a_raw,
+ AddzeSubfzeKind::Subfze => builder.ins().bxor_imm(a_raw, !0i64),
+ };
+ let zero = builder.ins().iconst(types::I32, 0);
+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);
+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, zero, cin);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum AddmeSubfmeKind {
+ Addme,
+ Subfme,
+}
+
+pub(crate) fn emit_addme_subfme<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ kind: AddmeSubfmeKind,
+) -> Value {
+ let a_raw = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let a = match kind {
+ AddmeSubfmeKind::Addme => a_raw,
+ AddmeSubfmeKind::Subfme => builder.ins().bxor_imm(a_raw, !0i64),
+ };
+ let neg_one = builder.ins().iconst(types::I32, -1i64);
+ let cin = read_xer_carry::<SYSTEM>(builder, ctx_ptr);
+ let r = add_with_carry_in::<SYSTEM>(builder, ctx_ptr, a, neg_one, cin);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ r
+}
+
+pub(crate) fn emit_mulli<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let r = builder.ins().imul_imm(a, instr.simm() as i64);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+}
+
+pub(crate) fn emit_srawi<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+
+ let result = if instr.rb() == 0 {
+ s
+ } else {
+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);
+ builder.ins().sshr(s, amt)
+ };
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), result);
+
+ let ca: Value = if instr.rb() == 0 {
+ builder.ins().iconst(types::I32, 0)
+ } else {
+ let mask = ((1u32 << instr.rb()) - 1) as i64;
+ let lower = builder.ins().band_imm(s, mask);
+ let lower_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, lower, 0);
+ let neg = builder.ins().icmp_imm(IntCC::SignedLessThan, s, 0);
+ let both = builder.ins().band(neg, lower_nonzero);
+ builder.ins().uextend(types::I32, both)
+ };
+
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);
+ let positioned = builder.ins().ishl_imm(ca, 29);
+ let merged = builder.ins().bor(cleared, positioned);
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);
+
+ result
+}
+
+pub(crate) fn emit_sraw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+
+ let count = builder.ins().band_imm(b, 0x3F);
+ let big = builder.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, count, 32);
+ let count_low5 = builder.ins().band_imm(count, 0x1F);
+
+ let thirtyone = builder.ins().iconst(types::I32, 31);
+ let eff_shift = builder.ins().select(big, thirtyone, count_low5);
+ let result = builder.ins().sshr(s, eff_shift);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), result);
+
+ let s_neg = builder.ins().icmp_imm(IntCC::SignedLessThan, s, 0);
+ let s_neg_v = builder.ins().uextend(types::I32, s_neg);
+
+ let one = builder.ins().iconst(types::I32, 1);
+ let shifted_one = builder.ins().ishl(one, count_low5);
+ let mask = builder.ins().iadd_imm(shifted_one, -1);
+ let lower = builder.ins().band(s, mask);
+ let lower_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, lower, 0);
+ let lower_nonzero_v = builder.ins().uextend(types::I32, lower_nonzero);
+ let small_ca = builder.ins().band(s_neg_v, lower_nonzero_v);
+
+ let ca = builder.ins().select(big, s_neg_v, small_ca);
+
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(xer, !0x2000_0000u32 as i64);
+ let positioned = builder.ins().ishl_imm(ca, 29);
+ let merged = builder.ins().bor(cleared, positioned);
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), merged, ctx_ptr, off);
+
+ result
+}
+
+pub(crate) fn emit_shift_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ kind: ShiftKind,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let count = builder.ins().band_imm(b, 0x3F);
+ let count_low5 = builder.ins().band_imm(count, 0x1F);
+ let shifted = match kind {
+ ShiftKind::Slw => builder.ins().ishl(s, count_low5),
+ ShiftKind::Srw => builder.ins().ushr(s, count_low5),
+ };
+ let high_bit = builder.ins().band_imm(count, 0x20);
+ let out_of_range = builder.ins().icmp_imm(IntCC::NotEqual, high_bit, 0);
+ let zero = builder.ins().iconst(types::I32, 0);
+ let r = builder.ins().select(out_of_range, zero, shifted);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), r);
+ r
+}
+
+pub(crate) fn emit_mfspr<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+) -> bool {
+ let spr_num = instr.spr_swapped() as u16;
+ if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {
+ let val = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+ } else {
+ let num_v = builder.ins().iconst(types::I32, spr_num as i64);
+ let call = builder.ins().call(local.read_spr, &[ctx_ptr, num_v]);
+ let val = builder.inst_results(call)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+ }
+ true
+}
+
+pub(crate) fn emit_mtspr<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+) -> bool {
+ let spr_num = instr.spr_swapped() as u16;
+ let val = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+ } else {
+ let num_v = builder.ins().iconst(types::I32, spr_num as i64);
+ builder.ins().call(local.write_spr, &[ctx_ptr, num_v, val]);
+ }
+ true
+}
+
+fn rlw_mask(mb: u8, me: u8) -> u32 {
+ debug_assert!(mb < 32 && me < 32);
+
+ let begin = 31u8 - mb;
+ let end = 31u8 - me;
+ if begin >= end {
+ let width = (begin - end + 1) as u32;
+
+ if width >= 32 {
+ 0xFFFF_FFFF
+ } else {
+ (((1u64 << width) - 1) as u32) << end
+ }
+ } else {
+ let lo_part = ((1u64 << (begin + 1)) - 1) as u32;
+ let hi_part = !(((1u64 << end) - 1) as u32);
+ lo_part | hi_part
+ }
+}
+
+pub(crate) fn emit_rlwnm<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let mask = rlw_mask(instr.mb(), instr.me());
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let amt = builder.ins().band_imm(b, 0x1F);
+ let rotated = builder.ins().rotl(s, amt);
+ let masked = builder.ins().band_imm(rotated, mask as i64);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), masked);
+ masked
+}
+
+pub(crate) fn emit_rlwimi<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let mask = rlw_mask(instr.mb(), instr.me());
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+
+ let rotated = if instr.rb() == 0 {
+ s
+ } else {
+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);
+ builder.ins().rotl(s, amt)
+ };
+
+ let inserted_bits = builder.ins().band_imm(rotated, mask as i64);
+ let kept_bits = builder.ins().band_imm(a, !mask as i64);
+ let val = builder.ins().bor(kept_bits, inserted_bits);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+
+ val
+}
+
+pub(crate) fn emit_rlwinm<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let mask = rlw_mask(instr.mb(), instr.me());
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let rotated = if instr.rb() == 0 {
+ s
+ } else {
+ let amt = builder.ins().iconst(types::I32, instr.rb() as i64);
+ builder.ins().rotl(s, amt)
+ };
+
+ let masked = builder.ins().band_imm(rotated, mask as i64);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), masked);
+
+ masked
+}
+
+fn cr_bit_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, bi: u8) -> Value {
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let shifted = builder.ins().ushr_imm(cr, (31 - bi) as i64);
+ builder.ins().band_imm(shifted, 1)
+}
+
+pub(crate) fn ctr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let off = abi::ctr_offset::<SYSTEM>() as i32;
+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn ctr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {
+ let off = abi::ctr_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+pub(crate) fn lr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {
+ let off = abi::lr_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+pub(crate) fn emit_b<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+) -> Value {
+ let target: u32 = if instr.aa() {
+ instr.li() as u32
+ } else {
+ pc.wrapping_add_signed(instr.li())
+ };
+ if instr.lk() {
+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);
+ }
+ builder.ins().iconst(types::I32, target as i64)
+}
+
+pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ taken_slot: Option<i64>,
+ fall_slot: Option<i64>,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+) -> TermEmit {
+ if taken_slot.is_none() && fall_slot.is_none() {
+ return TermEmit::HandledNia(emit_bc::<SYSTEM>(builder, ctx_ptr, instr, pc));
+ }
+
+ let bo = (instr.bo() as u32) as u8;
+
+ let taken_target: u32 = if instr.aa() {
+ instr.bd() as u32
+ } else {
+ pc.wrapping_add_signed(instr.bd())
+ };
+ let fall_target: u32 = pc.wrapping_add(4);
+
+ let decrement_ctr = (bo & 0x04) == 0;
+ let mut ctr_after: Option<Value> = None;
+ if decrement_ctr {
+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);
+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);
+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);
+ ctr_after = Some(ctr_minus_1);
+ }
+
+ let ctr_ok: Value = if decrement_ctr {
+ let ctr = ctr_after.unwrap();
+ let cc = if (bo & 0x02) != 0 {
+ IntCC::Equal
+ } else {
+ IntCC::NotEqual
+ };
+ let r = builder.ins().icmp_imm(cc, ctr, 0);
+ builder.ins().uextend(types::I32, r)
+ } else {
+ builder.ins().iconst(types::I32, 1)
+ };
+
+ let cond_ok: Value = if (bo & 0x10) != 0 {
+ builder.ins().iconst(types::I32, 1)
+ } else {
+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());
+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };
+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);
+ builder.ins().uextend(types::I32, r)
+ };
+
+ let take = builder.ins().band(ctr_ok, cond_ok);
+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);
+
+ let taken_block = builder.create_block();
+ let fall_block = builder.create_block();
+ builder.ins().brif(take_b, taken_block, &[], fall_block, &[]);
+
+ builder.switch_to_block(taken_block);
+ builder.seal_block(taken_block);
+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);
+ if instr.lk() {
+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);
+ }
+ let taken_v = builder.ins().iconst(types::I32, taken_target as i64);
+ if let Some(slot_addr) = taken_slot {
+ emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, taken_v, exit_block);
+ } else {
+ builder.ins().jump(exit_block, &[taken_v.into()]);
+ }
+
+ builder.switch_to_block(fall_block);
+ builder.seal_block(fall_block);
+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);
+
+ let fall_v = builder.ins().iconst(types::I32, fall_target as i64);
+ if let Some(slot_addr) = fall_slot {
+ emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, fall_v, exit_block);
+ } else {
+ builder.ins().jump(exit_block, &[fall_v.into()]);
+ }
+
+ TermEmit::HandledChained
+}
+
+pub(crate) fn emit_bc<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+) -> Value {
+ let bo = (instr.bo() as u32) as u8;
+
+ let target: u32 = if instr.aa() {
+ instr.bd() as u32
+ } else {
+ pc.wrapping_add_signed(instr.bd())
+ };
+ let target_v = builder.ins().iconst(types::I32, target as i64);
+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+
+ let decrement_ctr = (bo & 0x04) == 0;
+ let mut ctr_after: Option<Value> = None;
+ if decrement_ctr {
+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);
+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);
+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);
+ ctr_after = Some(ctr_minus_1);
+ }
+
+ let ctr_ok: Value = if decrement_ctr {
+ let ctr = ctr_after.unwrap();
+ if (bo & 0x02) != 0 {
+ let r = builder.ins().icmp_imm(IntCC::Equal, ctr, 0);
+ builder.ins().uextend(types::I32, r)
+ } else {
+ let r = builder.ins().icmp_imm(IntCC::NotEqual, ctr, 0);
+ builder.ins().uextend(types::I32, r)
+ }
+ } else {
+ builder.ins().iconst(types::I32, 1)
+ };
+
+ let cond_ok: Value = if (bo & 0x10) != 0 {
+ builder.ins().iconst(types::I32, 1)
+ } else {
+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());
+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };
+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);
+ builder.ins().uextend(types::I32, r)
+ };
+
+ let take = builder.ins().band(ctr_ok, cond_ok);
+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);
+
+ if instr.lk() {
+ let lk_block = builder.create_block();
+ let after_lk = builder.create_block();
+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);
+ builder.switch_to_block(lk_block);
+ builder.seal_block(lk_block);
+
+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);
+
+ builder.ins().jump(after_lk, &[]);
+ builder.switch_to_block(after_lk);
+ builder.seal_block(after_lk);
+ }
+
+ builder.ins().select(take_b, target_v, fall_v)
+}
+
+pub(crate) fn lr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let off = abi::lr_offset::<SYSTEM>() as i32;
+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn emit_bclr_dispatch<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ lookup_table_addr: i64,
+) -> TermEmit {
+ let bo = (instr.bo() as u32) as u8;
+
+ let unconditional = (bo & 0x14) == 0x14;
+ if unconditional && !instr.lk() {
+ emit_blr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)
+ } else {
+ TermEmit::HandledNia(emit_bclr::<SYSTEM>(builder, ctx_ptr, instr, pc))
+ }
+}
+
+pub(crate) fn emit_bcctr_dispatch<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ lookup_table_addr: i64,
+) -> TermEmit {
+ let bo = (instr.bo() as u32) as u8;
+ let unconditional = (bo & 0x14) == 0x14;
+ if unconditional && !instr.lk() {
+ emit_bctr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)
+ } else {
+ TermEmit::HandledNia(emit_bcctr::<SYSTEM>(builder, ctx_ptr, instr, pc))
+ }
+}
+
+pub(crate) fn emit_blr_with_ic<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ _pc: u32,
+ exit_block: Block,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ lookup_table_addr: i64,
+) -> TermEmit {
+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);
+
+ let lr = lr_load::<SYSTEM>(builder, ctx_ptr);
+ let target_pc = builder.ins().band_imm(lr, !3i64);
+ emit_inline_cache_dispatch::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ target_pc,
+ exit_block,
+ block_sig_ref,
+ lookup_table_addr,
+ );
+ TermEmit::HandledChained
+}
+
+pub(crate) fn emit_bctr_with_ic<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ _pc: u32,
+ exit_block: Block,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ lookup_table_addr: i64,
+) -> TermEmit {
+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);
+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);
+ let target_pc = builder.ins().band_imm(ctr, !3i64);
+ emit_inline_cache_dispatch::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ target_pc,
+ exit_block,
+ block_sig_ref,
+ lookup_table_addr,
+ );
+ TermEmit::HandledChained
+}
+
+pub(crate) fn emit_inline_cache_dispatch<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ target_pc: Value,
+ exit_block: Block,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ lookup_table_addr: i64,
+) {
+ let mask_v = builder.ins().iconst(types::I32, super::BLOCK_LOOKUP_TABLE_MASK as i64);
+ let idx_word = builder.ins().ushr_imm(target_pc, 2);
+ let idx = builder.ins().band(idx_word, mask_v);
+ let idx64 = builder.ins().uextend(types::I64, idx);
+ let entry_size = std::mem::size_of::<super::BlockLookupSlot>() as i64;
+ let off64 = builder.ins().imul_imm(idx64, entry_size);
+ let table_base = builder.ins().iconst(types::I64, lookup_table_addr);
+ let slot_addr = builder.ins().iadd(table_base, off64);
+
+ let slot_pc = builder.ins().load(types::I32, MemFlags::trusted(), slot_addr, 0);
+ let slot_entry = builder.ins().load(types::I64, MemFlags::trusted(), slot_addr, 8);
+ let pc_eq = builder.ins().icmp(IntCC::Equal, slot_pc, target_pc);
+
+ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;
+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;
+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);
+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);
+ let in_budget = builder.ins().icmp(IntCC::UnsignedLessThan, cycles, deadline);
+
+ let pc_eq_i8 = pc_eq;
+ let budget_i8 = in_budget;
+ let nonzero_i8 = builder.ins().icmp_imm(IntCC::NotEqual, slot_entry, 0);
+ let pc_and_budget = builder.ins().band(pc_eq_i8, budget_i8);
+ let ok = builder.ins().band(pc_and_budget, nonzero_i8);
+
+ let hit_block = builder.create_block();
+ let miss_block = builder.create_block();
+ builder.ins().brif(ok, hit_block, &[], miss_block, &[]);
+
+ builder.switch_to_block(miss_block);
+ builder.seal_block(miss_block);
+ builder.ins().jump(exit_block, &[target_pc.into()]);
+
+ builder.switch_to_block(hit_block);
+ builder.seal_block(hit_block);
+ builder
+ .ins()
+ .return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr]);
+}
+
+pub(crate) fn emit_bclr<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+) -> Value {
+ let bo = (instr.bo() as u32) as u8;
+
+ let lr = lr_load::<SYSTEM>(builder, ctx_ptr);
+ let target_v = builder.ins().band_imm(lr, !3i64);
+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+
+ let decrement_ctr = (bo & 0x04) == 0;
+ let mut ctr_after: Option<Value> = None;
+ if decrement_ctr {
+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);
+ let ctr_minus_1 = builder.ins().iadd_imm(ctr, -1);
+ ctr_store::<SYSTEM>(builder, ctx_ptr, ctr_minus_1);
+ ctr_after = Some(ctr_minus_1);
+ }
+
+ let ctr_ok: Value = if decrement_ctr {
+ let ctr = ctr_after.unwrap();
+ let cc = if (bo & 0x02) != 0 {
+ IntCC::Equal
+ } else {
+ IntCC::NotEqual
+ };
+ let r = builder.ins().icmp_imm(cc, ctr, 0);
+ builder.ins().uextend(types::I32, r)
+ } else {
+ builder.ins().iconst(types::I32, 1)
+ };
+
+ let cond_ok: Value = if (bo & 0x10) != 0 {
+ builder.ins().iconst(types::I32, 1)
+ } else {
+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());
+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };
+ let r = builder.ins().icmp_imm(IntCC::Equal, bit, want);
+ builder.ins().uextend(types::I32, r)
+ };
+
+ let take = builder.ins().band(ctr_ok, cond_ok);
+ let take_b = builder.ins().icmp_imm(IntCC::NotEqual, take, 0);
+
+ if instr.lk() {
+ let lk_block = builder.create_block();
+ let after_lk = builder.create_block();
+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);
+ builder.switch_to_block(lk_block);
+ builder.seal_block(lk_block);
+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);
+ builder.ins().jump(after_lk, &[]);
+ builder.switch_to_block(after_lk);
+ builder.seal_block(after_lk);
+ }
+
+ builder.ins().select(take_b, target_v, fall_v)
+}
+
+pub(crate) fn emit_bcctr<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+) -> Value {
+ let bo = (instr.bo() as u32) as u8;
+
+ let ctr = ctr_load::<SYSTEM>(builder, ctx_ptr);
+ let target_v = builder.ins().band_imm(ctr, !3i64);
+ let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+
+ let unconditional = (bo & 0x10) != 0;
+ let take_b = if unconditional {
+ builder.ins().iconst(types::I8, 1)
+ } else {
+ let bit = cr_bit_load::<SYSTEM>(builder, ctx_ptr, instr.bi());
+ let want = if (bo & 0x08) != 0 { 1 } else { 0 };
+ builder.ins().icmp_imm(IntCC::Equal, bit, want)
+ };
+
+ if instr.lk() {
+ let lk_block = builder.create_block();
+ let after_lk = builder.create_block();
+ builder.ins().brif(take_b, lk_block, &[], after_lk, &[]);
+ builder.switch_to_block(lk_block);
+ builder.seal_block(lk_block);
+ let lr_val = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);
+ builder.ins().jump(after_lk, &[]);
+ builder.switch_to_block(after_lk);
+ builder.seal_block(after_lk);
+ }
+
+ builder.ins().select(take_b, target_v, fall_v)
+}
+
+pub(crate) fn emit_cycles_add<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, n: i64) {
+ let off = abi::cycles_offset::<SYSTEM>() as i64;
+ let addr = builder.ins().iadd_imm(ctx_ptr, off);
+ let now = builder.ins().load(types::I64, MemFlags::trusted(), addr, 0);
+ let next = builder.ins().iadd_imm(now, n);
+ builder.ins().store(MemFlags::trusted(), next, addr, 0);
+}
+
+pub(crate) fn emit_mcrf<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let src_shift = 28 - 4 * (instr.crfs() as u32);
+ let dst_shift = 28 - 4 * (instr.crfd() as u32);
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let src_field = builder.ins().ushr_imm(cr, src_shift as i64);
+ let src_nibble = builder.ins().band_imm(src_field, 0xF);
+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));
+ let positioned = builder.ins().ishl_imm(src_nibble, dst_shift as i64);
+ let new_cr = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);
+}
+
+pub(crate) fn emit_cr_bit_op<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ xo: u32,
+) {
+ let bit_a = 31 - (instr.crba() as u32);
+ let bit_b = 31 - (instr.crbb() as u32);
+ let bit_d = 31 - (instr.crbd() as u32);
+
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let a_shifted = builder.ins().ushr_imm(cr, bit_a as i64);
+ let a = builder.ins().band_imm(a_shifted, 1);
+ let b_shifted = builder.ins().ushr_imm(cr, bit_b as i64);
+ let b = builder.ins().band_imm(b_shifted, 1);
+
+ let result = match xo {
+ 33 => {
+ let or_v = builder.ins().bor(a, b);
+ let inv = builder.ins().bxor_imm(or_v, 1);
+ inv
+ }
+ 129 => {
+ let not_b = builder.ins().bxor_imm(b, 1);
+ builder.ins().band(a, not_b)
+ }
+ 193 => builder.ins().bxor(a, b),
+ 225 => {
+ let and_v = builder.ins().band(a, b);
+ builder.ins().bxor_imm(and_v, 1)
+ }
+ 257 => builder.ins().band(a, b),
+ 289 => {
+ let xor_v = builder.ins().bxor(a, b);
+ builder.ins().bxor_imm(xor_v, 1)
+ }
+ 417 => {
+ let not_b = builder.ins().bxor_imm(b, 1);
+ builder.ins().bor(a, not_b)
+ }
+ 449 => builder.ins().bor(a, b),
+ _ => unreachable!("unexpected CR-bit XO {xo}"),
+ };
+
+ let cleared = builder.ins().band_imm(cr, !(1i64 << bit_d));
+ let positioned = builder.ins().ishl_imm(result, bit_d as i64);
+ let new_cr = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum ShiftKind {
+ Slw,
+ Srw,
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum LogicalOp {
+ And,
+ Or,
+ Xor,
+}
+
+fn terminator_fall_pc(spec: &BlockSpec) -> Option<u32> {
+ if !matches!(spec.terminator, TermKind::BranchCond) {
+ return None;
+ }
+ let last_idx = spec.instrs.len().checked_sub(1)?;
+ let pc = spec.pc_of(last_idx);
+ Some(pc.wrapping_add(4))
+}
+
+fn terminator_static_taken_pc(spec: &BlockSpec) -> Option<u32> {
+ if spec.instrs.is_empty() {
+ return None;
+ }
+ let last_idx = spec.instrs.len() - 1;
+ let pc = spec.pc_of(last_idx);
+ let instr = Instruction(spec.instrs[last_idx]);
+ match spec.terminator {
+ TermKind::Branch => Some(if instr.aa() {
+ instr.li() as u32
+ } else {
+ pc.wrapping_add_signed(instr.li())
+ }),
+ TermKind::BranchCond => Some(if instr.aa() {
+ instr.bd() as u32
+ } else {
+ pc.wrapping_add_signed(instr.bd())
+ }),
+ _ => None,
+ }
+}
+
+pub(crate) enum TermEmit {
+ #[allow(dead_code)]
+ NotHandled,
+
+ HandledNia(Value),
+
+ HandledChained,
+}
+
+pub(crate) fn emit_chain_or_exit<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ slot_addr: i64,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ nia_if_exit: Value,
+ exit_block: Block,
+) {
+ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;
+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;
+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);
+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);
+ let exhausted = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cycles, deadline);
+
+ let chain_block = builder.create_block();
+ builder
+ .ins()
+ .brif(exhausted, exit_block, &[nia_if_exit.into()], chain_block, &[]);
+
+ builder.switch_to_block(chain_block);
+ builder.seal_block(chain_block);
+
+ let slot_const = builder.ins().iconst(types::I64, slot_addr);
+ let target_ptr = builder.ins().load(types::I64, MemFlags::trusted(), slot_const, 0);
+ let is_null = builder.ins().icmp_imm(IntCC::Equal, target_ptr, 0);
+ let dispatch_block = builder.create_block();
+ let tailcall_block = builder.create_block();
+ builder.ins().brif(is_null, dispatch_block, &[], tailcall_block, &[]);
+
+ builder.switch_to_block(dispatch_block);
+ builder.seal_block(dispatch_block);
+ builder.ins().jump(exit_block, &[nia_if_exit.into()]);
+
+ builder.switch_to_block(tailcall_block);
+ builder.seal_block(tailcall_block);
+ builder
+ .ins()
+ .return_call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);
+}
+
+pub(crate) fn emit_call_or_exit<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ slot_addr: i64,
+ block_sig_ref: cranelift_codegen::ir::SigRef,
+ nia_if_exit: Value,
+ exit_block: Block,
+) {
+ let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;
+ let dl_off = abi::next_deadline_offset::<SYSTEM>() as i32;
+ let cycles = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, cyc_off);
+ let deadline = builder.ins().load(types::I64, MemFlags::trusted(), ctx_ptr, dl_off);
+ let exhausted = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cycles, deadline);
+
+ let chain_block = builder.create_block();
+ builder
+ .ins()
+ .brif(exhausted, exit_block, &[nia_if_exit.into()], chain_block, &[]);
+
+ builder.switch_to_block(chain_block);
+ builder.seal_block(chain_block);
+ let slot_const = builder.ins().iconst(types::I64, slot_addr);
+ let target_ptr = builder.ins().load(types::I64, MemFlags::trusted(), slot_const, 0);
+ let is_null = builder.ins().icmp_imm(IntCC::Equal, target_ptr, 0);
+
+ let dispatch_block = builder.create_block();
+ let call_block = builder.create_block();
+ builder.ins().brif(is_null, dispatch_block, &[], call_block, &[]);
+
+ builder.switch_to_block(dispatch_block);
+ builder.seal_block(dispatch_block);
+ builder.ins().jump(exit_block, &[nia_if_exit.into()]);
+
+ builder.switch_to_block(call_block);
+ builder.seal_block(call_block);
+
+ let call = builder.ins().call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);
+ let returned_nia = builder.inst_results(call)[0];
+
+ builder.ins().jump(exit_block, &[returned_nia.into()]);
+}
+
+fn gpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {
+ debug_assert!(i < 32);
+ (abi::gpr_base_offset::<SYSTEM>() + 4 * i as usize) as i32
+}
+
+pub(crate) fn gpr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
+ let off = gpr_offset::<SYSTEM>(i);
+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn gpr_load_or_zero<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
+ if i == 0 {
+ builder.ins().iconst(types::I32, 0)
+ } else {
+ gpr_load::<SYSTEM>(builder, ctx_ptr, i)
+ }
+}
+
+pub(crate) fn gpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {
+ let off = gpr_offset::<SYSTEM>(i);
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+fn fpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {
+ debug_assert!(i < 32);
+ (abi::fpr_base_offset::<SYSTEM>() + 8 * i as usize) as i32
+}
+
+pub(crate) fn fpr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
+ let off = fpr_offset::<SYSTEM>(i);
+ builder.ins().load(types::F64, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn fpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {
+ let off = fpr_offset::<SYSTEM>(i);
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+fn ps1_offset<const SYSTEM: SystemId>(i: u8) -> i32 {
+ debug_assert!(i < 32);
+ (abi::ps1_base_offset::<SYSTEM>() + 8 * i as usize) as i32
+}
+
+pub(crate) fn ps1_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
+ let off = ps1_offset::<SYSTEM>(i);
+ builder.ins().load(types::F64, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn ps1_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, val: Value) {
+ let off = ps1_offset::<SYSTEM>(i);
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+pub(crate) fn round_to_single(builder: &mut FunctionBuilder, val: Value) -> Value {
+ let f32v = builder.ins().fdemote(types::F32, val);
+ builder.ins().fpromote(types::F64, f32v)
+}
+
+pub(crate) fn f32_reciprocal(builder: &mut FunctionBuilder, b_f64: Value) -> Value {
+ let b32 = builder.ins().fdemote(types::F32, b_f64);
+ let one32 = builder.ins().f32const(1.0f32);
+ let q32 = builder.ins().fdiv(one32, b32);
+ builder.ins().fpromote(types::F64, q32)
+}
+
+pub(crate) fn f32_rsqrte(builder: &mut FunctionBuilder, b_f64: Value) -> Value {
+ let b32 = builder.ins().fdemote(types::F32, b_f64);
+ let s32 = builder.ins().sqrt(b32);
+ let one32 = builder.ins().f32const(1.0f32);
+ let q32 = builder.ins().fdiv(one32, s32);
+ builder.ins().fpromote(types::F64, q32)
+}
+
+pub(crate) fn emit_recompute_fpscr_summary<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) {
+ const VX_MASK: u32 = 0x01F8_0700;
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let fpscr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+
+ let vx_in = builder.ins().band_imm(fpscr, VX_MASK as i64);
+ let vx_b = builder.ins().icmp_imm(IntCC::NotEqual, vx_in, 0);
+ let vx_v = builder.ins().uextend(types::I32, vx_b);
+ let vx_pos = builder.ins().ishl_imm(vx_v, 29);
+ let cleared_vx = builder.ins().band_imm(fpscr, !(1i64 << 29));
+ let with_vx = builder.ins().bor(cleared_vx, vx_pos);
+
+ let upper = builder.ins().ushr_imm(with_vx, 22);
+ let upper_mask = builder.ins().band_imm(upper, 0xF8);
+ let lower_mask = builder.ins().band_imm(with_vx, 0xF8);
+ let pairs = builder.ins().band(upper_mask, lower_mask);
+ let fex_b = builder.ins().icmp_imm(IntCC::NotEqual, pairs, 0);
+ let fex_v = builder.ins().uextend(types::I32, fex_b);
+ let fex_pos = builder.ins().ishl_imm(fex_v, 30);
+ let cleared_fex = builder.ins().band_imm(with_vx, !(1i64 << 30));
+ let new_fpscr = builder.ins().bor(cleared_fex, fex_pos);
+
+ builder.ins().store(MemFlags::trusted(), new_fpscr, ctx_ptr, off);
+}
+
+pub(crate) fn fpr_store_paired<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ i: u8,
+ val: Value,
+) {
+ fpr_store::<SYSTEM>(builder, ctx_ptr, i, val);
+ ps1_store::<SYSTEM>(builder, ctx_ptr, i, val);
+}
+
+pub(crate) fn cr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let off = abi::cr_offset::<SYSTEM>() as i32;
+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)
+}
+
+pub(crate) fn cr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {
+ let off = abi::cr_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), val, ctx_ptr, off);
+}
+
+pub(crate) fn emit_update_cr1_from_fpscr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) {
+ let fpscr = builder.ins().load(
+ types::I32,
+ MemFlags::trusted(),
+ ctx_ptr,
+ abi::fpscr_offset::<SYSTEM>() as i32,
+ );
+
+ let top4 = builder.ins().ushr_imm(fpscr, 28);
+ let positioned = builder.ins().ishl_imm(top4, 24);
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << 24));
+ let new_cr = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);
+}
+
+pub(crate) fn xer_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let off = abi::xer_offset::<SYSTEM>() as i32;
+ builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off)
+}
+
+fn xer_so_bit<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let shifted = builder.ins().ushr_imm(xer, 31);
+ builder.ins().band_imm(shifted, 1)
+}
+
+fn cr_set_field<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, crfd: u8, nibble: Value) {
+ debug_assert!(crfd < 8);
+ let shift = 28 - 4 * crfd as i64;
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let mask = !(0xFu32 << shift) as i64 as u64 as i64;
+ let cleared = builder.ins().band_imm(cr, mask);
+ let nibble_only = builder.ins().band_imm(nibble, 0xF);
+ let positioned = builder.ins().ishl_imm(nibble_only, shift);
+ let merged = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, merged);
+}
+
+fn build_cr_nibble(builder: &mut FunctionBuilder, lt: Value, gt: Value, eq: Value, so: Value) -> Value {
+ let lt8 = builder.ins().ishl_imm(lt, 3);
+ let gt4 = builder.ins().ishl_imm(gt, 2);
+ let eq2 = builder.ins().ishl_imm(eq, 1);
+ let a = builder.ins().bor(lt8, gt4);
+ let b = builder.ins().bor(eq2, so);
+ builder.ins().bor(a, b)
+}
+
+fn cmp_to_i32(builder: &mut FunctionBuilder, cc: IntCC, a: Value, b: Value) -> Value {
+ let r = builder.ins().icmp(cc, a, b);
+ builder.ins().uextend(types::I32, r)
+}
+
+pub(crate) fn emit_update_cr0<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, val: Value) {
+ let zero = builder.ins().iconst(types::I32, 0);
+ let lt = cmp_to_i32(builder, IntCC::SignedLessThan, val, zero);
+ let gt = cmp_to_i32(builder, IntCC::SignedGreaterThan, val, zero);
+ let eq = cmp_to_i32(builder, IntCC::Equal, val, zero);
+ let so = xer_so_bit::<SYSTEM>(builder, ctx_ptr);
+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);
+ cr_set_field::<SYSTEM>(builder, ctx_ptr, 0, nibble);
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum MemSize {
+ U8,
+ U16,
+ U32,
+}
+
+impl MemSize {
+ fn ir_type(self) -> cranelift_codegen::ir::Type {
+ match self {
+ MemSize::U8 => types::I8,
+ MemSize::U16 => types::I16,
+ MemSize::U32 => types::I32,
+ }
+ }
+}
+
+pub(crate) fn emit_d_form_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ builder.ins().iadd_imm(base, instr.simm() as i64)
+}
+
+pub(crate) fn emit_x_form_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ builder.ins().iadd(base, b)
+}
+
+pub(crate) fn emit_fastmem_lookup<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+) -> (Value, Value) {
+ let lut_base = {
+ let off = abi::fastmem_lut_ptr_offset::<SYSTEM>() as i32;
+ builder
+ .ins()
+ .load(types::I64, MemFlags::trusted().with_readonly(), ctx_ptr, off)
+ };
+
+ let idx_u32 = builder.ins().ushr_imm(ea, crate::mmio::FASTMEM_PAGE_SHIFT as i64);
+ let idx = builder.ins().uextend(types::I64, idx_u32);
+ let entry_off = builder.ins().ishl_imm(idx, 3);
+ let entry_addr = builder.ins().iadd(lut_base, entry_off);
+ let host_ptr = builder.ins().load(types::I64, MemFlags::trusted(), entry_addr, 0);
+
+ let page_off_u32 = builder.ins().band_imm(ea, crate::mmio::FASTMEM_PAGE_MASK as i64);
+ let page_off = builder.ins().uextend(types::I64, page_off_u32);
+
+ (host_ptr, page_off)
+}
+
+pub(crate) fn emit_load<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rd(), instr.ra(), size, slow, update);
+}
+
+pub(crate) fn emit_lha_d_form<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_val = builder.ins().load(types::I16, MemFlags::new(), addr, 0);
+ let swapped = builder.ins().bswap(raw_val);
+
+ let val32 = builder.ins().sextend(types::I32, swapped);
+ builder.ins().jump(merge, &[val32.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);
+ let raw_u16_in_u32 = builder.inst_results(call)[0];
+
+ let trunc = builder.ins().ireduce(types::I16, raw_u16_in_u32);
+ let sext = builder.ins().sextend(types::I32, trunc);
+ builder.ins().jump(merge, &[sext.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+ }
+}
+
+pub(crate) fn emit_lfx_update<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ single: bool,
+ local: &LocalFuncs,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ if single {
+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f32);
+ } else {
+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f64);
+ }
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+}
+
+pub(crate) fn emit_stfx_update<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ single: bool,
+ local: &LocalFuncs,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ if single {
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);
+ } else {
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);
+ }
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+}
+
+pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_u32: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let f = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let u64v = builder.ins().bitcast(types::I64, MemFlags::new(), f);
+ let u32v = builder.ins().ireduce(types::I32, u64v);
+
+ emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32);
+}
+
+pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ val: Value,
+ slow: FuncRef,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let bswapped = builder.ins().bswap(val);
+ builder.ins().store(MemFlags::new(), bswapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow, &[ctx_ptr, ea, val]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+ store: bool,
+ update: bool,
+) {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let ea = builder.ins().iadd(base, b);
+
+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_22_24(), store);
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ if store {
+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);
+ if !instr.w_21_21() {
+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);
+ }
+ } else {
+ let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
+ let ps1 = if instr.w_21_21() {
+ builder.ins().f64const(1.0)
+ } else {
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)
+ };
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);
+ ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);
+ }
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ if store {
+ emit_psq_store_quantized::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rd(),
+ instr.w_21_21(),
+ instr.i_22_24(),
+ merge,
+ local,
+ );
+ } else {
+ emit_psq_load_quantized::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rd(),
+ instr.w_21_21(),
+ instr.i_22_24(),
+ merge,
+ local,
+ );
+ }
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+ }
+}
+
+pub(crate) fn emit_lwarx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_load_at_ea::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rd(),
+ instr.ra(),
+ MemSize::U32,
+ local.read_u32,
+ false,
+ );
+ let reserve_off = abi::reserve_addr_offset::<SYSTEM>() as i32;
+ builder.ins().store(MemFlags::trusted(), ea, ctx_ptr, reserve_off);
+}
+
+pub(crate) fn emit_stwcx_dot<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let reserve_off = abi::reserve_addr_offset::<SYSTEM>() as i32;
+ let reserved = builder
+ .ins()
+ .load(types::I32, MemFlags::trusted(), ctx_ptr, reserve_off);
+ let matched_b = builder.ins().icmp(IntCC::Equal, reserved, ea);
+ let matched = builder.ins().uextend(types::I32, matched_b);
+
+ let store_block = builder.create_block();
+ let cr_block = builder.create_block();
+ builder.append_block_param(cr_block, types::I32);
+ builder
+ .ins()
+ .brif(matched_b, store_block, &[], cr_block, &[matched.into()]);
+
+ builder.switch_to_block(store_block);
+ builder.seal_block(store_block);
+
+ let no_reservation = builder
+ .ins()
+ .iconst(types::I32, crate::gekko::Gekko::NO_RESERVATION as i64);
+ builder
+ .ins()
+ .store(MemFlags::trusted(), no_reservation, ctx_ptr, reserve_off);
+ emit_store_at_ea::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rs(),
+ 0,
+ MemSize::U32,
+ local.write_u32,
+ false,
+ );
+ builder.ins().jump(cr_block, &[matched.into()]);
+
+ builder.switch_to_block(cr_block);
+ builder.seal_block(cr_block);
+ let eq_flag = builder.block_params(cr_block)[0];
+
+ let xer = xer_load::<SYSTEM>(builder, ctx_ptr);
+ let so = builder.ins().ushr_imm(xer, 31);
+ let so_clean = builder.ins().band_imm(so, 1);
+ let eq_shifted = builder.ins().ishl_imm(eq_flag, 1);
+ let nibble = builder.ins().bor(eq_shifted, so_clean);
+ let positioned = builder.ins().ishl_imm(nibble, 28);
+ let cr = cr_load::<SYSTEM>(builder, ctx_ptr);
+ let cleared = builder.ins().band_imm(cr, !(0xFi64 << 28));
+ let new_cr = builder.ins().bor(cleared, positioned);
+ cr_store::<SYSTEM>(builder, ctx_ptr, new_cr);
+}
+
+pub(crate) fn emit_lmw_stmw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ local: &LocalFuncs,
+ store: bool,
+) {
+ let ea_base = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ for r in instr.rd()..32 {
+ let off = ((r - instr.rd()) as i32) * 4;
+ let ea = if off == 0 {
+ ea_base
+ } else {
+ builder.ins().iadd_imm(ea_base, off as i64)
+ };
+ if store {
+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.write_u32, false);
+ } else {
+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.read_u32, false);
+ }
+ }
+}
+
+pub(crate) fn emit_lha_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_val = builder.ins().load(types::I16, MemFlags::new(), addr, 0);
+ let swapped = builder.ins().bswap(raw_val);
+ let val32 = builder.ins().sextend(types::I32, swapped);
+ builder.ins().jump(merge, &[val32.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);
+ let raw_u16_in_u32 = builder.inst_results(call)[0];
+ let trunc = builder.ins().ireduce(types::I16, raw_u16_in_u32);
+ let sext = builder.ins().sextend(types::I32, trunc);
+ builder.ins().jump(merge, &[sext.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+ }
+}
+
+pub(crate) fn emit_load_xform_brx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_val = builder.ins().load(size.ir_type(), MemFlags::new(), addr, 0);
+ let val32 = match size {
+ MemSize::U32 => raw_val,
+ MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, raw_val),
+ };
+ builder.ins().jump(merge, &[val32.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);
+ let raw_be_in_u32 = builder.inst_results(call)[0];
+
+ let swapped = match size {
+ MemSize::U32 => builder.ins().bswap(raw_be_in_u32),
+ MemSize::U16 => {
+ let trunc = builder.ins().ireduce(types::I16, raw_be_in_u32);
+ let sw = builder.ins().bswap(trunc);
+ builder.ins().uextend(types::I32, sw)
+ }
+ MemSize::U8 => raw_be_in_u32,
+ };
+ builder.ins().jump(merge, &[swapped.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+}
+
+pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+
+ let truncated = match size {
+ MemSize::U32 => val32,
+ MemSize::U16 => builder.ins().ireduce(types::I16, val32),
+ MemSize::U8 => builder.ins().ireduce(types::I8, val32),
+ };
+ builder.ins().store(MemFlags::new(), truncated, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+
+ let to_write = match size {
+ MemSize::U32 => builder.ins().bswap(val32),
+ MemSize::U16 => {
+ let trunc = builder.ins().ireduce(types::I16, val32);
+ let sw = builder.ins().bswap(trunc);
+ builder.ins().uextend(types::I32, sw)
+ }
+ MemSize::U8 => val32,
+ };
+ builder.ins().call(slow, &[ctx_ptr, ea, to_write]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_exception_dispatch<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ srr0_value: Value,
+ srr1_extra_bits: u32,
+ vector_offset: u32,
+) -> Value {
+ const MSR_CLEAR_MASK: i64 = 0x4_EF37;
+
+ const MSR_PRESERVE_MASK: i64 = 0x87C0_FFFF;
+
+ let msr_off = abi::msr_offset::<SYSTEM>() as i32;
+ let srr0_off = abi::spr_field_offset::<SYSTEM>(26).unwrap() as i32;
+ let srr1_off = abi::spr_field_offset::<SYSTEM>(27).unwrap() as i32;
+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;
+
+ let msr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, msr_off);
+
+ builder.ins().store(MemFlags::trusted(), srr0_value, ctx_ptr, srr0_off);
+
+ let preserved = builder.ins().band_imm(msr, MSR_PRESERVE_MASK);
+ let srr1 = if srr1_extra_bits != 0 {
+ builder.ins().bor_imm(preserved, srr1_extra_bits as i64)
+ } else {
+ preserved
+ };
+ builder.ins().store(MemFlags::trusted(), srr1, ctx_ptr, srr1_off);
+
+ let ip_bit = builder.ins().band_imm(msr, 0x40);
+ let want_high = builder.ins().icmp_imm(IntCC::NotEqual, ip_bit, 0);
+ let high_base = builder.ins().iconst(types::I32, 0xFFF0_0000i64);
+ let zero = builder.ins().iconst(types::I32, 0);
+ let base = builder.ins().select(want_high, high_base, zero);
+ let nia = builder.ins().bor_imm(base, vector_offset as i64);
+ builder.ins().store(MemFlags::trusted(), nia, ctx_ptr, nia_off);
+
+ let msr_cleared = builder.ins().band_imm(msr, !MSR_CLEAR_MASK & 0xFFFF_FFFFi64);
+ let ile_shifted = builder.ins().ushr_imm(msr, 16);
+ let ile_bit = builder.ins().band_imm(ile_shifted, 1);
+ let new_msr = builder.ins().bor(msr_cleared, ile_bit);
+ builder.ins().store(MemFlags::trusted(), new_msr, ctx_ptr, msr_off);
+
+ nia
+}
+
+pub(crate) fn emit_msr_fp_guard<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ _instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+) {
+ if local.fp_guard_emitted.get() {
+ return;
+ }
+ local.fp_guard_emitted.set(true);
+
+ let msr_off = abi::msr_offset::<SYSTEM>() as i32;
+ let msr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, msr_off);
+ let fp_bit = builder.ins().band_imm(msr, 0x2000);
+ let fp_disabled = builder.ins().icmp_imm(IntCC::Equal, fp_bit, 0);
+ let trap_block = builder.create_block();
+ let cont_block = builder.create_block();
+ builder.ins().brif(fp_disabled, trap_block, &[], cont_block, &[]);
+
+ builder.switch_to_block(trap_block);
+ builder.seal_block(trap_block);
+
+ let _ = local;
+ let cia_off = abi::cia_offset::<SYSTEM>() as i32;
+ let pc_const = builder.ins().iconst(types::I32, pc as i64);
+ builder.ins().store(MemFlags::trusted(), pc_const, ctx_ptr, cia_off);
+ let nia_off = abi::nia_offset::<SYSTEM>() as i32;
+ let pc_plus4 = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);
+ builder.ins().store(MemFlags::trusted(), pc_plus4, ctx_ptr, nia_off);
+ let nia = emit_exception_dispatch::<SYSTEM>(builder, ctx_ptr, pc_const, 0, 0x0000_0800);
+ emit_cycles_add::<SYSTEM>(builder, ctx_ptr, CYCLES_PER_INSTR);
+ builder.ins().jump(exit_block, &[nia.into()]);
+
+ builder.switch_to_block(cont_block);
+ builder.seal_block(cont_block);
+}
+
+pub(crate) fn emit_lf_d_form_update<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ single: bool,
+ local: &LocalFuncs,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ if single {
+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f32);
+ } else {
+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, local.read_f64);
+ }
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+}
+
+pub(crate) fn emit_stf_d_form_update<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ single: bool,
+ local: &LocalFuncs,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ if single {
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);
+ } else {
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);
+ }
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+}
+
+pub(crate) fn emit_lfs<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f32: FuncRef,
+ _slow_u32: FuncRef,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);
+ let be_u32 = builder.ins().bswap(raw_u32);
+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);
+ let f64v = builder.ins().fpromote(types::F64, f32v);
+ builder.ins().jump(merge, &[f64v.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);
+ let f64v_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[f64v_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+}
+
+pub(crate) fn emit_lfd<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f64: FuncRef,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_u64 = builder.ins().load(types::I64, MemFlags::new(), addr, 0);
+ let be_u64 = builder.ins().bswap(raw_u64);
+ let f64v = builder.ins().bitcast(types::F64, MemFlags::new(), be_u64);
+ builder.ins().jump(merge, &[f64v.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow_f64, &[ctx_ptr, ea]);
+ let f64v_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[f64v_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+}
+
+pub(crate) fn emit_stfs<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f32: FuncRef,
+ _slow_u32: FuncRef,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let f32v = builder.ins().fdemote(types::F32, f64v);
+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
+ let swapped = builder.ins().bswap(bits);
+ builder.ins().store(MemFlags::new(), swapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_stfd<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f64: FuncRef,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);
+ let swapped = builder.ins().bswap(bits);
+ builder.ins().store(MemFlags::new(), swapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow_f64, &[ctx_ptr, ea, f64v]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_lfsx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f32: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_lfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, slow_f32);
+}
+
+pub(crate) fn emit_lfdx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f64: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_lfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rd(), ea, slow_f64);
+}
+
+pub(crate) fn emit_stfsx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f32: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32);
+}
+
+pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ slow_f64: FuncRef,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64);
+}
+
+pub(crate) fn emit_lfs_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ rd: u8,
+ ea: Value,
+ slow_f32: FuncRef,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);
+ let be_u32 = builder.ins().bswap(raw_u32);
+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);
+ let f64v = builder.ins().fpromote(types::F64, f32v);
+ builder.ins().jump(merge, &[f64v.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);
+ let f64v_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[f64v_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, rd, val);
+}
+
+pub(crate) fn emit_lfd_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ rd: u8,
+ ea: Value,
+ slow_f64: FuncRef,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_u64 = builder.ins().load(types::I64, MemFlags::new(), addr, 0);
+ let be_u64 = builder.ins().bswap(raw_u64);
+ let f64v = builder.ins().bitcast(types::F64, MemFlags::new(), be_u64);
+ builder.ins().jump(merge, &[f64v.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow_f64, &[ctx_ptr, ea]);
+ let f64v_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[f64v_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ fpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);
+}
+
+pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ rs: u8,
+ ea: Value,
+ slow_f32: FuncRef,
+) {
+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let f32v = builder.ins().fdemote(types::F32, f64v);
+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
+ let swapped = builder.ins().bswap(bits);
+ builder.ins().store(MemFlags::new(), swapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ rs: u8,
+ ea: Value,
+ slow_f64: FuncRef,
+) {
+ let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);
+ let swapped = builder.ins().bswap(bits);
+ builder.ins().store(MemFlags::new(), swapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow_f64, &[ctx_ptr, ea, f64v]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+ update: bool,
+) {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);
+
+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), false);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
+ let ps1 = if instr.w_16_16() {
+ builder.ins().f64const(1.0)
+ } else {
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)
+ };
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);
+ ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let _ = pc;
+ let _ = exit_block;
+ emit_psq_load_quantized::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rd(),
+ instr.w_16_16(),
+ instr.i_17_19(),
+ merge,
+ local,
+ );
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+ }
+}
+
+pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ pc: u32,
+ exit_block: Block,
+ local: &LocalFuncs,
+ update: bool,
+) {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);
+
+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), true);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);
+ if !instr.w_16_16() {
+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);
+ }
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let _ = pc;
+ let _ = exit_block;
+ emit_psq_store_quantized::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rs(),
+ instr.w_16_16(),
+ instr.i_17_19(),
+ merge,
+ local,
+ );
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
+ }
+}
+
+pub(crate) fn emit_gqr_type_zero_check<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ gqr_idx: u8,
+ store: bool,
+) -> Value {
+ let off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;
+ let gqr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let mask = if store { 0x7 } else { 0x70000 };
+ let bits = builder.ins().band_imm(gqr, mask);
+ builder.ins().icmp_imm(IntCC::Equal, bits, 0)
+}
+
+pub(crate) fn emit_psq_dequant_element(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ type_v: Value,
+ scale_f32: Value,
+ read_u8_fn: FuncRef,
+ read_u16_fn: FuncRef,
+) -> Value {
+ let half_bit = builder.ins().band_imm(type_v, 1);
+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);
+ let signed_bit = builder.ins().band_imm(type_v, 2);
+ let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);
+
+ let byte_block = builder.create_block();
+ let half_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F32);
+
+ builder.ins().brif(is_half, half_block, &[], byte_block, &[]);
+
+ builder.switch_to_block(byte_block);
+ builder.seal_block(byte_block);
+ let call = builder.ins().call(read_u8_fn, &[ctx_ptr, ea]);
+ let raw_byte = builder.inst_results(call)[0];
+ let trunc = builder.ins().ireduce(types::I8, raw_byte);
+ let sext = builder.ins().sextend(types::I32, trunc);
+ let val_i32 = builder.ins().select(is_signed, sext, raw_byte);
+ let val_f32 = builder.ins().fcvt_from_sint(types::F32, val_i32);
+ let scaled = builder.ins().fmul(val_f32, scale_f32);
+ builder.ins().jump(merge, &[scaled.into()]);
+
+ builder.switch_to_block(half_block);
+ builder.seal_block(half_block);
+ let call = builder.ins().call(read_u16_fn, &[ctx_ptr, ea]);
+ let raw_half = builder.inst_results(call)[0];
+ let trunc = builder.ins().ireduce(types::I16, raw_half);
+ let sext = builder.ins().sextend(types::I32, trunc);
+ let val_i32 = builder.ins().select(is_signed, sext, raw_half);
+ let val_f32 = builder.ins().fcvt_from_sint(types::F32, val_i32);
+ let scaled = builder.ins().fmul(val_f32, scale_f32);
+ builder.ins().jump(merge, &[scaled.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let scaled_f32 = builder.block_params(merge)[0];
+ builder.ins().fpromote(types::F64, scaled_f32)
+}
+
+pub(crate) fn emit_psq_quant_element(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ value_f64: Value,
+ type_v: Value,
+ scale_f32: Value,
+ write_u8_fn: FuncRef,
+ write_u16_fn: FuncRef,
+) {
+ let val_f32 = builder.ins().fdemote(types::F32, value_f64);
+ let scaled_f32 = builder.ins().fmul(val_f32, scale_f32);
+
+ let half_bit = builder.ins().band_imm(type_v, 1);
+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);
+ let signed_bit = builder.ins().band_imm(type_v, 2);
+ let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);
+
+ let zero_f = builder.ins().f32const(0.0_f32);
+ let u8_max = builder.ins().f32const(255.0_f32);
+ let u16_max = builder.ins().f32const(65535.0_f32);
+ let i8_min = builder.ins().f32const(-128.0_f32);
+ let i8_max = builder.ins().f32const(127.0_f32);
+ let i16_min = builder.ins().f32const(-32768.0_f32);
+ let i16_max = builder.ins().f32const(32767.0_f32);
+
+ let signed_min = builder.ins().select(is_half, i16_min, i8_min);
+ let min_f32 = builder.ins().select(is_signed, signed_min, zero_f);
+ let signed_max = builder.ins().select(is_half, i16_max, i8_max);
+ let unsigned_max = builder.ins().select(is_half, u16_max, u8_max);
+ let max_f32 = builder.ins().select(is_signed, signed_max, unsigned_max);
+
+ let lo_clamped = builder.ins().fmax(scaled_f32, min_f32);
+ let clamped = builder.ins().fmin(lo_clamped, max_f32);
+
+ let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, clamped);
+
+ let mask_byte = builder.ins().iconst(types::I32, 0xFFi64);
+ let mask_half = builder.ins().iconst(types::I32, 0xFFFFi64);
+ let mask = builder.ins().select(is_half, mask_half, mask_byte);
+ let val_to_write = builder.ins().band(val_i32, mask);
+
+ let byte_block = builder.create_block();
+ let half_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.ins().brif(is_half, half_block, &[], byte_block, &[]);
+
+ builder.switch_to_block(byte_block);
+ builder.seal_block(byte_block);
+ builder.ins().call(write_u8_fn, &[ctx_ptr, ea, val_to_write]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(half_block);
+ builder.seal_block(half_block);
+ builder.ins().call(write_u16_fn, &[ctx_ptr, ea, val_to_write]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_psq_elem_size(builder: &mut FunctionBuilder, type_v: Value) -> Value {
+ let half_bit = builder.ins().band_imm(type_v, 1);
+ let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);
+ let two = builder.ins().iconst(types::I32, 2);
+ let one = builder.ins().iconst(types::I32, 1);
+ builder.ins().select(is_half, two, one)
+}
+
+pub(crate) fn emit_psq_load_quantized<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ fd: u8,
+ w: bool,
+ gqr_idx: u8,
+ merge: Block,
+ local: &LocalFuncs,
+) {
+ let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;
+ let gqr_v = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, gqr_off);
+ let type_shifted = builder.ins().ushr_imm(gqr_v, 16);
+ let type_v = builder.ins().band_imm(type_shifted, 0x7);
+
+ let q_bit = builder.ins().band_imm(type_v, 0x4);
+ let is_quant = builder.ins().icmp_imm(IntCC::NotEqual, q_bit, 0);
+ let inline_block = builder.create_block();
+ let tramp_block = builder.create_block();
+ builder.ins().brif(is_quant, inline_block, &[], tramp_block, &[]);
+
+ builder.switch_to_block(inline_block);
+ builder.seal_block(inline_block);
+ let scale_idx_shifted = builder.ins().ushr_imm(gqr_v, 24);
+ let scale_idx = builder.ins().band_imm(scale_idx_shifted, 0x3F);
+ let scale_idx_64 = builder.ins().uextend(types::I64, scale_idx);
+ let scale_offset = builder.ins().ishl_imm(scale_idx_64, 2);
+ let table_addr = builder
+ .ins()
+ .iconst(types::I64, crate::gekko::jit::runtime::DEQUANT_TABLE.as_ptr() as i64);
+ let scale_addr = builder.ins().iadd(table_addr, scale_offset);
+ let scale_f32 = builder.ins().load(types::F32, MemFlags::trusted(), scale_addr, 0);
+
+ let ps0 = emit_psq_dequant_element(builder, ctx_ptr, ea, type_v, scale_f32, local.read_u8, local.read_u16);
+ let ps1 = if w {
+ builder.ins().f64const(1.0)
+ } else {
+ let elem_size = emit_psq_elem_size(builder, type_v);
+ let ea1 = builder.ins().iadd(ea, elem_size);
+ emit_psq_dequant_element(builder, ctx_ptr, ea1, type_v, scale_f32, local.read_u8, local.read_u16)
+ };
+ fpr_store::<SYSTEM>(builder, ctx_ptr, fd, ps0);
+ ps1_store::<SYSTEM>(builder, ctx_ptr, fd, ps1);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(tramp_block);
+ builder.seal_block(tramp_block);
+ let fd_v = builder.ins().iconst(types::I32, fd as i64);
+ let w_v = builder.ins().iconst(types::I32, if w { 1 } else { 0 });
+ let gqr_const = builder.ins().iconst(types::I32, gqr_idx as i64);
+ builder
+ .ins()
+ .call(local.do_psq_load, &[ctx_ptr, fd_v, ea, w_v, gqr_const]);
+ builder.ins().jump(merge, &[]);
+}
+
+pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ fs: u8,
+ w: bool,
+ gqr_idx: u8,
+ merge: Block,
+ local: &LocalFuncs,
+) {
+ let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;
+ let gqr_v = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, gqr_off);
+ let type_v = builder.ins().band_imm(gqr_v, 0x7);
+
+ let q_bit = builder.ins().band_imm(type_v, 0x4);
+ let is_quant = builder.ins().icmp_imm(IntCC::NotEqual, q_bit, 0);
+ let inline_block = builder.create_block();
+ let tramp_block = builder.create_block();
+ builder.ins().brif(is_quant, inline_block, &[], tramp_block, &[]);
+
+ builder.switch_to_block(inline_block);
+ builder.seal_block(inline_block);
+ let scale_idx_shifted = builder.ins().ushr_imm(gqr_v, 8);
+ let scale_idx = builder.ins().band_imm(scale_idx_shifted, 0x3F);
+ let scale_idx_64 = builder.ins().uextend(types::I64, scale_idx);
+ let scale_offset = builder.ins().ishl_imm(scale_idx_64, 2);
+ let table_addr = builder
+ .ins()
+ .iconst(types::I64, crate::gekko::jit::runtime::QUANT_TABLE.as_ptr() as i64);
+ let scale_addr = builder.ins().iadd(table_addr, scale_offset);
+ let scale_f32 = builder.ins().load(types::F32, MemFlags::trusted(), scale_addr, 0);
+
+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, fs);
+ emit_psq_quant_element(
+ builder,
+ ctx_ptr,
+ ea,
+ ps0,
+ type_v,
+ scale_f32,
+ local.write_u8,
+ local.write_u16,
+ );
+ if !w {
+ let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, fs);
+ let elem_size = emit_psq_elem_size(builder, type_v);
+ let ea1 = builder.ins().iadd(ea, elem_size);
+ emit_psq_quant_element(
+ builder,
+ ctx_ptr,
+ ea1,
+ ps1,
+ type_v,
+ scale_f32,
+ local.write_u8,
+ local.write_u16,
+ );
+ }
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(tramp_block);
+ builder.seal_block(tramp_block);
+ let fs_v = builder.ins().iconst(types::I32, fs as i64);
+ let w_v = builder.ins().iconst(types::I32, if w { 1 } else { 0 });
+ let gqr_const = builder.ins().iconst(types::I32, gqr_idx as i64);
+ builder
+ .ins()
+ .call(local.do_psq_store, &[ctx_ptr, fs_v, ea, w_v, gqr_const]);
+ builder.ins().jump(merge, &[]);
+}
+
+pub(crate) fn emit_read_f32<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ slow_f32: FuncRef,
+) -> Value {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw_u32 = builder.ins().load(types::I32, MemFlags::new(), addr, 0);
+ let be_u32 = builder.ins().bswap(raw_u32);
+ let f32v = builder.ins().bitcast(types::F32, MemFlags::new(), be_u32);
+ let f64v = builder.ins().fpromote(types::F64, f32v);
+ builder.ins().jump(merge, &[f64v.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow_f32, &[ctx_ptr, ea]);
+ let f64v_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[f64v_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ builder.block_params(merge)[0]
+}
+
+pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ f64v: Value,
+ slow_f32: FuncRef,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let f32v = builder.ins().fdemote(types::F32, f64v);
+ let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
+ let swapped = builder.ins().bswap(bits);
+ builder.ins().store(MemFlags::new(), swapped, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow_f32, &[ctx_ptr, ea, f64v]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
+pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> bool {
+ use crate::gekko::jit::lut;
+
+ match OP {
+ lut::OP_PS_ADD => {
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s0 = builder.ins().fadd(a0, b0);
+ let r0 = round_to_single(builder, s0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s1 = builder.ins().fadd(a1, b1);
+ let r1 = round_to_single(builder, s1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_SUB => {
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s0 = builder.ins().fsub(a0, b0);
+ let r0 = round_to_single(builder, s0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s1 = builder.ins().fsub(a1, b1);
+ let r1 = round_to_single(builder, s1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MUL => {
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let m0 = builder.ins().fmul(a0, c0);
+ let r0 = round_to_single(builder, m0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let m1 = builder.ins().fmul(a1, c1);
+ let r1 = round_to_single(builder, m1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_DIV => {
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let q0 = builder.ins().fdiv(a0, b0);
+ let r0 = round_to_single(builder, q0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let q1 = builder.ins().fdiv(a1, b1);
+ let r1 = round_to_single(builder, q1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MADD => {
+ let r0 = ps_fma::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ instr.ra(),
+ instr.fc(),
+ instr.rb(),
+ FmaSign::Pos,
+ false,
+ );
+ let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MSUB => {
+ let r0 = ps_fma::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ instr.ra(),
+ instr.fc(),
+ instr.rb(),
+ FmaSign::Neg,
+ false,
+ );
+ let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_NMADD => {
+ let v0 = ps_fma::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ instr.ra(),
+ instr.fc(),
+ instr.rb(),
+ FmaSign::Pos,
+ false,
+ );
+ let r0 = builder.ins().fneg(v0);
+ let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);
+ let r1 = builder.ins().fneg(v1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_NMSUB => {
+ let v0 = ps_fma::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ instr.ra(),
+ instr.fc(),
+ instr.rb(),
+ FmaSign::Neg,
+ false,
+ );
+ let r0 = builder.ins().fneg(v0);
+ let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);
+ let r1 = builder.ins().fneg(v1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MULS0 => {
+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m0 = builder.ins().fmul(a0, c0);
+ let r0 = round_to_single(builder, m0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m1 = builder.ins().fmul(a1, c0);
+ let r1 = round_to_single(builder, m1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MULS1 => {
+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m0 = builder.ins().fmul(a0, c1);
+ let r0 = round_to_single(builder, m0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m1 = builder.ins().fmul(a1, c1);
+ let r1 = round_to_single(builder, m1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MADDS0 => {
+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let m0 = builder.ins().fmul(a0, c0);
+ let s0 = builder.ins().fadd(m0, b0);
+ let r0 = round_to_single(builder, s0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let m1 = builder.ins().fmul(a1, c0);
+ let s1 = builder.ins().fadd(m1, b1);
+ let r1 = round_to_single(builder, s1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MADDS1 => {
+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let m0 = builder.ins().fmul(a0, c1);
+ let s0 = builder.ins().fadd(m0, b0);
+ let r0 = round_to_single(builder, s0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let m1 = builder.ins().fmul(a1, c1);
+ let s1 = builder.ins().fadd(m1, b1);
+ let r1 = round_to_single(builder, s1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_SUM0 => {
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fadd(a, b);
+ let r0 = round_to_single(builder, s);
+ let r1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_SUM1 => {
+ let r0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fadd(a, b);
+ let r1 = round_to_single(builder, s);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_SEL => {
+ use cranelift_codegen::ir::condcodes::FloatCC;
+ let zero = builder.ins().f64const(0.0);
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let cond0 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a0, zero);
+ let r0 = builder.ins().select(cond0, c0, b0);
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let cond1 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a1, zero);
+ let r1 = builder.ins().select(cond1, c1, b1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_RES => {
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r0 = f32_reciprocal(builder, b0);
+ let r1 = f32_reciprocal(builder, b1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_RSQRTE => {
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r0 = f32_rsqrte(builder, b0);
+ let r1 = f32_rsqrte(builder, b1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
+ return true;
+ }
+ lut::OP_PS_MR => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ true
+ }
+ lut::OP_PS_NEG => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let n0 = builder.ins().fneg(p0);
+ let n1 = builder.ins().fneg(p1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);
+ true
+ }
+ lut::OP_PS_ABS => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a0 = builder.ins().fabs(p0);
+ let a1 = builder.ins().fabs(p1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), a0, a1);
+ true
+ }
+ lut::OP_PS_NABS => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a0 = builder.ins().fabs(p0);
+ let a1 = builder.ins().fabs(p1);
+ let n0 = builder.ins().fneg(a0);
+ let n1 = builder.ins().fneg(a1);
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);
+ true
+ }
+ lut::OP_PS_MERGE00 => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let p1 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ true
+ }
+ lut::OP_PS_MERGE01 => {
+ let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ true
+ }
+ lut::OP_PS_MERGE10 => {
+ let p0 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let p1 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ true
+ }
+ lut::OP_PS_MERGE11 => {
+ let p0 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ true
+ }
+ lut::OP_PS_CMPU0 | lut::OP_PS_CMPO0 => {
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);
+ true
+ }
+ lut::OP_PS_CMPU1 | lut::OP_PS_CMPO1 => {
+ let a = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);
+ true
+ }
+ _ => false,
+ }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum FmaSign {
+ Pos,
+ Neg,
+}
+
+pub(crate) fn ps_fma<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ fra: u8,
+ frc: u8,
+ frb: u8,
+ sign: FmaSign,
+ pick_ps1: bool,
+) -> Value {
+ let a = if pick_ps1 {
+ ps1_load::<SYSTEM>(builder, ctx_ptr, fra)
+ } else {
+ fpr_load::<SYSTEM>(builder, ctx_ptr, fra)
+ };
+ let c = if pick_ps1 {
+ ps1_load::<SYSTEM>(builder, ctx_ptr, frc)
+ } else {
+ fpr_load::<SYSTEM>(builder, ctx_ptr, frc)
+ };
+ let b = if pick_ps1 {
+ ps1_load::<SYSTEM>(builder, ctx_ptr, frb)
+ } else {
+ fpr_load::<SYSTEM>(builder, ctx_ptr, frb)
+ };
+ let m = builder.ins().fmul(a, c);
+ let r = match sign {
+ FmaSign::Pos => builder.ins().fadd(m, b),
+ FmaSign::Neg => builder.ins().fsub(m, b),
+ };
+ round_to_single(builder, r)
+}
+
+pub(crate) fn ps_write<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ rd: u8,
+ ps0: Value,
+ ps1: Value,
+) {
+ fpr_store::<SYSTEM>(builder, ctx_ptr, rd, ps0);
+ ps1_store::<SYSTEM>(builder, ctx_ptr, rd, ps1);
+}
+
+pub(crate) fn emit_fp_compare<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ crfd: u8,
+ a: Value,
+ b: Value,
+) {
+ use cranelift_codegen::ir::condcodes::FloatCC;
+
+ let a_nan = builder.ins().fcmp(FloatCC::NotEqual, a, a);
+ let b_nan = builder.ins().fcmp(FloatCC::NotEqual, b, b);
+ let so_b = builder.ins().bor(a_nan, b_nan);
+ let so = builder.ins().uextend(types::I32, so_b);
+
+ let lt_b = builder.ins().fcmp(FloatCC::LessThan, a, b);
+ let gt_b = builder.ins().fcmp(FloatCC::GreaterThan, a, b);
+ let eq_b = builder.ins().fcmp(FloatCC::Equal, a, b);
+ let lt = builder.ins().uextend(types::I32, lt_b);
+ let gt = builder.ins().uextend(types::I32, gt_b);
+ let eq = builder.ins().uextend(types::I32, eq_b);
+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);
+ cr_set_field::<SYSTEM>(builder, ctx_ptr, crfd, nibble);
+}
+
+pub(crate) fn emit_fp_arith<const OP: u32, const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> bool {
+ use crate::gekko::jit::lut;
+
+ let single = matches!(
+ OP,
+ lut::OP_FADDSX
+ | lut::OP_FSUBSX
+ | lut::OP_FMULSX
+ | lut::OP_FDIVSX
+ | lut::OP_FMADDSX
+ | lut::OP_FMSUBSX
+ | lut::OP_FNMADDSX
+ | lut::OP_FNMSUBSX
+ | lut::OP_FSQRTSX
+ | lut::OP_FRESX
+ );
+
+ match OP {
+ lut::OP_FADDSX | lut::OP_FADDX => {
+ emit_fp_binop::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ FpBinOp::Add,
+ instr.rd(),
+ instr.ra(),
+ instr.rb(),
+ single,
+ );
+ return true;
+ }
+ lut::OP_FSUBSX | lut::OP_FSUBX => {
+ emit_fp_binop::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ FpBinOp::Sub,
+ instr.rd(),
+ instr.ra(),
+ instr.rb(),
+ single,
+ );
+ return true;
+ }
+ lut::OP_FMULSX | lut::OP_FMULX => {
+ emit_fp_binop::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ FpBinOp::Mul,
+ instr.rd(),
+ instr.ra(),
+ instr.fc(),
+ single,
+ );
+ return true;
+ }
+ lut::OP_FDIVSX | lut::OP_FDIVX => {
+ emit_fp_binop::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ FpBinOp::Div,
+ instr.rd(),
+ instr.ra(),
+ instr.rb(),
+ single,
+ );
+ return true;
+ }
+ lut::OP_FMADDSX
+ | lut::OP_FMADDX
+ | lut::OP_FMSUBSX
+ | lut::OP_FMSUBX
+ | lut::OP_FNMADDSX
+ | lut::OP_FNMADDX
+ | lut::OP_FNMSUBSX
+ | lut::OP_FNMSUBX => {
+ let is_add = matches!(
+ OP,
+ lut::OP_FMADDSX | lut::OP_FMADDX | lut::OP_FNMADDSX | lut::OP_FNMADDX
+ );
+ let neg = matches!(
+ OP,
+ lut::OP_FNMADDSX | lut::OP_FNMADDX | lut::OP_FNMSUBSX | lut::OP_FNMSUBX
+ );
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let m = builder.ins().fmul(a, c);
+ let core = if is_add {
+ builder.ins().fadd(m, b)
+ } else {
+ builder.ins().fsub(m, b)
+ };
+ let signed = if neg { builder.ins().fneg(core) } else { core };
+ let stored = if single {
+ round_to_single(builder, signed)
+ } else {
+ signed
+ };
+ if single {
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);
+ } else {
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);
+ }
+ return true;
+ }
+ lut::OP_FSQRTSX | lut::OP_FSQRTX => {
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().sqrt(b);
+ let stored = if single { round_to_single(builder, r) } else { r };
+ if single {
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);
+ } else {
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);
+ }
+ return true;
+ }
+ lut::OP_FRESX => {
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = f32_reciprocal(builder, b);
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FRSQRTEX => {
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().sqrt(b);
+ let one = builder.ins().f64const(1.0);
+ let r = builder.ins().fdiv(one, s);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FSELX => {
+ use cranelift_codegen::ir::condcodes::FloatCC;
+ let zero = builder.ins().f64const(0.0);
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let cond = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a, zero);
+ let r = builder.ins().select(cond, c, b);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FMRX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);
+ return true;
+ }
+ lut::OP_FNEGX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().fneg(v);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FABSX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = builder.ins().fabs(v);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FNABSX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a = builder.ins().fabs(v);
+ let r = builder.ins().fneg(a);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FRSPX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let r = round_to_single(builder, v);
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FCTIWZX | lut::OP_FCTIWX => {
+ let v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let i = builder.ins().fcvt_to_sint_sat(types::I32, v);
+ let i64v = builder.ins().uextend(types::I64, i);
+ let f = builder.ins().bitcast(types::F64, MemFlags::new(), i64v);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), f);
+ return true;
+ }
+ lut::OP_MTFSB1X => {
+ const FPSCR_ANY_X: u32 = 0x1FF8_0700;
+ let bit_pos = 31 - (instr.crbd() as u32);
+ let bit_value = 1u32 << bit_pos;
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let mut new_v = builder.ins().bor_imm(cur, bit_value as i64);
+ if (bit_value & FPSCR_ANY_X) != 0 {
+ let was_set = builder.ins().band_imm(cur, bit_value as i64);
+ let was_unset = builder.ins().icmp_imm(IntCC::Equal, was_set, 0);
+ let was_unset_v = builder.ins().uextend(types::I32, was_unset);
+ let fx_bit = builder.ins().ishl_imm(was_unset_v, 31);
+ new_v = builder.ins().bor(new_v, fx_bit);
+ }
+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);
+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);
+ if instr.rc() {
+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);
+ }
+ true
+ }
+ lut::OP_MTFSB0X => {
+ let bit_pos = 31 - (instr.crbd() as u32);
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let new_v = builder.ins().band_imm(cur, !(1i64 << bit_pos));
+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);
+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);
+ if instr.rc() {
+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);
+ }
+ true
+ }
+ lut::OP_MTFSFIX => {
+ let dst_shift = 28 - 4 * (instr.crfd() as u32);
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let cleared = builder.ins().band_imm(cur, !(0xFi64 << dst_shift));
+ let new_v = builder.ins().bor_imm(cleared, (instr.imm() as i64) << dst_shift);
+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);
+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);
+ if instr.rc() {
+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);
+ }
+ true
+ }
+ lut::OP_MFFSX => {
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let cur_64 = builder.ins().uextend(types::I64, cur);
+ let with_nan_hi = builder.ins().bor_imm(cur_64, 0xFFF8_0000_0000_0000u64 as i64);
+ let f = builder.ins().bitcast(types::F64, MemFlags::new(), with_nan_hi);
+ fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), f);
+ if instr.rc() {
+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);
+ }
+ true
+ }
+ lut::OP_MTFSFX => {
+ let mut mask: u32 = 0;
+ for i in 0..8 {
+ if (instr.fm() as u32) & (1 << (7 - i)) != 0 {
+ mask |= 0xFu32 << (28 - 4 * i);
+ }
+ }
+ let off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let frb_v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let frb_bits = builder.ins().bitcast(types::I64, MemFlags::new(), frb_v);
+ let frb_low = builder.ins().ireduce(types::I32, frb_bits);
+ let cur = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, off);
+ let cleared = builder.ins().band_imm(cur, !(mask as i64) & 0xFFFF_FFFF);
+ let from_src = builder.ins().band_imm(frb_low, mask as i64);
+ let new_v = builder.ins().bor(cleared, from_src);
+ builder.ins().store(MemFlags::trusted(), new_v, ctx_ptr, off);
+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);
+ if instr.rc() {
+ emit_update_cr1_from_fpscr::<SYSTEM>(builder, ctx_ptr);
+ }
+ true
+ }
+ lut::OP_FCMPU | lut::OP_FCMPO => {
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ emit_fp_compare::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b);
+ true
+ }
+ lut::OP_MCRFS => {
+ let src_shift = (7 - (instr.crfs() as u32)) * 4;
+ let dst_shift = 28 - 4 * (instr.crfd() as u32);
+
+ const FX_ANY_X: u32 = 0x9FF8_0700;
+ let fpscr_off = abi::fpscr_offset::<SYSTEM>() as i32;
+ let fpscr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, fpscr_off);
+
+ let nibble = builder.ins().ushr_imm(fpscr, src_shift as i64);
+ let nibble = builder.ins().band_imm(nibble, 0xF);
+
+ let cr_off = abi::cr_offset::<SYSTEM>() as i32;
+ let cr = builder.ins().load(types::I32, MemFlags::trusted(), ctx_ptr, cr_off);
+ let cr_cleared = builder.ins().band_imm(cr, !(0xFi64 << dst_shift));
+ let positioned = builder.ins().ishl_imm(nibble, dst_shift as i64);
+ let cr_new = builder.ins().bor(cr_cleared, positioned);
+ builder.ins().store(MemFlags::trusted(), cr_new, ctx_ptr, cr_off);
+
+ let clear_mask = (0xFu32 << src_shift) & FX_ANY_X;
+ if clear_mask != 0 {
+ let fpscr_new = builder.ins().band_imm(fpscr, !(clear_mask as i64) & 0xFFFF_FFFF);
+ builder.ins().store(MemFlags::trusted(), fpscr_new, ctx_ptr, fpscr_off);
+ }
+ emit_recompute_fpscr_summary::<SYSTEM>(builder, ctx_ptr);
+ true
+ }
+ _ => false,
+ }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum FpBinOp {
+ Add,
+ Sub,
+ Mul,
+ Div,
+}
+
+pub(crate) fn emit_fp_binop<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ op: FpBinOp,
+ frd: u8,
+ fra: u8,
+ frb_or_c: u8,
+ single: bool,
+) {
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, fra);
+ let b = fpr_load::<SYSTEM>(builder, ctx_ptr, frb_or_c);
+
+ let res = match op {
+ FpBinOp::Add => builder.ins().fadd(a, b),
+ FpBinOp::Sub => builder.ins().fsub(a, b),
+ FpBinOp::Mul => builder.ins().fmul(a, b),
+ FpBinOp::Div => builder.ins().fdiv(a, b),
+ };
+ let stored = if single {
+ let demoted = builder.ins().fdemote(types::F32, res);
+ builder.ins().fpromote(types::F64, demoted)
+ } else {
+ res
+ };
+ if single {
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, frd, stored);
+ } else {
+ fpr_store::<SYSTEM>(builder, ctx_ptr, frd, stored);
+ }
+}
+
+pub(crate) fn emit_load_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rd(), instr.ra(), size, slow, update);
+}
+
+pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ rd: u8,
+ ra: u8,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::I32);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let flags = MemFlags::new();
+ let raw_val = builder.ins().load(size.ir_type(), flags, addr, 0);
+ let val_swapped = match size {
+ MemSize::U8 => raw_val,
+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw_val),
+ };
+ let val32 = match size {
+ MemSize::U32 => val_swapped,
+ MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, val_swapped),
+ };
+ builder.ins().jump(merge, &[val32.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);
+ let val32_slow = builder.inst_results(call)[0];
+ builder.ins().jump(merge, &[val32_slow.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ let val = builder.block_params(merge)[0];
+ gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
+ }
+}
+
+pub(crate) fn emit_store<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);
+}
+
+pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
+ emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);
+}
+
+pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ rs: u8,
+ ra: u8,
+ size: MemSize,
+ slow: FuncRef,
+ update: bool,
+) {
+ let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let flags = MemFlags::new();
+ let truncated = match size {
+ MemSize::U32 => val32,
+ MemSize::U16 => builder.ins().ireduce(types::I16, val32),
+ MemSize::U8 => builder.ins().ireduce(types::I8, val32),
+ };
+ let val_to_store = match size {
+ MemSize::U8 => truncated,
+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),
+ };
+ builder.ins().store(flags, val_to_store, addr, 0);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.ins().call(slow, &[ctx_ptr, ea, val32]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
+ }
+}
+
+pub(crate) fn emit_addi<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let val = builder.ins().iadd_imm(base, instr.simm() as i64);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+}
+
+pub(crate) fn emit_addis<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let val = builder.ins().iadd_imm(base, (instr.simm() as i64) << 16);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), val);
+}
+
+pub(crate) fn emit_ori<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let uimm = (instr.uimm() as u32) as i64;
+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let val = builder.ins().bor_imm(src, uimm);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum ImmLogical {
+ And,
+ Xor,
+}
+
+pub(crate) fn emit_imm_logical<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ op: ImmLogical,
+ shift_high: bool,
+) -> Value {
+ let uimm = (instr.uimm() as u32) as i64;
+ let imm = if shift_high { uimm << 16 } else { uimm };
+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let val = match op {
+ ImmLogical::And => builder.ins().band_imm(src, imm),
+ ImmLogical::Xor => builder.ins().bxor_imm(src, imm),
+ };
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+ val
+}
+
+pub(crate) fn emit_oris<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
+ let uimm = (instr.uimm() as u32) as i64;
+ let src = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let val = builder.ins().bor_imm(src, uimm << 16);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+}
+
+pub(crate) fn emit_cmp_imm<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ signed: bool,
+) {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let imm_val: i64 = if signed {
+ instr.simm() as i64
+ } else {
+ (instr.uimm() as u32) as i64
+ };
+ let imm = builder.ins().iconst(types::I32, imm_val);
+ emit_cmp_common::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, imm, signed);
+}
+
+pub(crate) fn emit_cmp_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ signed: bool,
+) {
+ let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ emit_cmp_common::<SYSTEM>(builder, ctx_ptr, instr.crfd(), a, b, signed);
+}
+
+pub(crate) fn emit_cmp_common<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ crfd: u8,
+ a: Value,
+ b: Value,
+ signed: bool,
+) {
+ let (lt_cc, gt_cc) = if signed {
+ (IntCC::SignedLessThan, IntCC::SignedGreaterThan)
+ } else {
+ (IntCC::UnsignedLessThan, IntCC::UnsignedGreaterThan)
+ };
+ let lt = cmp_to_i32(builder, lt_cc, a, b);
+ let gt = cmp_to_i32(builder, gt_cc, a, b);
+ let eq = cmp_to_i32(builder, IntCC::Equal, a, b);
+ let so = xer_so_bit::<SYSTEM>(builder, ctx_ptr);
+ let nibble = build_cr_nibble(builder, lt, gt, eq, so);
+ cr_set_field::<SYSTEM>(builder, ctx_ptr, crfd, nibble);
+}
+
+pub(crate) fn emit_cntlzw<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let r = builder.ins().clz(s);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), r);
+ r
+}
+
+pub(crate) fn emit_extend<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ from_ty: cranelift_codegen::ir::Type,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let truncated = builder.ins().ireduce(from_ty, s);
+ let extended = builder.ins().sextend(types::I32, truncated);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), extended);
+ extended
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum LogicalFullOp {
+ Nor,
+ Nand,
+ Andc,
+ Orc,
+ Eqv,
+}
+
+pub(crate) fn emit_logical_xform_full<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ op: LogicalFullOp,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+
+ let val = match op {
+ LogicalFullOp::Nor => {
+ let or = builder.ins().bor(s, b);
+ builder.ins().bnot(or)
+ }
+ LogicalFullOp::Nand => {
+ let and = builder.ins().band(s, b);
+ builder.ins().bnot(and)
+ }
+ LogicalFullOp::Eqv => {
+ let xor = builder.ins().bxor(s, b);
+ builder.ins().bnot(xor)
+ }
+ LogicalFullOp::Andc => {
+ let nb = builder.ins().bnot(b);
+ builder.ins().band(s, nb)
+ }
+ LogicalFullOp::Orc => {
+ let nb = builder.ins().bnot(b);
+ builder.ins().bor(s, nb)
+ }
+ };
+
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+ val
+}
+
+pub(crate) fn emit_logical_xform<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ instr: Instruction,
+ op: LogicalOp,
+) -> Value {
+ let s = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+
+ let val = match op {
+ LogicalOp::And => builder.ins().band(s, b),
+ LogicalOp::Or => builder.ins().bor(s, b),
+ LogicalOp::Xor => builder.ins().bxor(s, b),
+ };
+
+ gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), val);
+ val
+}
diff --git a/crates/gecko/src/idle.rs b/crates/gecko/src/idle.rs
deleted file mode 100644
index 45e2b8f..0000000
--- a/crates/gecko/src/idle.rs
+++ /dev/null
@@ -1,214 +0,0 @@
-/// Maximum backward branch distance (in bytes) considered a potential polling loop.
-pub const IDLE_LOOP_MAX_SIZE: u32 = 10 * 4;
-pub const IDLE_LOOP_MAX_INSTRS: usize = (IDLE_LOOP_MAX_SIZE / 4 + 1) as usize;
-/// Number of consecutive loop iterations before we attempt to skip.
-const IDLE_LOOP_THRESHOLD: u32 = 10;
-
-const MMIO_VIRT_START: u32 = 0xCC00_0000;
-const MMIO_VIRT_END: u32 = 0xCC00_FFFF;
-
-pub enum IdleCheck {
- /// Nothing special, keep executing normally.
- Continue,
- /// Branch-to-self detected, always safe to skip to the next event.
- Skip,
- /// A backward-branch loop hit the threshold, caller must validate the loop
- /// body and call [`IdleDetector::set_validated`] with the result.
- Validate { start: u32, end: u32 },
-}
-
-pub struct IdleDetector {
- loop_pc: Option<u32>,
- loop_end: u32,
- loop_count: u32,
- validated: LoopStatus,
-}
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum LoopStatus {
- /// Not yet checked.
- Unknown,
- /// Safe polling loop, skip allowed.
- Safe,
- /// Has side-effects, never skip.
- Unsafe,
-}
-
-impl IdleDetector {
- pub fn new() -> Self {
- Self {
- loop_pc: None,
- loop_end: 0,
- loop_count: 0,
- validated: LoopStatus::Unknown,
- }
- }
-
- #[inline(always)]
- pub fn check(&mut self, cia: u32, nia: u32) -> IdleCheck {
- // Branching to current is always a safe idle loop
- if nia == cia {
- return IdleCheck::Skip;
- }
-
- // Short backward branch, potential polling loop
- if nia < cia && cia.wrapping_sub(nia) <= IDLE_LOOP_MAX_SIZE {
- return self.track_backward_branch(nia, cia);
- }
-
- self.try_reset(cia);
- IdleCheck::Continue
- }
-
- /// Record that the current loop was validated (or not) as a safe polling
- /// loop. The result is cached until the loop changes.
- pub fn set_validated(&mut self, safe: bool) {
- self.validated = if safe { LoopStatus::Safe } else { LoopStatus::Unsafe };
- }
-
- fn track_backward_branch(&mut self, target: u32, branch_pc: u32) -> IdleCheck {
- if self.loop_pc == Some(target) {
- self.loop_count += 1;
- if self.loop_count >= IDLE_LOOP_THRESHOLD {
- self.loop_count = 0;
- return match self.validated {
- LoopStatus::Safe => IdleCheck::Skip,
- LoopStatus::Unsafe => IdleCheck::Continue,
- LoopStatus::Unknown => IdleCheck::Validate {
- start: target,
- end: branch_pc,
- },
- };
- }
- } else {
- // New loop
- self.loop_pc = Some(target);
- self.loop_end = branch_pc;
- self.loop_count = 1;
- self.validated = LoopStatus::Unknown;
- }
- IdleCheck::Continue
- }
-
- fn try_reset(&mut self, cia: u32) {
- if let Some(start) = self.loop_pc {
- if cia < start || cia > self.loop_end {
- self.loop_pc = None;
- self.loop_count = 0;
- self.validated = LoopStatus::Unknown;
- }
- }
- }
-}
-
-/// Analyse a small backward-branch loop and decide whether it is a pure
-/// MMIO-polling loop that is safe to skip.
-///
-/// Requirements:
-/// * Every instruction is side effect free (no stores, no calls, no
-/// supervisor stuff).
-/// * At least one load targets the hardware register address range
-/// (`0xCC00_0000 ..= 0xCC00_FFFF`).
-pub fn validate_polling_loop(instrs: &[u32], gprs: &[u32; 32]) -> bool {
- let mut has_mmio_load = false;
-
- for &raw in instrs {
- let opcode = raw >> 26;
- match opcode {
- // D-form integer loads
- // lwz, lwzu, lbz, lbzu, lhz, lhzu, lha, lhau
- 32 | 33 | 34 | 35 | 40 | 41 | 42 | 43 => {
- has_mmio_load |= d_form_targets_mmio(raw, gprs);
- }
- // D-form FP loads (lfs, lfsu, lfd, lfdu)
- 48 | 49 | 50 | 51 => {}
- // Load multiple word
- 46 => {}
- // ALU immediate / compares
- // mulli, subfic, cmpli, cmpi, addic, addic., addi, addis
- 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 => {}
- // ori, oris, xori, xoris, andi., andis.
- 24 | 25 | 26 | 27 | 28 | 29 => {}
- // rlwimi, rlwinm, rlwnm
- 20 | 21 | 23 => {}
- // FP arith (single / double)
- 59 | 63 => {}
- // Branches (without link)
- 16 | 18 => {
- if raw & 1 != 0 {
- return false; // bl / bcl = function call
- }
- }
- // Extended integer ops (opcode 31)
- 31 => {
- let xo = (raw >> 1) & 0x3FF;
- if is_unsafe_xo_31(xo) {
- return false;
- }
- if is_indexed_load_xo(xo) {
- has_mmio_load |= x_form_targets_mmio(raw, gprs);
- }
- }
- // Rest
- _ => return false,
- }
- }
-
- has_mmio_load
-}
-
-/// Check the effective address of a D-form load (rA + sign-ext d).
-fn d_form_targets_mmio(raw: u32, gprs: &[u32; 32]) -> bool {
- let ra = ((raw >> 16) & 0x1F) as usize;
- let d = (raw & 0xFFFF) as i16 as i32;
- let base = if ra == 0 { 0u32 } else { gprs[ra] };
- let ea = (base as i32).wrapping_add(d) as u32;
- (MMIO_VIRT_START..=MMIO_VIRT_END).contains(&ea)
-}
-
-/// Check the effective address of an X-form indexed load (rA + rB).
-fn x_form_targets_mmio(raw: u32, gprs: &[u32; 32]) -> bool {
- let ra = ((raw >> 16) & 0x1F) as usize;
- let rb = ((raw >> 11) & 0x1F) as usize;
- let base = if ra == 0 { 0u32 } else { gprs[ra] };
- let ea = base.wrapping_add(gprs[rb]);
- (MMIO_VIRT_START..=MMIO_VIRT_END).contains(&ea)
-}
-
-/// Opcode-31 extended ops that have side-effects (stores, supervisor writes,
-/// I/O writes, cache-invalidate).
-fn is_unsafe_xo_31(xo: u32) -> bool {
- matches!(
- xo,
- // Integer stores
- 150 | 151 | 183 | 215 | 247 | 407 | 439 | 661 | 662 | 725 | 918
- // FP stores
- | 663 | 695 | 727 | 759 | 983
- // dcbz
- | 1014
- // Supervisor / SPR writes
- | 146 // mtmsr
- | 210 // mtsr
- | 242 // mtsrin
- | 467 // mtspr
- // I/O write
- | 438 // ecowx
- )
-}
-
-/// Opcode-31 X-form indexed loads.
-fn is_indexed_load_xo(xo: u32) -> bool {
- matches!(
- xo,
- 23 // lwzx
- | 55 // lwzux
- | 87 // lbzx
- | 119 // lbzux
- | 279 // lhzx
- | 311 // lhzux
- | 343 // lhax
- | 375 // lhaux
- | 534 // lwbrx
- | 790 // lhbrx
- )
-}
diff --git a/crates/gecko/src/lib.rs b/crates/gecko/src/lib.rs
index 064a158..6c98a21 100644
--- a/crates/gecko/src/lib.rs
+++ b/crates/gecko/src/lib.rs
@@ -6,7 +6,6 @@ pub mod gamecube;
pub mod gekko;
pub mod hollywood;
pub mod host;
-pub mod idle;
pub mod input;
pub mod ipl;
pub mod mmio;
diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rs
index 22b3339..37fafbc 100644
--- a/crates/gecko/src/mmio.rs
+++ b/crates/gecko/src/mmio.rs
@@ -7,6 +7,11 @@ pub mod traits;
use crate::system::{SystemId, WII};
use constants::*;
+pub const FASTMEM_LUT_PAGES: usize = 1 << 15;
+pub const FASTMEM_PAGE_BYTES: usize = 1 << 17;
+pub const FASTMEM_PAGE_MASK: u32 = (FASTMEM_PAGE_BYTES as u32) - 1;
+pub const FASTMEM_PAGE_SHIFT: u32 = 17;
+
pub struct Mmio<const SYSTEM: SystemId> {
pub ram: Vec<u8>,
pub efb: Vec<u8>,
@@ -14,6 +19,9 @@ pub struct Mmio<const SYSTEM: SystemId> {
pub ipl: Vec<u8>,
pub lcache: Vec<u8>,
pub mem2: Vec<u8>,
+ pub ram_ptr: usize,
+ pub fastmem_lut: Vec<usize>,
+ pub fastmem_lut_ptr: usize,
}
/// Read-only view over MEM1 plus (on Wii) MEM2, addressed by physical
@@ -75,13 +83,55 @@ impl<'a> RamViewMut<'a> {
impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
pub fn new() -> Self {
+ let ram = vec![0u8; RAM_SIZE];
+ let ram_ptr = ram.as_ptr() as usize;
+ let mem2 = if SYSTEM == WII { vec![0; MEM2_SIZE] } else { Vec::new() };
+
+ let mut fastmem_lut = vec![0usize; FASTMEM_LUT_PAGES];
+
+ // Map MEM1 (24 MiB) at all three aliases:
+ // * physical 0x0000_0000..0x017F_FFFF
+ // * cached 0x8000_0000..0x817F_FFFF
+ // * uncached 0xC000_0000..0xC17F_FFFF
+ let mem1_pages = (RAM_SIZE / FASTMEM_PAGE_BYTES) as u32;
+ let bases = [0x0000_0000u32, 0x8000_0000, 0xC000_0000];
+ for base in bases {
+ let lut_base = (base >> FASTMEM_PAGE_SHIFT) as usize;
+ for i in 0..mem1_pages {
+ let host_addr = ram_ptr + (i as usize) * FASTMEM_PAGE_BYTES;
+ fastmem_lut[lut_base + i as usize] = host_addr;
+ }
+ }
+
+ // Wii: also map MEM2 (64 MiB) at its three aliases:
+ // * physical 0x1000_0000..0x13FF_FFFF
+ // * cached 0x9000_0000..0x93FF_FFFF
+ // * uncached 0xD000_0000..0xD3FF_FFFF
+ if SYSTEM == WII {
+ let mem2_pages = (MEM2_SIZE / FASTMEM_PAGE_BYTES) as u32;
+ let mem2_ptr = mem2.as_ptr() as usize;
+ let bases = [0x1000_0000u32, 0x9000_0000, 0xD000_0000];
+ for base in bases {
+ let lut_base = (base >> FASTMEM_PAGE_SHIFT) as usize;
+ for i in 0..mem2_pages {
+ let host_addr = mem2_ptr + (i as usize) * FASTMEM_PAGE_BYTES;
+ fastmem_lut[lut_base + i as usize] = host_addr;
+ }
+ }
+ }
+
+ let fastmem_lut_ptr = fastmem_lut.as_ptr() as usize;
+
Mmio {
- ram: vec![0; RAM_SIZE],
+ ram,
efb: vec![0; EFB_SIZE],
hwr: vec![0; HW_REG_SIZE],
ipl: vec![0; 0],
lcache: vec![0; LCACHE_SIZE],
- mem2: if SYSTEM == WII { vec![0; MEM2_SIZE] } else { Vec::new() },
+ mem2,
+ ram_ptr,
+ fastmem_lut,
+ fastmem_lut_ptr,
}
}
diff --git a/crates/gecko/src/scheduler.rs b/crates/gecko/src/scheduler.rs
index b4b1132..bf3a770 100644
--- a/crates/gecko/src/scheduler.rs
+++ b/crates/gecko/src/scheduler.rs
@@ -41,8 +41,8 @@ struct ScheduledEvent<const SYSTEM: SystemId> {
pub struct Scheduler<const SYSTEM: SystemId> {
pub cycles: u64,
- next_deadline: u64,
- timebase_offset: i64,
+ pub(crate) next_deadline: u64,
+ pub(crate) timebase_offset: i64,
events: VecDeque<ScheduledEvent<SYSTEM>>,
}
diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rs
index aac6bf7..98ecd0b 100644
--- a/crates/gecko/src/system.rs
+++ b/crates/gecko/src/system.rs
@@ -16,8 +16,6 @@ use crate::hollywood::Hollywood;
#[cfg(feature = "hooks")]
use crate::hooks::{HookFilters, HookFlags, HookState, Host};
use crate::host::{EmptyRenderSink, RenderSink};
-#[cfg(feature = "idle-skip")]
-use crate::idle::{IDLE_LOOP_MAX_INSTRS, IdleCheck, IdleDetector};
use crate::mmio::Mmio;
use crate::scheduler::Scheduler;
use crate::starlet::Starlet;
@@ -55,15 +53,15 @@ pub struct System<const SYSTEM: SystemId> {
/// AID DMA pushes 8-frame stereo s16 blocks here.
pub audio_sink: Box<dyn AudioSink>,
- #[cfg(feature = "idle-skip")]
- pub(crate) idle: IdleDetector,
-
#[cfg(feature = "hooks")]
pub hook_host: Option<Box<dyn Host<SYSTEM> + Send>>,
#[cfg(feature = "hooks")]
pub hook_flags: HookFlags,
#[cfg(feature = "hooks")]
pub hook_filters: HookFilters,
+
+ #[cfg(feature = "jit")]
+ pub jit: Option<Box<crate::gekko::jit::JitEngine<SYSTEM>>>,
}
impl<const SYSTEM: SystemId> System<SYSTEM> {
@@ -91,15 +89,15 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
render_sink: Box::new(EmptyRenderSink),
audio_sink: Box::new(EmptyAudioSink),
- #[cfg(feature = "idle-skip")]
- idle: IdleDetector::new(),
-
#[cfg(feature = "hooks")]
hook_host: None,
#[cfg(feature = "hooks")]
hook_flags: HookFlags::empty(),
#[cfg(feature = "hooks")]
hook_filters: HookFilters::default(),
+
+ #[cfg(feature = "jit")]
+ jit: None,
}
}
@@ -153,21 +151,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
}
}
- #[cfg(feature = "idle-skip")]
- match self.idle.check(self.gekko.cia, self.gekko.nia) {
- IdleCheck::Skip => {
- self.scheduler.cycles = self.scheduler.next_deadline();
- }
- IdleCheck::Validate { start, end } => {
- let safe = self.is_polling_loop(start, end);
- self.idle.set_validated(safe);
- if safe {
- self.scheduler.cycles = self.scheduler.next_deadline();
- }
- }
- IdleCheck::Continue => {}
- }
-
self.gekko.pc = self.gekko.nia;
}
@@ -195,26 +178,50 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
self.prepare_frame();
while !self.vsync_pending {
self.scheduler.refresh_deadline();
- // Execute a slice of CPU instructions until the next event deadline
- while self.scheduler.cycles < self.scheduler.next_deadline() {
- self.step_cpu();
+ #[cfg(feature = "jit")]
+ {
+ self.run_until_deadline_jit();
+ }
+ #[cfg(not(feature = "jit"))]
+ {
+ while self.scheduler.cycles < self.scheduler.next_deadline() {
+ self.step_cpu();
+ }
}
// Drain all events that are now due
self.drain_events();
}
}
- /// Read the instructions in `[start, end]` and check whether the loop is a
- /// side effect free MMIO polling loop that can safely be skipped.
- #[cfg(feature = "idle-skip")]
- #[inline(always)]
- fn is_polling_loop(&self, start: u32, end: u32) -> bool {
- let count = ((end - start) / 4 + 1) as usize;
- let mut buf = [0u32; IDLE_LOOP_MAX_INSTRS];
- for i in 0..count.min(buf.len()) {
- buf[i] = self.mmio.fetch_instruction(start + (i as u32) * 4);
+ /// JIT inner loop: runs compiled blocks back-to-back until
+ /// `scheduler.cycles >= next_deadline`. Interrupts are checked at block
+ /// boundaries.
+ #[cfg(feature = "jit")]
+ fn run_until_deadline_jit(&mut self) {
+ let mut jit = match self.jit.take() {
+ Some(jit) => jit,
+ None => Box::new(crate::gekko::jit::JitEngine::<SYSTEM>::new()),
+ };
+
+ while self.scheduler.cycles < self.scheduler.next_deadline() {
+ if self.gekko.msr.external_interrupt_enable() {
+ if self.pi.interrupt_pending() {
+ self.cause_external_interrupt();
+ self.scheduler.cycles += 2;
+ continue;
+ }
+
+ if self.gekko.dec.interrupt_pending() {
+ self.cause_decrementer_interrupt();
+ self.scheduler.cycles += 2;
+ continue;
+ }
+ }
+
+ jit.run_block(self);
}
- crate::idle::validate_polling_loop(&buf[..count.min(buf.len())], &self.gekko.gprs)
+
+ self.jit = Some(jit);
}
pub fn frame_size(&self) -> (usize, usize) {
diff --git a/crates/gecko/wii_gekko_jit.bindings.chipi b/crates/gecko/wii_gekko_jit.bindings.chipi
new file mode 100644
index 0000000..eedbd87
--- /dev/null
+++ b/crates/gecko/wii_gekko_jit.bindings.chipi
@@ -0,0 +1,276 @@
+include "../../submodules/chipi-spec/gamecube/gekko.chipi"
+
+target rust {
+ dispatch Gekko {
+ output "$OUT_DIR/gekko_jit_lut_wii.rs"
+
+ context crate::gekko::jit::translator::JitTranslator
+ handlers crate::gekko::jit::handlers
+ handler_const crate::system::WII
+ strategy fn_ptr_lut
+
+ invalid_handler crate::gekko::jit::handlers::invalid
+
+ instruction_type crate::gekko::instruction::Instruction {
+ output "$OUT_DIR/gekko_jit_instr_wii.rs"
+ }
+
+ handler branch {
+ bx
+ bcx
+ bclrx
+ bcctrx
+ }
+
+ handler alu {
+ ori
+ oris
+ addx
+ addi
+ addis
+ xori
+ xoris
+ andi_dot
+ andis_dot
+ subfx
+ negx
+ addcx
+ subfcx
+ addex
+ subfex
+ addzex
+ subfzex
+ addmex
+ subfmex
+ mullwx
+ divwux
+ divwx
+ mulhwux
+ mulhwx
+ subfic
+ addic
+ addic_dot
+ mulli
+ }
+
+ handler rotate {
+ rlwinmx
+ rlwimix
+ rlwnmx
+ }
+
+ handler msr {
+ mtmsr
+ mfmsr
+ rfi
+ }
+
+ handler spr {
+ mtspr
+ mfspr
+ }
+
+ handler segment {
+ mtsr
+ mfsr
+ }
+
+ handler store_load {
+ stw
+ stwu
+ sth
+ sthu
+ lwz
+ lwzu
+ lbz
+ lbzu
+ stb
+ stbu
+ lhz
+ lhzu
+ lha
+ lhau
+ lmw
+ stmw
+ lwzx
+ lbzx
+ lhzx
+ lhax
+ stwx
+ stbx
+ sthx
+ lwzux
+ lbzux
+ lhzux
+ lhaux
+ stwux
+ stbux
+ sthux
+ lwbrx
+ lhbrx
+ stwbrx
+ sthbrx
+ }
+
+ handler store_load_fp {
+ lfd
+ lfdu
+ stfd
+ stfdu
+ lfs
+ lfsu
+ stfs
+ stfsu
+ lfsx
+ lfsux
+ lfdx
+ lfdux
+ stfsx
+ stfsux
+ stfdx
+ stfdux
+ stfiwx
+ }
+
+ handler store_load_psq {
+ psq_l
+ psq_lu
+ psq_st
+ psq_stu
+ psq_lx
+ psq_lux
+ psq_stx
+ psq_stux
+ }
+
+ handler compare {
+ cmp
+ cmpi
+ cmpli
+ cmpl
+ }
+
+ handler nop {
+ isync
+ sync
+ eieio
+ dcbf
+ dcbi
+ dcbst
+ dcbt
+ dcbtst
+ dcba
+ dcbz
+ dcbz_l
+ icbi
+ tlbie
+ tlbia
+ tlbsync
+ tlbld
+ tlbli
+ }
+
+ handler logical {
+ andx
+ orx
+ xorx
+ nandx
+ norx
+ andcx
+ orcx
+ eqvx
+ slwx
+ srwx
+ srawx
+ srawix
+ cntlzwx
+ extshx
+ extsbx
+ }
+
+ handler cr_ops {
+ mtcrf
+ mfcr
+ crxor
+ cror
+ crand
+ creqv
+ crnor
+ crnand
+ crandc
+ crorc
+ mcrf
+ }
+
+ handler fp_ops {
+ mtfsfx
+ mffsx
+ mtfsb0x
+ mtfsb1x
+ mtfsfix
+ mcrfs
+ fmrx
+ fnegx
+ fabsx
+ fnabsx
+ frspx
+ fctiwx
+ fctiwzx
+ fcmpu
+ fcmpo
+ faddx
+ fsubx
+ fmulx
+ fdivx
+ fmaddx
+ fmsubx
+ fnmaddx
+ fnmsubx
+ faddsx
+ fsubsx
+ fmulsx
+ fdivsx
+ fmaddsx
+ fmsubsx
+ fnmaddsx
+ fnmsubsx
+ fsqrtsx
+ fresx
+ fsqrtx
+ fselx
+ frsqrtex
+ }
+
+ handler ps_ops {
+ ps_cmpu0
+ ps_cmpo0
+ ps_cmpu1
+ ps_cmpo1
+ ps_res
+ ps_rsqrte
+ ps_neg
+ ps_mr
+ ps_nabs
+ ps_abs
+ ps_merge00
+ ps_merge01
+ ps_merge10
+ ps_merge11
+ ps_sum0
+ ps_sum1
+ ps_muls0
+ ps_muls1
+ ps_madds0
+ ps_madds1
+ ps_div
+ ps_sub
+ ps_add
+ ps_sel
+ ps_mul
+ ps_msub
+ ps_madd
+ ps_nmsub
+ ps_nmadd
+ }
+ }
+}
diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml
index beda2ca..db20d7b 100644
--- a/crates/web/Cargo.toml
+++ b/crates/web/Cargo.toml
@@ -10,7 +10,7 @@ crate-type = ["cdylib"]
debug = ["dep:dbglib", "dep:egui-phosphor"]
[dependencies]
-gecko = { path = "../gecko" }
+gecko = { path = "../gecko", default-features = false }
image = { path = "../image", default-features = false }
backend-wgpu = { path = "../backend-wgpu" }
dbglib = { path = "../dbglib", optional = true }