A GameCube/Wii emulator and debugger written in Rust.
dsp improvements
Laylef84db6f2bd6c
diff --git a/crates/dsptestrunner/Cargo.toml b/crates/dsptestrunner/Cargo.tomlindex 1c6bb3e..a652a4a 100644--- a/crates/dsptestrunner/Cargo.toml+++ b/crates/dsptestrunner/Cargo.toml@@ -5,5 +5,5 @@ edition = "2024"[dependencies]image = { path = "../image" }-gecko = { path = "../gecko", features = ["hooks"] }+gecko = { path = "../gecko", features = ["hooks", "jit"] }disasm = { path = "../disasm" }\ No newline at end of filediff --git a/crates/dsptestrunner/src/bin/zayd.rs b/crates/dsptestrunner/src/bin/zayd.rsindex 013af38..d9eab79 100644--- a/crates/dsptestrunner/src/bin/zayd.rs+++ b/crates/dsptestrunner/src/bin/zayd.rs@@ -435,22 +435,37 @@ struct RunResult {failed: u32,}-fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunResult, String> {- let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?;- let cases = parse_test_file(&bytes)?;- let ignore_flags = is_prod_test(test_name);-- let mut passed = 0u32;- let mut failed = 0u32;+fn run_case(emu: &mut GameCube, case: &TestCase, jit: bool) -> (TestState, bool) {+ reset_dsp(&mut emu.dsp);+ apply_initial_state(&mut emu.dsp, &case.initial);+ load_instructions(&mut emu.dsp, &case.instructions);+ emu.dsp.registers.pc = TEST_PC;+ emu.dsp.rebuild_wait_table();++ let mut runaway = false;++ if jit {+ emu.execution_mode = gecko::system::ExecutionMode::Jit;+ if let Some(j) = emu.dsp.jit.as_mut() {+ j.flush();+ }- for (idx, case) in cases.iter().enumerate() {- reset_dsp(&mut emu.dsp);- apply_initial_state(&mut emu.dsp, &case.initial);- load_instructions(&mut emu.dsp, &case.instructions);- emu.dsp.registers.pc = TEST_PC;+ let mut steps = 0u64;+ loop {+ if emu.dsp.csr.halt() {+ break;+ }+ if steps >= MAX_STEPS {+ runaway = true;+ break;+ }+ emu.execute_dsp_batch();+ steps += 1024;+ }+ } else {+ emu.execution_mode = gecko::system::ExecutionMode::Interpreter;let mut steps = 0u64;- let mut runaway = false;while !emu.dsp.csr.halt() {if !emu.step_dsp_instruction() {break;@@ -461,40 +476,137 @@ fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunRbreak;}}+ }++ (read_actual_state(&mut emu.dsp), runaway)+}++const SEQ_CHUNK: usize = 8;- let actual = read_actual_state(&mut emu.dsp);- let diff = diff_state(&case.initial, &case.expected, &actual, ignore_flags);+fn run_sequences(emu: &mut GameCube, test_name: &str, cases: &[TestCase]) -> (u32, u32) {+ let mut passed = 0u32;+ let mut failed = 0u32;- if runaway || diff.any_failure() {+ let eligible: Vec<&TestCase> = cases+ .iter()+ .filter(|c| {+ !c.instructions+ .iter()+ .any(|&w| gecko::flipper::dsp::jit::block::is_terminator_word(w))+ })+ .collect();++ for chunk in eligible.chunks(SEQ_CHUNK) {+ if chunk.len() < 2 {+ continue;+ }++ let mut program = Vec::new();+ for c in chunk {+ program.extend_from_slice(&c.instructions);+ }+ if (TEST_PC as usize + program.len() + 1) * 2 > emu.dsp.iram.len() {+ continue;+ }++ let synth = TestCase {+ instructions: program,+ expected: TestState::default(),+ initial: chunk[0].initial,+ };++ let dram_snapshot = emu.dsp.dram.clone();+ let ifx_snapshot = emu.dsp.ifx.clone();+ let (interp_state, interp_runaway) = run_case(emu, &synth, false);++ emu.dsp.dram.copy_from_slice(&dram_snapshot[..]);+ emu.dsp.ifx.copy_from_slice(&ifx_snapshot[..]);+ let (jit_state, jit_runaway) = run_case(emu, &synth, true);++ let mismatches: Vec<usize> = (0..31)+ .filter(|&slot| interp_state.reg[slot] != jit_state.reg[slot])+ .collect();++ if interp_runaway || jit_runaway || !mismatches.is_empty() {failed += 1;- println!("===== DSP Test {test_name} Failed =====");- println!("Test case: {idx}");+ println!("===== DSP Seq {test_name} interp/jit mismatch =====");++ if interp_runaway || jit_runaway {+ println!("(runaway: interp={interp_runaway} jit={jit_runaway})");+ }- if runaway {+ for &slot in &mismatches {+ let reg = slot_to_dsp_reg(slot);println!(- "(runaway: >= {MAX_STEPS} steps without halt, pc={:04x})",- emu.dsp.registers.pc+ "reg {:2}: interp={:04x} jit={:04x}",+ reg, interp_state.reg[slot], jit_state.reg[slot]);}- println!("Instructions: {}", format_instructions_hex(&case.instructions));+ println!("Instructions: {}", format_instructions_hex(&synth.instructions));- for line in disassemble_instructions(&case.instructions) {+ for line in disassemble_instructions(&synth.instructions) {println!("{line}");}- println!("Initial state:");- pretty_print(&case.initial, &diff);- println!("Expected:");- pretty_print(&case.expected, &diff);- println!("Actual:");- pretty_print(&actual, &diff);println!();} else {passed += 1;}}+ (passed, failed)+}++fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunResult, String> {+ let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?;+ let cases = parse_test_file(&bytes)?;+ let ignore_flags = is_prod_test(test_name);++ let mut passed = 0u32;+ let mut failed = 0u32;++ for (idx, case) in cases.iter().enumerate() {+ for jit in [false, true] {+ let (actual, runaway) = run_case(emu, case, jit);+ let diff = diff_state(&case.initial, &case.expected, &actual, ignore_flags);+ let mode = if jit { "jit" } else { "interp" };++ if runaway || diff.any_failure() {+ failed += 1;+ println!("===== DSP Test {test_name} Failed [{mode}] =====");+ println!("Test case: {idx}");++ if runaway {+ println!(+ "(runaway: >= {MAX_STEPS} steps without halt, pc={:04x})",+ emu.dsp.registers.pc+ );+ }++ println!("Instructions: {}", format_instructions_hex(&case.instructions));++ for line in disassemble_instructions(&case.instructions) {+ println!("{line}");+ }++ println!("Initial state:");+ pretty_print(&case.initial, &diff);+ println!("Expected:");+ pretty_print(&case.expected, &diff);+ println!("Actual:");+ pretty_print(&actual, &diff);+ println!();+ } else {+ passed += 1;+ }+ }+ }++ let (seq_passed, seq_failed) = self::run_sequences(emu, test_name, &cases);+ passed += seq_passed;+ failed += seq_failed;+Ok(RunResult { passed, failed })}diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rsindex 56e3b5c..9cd3938 100644--- a/crates/gecko/src/flipper/dsp.rs+++ b/crates/gecko/src/flipper/dsp.rs@@ -24,7 +24,7 @@ use crate::mmio::Mmio;use crate::system::{ExecutionMode, System, SystemId};#[cfg(feature = "jit")]-pub const DSP_JIT_CHAIN_BUDGET: u32 = 16;+pub const DSP_JIT_CHAIN_BUDGET: u32 = 64;pub struct Dsp {pub registers: core::Registers,@@ -133,9 +133,10 @@ impl Dsp {}#[inline(always)]- pub fn mailbox_wait_state(&self) -> (bool, bool) {- let b = self.wait_table[self.registers.pc as usize];- (b & 1 != 0, b & 2 != 0)+ pub fn parked_in_mailbox_wait(&self) -> bool {+ let cpu_mail_quiet = !self.mailbox_to_dsp_hi.busy();+ let dsp_mail_full = self.mailbox_to_cpu_hi.busy();+ (cpu_mail_quiet && self.is_waiting_for_cpu_mail()) || (dsp_mail_full && self.is_waiting_for_dsp_mail())}pub fn rebuild_wait_table(&mut self) {@@ -358,7 +359,39 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}#[cfg(feature = "jit")]+ fn dsp_jit_step(&mut self, iram: &[u8], irom: &[u8]) -> u64 {+ let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;++ if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {+ self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);+ self.dsp.registers.call_stack.push(self.dsp.registers.pc);+ self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());+ self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);+ self.dsp.registers.pc = 0x000E;+ }++ let start_pc = self.dsp.registers.pc;+ self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;+ self.dsp.instr_count = 0;++ let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);+ self.dsp.registers.pc = next_pc;++ #[cfg(feature = "jit-stats")]+ {+ let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;+ self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);+ }++ (self.dsp.instr_count as u64).max(1)+ }++ #[cfg(feature = "jit")]+ #[cfg_attr(feature = "hotpath", hotpath::measure(label = "dsp_batch"))]fn execute_dsp_batch_jit(&mut self) {+ #[cfg(feature = "jit-stats")]+ let batch_start = std::time::Instant::now();+if self.dsp.csr.reset() || self.dsp.csr.halt() {self::refresh_interrupts(self);return;@@ -368,7 +401,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));}- let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;let iram_ptr = self.dsp.iram.as_ptr();let irom_ptr = self.dsp.irom.as_ptr();let iram_len = self.dsp.iram.len();@@ -378,12 +410,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let mut budget = crate::scheduler::DSP_BATCH_SIZE as u64;while budget > 0 {- let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();- let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();-- if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())- || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())- {+ if self.dsp.parked_in_mailbox_wait() {break;}@@ -391,26 +418,17 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {break;}- if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {- self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);- self.dsp.registers.call_stack.push(self.dsp.registers.pc);- self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());- self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);- self.dsp.registers.pc = 0x000E;- }-- let start_pc = self.dsp.registers.pc;- self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;- self.dsp.instr_count = 0;-- let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);- self.dsp.registers.pc = next_pc;-- let consumed = (self.dsp.instr_count as u64).max(1);+ let consumed = self.dsp_jit_step(iram, irom);budget = budget.saturating_sub(consumed);+ }- let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;- self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);+ #[cfg(feature = "jit-stats")]+ {+ use std::sync::atomic::Ordering;++ DSP_BATCH_CALLS.fetch_add(1, Ordering::Relaxed);+ DSP_BATCH_STEPS.fetch_add(crate::scheduler::DSP_BATCH_SIZE - budget, Ordering::Relaxed);+ DSP_BATCH_NANOS.fetch_add(batch_start.elapsed().as_nanos() as u64, Ordering::Relaxed);}self::refresh_interrupts(self);@@ -427,10 +445,20 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}#[cfg(feature = "jit")]+ #[cfg_attr(feature = "hotpath", hotpath::measure(label = "dsp_drain"))]fn drain_dsp_synchronous_jit(&mut self, max_steps: u32) {+ #[cfg(feature = "jit-stats")]+ let drain_start = std::time::Instant::now();+let already_busy = self.dsp.mailbox_to_cpu_hi.busy();if self.dsp.csr.reset() || self.dsp.csr.halt() {+ #[cfg(feature = "jit-stats")]+ {+ DSP_DRAIN_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ DSP_DRAIN_EXIT_HALT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ }+self::refresh_interrupts(self);return;}@@ -439,7 +467,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));}- let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;let iram_ptr = self.dsp.iram.as_ptr();let irom_ptr = self.dsp.irom.as_ptr();let iram_len = self.dsp.iram.len();@@ -448,6 +475,10 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let irom = unsafe { ::core::slice::from_raw_parts(irom_ptr, irom_len) };let mut budget = max_steps as u64;++ #[cfg(feature = "jit-stats")]+ let mut blocks: u64 = 0;+while budget > 0 {if self.dsp.csr.reset() || self.dsp.csr.halt() {break;@@ -457,34 +488,38 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {break;}- let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();- let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();- if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())- || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())- {+ if self.dsp.parked_in_mailbox_wait() {break;}- if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {- self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);- self.dsp.registers.call_stack.push(self.dsp.registers.pc);- self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());- self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);- self.dsp.registers.pc = 0x000E;- }-- let start_pc = self.dsp.registers.pc;- self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;- self.dsp.instr_count = 0;-- let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);- self.dsp.registers.pc = next_pc;-- let consumed = (self.dsp.instr_count as u64).max(1);+ let consumed = self.dsp_jit_step(iram, irom);budget = budget.saturating_sub(consumed);- let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;- self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);+ #[cfg(feature = "jit-stats")]+ {+ blocks += 1;+ }+ }++ #[cfg(feature = "jit-stats")]+ {+ use std::sync::atomic::Ordering;++ DSP_DRAIN_CALLS.fetch_add(1, Ordering::Relaxed);+ DSP_DRAIN_STEPS.fetch_add(max_steps as u64 - budget, Ordering::Relaxed);+ DSP_DRAIN_BLOCKS.fetch_add(blocks, Ordering::Relaxed);+ DSP_DRAIN_NANOS.fetch_add(drain_start.elapsed().as_nanos() as u64, Ordering::Relaxed);++ let exit = if self.dsp.csr.reset() || self.dsp.csr.halt() {+ &DSP_DRAIN_EXIT_HALT+ } else if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {+ &DSP_DRAIN_EXIT_ANSWERED+ } else if budget > 0 {+ &DSP_DRAIN_EXIT_WAIT+ } else {+ &DSP_DRAIN_EXIT_BUDGET+ };+ exit.fetch_add(1, Ordering::Relaxed);}self::refresh_interrupts(self);@@ -501,6 +536,10 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {break;}++ if self.dsp.parked_in_mailbox_wait() {+ break;+ }}self::refresh_interrupts(self);@@ -578,6 +617,28 @@ pub fn refresh_interrupts<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {pub static DSP_SUSPEND_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);#[cfg(feature = "jit-stats")]pub static DSP_WAKE_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_STEPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_BLOCKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_EXIT_HALT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_EXIT_ANSWERED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_EXIT_WAIT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_DRAIN_EXIT_BUDGET: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_BATCH_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_BATCH_STEPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+#[cfg(feature = "jit-stats")]+pub static DSP_BATCH_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);#[inline]pub fn wake_dsp_scheduler<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {@@ -793,7 +854,10 @@ pub trait DspJitHandle {#[cfg(feature = "jit")]impl<const SYSTEM: SystemId> DspJitHandle for jit::JitEngine<SYSTEM> {fn run_block(&mut self, ctx_ptr: *mut ::core::ffi::c_void, iram: &[u8], irom: &[u8], start_pc: u16) -> u16 {- let entry = self.lookup_or_compile(iram, irom, start_pc);+ let entry = match self.lookup_block_fast(start_pc) {+ Some(entry) => entry,+ None => self.lookup_or_compile(iram, irom, start_pc),+ };Self::run_block(self, ctx_ptr, entry)}diff --git a/crates/gecko/src/flipper/dsp/jit.rs b/crates/gecko/src/flipper/dsp/jit.rsindex 02e02d7..8c4335e 100644--- a/crates/gecko/src/flipper/dsp/jit.rs+++ b/crates/gecko/src/flipper/dsp/jit.rs@@ -45,6 +45,8 @@ pub struct JitEngine<const SYSTEM: SystemId> {trampoline_fn: TrampolineFn,block_seq: u64,block_lookup_table_addr: usize,+ loop_end_table: Box<[u8; 0x10000]>,+ loop_end_table_built: bool,chain_depth_total: u64,dispatcher_entries_total: u64,chain_depth_histogram: [u64; CHAIN_DEPTH_HISTOGRAM_LEN],@@ -81,12 +83,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());struct Syms {- cache_ext_ac: (&'static str, *const u8),loop_tail: (&'static str, *const u8),- update_flags_logic: (&'static str, *const u8),- update_flags_add: (&'static str, *const u8),- update_flags_sub: (&'static str, *const u8),- update_flags_ac: (&'static str, *const u8),read_dmem: (&'static str, *const u8),write_dmem: (&'static str, *const u8),inc_ar: (&'static str, *const u8),@@ -95,7 +92,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {decrease_ar_ix: (&'static str, *const u8),dynamic_shift: (&'static str, *const u8),read_imem: (&'static str, *const u8),- write_ac_mid_sxm: (&'static str, *const u8),call_stack_push: (&'static str, *const u8),call_stack_pop: (&'static str, *const u8),data_stack_pop: (&'static str, *const u8),@@ -105,18 +101,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}let syms: Syms = match SYSTEM {GC => Syms {- cache_ext_ac: (- "gecko_dsp_jit_cache_ext_ac_gc",- runtime::dsp_cache_ext_ac_gc as *const u8,- ),loop_tail: ("gecko_dsp_jit_loop_tail_gc", runtime::dsp_loop_tail_gc as *const u8),- update_flags_logic: (- "gecko_dsp_jit_uflog_gc",- runtime::dsp_update_flags_logic_gc as *const u8,- ),- update_flags_add: ("gecko_dsp_jit_ufadd_gc", runtime::dsp_update_flags_add_gc as *const u8),- update_flags_sub: ("gecko_dsp_jit_ufsub_gc", runtime::dsp_update_flags_sub_gc as *const u8),- update_flags_ac: ("gecko_dsp_jit_ufac_gc", runtime::dsp_update_flags_ac_gc as *const u8),read_dmem: ("gecko_dsp_jit_rdmem_gc", runtime::dsp_read_dmem_gc as *const u8),write_dmem: ("gecko_dsp_jit_wdmem_gc", runtime::dsp_write_dmem_gc as *const u8),inc_ar: ("gecko_dsp_jit_incar_gc", runtime::dsp_increment_ar_gc as *const u8),@@ -125,7 +110,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {decrease_ar_ix: ("gecko_dsp_jit_decarix_gc", runtime::dsp_decrease_ar_ix_gc as *const u8),dynamic_shift: ("gecko_dsp_jit_dynshift_gc", runtime::dsp_dynamic_shift_gc as *const u8),read_imem: ("gecko_dsp_jit_rimem_gc", runtime::dsp_read_imem_gc as *const u8),- write_ac_mid_sxm: ("gecko_dsp_jit_wamsxm_gc", runtime::dsp_write_ac_mid_sxm_gc as *const u8),call_stack_push: ("gecko_dsp_jit_cspush_gc", runtime::dsp_call_stack_push_gc as *const u8),call_stack_pop: ("gecko_dsp_jit_cspop_gc", runtime::dsp_call_stack_pop_gc as *const u8),data_stack_pop: ("gecko_dsp_jit_dspop_gc", runtime::dsp_data_stack_pop_gc as *const u8),@@ -134,24 +118,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {loop_setup: ("gecko_dsp_jit_loopsetup_gc", runtime::dsp_loop_setup_gc as *const u8),},WII => Syms {- cache_ext_ac: (- "gecko_dsp_jit_cache_ext_ac_wii",- runtime::dsp_cache_ext_ac_wii as *const u8,- ),loop_tail: ("gecko_dsp_jit_loop_tail_wii", runtime::dsp_loop_tail_wii as *const u8),- update_flags_logic: (- "gecko_dsp_jit_uflog_wii",- runtime::dsp_update_flags_logic_wii as *const u8,- ),- update_flags_add: (- "gecko_dsp_jit_ufadd_wii",- runtime::dsp_update_flags_add_wii as *const u8,- ),- update_flags_sub: (- "gecko_dsp_jit_ufsub_wii",- runtime::dsp_update_flags_sub_wii as *const u8,- ),- update_flags_ac: ("gecko_dsp_jit_ufac_wii", runtime::dsp_update_flags_ac_wii as *const u8),read_dmem: ("gecko_dsp_jit_rdmem_wii", runtime::dsp_read_dmem_wii as *const u8),write_dmem: ("gecko_dsp_jit_wdmem_wii", runtime::dsp_write_dmem_wii as *const u8),inc_ar: ("gecko_dsp_jit_incar_wii", runtime::dsp_increment_ar_wii as *const u8),@@ -166,10 +133,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {runtime::dsp_dynamic_shift_wii as *const u8,),read_imem: ("gecko_dsp_jit_rimem_wii", runtime::dsp_read_imem_wii as *const u8),- write_ac_mid_sxm: (- "gecko_dsp_jit_wamsxm_wii",- runtime::dsp_write_ac_mid_sxm_wii as *const u8,- ),call_stack_push: ("gecko_dsp_jit_cspush_wii",runtime::dsp_call_stack_push_wii as *const u8,@@ -182,12 +145,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {},_ => unreachable!(),};- jit_builder.symbol(syms.cache_ext_ac.0, syms.cache_ext_ac.1);jit_builder.symbol(syms.loop_tail.0, syms.loop_tail.1);- jit_builder.symbol(syms.update_flags_logic.0, syms.update_flags_logic.1);- jit_builder.symbol(syms.update_flags_add.0, syms.update_flags_add.1);- jit_builder.symbol(syms.update_flags_sub.0, syms.update_flags_sub.1);- jit_builder.symbol(syms.update_flags_ac.0, syms.update_flags_ac.1);jit_builder.symbol(syms.read_dmem.0, syms.read_dmem.1);jit_builder.symbol(syms.write_dmem.0, syms.write_dmem.1);jit_builder.symbol(syms.inc_ar.0, syms.inc_ar.1);@@ -196,7 +154,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {jit_builder.symbol(syms.decrease_ar_ix.0, syms.decrease_ar_ix.1);jit_builder.symbol(syms.dynamic_shift.0, syms.dynamic_shift.1);jit_builder.symbol(syms.read_imem.0, syms.read_imem.1);- jit_builder.symbol(syms.write_ac_mid_sxm.0, syms.write_ac_mid_sxm.1);jit_builder.symbol(syms.call_stack_push.0, syms.call_stack_push.1);jit_builder.symbol(syms.call_stack_pop.0, syms.call_stack_pop.1);jit_builder.symbol(syms.data_stack_pop.0, syms.data_stack_pop.1);@@ -211,33 +168,15 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let block_sig = translator::block_signature(pointer_type);let void_thunk_sig = translator::void_thunk_signature(pointer_type, host_cc);- let flags_logic_sig = translator::flags_logic_signature(pointer_type, host_cc);- let flags_arith_sig = translator::flags_arith_signature(pointer_type, host_cc);- let flags_ac_sig = translator::flags_ac_signature(pointer_type, host_cc);let dmem_read_sig = translator::dmem_read_signature(pointer_type, host_cc);let dmem_write_sig = translator::dmem_write_signature(pointer_type, host_cc);let ar_unary_sig = translator::ar_unary_signature(pointer_type, host_cc);let ar_binary_sig = translator::ar_binary_signature(pointer_type, host_cc);let dynamic_shift_sig = translator::dynamic_shift_signature(pointer_type, host_cc);- let cache_ext_ac_id = module- .declare_function(syms.cache_ext_ac.0, Linkage::Import, &void_thunk_sig)- .expect("declare cache_ext_ac thunk");let loop_tail_id = module.declare_function(syms.loop_tail.0, Linkage::Import, &void_thunk_sig).expect("declare loop_tail thunk");- let update_flags_logic_id = module- .declare_function(syms.update_flags_logic.0, Linkage::Import, &flags_logic_sig)- .expect("declare update_flags_logic thunk");- let update_flags_add_id = module- .declare_function(syms.update_flags_add.0, Linkage::Import, &flags_arith_sig)- .expect("declare update_flags_add thunk");- let update_flags_sub_id = module- .declare_function(syms.update_flags_sub.0, Linkage::Import, &flags_arith_sig)- .expect("declare update_flags_sub thunk");- let update_flags_ac_id = module- .declare_function(syms.update_flags_ac.0, Linkage::Import, &flags_ac_sig)- .expect("declare update_flags_ac thunk");let read_dmem_id = module.declare_function(syms.read_dmem.0, Linkage::Import, &dmem_read_sig).expect("declare read_dmem thunk");@@ -262,9 +201,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let read_imem_id = module.declare_function(syms.read_imem.0, Linkage::Import, &dmem_read_sig).expect("declare read_imem thunk");- let write_ac_mid_sxm_id = module- .declare_function(syms.write_ac_mid_sxm.0, Linkage::Import, &dmem_write_sig)- .expect("declare write_ac_mid_sxm thunk");let call_stack_push_sig = {let mut sig = cranelift_codegen::ir::Signature::new(host_cc);@@ -301,12 +237,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {.expect("declare loop_setup thunk");let extern_funcs = translator::ExternFuncs {- cache_ext_ac: cache_ext_ac_id,loop_tail: loop_tail_id,- update_flags_logic: update_flags_logic_id,- update_flags_add: update_flags_add_id,- update_flags_sub: update_flags_sub_id,- update_flags_ac: update_flags_ac_id,read_dmem: read_dmem_id,write_dmem: write_dmem_id,inc_ar: inc_ar_id,@@ -315,7 +246,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {decrease_ar_ix: decrease_ar_ix_id,dynamic_shift: dynamic_shift_id,read_imem: read_imem_id,- write_ac_mid_sxm: write_ac_mid_sxm_id,call_stack_push: call_stack_push_id,call_stack_pop: call_stack_pop_id,data_stack_pop: data_stack_pop_id,@@ -348,6 +278,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {trampoline_fn,block_seq: 0,block_lookup_table_addr,+ loop_end_table: unsafe { Box::<[u8; 0x10000]>::new_zeroed().assume_init() },+ loop_end_table_built: false,chain_depth_total: 0,dispatcher_entries_total: 0,chain_depth_histogram: [0; CHAIN_DEPTH_HISTOGRAM_LEN],@@ -377,9 +309,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.chain_depth_histogram[bucket] = self.chain_depth_histogram[bucket].wrapping_add(1);}+ fn lookup_table_index(pc: u16) -> usize {+ let pc_u32 = pc as u32;+ ((pc_u32 & 0xFFF) | ((pc_u32 >> 3) & 0x1000)) as usize+ }+fn register_in_lookup_table(&self, pc: u16, entry: usize) {let pc_u32 = pc as u32;- let idx = ((pc_u32 & 0xFFF) | ((pc_u32 >> 3) & 0x1000)) as usize;+ let idx = Self::lookup_table_index(pc);unsafe {let table = self.block_lookup_table_addr as *mut DspBlockLookupSlot;(*table.add(idx)).pc = pc_u32;@@ -397,6 +334,51 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}}+ fn rebuild_loop_end_table(&mut self, iram: &[u8], irom: &[u8]) {+ self.loop_end_table.fill(0);++ for (base, mem) in [(0x0000u32, iram), (0x8000u32, irom)] {+ let words = mem.len() / 2;+ for i in 0..words {+ let w = u16::from_be_bytes([mem[i * 2], mem[i * 2 + 1]]);++ if (w & 0xFFE0) == 0x0040 || (w & 0xFF00) == 0x1000 {+ let end = (base + i as u32 + 1) as u16;+ self.loop_end_table[end as usize] = 1;+ }++ if ((w & 0xFFE0) == 0x0060 || (w & 0xFF00) == 0x1100) && i + 1 < words {+ let end = u16::from_be_bytes([mem[i * 2 + 2], mem[i * 2 + 3]]);+ self.loop_end_table[end as usize] = 1;+ }+ }+ }++ self.loop_end_table_built = true;+ }++ pub fn lookup_block_fast(&mut self, pc: u16) -> Option<BlockEntry> {+ let pc_u32 = pc as u32;+ let idx = Self::lookup_table_index(pc);++ let entry = unsafe {+ let table = self.block_lookup_table_addr as *const DspBlockLookupSlot;+ let slot = &*table.add(idx);+ if slot.pc == pc_u32 && slot.entry != 0 {+ Some(slot.entry)+ } else {+ None+ }+ };++ #[cfg(feature = "jit-stats")]+ if entry.is_some() {+ *self.hits.entry(pc).or_insert(0) += 1;+ }++ entry+ }+pub fn lookup_or_compile(&mut self, iram: &[u8], irom: &[u8], start_pc: u16) -> BlockEntry {if let Some(&entry) = self.cache.get(&start_pc) {#[cfg(feature = "jit-stats")]@@ -405,6 +387,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}return entry;}++ if !self.loop_end_table_built {+ self.rebuild_loop_end_table(iram, irom);+ }let spec = block::discover(iram, irom, start_pc);let entry = self.compile(&spec);self.cache.insert(start_pc, entry);@@ -489,6 +475,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {#[cfg(feature = "jit-stats")]pub fn dump_block_clif(&mut self, start_pc: u16, iram: &[u8], irom: &[u8]) {+ if !self.loop_end_table_built {+ self.rebuild_loop_end_table(iram, irom);+ }+let spec = block::discover(iram, irom, start_pc);self.ctx.clear();@@ -501,6 +491,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {&spec,self.block_lookup_table_addr as i64,None,+ &self.loop_end_table,);tracing::info!("CLIF for DSP block pc={:04X} (len={}, term={:?})",@@ -531,6 +522,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {spec,self.block_lookup_table_addr as i64,entry_counter_addr,+ &self.loop_end_table,);self.module@@ -548,6 +540,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {pub fn flush(&mut self) {self.cache.clear();self.block_func_ids.clear();+ self.loop_end_table_built = false;self.clear_lookup_table();}diff --git a/crates/gecko/src/flipper/dsp/jit/block.rs b/crates/gecko/src/flipper/dsp/jit/block.rsindex 715adb2..0637865 100644--- a/crates/gecko/src/flipper/dsp/jit/block.rs+++ b/crates/gecko/src/flipper/dsp/jit/block.rs@@ -35,16 +35,40 @@ fn read_imem_word(iram: &[u8], irom: &[u8], addr: u16) -> Option<u16> {}}+pub fn is_terminator_word(primary: u16) -> bool {+ self::classify(primary).is_some()+}+fn classify(primary: u16) -> Option<TermKind> {- let nibble = (primary >> 12) & 0xF;- match nibble {- 0x0 | 0x1 => {- if primary == 0x0021 {- Some(TermKind::Halt)- } else {- Some(TermKind::Jump)- }- }+ match primary {+ 0x0021 => Some(TermKind::Halt),++ 0x0040..=0x007F | 0x1000..=0x11FF => Some(TermKind::LoopSetup),++ 0x0270..=0x027F => Some(TermKind::IfCc),+ 0x0290..=0x029F => Some(TermKind::Jump),+ 0x02B0..=0x02BF => Some(TermKind::Call),+ 0x02D0..=0x02DF | 0x02F0..=0x02FF => Some(TermKind::Ret),+ _ if (primary & 0xFF10) == 0x1700 => Some(TermKind::Jump),+ _ if (primary & 0xFF10) == 0x1710 => Some(TermKind::Call),++ 0x0000 | 0x0004..=0x001F => None,+ 0x0080..=0x009F | 0x00C0..=0x00FF => None,+ 0x02CA | 0x02CB => None,+ 0x0400..=0x0FFF => None,+ 0x1200..=0x1207 | 0x1300..=0x1307 => None,+ 0x1400..=0x16FF | 0x1800..=0x1FFF => None,+ _ if (primary & 0xFEFF) == 0x0200 => None,+ _ if (primary & 0xFEFF) == 0x0220 => None,+ _ if (primary & 0xFEFF) == 0x0240 => None,+ _ if (primary & 0xFEFF) == 0x0260 => None,+ _ if (primary & 0xFEFF) == 0x0280 => None,+ _ if (primary & 0xFEFF) == 0x02A0 => None,+ _ if (primary & 0xFEFF) == 0x02C0 => None,+ _ if (primary & 0xFEF0) == 0x0210 => None,++ _ if (primary >> 12) <= 0x1 => Some(TermKind::Jump),+_ => None,}}diff --git a/crates/gecko/src/flipper/dsp/jit/runtime.rs b/crates/gecko/src/flipper/dsp/jit/runtime.rsindex afb2f8d..478b754 100644--- a/crates/gecko/src/flipper/dsp/jit/runtime.rs+++ b/crates/gecko/src/flipper/dsp/jit/runtime.rs@@ -44,62 +44,6 @@ pub extern "C" fn dsp_loop_tail_wii(sys: *mut core::ffi::c_void) {loop_tail::<WII>(sys.cast());}-#[inline(always)]-fn update_flags_logic<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, result16: u32, ac_full: i64) {- let sys = unsafe { &mut *sys };- sys.dsp.registers.update_flags_logic(result16 as u16, ac_full);-}--pub extern "C" fn dsp_update_flags_logic_gc(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {- update_flags_logic::<GC>(sys.cast(), result, ac_full);-}--pub extern "C" fn dsp_update_flags_logic_wii(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {- update_flags_logic::<WII>(sys.cast(), result, ac_full);-}--#[inline(always)]-fn update_flags_add<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {- let sys = unsafe { &mut *sys };- sys.dsp.registers.update_flags_add(a, b, result);-}--pub extern "C" fn dsp_update_flags_add_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {- update_flags_add::<GC>(sys.cast(), a, b, result);-}--pub extern "C" fn dsp_update_flags_add_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {- update_flags_add::<WII>(sys.cast(), a, b, result);-}--#[inline(always)]-fn update_flags_sub<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {- let sys = unsafe { &mut *sys };- sys.dsp.registers.update_flags_sub(a, b, result);-}--pub extern "C" fn dsp_update_flags_sub_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {- update_flags_sub::<GC>(sys.cast(), a, b, result);-}--pub extern "C" fn dsp_update_flags_sub_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {- update_flags_sub::<WII>(sys.cast(), a, b, result);-}--#[inline(always)]-fn update_flags_ac<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ac_full: i64) {- let sys = unsafe { &mut *sys };- sys.dsp.registers.update_flags_ac(ac_full);-}--pub extern "C" fn dsp_update_flags_ac_gc(sys: *mut core::ffi::c_void, ac_full: i64) {- update_flags_ac::<GC>(sys.cast(), ac_full);-}--pub extern "C" fn dsp_update_flags_ac_wii(sys: *mut core::ffi::c_void, ac_full: i64) {- update_flags_ac::<WII>(sys.cast(), ac_full);-}-pub extern "C" fn dsp_read_dmem_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {let sys = unsafe { &mut *(sys as *mut System<GC>) };dsp::read_dmem(sys, addr as u16) as u32@@ -215,20 +159,6 @@ pub extern "C" fn dsp_read_imem_wii(sys: *mut core::ffi::c_void, addr: u32) -> uread_imem::<WII>(sys.cast(), addr)}-#[inline(always)]-fn write_ac_mid_sxm<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32, value: u32) {- let sys = unsafe { &mut *sys };- sys.dsp.registers.write::<true>(30 + idx as u8, value as u16);-}--pub extern "C" fn dsp_write_ac_mid_sxm_gc(sys: *mut core::ffi::c_void, idx: u32, value: u32) {- write_ac_mid_sxm::<GC>(sys.cast(), idx, value);-}--pub extern "C" fn dsp_write_ac_mid_sxm_wii(sys: *mut core::ffi::c_void, idx: u32, value: u32) {- write_ac_mid_sxm::<WII>(sys.cast(), idx, value);-}-#[inline(always)]fn call_stack_push<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, value: u32) {let sys = unsafe { &mut *sys };diff --git a/crates/gecko/src/flipper/dsp/jit/translate.rs b/crates/gecko/src/flipper/dsp/jit/translate.rsindex ce7331b..4a11363 100644--- a/crates/gecko/src/flipper/dsp/jit/translate.rs+++ b/crates/gecko/src/flipper/dsp/jit/translate.rs@@ -446,33 +446,132 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {self.store_u16(val, m_off);}+ fn store_status_bits(&mut self, bits: Value, clear_mask: i64) {+ let off = abi::dsp_status_offset() as i32;+ let sr = self.load_u16(off);+ let cleared = self.builder.ins().band_imm(sr, clear_mask);+ let new_sr = self.builder.ins().bor(cleared, bits);+ self.store_u16(new_sr, off);+ }++ fn flags40_as32_pos(&mut self, r40: Value) -> Value {+ use cranelift_codegen::ir::condcodes::IntCC;++ let upper9_raw = self.builder.ins().ushr_imm(r40, 31);+ let upper9 = self.builder.ins().band_imm(upper9_raw, 0x1FF);+ let ne0 = self.builder.ins().icmp_imm(IntCC::NotEqual, upper9, 0);+ let ne1ff = self.builder.ins().icmp_imm(IntCC::NotEqual, upper9, 0x1FF);+ let as32 = self.builder.ins().band(ne0, ne1ff);+ let as32_16 = self.builder.ins().uextend(types::I16, as32);+ self.builder.ins().ishl_imm(as32_16, 4)+ }++ fn flags40_szat(&mut self, r40: Value) -> Value {+ use cranelift_codegen::ir::condcodes::IntCC;++ let sign64 = self.builder.ins().ushr_imm(r40, 39);+ let sign16 = self.builder.ins().ireduce(types::I16, sign64);+ let s_pos = self.builder.ins().ishl_imm(sign16, 3);++ let z = self.builder.ins().icmp_imm(IntCC::Equal, r40, 0);+ let z16 = self.builder.ins().uextend(types::I16, z);+ let z_pos = self.builder.ins().ishl_imm(z16, 2);++ let top2 = self.builder.ins().band_imm(r40, 0xC000_0000_i64);+ let tb0 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0);+ let tb3 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0xC000_0000_i64);+ let tb = self.builder.ins().bor(tb0, tb3);+ let tb16 = self.builder.ins().uextend(types::I16, tb);+ let tb_pos = self.builder.ins().ishl_imm(tb16, 5);++ let as32_pos = self.flags40_as32_pos(r40);++ let sz = self.builder.ins().bor(s_pos, z_pos);+ let at = self.builder.ins().bor(as32_pos, tb_pos);+ self.builder.ins().bor(sz, at)+ }++ fn flags40_carry_overflow(&mut self, a: Value, b: Value, a40: Value, r40: Value, sub: bool) -> Value {+ use cranelift_codegen::ir::condcodes::IntCC;++ let carry = if sub {+ self.builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, a40, r40)+ } else {+ self.builder.ins().icmp(IntCC::UnsignedLessThan, r40, a40)+ };+ let c_pos = self.builder.ins().uextend(types::I16, carry);++ let a_shift = self.builder.ins().ushr_imm(a, 39);+ let a_sign = self.builder.ins().band_imm(a_shift, 1);+ let b_shift = self.builder.ins().ushr_imm(b, 39);+ let b_sign = self.builder.ins().band_imm(b_shift, 1);+ let r_sign = self.builder.ins().ushr_imm(r40, 39);++ let ab = if sub {+ self.builder.ins().icmp(IntCC::NotEqual, a_sign, b_sign)+ } else {+ self.builder.ins().icmp(IntCC::Equal, a_sign, b_sign)+ };+ let ra = self.builder.ins().icmp(IntCC::NotEqual, r_sign, a_sign);+ let overflow = self.builder.ins().band(ab, ra);+ let o16 = self.builder.ins().uextend(types::I16, overflow);+ let o_pos = self.builder.ins().ishl_imm(o16, 1);+ let os_pos = self.builder.ins().ishl_imm(o16, 7);++ let co = self.builder.ins().bor(c_pos, o_pos);+ self.builder.ins().bor(co, os_pos)+ }+pub fn emit_update_flags_logic(&mut self, result16: Value, ac_full: Value) {- let result32 = self.builder.ins().uextend(types::I32, result16);- let f = self- .module- .declare_func_in_func(self.extern_funcs.update_flags_logic, self.builder.func);- self.builder.ins().call(f, &[self.sys_ptr, result32, ac_full]);+ use cranelift_codegen::ir::condcodes::IntCC;++ let sign = self.builder.ins().ushr_imm(result16, 15);+ let s_pos = self.builder.ins().ishl_imm(sign, 3);++ let z = self.builder.ins().icmp_imm(IntCC::Equal, result16, 0);+ let z16 = self.builder.ins().uextend(types::I16, z);+ let z_pos = self.builder.ins().ishl_imm(z16, 2);++ let top2 = self.builder.ins().ushr_imm(result16, 14);+ let tb0 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0);+ let tb3 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 3);+ let tb = self.builder.ins().bor(tb0, tb3);+ let tb16 = self.builder.ins().uextend(types::I16, tb);+ let tb_pos = self.builder.ins().ishl_imm(tb16, 5);++ let r40 = self.builder.ins().band_imm(ac_full, 0xFF_FFFF_FFFF_i64);+ let as32_pos = self.flags40_as32_pos(r40);++ let sz = self.builder.ins().bor(s_pos, z_pos);+ let at = self.builder.ins().bor(as32_pos, tb_pos);+ let bits = self.builder.ins().bor(sz, at);+ self.store_status_bits(bits, !0x003C_i64);}pub fn emit_update_flags_add(&mut self, a: Value, b: Value, result: Value) {- let f = self- .module- .declare_func_in_func(self.extern_funcs.update_flags_add, self.builder.func);- self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);+ let r40 = self.builder.ins().band_imm(result, 0xFF_FFFF_FFFF_i64);+ let a40 = self.builder.ins().band_imm(a, 0xFF_FFFF_FFFF_i64);++ let szat = self.flags40_szat(r40);+ let co = self.flags40_carry_overflow(a, b, a40, r40, false);+ let bits = self.builder.ins().bor(szat, co);+ self.store_status_bits(bits, !0x003F_i64);}pub fn emit_update_flags_sub(&mut self, a: Value, b: Value, result: Value) {- let f = self- .module- .declare_func_in_func(self.extern_funcs.update_flags_sub, self.builder.func);- self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);+ let r40 = self.builder.ins().band_imm(result, 0xFF_FFFF_FFFF_i64);+ let a40 = self.builder.ins().band_imm(a, 0xFF_FFFF_FFFF_i64);++ let szat = self.flags40_szat(r40);+ let co = self.flags40_carry_overflow(a, b, a40, r40, true);+ let bits = self.builder.ins().bor(szat, co);+ self.store_status_bits(bits, !0x003F_i64);}pub fn emit_update_flags_ac(&mut self, ac_full: Value) {- let f = self- .module- .declare_func_in_func(self.extern_funcs.update_flags_ac, self.builder.func);- self.builder.ins().call(f, &[self.sys_ptr, ac_full]);+ let r40 = self.builder.ins().band_imm(ac_full, 0xFF_FFFF_FFFF_i64);+ let bits = self.flags40_szat(r40);+ self.store_status_bits(bits, !0x003C_i64);}pub fn emit_read_dmem(&mut self, addr_u32: Value) -> Value {diff --git a/crates/gecko/src/flipper/dsp/jit/translator.rs b/crates/gecko/src/flipper/dsp/jit/translator.rsindex 5611b24..3972116 100644--- a/crates/gecko/src/flipper/dsp/jit/translator.rs+++ b/crates/gecko/src/flipper/dsp/jit/translator.rs@@ -5,15 +5,10 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};use cranelift_jit::JITModule;use cranelift_module::{FuncId, Module};-use super::block::BlockSpec;+use super::block::{BlockSpec, TermKind};#[derive(Clone, Copy)]pub struct ExternFuncs {- pub cache_ext_ac: FuncId,pub loop_tail: FuncId,- pub update_flags_logic: FuncId,- pub update_flags_add: FuncId,- pub update_flags_sub: FuncId,- pub update_flags_ac: FuncId,pub read_dmem: FuncId,pub write_dmem: FuncId,pub inc_ar: FuncId,@@ -22,7 +17,6 @@ pub struct ExternFuncs {pub decrease_ar_ix: FuncId,pub dynamic_shift: FuncId,pub read_imem: FuncId,- pub write_ac_mid_sxm: FuncId,pub call_stack_push: FuncId,pub call_stack_pop: FuncId,pub data_stack_pop: FuncId,@@ -39,6 +33,7 @@ pub fn translate(spec: &BlockSpec,block_lookup_table_addr: i64,entry_counter_addr: Option<usize>,+ loop_end_table: &[u8; 0x10000],) {let pointer_type = module.target_config().pointer_type();@@ -65,7 +60,9 @@ pub fn translate(builder.ins().store(MemFlagsData::trusted(), next, slot_v, 0);}- for entry in &spec.instrs {+ let block_sig_ref = builder.import_signature(block_signature(pointer_type));++ for (idx, entry) in spec.instrs.iter().enumerate() {let natural_nia = entry.pc.wrapping_add(entry.size as u16);let nia_v = builder.ins().iconst(types::I16, natural_nia as i64);builder.ins().store(MemFlagsData::trusted(), nia_v, ctx_ptr, nia_offset);@@ -104,6 +101,16 @@ pub fn translate(jit_lut::dispatch_gc_dsp_ext(&mut tctx2, crate::flipper::dsp::instruction::GcDspExt(ext_byte as u8));}+ let is_last = idx + 1 == spec.instrs.len();+ let dynamic_nia = is_last && spec.terminator != TermKind::LengthLimit;+ let marked_loop_end = loop_end_table[entry.pc as usize] != 0;++ if !dynamic_nia && !marked_loop_end {+ let pc_v = builder.ins().iconst(types::I16, natural_nia as i64);+ builder.ins().store(MemFlagsData::trusted(), pc_v, ctx_ptr, pc_offset);+ continue;+ }+let loop_ptr = builder.ins().load(types::I8, MemFlagsData::trusted(), ctx_ptr, loop_addr_ptr_offset);@@ -141,14 +148,18 @@ pub fn translate(builder.switch_to_block(exit_block);builder.seal_block(exit_block);- let pc_u32 = builder.ins().uextend(types::I32, pc_after);- builder.ins().return_(&[pc_u32]);+ emit_block_tail_chain(+ &mut builder,+ ctx_ptr,+ block_sig_ref,+ block_lookup_table_addr,+ idx as i64 + 1,+ );builder.switch_to_block(continue_block);builder.seal_block(continue_block);}- let block_sig_ref = builder.import_signature(block_signature(pointer_type));emit_block_tail_chain(&mut builder,ctx_ptr,@@ -324,30 +335,6 @@ pub fn void_thunk_signature(pointer_type: cranelift_codegen::ir::Type, host_cc:sig}-pub fn flags_logic_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {- let mut sig = Signature::new(host_cc);- sig.params.push(AbiParam::new(pointer_type));- sig.params.push(AbiParam::new(types::I32));- sig.params.push(AbiParam::new(types::I64));- sig-}--pub fn flags_arith_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {- let mut sig = Signature::new(host_cc);- sig.params.push(AbiParam::new(pointer_type));- sig.params.push(AbiParam::new(types::I64));- sig.params.push(AbiParam::new(types::I64));- sig.params.push(AbiParam::new(types::I64));- sig-}--pub fn flags_ac_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {- let mut sig = Signature::new(host_cc);- sig.params.push(AbiParam::new(pointer_type));- sig.params.push(AbiParam::new(types::I64));- sig-}-pub fn dmem_read_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {let mut sig = Signature::new(host_cc);sig.params.push(AbiParam::new(pointer_type));diff --git a/crates/gecko/src/scheduler.rs b/crates/gecko/src/scheduler.rsindex 3253a0d..069f981 100644--- a/crates/gecko/src/scheduler.rs+++ b/crates/gecko/src/scheduler.rs@@ -197,10 +197,7 @@ pub fn dsp_batch_handler<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {return;}- let cpu_mail_quiet = !sys.dsp.mailbox_to_dsp_hi.busy();- let dsp_mail_full = sys.dsp.mailbox_to_cpu_hi.busy();- let (waits_cpu, waits_dsp) = sys.dsp.mailbox_wait_state();- let in_idle_wait = (cpu_mail_quiet && waits_cpu) || (dsp_mail_full && waits_dsp);+ let in_idle_wait = sys.dsp.parked_in_mailbox_wait();let pending_interrupt = sys.dsp.csr.pi_interrupt() && sys.dsp.registers.status.external_interrupt_enable();if in_idle_wait && !pending_interrupt {diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rsindex 43a3cb1..db2242b 100644--- a/crates/gecko/src/system.rs+++ b/crates/gecko/src/system.rs@@ -457,6 +457,39 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let dsp_suspends = crate::flipper::dsp::DSP_SUSPEND_COUNT.load(Ordering::Relaxed);let dsp_wakes = crate::flipper::dsp::DSP_WAKE_COUNT.load(Ordering::Relaxed);+ let drain_calls = crate::flipper::dsp::DSP_DRAIN_CALLS.load(Ordering::Relaxed);+ let drain_steps = crate::flipper::dsp::DSP_DRAIN_STEPS.load(Ordering::Relaxed);+ let drain_blocks = crate::flipper::dsp::DSP_DRAIN_BLOCKS.load(Ordering::Relaxed);+ let drain_nanos = crate::flipper::dsp::DSP_DRAIN_NANOS.load(Ordering::Relaxed);+ let drain_halt = crate::flipper::dsp::DSP_DRAIN_EXIT_HALT.load(Ordering::Relaxed);+ let drain_answered = crate::flipper::dsp::DSP_DRAIN_EXIT_ANSWERED.load(Ordering::Relaxed);+ let drain_wait = crate::flipper::dsp::DSP_DRAIN_EXIT_WAIT.load(Ordering::Relaxed);+ let drain_budget = crate::flipper::dsp::DSP_DRAIN_EXIT_BUDGET.load(Ordering::Relaxed);+ let batch_calls = crate::flipper::dsp::DSP_BATCH_CALLS.load(Ordering::Relaxed);+ let batch_steps = crate::flipper::dsp::DSP_BATCH_STEPS.load(Ordering::Relaxed);+ let batch_nanos = crate::flipper::dsp::DSP_BATCH_NANOS.load(Ordering::Relaxed);++ let drain_avg_steps = if drain_calls > 0 {+ drain_steps as f64 / drain_calls as f64+ } else {+ 0.0+ };+ let drain_avg_block = if drain_blocks > 0 {+ drain_steps as f64 / drain_blocks as f64+ } else {+ 0.0+ };+ let drain_ns_per_step = if drain_steps > 0 {+ drain_nanos as f64 / drain_steps as f64+ } else {+ 0.0+ };+ let batch_ns_per_step = if batch_steps > 0 {+ batch_nanos as f64 / batch_steps as f64+ } else {+ 0.0+ };+let event_breakdown = self.event_breakdown_top_n(20);let path = self.heatmap.out_dir.join("idle-skip.txt");@@ -467,6 +500,33 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.vsync_count, calls, cycles, avg, dsp_suspends, dsp_wakes)?;+ writeln!(+ f,+ "dsp_drain_calls={}\ndsp_drain_steps={}\ndsp_drain_blocks={}\ndsp_drain_ms={:.1}\ndsp_drain_avg_steps={:.1}\ndsp_drain_avg_steps_per_block={:.1}\ndsp_drain_ns_per_step={:.1}",+ drain_calls,+ drain_steps,+ drain_blocks,+ drain_nanos as f64 / 1e6,+ drain_avg_steps,+ drain_avg_block,+ drain_ns_per_step+ )?;++ writeln!(+ f,+ "dsp_drain_exit_halt={}\ndsp_drain_exit_answered={}\ndsp_drain_exit_wait={}\ndsp_drain_exit_budget={}",+ drain_halt, drain_answered, drain_wait, drain_budget+ )?;++ writeln!(+ f,+ "dsp_batch_calls={}\ndsp_batch_steps={}\ndsp_batch_ms={:.1}\ndsp_batch_ns_per_step={:.1}",+ batch_calls,+ batch_steps,+ batch_nanos as f64 / 1e6,+ batch_ns_per_step+ )?;+writeln!(f, "\n--- top scheduler events by fire count ---")?;for (name, count) in &event_breakdown {