Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

dsp improvements

Laylef84db6f2bd6c
diff --git a/crates/dsptestrunner/Cargo.toml b/crates/dsptestrunner/Cargo.toml
index 1c6bb3e..a652a4a 100644
--- a/crates/dsptestrunner/Cargo.toml
+++ b/crates/dsptestrunner/Cargo.toml
@@ -5,5 +5,5 @@ edition = "2024"
[dependencies]
image = { path = "../image" }
-gecko = { path = "../gecko", features = ["hooks"] }
+gecko = { path = "../gecko", features = ["hooks", "jit"] }
disasm = { path = "../disasm" }
\ No newline at end of file
diff --git a/crates/dsptestrunner/src/bin/zayd.rs b/crates/dsptestrunner/src/bin/zayd.rs
index 013af38..d9eab79 100644
--- a/crates/dsptestrunner/src/bin/zayd.rs
+++ b/crates/dsptestrunner/src/bin/zayd.rs
@@ -435,22 +435,37 @@ struct RunResult {
failed: u32,
}
-fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunResult, String> {
- let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?;
- let cases = parse_test_file(&bytes)?;
- let ignore_flags = is_prod_test(test_name);
-
- let mut passed = 0u32;
- let mut failed = 0u32;
+fn run_case(emu: &mut GameCube, case: &TestCase, jit: bool) -> (TestState, bool) {
+ reset_dsp(&mut emu.dsp);
+ apply_initial_state(&mut emu.dsp, &case.initial);
+ load_instructions(&mut emu.dsp, &case.instructions);
+ emu.dsp.registers.pc = TEST_PC;
+ emu.dsp.rebuild_wait_table();
+
+ let mut runaway = false;
+
+ if jit {
+ emu.execution_mode = gecko::system::ExecutionMode::Jit;
+ if let Some(j) = emu.dsp.jit.as_mut() {
+ j.flush();
+ }
- for (idx, case) in cases.iter().enumerate() {
- reset_dsp(&mut emu.dsp);
- apply_initial_state(&mut emu.dsp, &case.initial);
- load_instructions(&mut emu.dsp, &case.instructions);
- emu.dsp.registers.pc = TEST_PC;
+ let mut steps = 0u64;
+ loop {
+ if emu.dsp.csr.halt() {
+ break;
+ }
+ if steps >= MAX_STEPS {
+ runaway = true;
+ break;
+ }
+ emu.execute_dsp_batch();
+ steps += 1024;
+ }
+ } else {
+ emu.execution_mode = gecko::system::ExecutionMode::Interpreter;
let mut steps = 0u64;
- let mut runaway = false;
while !emu.dsp.csr.halt() {
if !emu.step_dsp_instruction() {
break;
@@ -461,40 +476,137 @@ fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunR
break;
}
}
+ }
+
+ (read_actual_state(&mut emu.dsp), runaway)
+}
+
+const SEQ_CHUNK: usize = 8;
- let actual = read_actual_state(&mut emu.dsp);
- let diff = diff_state(&case.initial, &case.expected, &actual, ignore_flags);
+fn run_sequences(emu: &mut GameCube, test_name: &str, cases: &[TestCase]) -> (u32, u32) {
+ let mut passed = 0u32;
+ let mut failed = 0u32;
- if runaway || diff.any_failure() {
+ let eligible: Vec<&TestCase> = cases
+ .iter()
+ .filter(|c| {
+ !c.instructions
+ .iter()
+ .any(|&w| gecko::flipper::dsp::jit::block::is_terminator_word(w))
+ })
+ .collect();
+
+ for chunk in eligible.chunks(SEQ_CHUNK) {
+ if chunk.len() < 2 {
+ continue;
+ }
+
+ let mut program = Vec::new();
+ for c in chunk {
+ program.extend_from_slice(&c.instructions);
+ }
+ if (TEST_PC as usize + program.len() + 1) * 2 > emu.dsp.iram.len() {
+ continue;
+ }
+
+ let synth = TestCase {
+ instructions: program,
+ expected: TestState::default(),
+ initial: chunk[0].initial,
+ };
+
+ let dram_snapshot = emu.dsp.dram.clone();
+ let ifx_snapshot = emu.dsp.ifx.clone();
+ let (interp_state, interp_runaway) = run_case(emu, &synth, false);
+
+ emu.dsp.dram.copy_from_slice(&dram_snapshot[..]);
+ emu.dsp.ifx.copy_from_slice(&ifx_snapshot[..]);
+ let (jit_state, jit_runaway) = run_case(emu, &synth, true);
+
+ let mismatches: Vec<usize> = (0..31)
+ .filter(|&slot| interp_state.reg[slot] != jit_state.reg[slot])
+ .collect();
+
+ if interp_runaway || jit_runaway || !mismatches.is_empty() {
failed += 1;
- println!("===== DSP Test {test_name} Failed =====");
- println!("Test case: {idx}");
+ println!("===== DSP Seq {test_name} interp/jit mismatch =====");
+
+ if interp_runaway || jit_runaway {
+ println!("(runaway: interp={interp_runaway} jit={jit_runaway})");
+ }
- if runaway {
+ for &slot in &mismatches {
+ let reg = slot_to_dsp_reg(slot);
println!(
- "(runaway: >= {MAX_STEPS} steps without halt, pc={:04x})",
- emu.dsp.registers.pc
+ "reg {:2}: interp={:04x} jit={:04x}",
+ reg, interp_state.reg[slot], jit_state.reg[slot]
);
}
- println!("Instructions: {}", format_instructions_hex(&case.instructions));
+ println!("Instructions: {}", format_instructions_hex(&synth.instructions));
- for line in disassemble_instructions(&case.instructions) {
+ for line in disassemble_instructions(&synth.instructions) {
println!("{line}");
}
- println!("Initial state:");
- pretty_print(&case.initial, &diff);
- println!("Expected:");
- pretty_print(&case.expected, &diff);
- println!("Actual:");
- pretty_print(&actual, &diff);
println!();
} else {
passed += 1;
}
}
+ (passed, failed)
+}
+
+fn run_one_test(emu: &mut GameCube, test_name: &str, path: &Path) -> Result<RunResult, String> {
+ let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?;
+ let cases = parse_test_file(&bytes)?;
+ let ignore_flags = is_prod_test(test_name);
+
+ let mut passed = 0u32;
+ let mut failed = 0u32;
+
+ for (idx, case) in cases.iter().enumerate() {
+ for jit in [false, true] {
+ let (actual, runaway) = run_case(emu, case, jit);
+ let diff = diff_state(&case.initial, &case.expected, &actual, ignore_flags);
+ let mode = if jit { "jit" } else { "interp" };
+
+ if runaway || diff.any_failure() {
+ failed += 1;
+ println!("===== DSP Test {test_name} Failed [{mode}] =====");
+ println!("Test case: {idx}");
+
+ if runaway {
+ println!(
+ "(runaway: >= {MAX_STEPS} steps without halt, pc={:04x})",
+ emu.dsp.registers.pc
+ );
+ }
+
+ println!("Instructions: {}", format_instructions_hex(&case.instructions));
+
+ for line in disassemble_instructions(&case.instructions) {
+ println!("{line}");
+ }
+
+ println!("Initial state:");
+ pretty_print(&case.initial, &diff);
+ println!("Expected:");
+ pretty_print(&case.expected, &diff);
+ println!("Actual:");
+ pretty_print(&actual, &diff);
+ println!();
+ } else {
+ passed += 1;
+ }
+ }
+ }
+
+ let (seq_passed, seq_failed) = self::run_sequences(emu, test_name, &cases);
+ passed += seq_passed;
+ failed += seq_failed;
+
Ok(RunResult { passed, failed })
}
diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rs
index 56e3b5c..9cd3938 100644
--- a/crates/gecko/src/flipper/dsp.rs
+++ b/crates/gecko/src/flipper/dsp.rs
@@ -24,7 +24,7 @@ use crate::mmio::Mmio;
use crate::system::{ExecutionMode, System, SystemId};
#[cfg(feature = "jit")]
-pub const DSP_JIT_CHAIN_BUDGET: u32 = 16;
+pub const DSP_JIT_CHAIN_BUDGET: u32 = 64;
pub struct Dsp {
pub registers: core::Registers,
@@ -133,9 +133,10 @@ impl Dsp {
}
#[inline(always)]
- pub fn mailbox_wait_state(&self) -> (bool, bool) {
- let b = self.wait_table[self.registers.pc as usize];
- (b & 1 != 0, b & 2 != 0)
+ pub fn parked_in_mailbox_wait(&self) -> bool {
+ let cpu_mail_quiet = !self.mailbox_to_dsp_hi.busy();
+ let dsp_mail_full = self.mailbox_to_cpu_hi.busy();
+ (cpu_mail_quiet && self.is_waiting_for_cpu_mail()) || (dsp_mail_full && self.is_waiting_for_dsp_mail())
}
pub fn rebuild_wait_table(&mut self) {
@@ -358,7 +359,39 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
}
#[cfg(feature = "jit")]
+ fn dsp_jit_step(&mut self, iram: &[u8], irom: &[u8]) -> u64 {
+ let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;
+
+ if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {
+ self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);
+ self.dsp.registers.call_stack.push(self.dsp.registers.pc);
+ self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());
+ self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);
+ self.dsp.registers.pc = 0x000E;
+ }
+
+ let start_pc = self.dsp.registers.pc;
+ self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;
+ self.dsp.instr_count = 0;
+
+ let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);
+ self.dsp.registers.pc = next_pc;
+
+ #[cfg(feature = "jit-stats")]
+ {
+ let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;
+ self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);
+ }
+
+ (self.dsp.instr_count as u64).max(1)
+ }
+
+ #[cfg(feature = "jit")]
+ #[cfg_attr(feature = "hotpath", hotpath::measure(label = "dsp_batch"))]
fn execute_dsp_batch_jit(&mut self) {
+ #[cfg(feature = "jit-stats")]
+ let batch_start = std::time::Instant::now();
+
if self.dsp.csr.reset() || self.dsp.csr.halt() {
self::refresh_interrupts(self);
return;
@@ -368,7 +401,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));
}
- let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;
let iram_ptr = self.dsp.iram.as_ptr();
let irom_ptr = self.dsp.irom.as_ptr();
let iram_len = self.dsp.iram.len();
@@ -378,12 +410,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
let mut budget = crate::scheduler::DSP_BATCH_SIZE as u64;
while budget > 0 {
- let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();
- let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();
-
- if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())
- || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())
- {
+ if self.dsp.parked_in_mailbox_wait() {
break;
}
@@ -391,26 +418,17 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
break;
}
- if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {
- self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);
- self.dsp.registers.call_stack.push(self.dsp.registers.pc);
- self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());
- self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);
- self.dsp.registers.pc = 0x000E;
- }
-
- let start_pc = self.dsp.registers.pc;
- self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;
- self.dsp.instr_count = 0;
-
- let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);
- self.dsp.registers.pc = next_pc;
-
- let consumed = (self.dsp.instr_count as u64).max(1);
+ let consumed = self.dsp_jit_step(iram, irom);
budget = budget.saturating_sub(consumed);
+ }
- let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;
- self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);
+ #[cfg(feature = "jit-stats")]
+ {
+ use std::sync::atomic::Ordering;
+
+ DSP_BATCH_CALLS.fetch_add(1, Ordering::Relaxed);
+ DSP_BATCH_STEPS.fetch_add(crate::scheduler::DSP_BATCH_SIZE - budget, Ordering::Relaxed);
+ DSP_BATCH_NANOS.fetch_add(batch_start.elapsed().as_nanos() as u64, Ordering::Relaxed);
}
self::refresh_interrupts(self);
@@ -427,10 +445,20 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
}
#[cfg(feature = "jit")]
+ #[cfg_attr(feature = "hotpath", hotpath::measure(label = "dsp_drain"))]
fn drain_dsp_synchronous_jit(&mut self, max_steps: u32) {
+ #[cfg(feature = "jit-stats")]
+ let drain_start = std::time::Instant::now();
+
let already_busy = self.dsp.mailbox_to_cpu_hi.busy();
if self.dsp.csr.reset() || self.dsp.csr.halt() {
+ #[cfg(feature = "jit-stats")]
+ {
+ DSP_DRAIN_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ DSP_DRAIN_EXIT_HALT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }
+
self::refresh_interrupts(self);
return;
}
@@ -439,7 +467,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));
}
- let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;
let iram_ptr = self.dsp.iram.as_ptr();
let irom_ptr = self.dsp.irom.as_ptr();
let iram_len = self.dsp.iram.len();
@@ -448,6 +475,10 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
let irom = unsafe { ::core::slice::from_raw_parts(irom_ptr, irom_len) };
let mut budget = max_steps as u64;
+
+ #[cfg(feature = "jit-stats")]
+ let mut blocks: u64 = 0;
+
while budget > 0 {
if self.dsp.csr.reset() || self.dsp.csr.halt() {
break;
@@ -457,34 +488,38 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
break;
}
- let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();
- let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();
- if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())
- || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())
- {
+ if self.dsp.parked_in_mailbox_wait() {
break;
}
- if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {
- self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);
- self.dsp.registers.call_stack.push(self.dsp.registers.pc);
- self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());
- self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);
- self.dsp.registers.pc = 0x000E;
- }
-
- let start_pc = self.dsp.registers.pc;
- self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;
- self.dsp.instr_count = 0;
-
- let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);
- self.dsp.registers.pc = next_pc;
-
- let consumed = (self.dsp.instr_count as u64).max(1);
+ let consumed = self.dsp_jit_step(iram, irom);
budget = budget.saturating_sub(consumed);
- let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;
- self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);
+ #[cfg(feature = "jit-stats")]
+ {
+ blocks += 1;
+ }
+ }
+
+ #[cfg(feature = "jit-stats")]
+ {
+ use std::sync::atomic::Ordering;
+
+ DSP_DRAIN_CALLS.fetch_add(1, Ordering::Relaxed);
+ DSP_DRAIN_STEPS.fetch_add(max_steps as u64 - budget, Ordering::Relaxed);
+ DSP_DRAIN_BLOCKS.fetch_add(blocks, Ordering::Relaxed);
+ DSP_DRAIN_NANOS.fetch_add(drain_start.elapsed().as_nanos() as u64, Ordering::Relaxed);
+
+ let exit = if self.dsp.csr.reset() || self.dsp.csr.halt() {
+ &DSP_DRAIN_EXIT_HALT
+ } else if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {
+ &DSP_DRAIN_EXIT_ANSWERED
+ } else if budget > 0 {
+ &DSP_DRAIN_EXIT_WAIT
+ } else {
+ &DSP_DRAIN_EXIT_BUDGET
+ };
+ exit.fetch_add(1, Ordering::Relaxed);
}
self::refresh_interrupts(self);
@@ -501,6 +536,10 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {
break;
}
+
+ if self.dsp.parked_in_mailbox_wait() {
+ break;
+ }
}
self::refresh_interrupts(self);
@@ -578,6 +617,28 @@ pub fn refresh_interrupts<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {
pub static DSP_SUSPEND_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
#[cfg(feature = "jit-stats")]
pub static DSP_WAKE_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_STEPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_BLOCKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_EXIT_HALT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_EXIT_ANSWERED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_EXIT_WAIT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_DRAIN_EXIT_BUDGET: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_BATCH_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_BATCH_STEPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+#[cfg(feature = "jit-stats")]
+pub static DSP_BATCH_NANOS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
#[inline]
pub fn wake_dsp_scheduler<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {
@@ -793,7 +854,10 @@ pub trait DspJitHandle {
#[cfg(feature = "jit")]
impl<const SYSTEM: SystemId> DspJitHandle for jit::JitEngine<SYSTEM> {
fn run_block(&mut self, ctx_ptr: *mut ::core::ffi::c_void, iram: &[u8], irom: &[u8], start_pc: u16) -> u16 {
- let entry = self.lookup_or_compile(iram, irom, start_pc);
+ let entry = match self.lookup_block_fast(start_pc) {
+ Some(entry) => entry,
+ None => self.lookup_or_compile(iram, irom, start_pc),
+ };
Self::run_block(self, ctx_ptr, entry)
}
diff --git a/crates/gecko/src/flipper/dsp/jit.rs b/crates/gecko/src/flipper/dsp/jit.rs
index 02e02d7..8c4335e 100644
--- a/crates/gecko/src/flipper/dsp/jit.rs
+++ b/crates/gecko/src/flipper/dsp/jit.rs
@@ -45,6 +45,8 @@ pub struct JitEngine<const SYSTEM: SystemId> {
trampoline_fn: TrampolineFn,
block_seq: u64,
block_lookup_table_addr: usize,
+ loop_end_table: Box<[u8; 0x10000]>,
+ loop_end_table_built: bool,
chain_depth_total: u64,
dispatcher_entries_total: u64,
chain_depth_histogram: [u64; CHAIN_DEPTH_HISTOGRAM_LEN],
@@ -81,12 +83,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());
struct Syms {
- cache_ext_ac: (&'static str, *const u8),
loop_tail: (&'static str, *const u8),
- update_flags_logic: (&'static str, *const u8),
- update_flags_add: (&'static str, *const u8),
- update_flags_sub: (&'static str, *const u8),
- update_flags_ac: (&'static str, *const u8),
read_dmem: (&'static str, *const u8),
write_dmem: (&'static str, *const u8),
inc_ar: (&'static str, *const u8),
@@ -95,7 +92,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
decrease_ar_ix: (&'static str, *const u8),
dynamic_shift: (&'static str, *const u8),
read_imem: (&'static str, *const u8),
- write_ac_mid_sxm: (&'static str, *const u8),
call_stack_push: (&'static str, *const u8),
call_stack_pop: (&'static str, *const u8),
data_stack_pop: (&'static str, *const u8),
@@ -105,18 +101,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
}
let syms: Syms = match SYSTEM {
GC => Syms {
- cache_ext_ac: (
- "gecko_dsp_jit_cache_ext_ac_gc",
- runtime::dsp_cache_ext_ac_gc as *const u8,
- ),
loop_tail: ("gecko_dsp_jit_loop_tail_gc", runtime::dsp_loop_tail_gc as *const u8),
- update_flags_logic: (
- "gecko_dsp_jit_uflog_gc",
- runtime::dsp_update_flags_logic_gc as *const u8,
- ),
- update_flags_add: ("gecko_dsp_jit_ufadd_gc", runtime::dsp_update_flags_add_gc as *const u8),
- update_flags_sub: ("gecko_dsp_jit_ufsub_gc", runtime::dsp_update_flags_sub_gc as *const u8),
- update_flags_ac: ("gecko_dsp_jit_ufac_gc", runtime::dsp_update_flags_ac_gc as *const u8),
read_dmem: ("gecko_dsp_jit_rdmem_gc", runtime::dsp_read_dmem_gc as *const u8),
write_dmem: ("gecko_dsp_jit_wdmem_gc", runtime::dsp_write_dmem_gc as *const u8),
inc_ar: ("gecko_dsp_jit_incar_gc", runtime::dsp_increment_ar_gc as *const u8),
@@ -125,7 +110,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
decrease_ar_ix: ("gecko_dsp_jit_decarix_gc", runtime::dsp_decrease_ar_ix_gc as *const u8),
dynamic_shift: ("gecko_dsp_jit_dynshift_gc", runtime::dsp_dynamic_shift_gc as *const u8),
read_imem: ("gecko_dsp_jit_rimem_gc", runtime::dsp_read_imem_gc as *const u8),
- write_ac_mid_sxm: ("gecko_dsp_jit_wamsxm_gc", runtime::dsp_write_ac_mid_sxm_gc as *const u8),
call_stack_push: ("gecko_dsp_jit_cspush_gc", runtime::dsp_call_stack_push_gc as *const u8),
call_stack_pop: ("gecko_dsp_jit_cspop_gc", runtime::dsp_call_stack_pop_gc as *const u8),
data_stack_pop: ("gecko_dsp_jit_dspop_gc", runtime::dsp_data_stack_pop_gc as *const u8),
@@ -134,24 +118,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
loop_setup: ("gecko_dsp_jit_loopsetup_gc", runtime::dsp_loop_setup_gc as *const u8),
},
WII => Syms {
- cache_ext_ac: (
- "gecko_dsp_jit_cache_ext_ac_wii",
- runtime::dsp_cache_ext_ac_wii as *const u8,
- ),
loop_tail: ("gecko_dsp_jit_loop_tail_wii", runtime::dsp_loop_tail_wii as *const u8),
- update_flags_logic: (
- "gecko_dsp_jit_uflog_wii",
- runtime::dsp_update_flags_logic_wii as *const u8,
- ),
- update_flags_add: (
- "gecko_dsp_jit_ufadd_wii",
- runtime::dsp_update_flags_add_wii as *const u8,
- ),
- update_flags_sub: (
- "gecko_dsp_jit_ufsub_wii",
- runtime::dsp_update_flags_sub_wii as *const u8,
- ),
- update_flags_ac: ("gecko_dsp_jit_ufac_wii", runtime::dsp_update_flags_ac_wii as *const u8),
read_dmem: ("gecko_dsp_jit_rdmem_wii", runtime::dsp_read_dmem_wii as *const u8),
write_dmem: ("gecko_dsp_jit_wdmem_wii", runtime::dsp_write_dmem_wii as *const u8),
inc_ar: ("gecko_dsp_jit_incar_wii", runtime::dsp_increment_ar_wii as *const u8),
@@ -166,10 +133,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
runtime::dsp_dynamic_shift_wii as *const u8,
),
read_imem: ("gecko_dsp_jit_rimem_wii", runtime::dsp_read_imem_wii as *const u8),
- write_ac_mid_sxm: (
- "gecko_dsp_jit_wamsxm_wii",
- runtime::dsp_write_ac_mid_sxm_wii as *const u8,
- ),
call_stack_push: (
"gecko_dsp_jit_cspush_wii",
runtime::dsp_call_stack_push_wii as *const u8,
@@ -182,12 +145,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
},
_ => unreachable!(),
};
- jit_builder.symbol(syms.cache_ext_ac.0, syms.cache_ext_ac.1);
jit_builder.symbol(syms.loop_tail.0, syms.loop_tail.1);
- jit_builder.symbol(syms.update_flags_logic.0, syms.update_flags_logic.1);
- jit_builder.symbol(syms.update_flags_add.0, syms.update_flags_add.1);
- jit_builder.symbol(syms.update_flags_sub.0, syms.update_flags_sub.1);
- jit_builder.symbol(syms.update_flags_ac.0, syms.update_flags_ac.1);
jit_builder.symbol(syms.read_dmem.0, syms.read_dmem.1);
jit_builder.symbol(syms.write_dmem.0, syms.write_dmem.1);
jit_builder.symbol(syms.inc_ar.0, syms.inc_ar.1);
@@ -196,7 +154,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
jit_builder.symbol(syms.decrease_ar_ix.0, syms.decrease_ar_ix.1);
jit_builder.symbol(syms.dynamic_shift.0, syms.dynamic_shift.1);
jit_builder.symbol(syms.read_imem.0, syms.read_imem.1);
- jit_builder.symbol(syms.write_ac_mid_sxm.0, syms.write_ac_mid_sxm.1);
jit_builder.symbol(syms.call_stack_push.0, syms.call_stack_push.1);
jit_builder.symbol(syms.call_stack_pop.0, syms.call_stack_pop.1);
jit_builder.symbol(syms.data_stack_pop.0, syms.data_stack_pop.1);
@@ -211,33 +168,15 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
let block_sig = translator::block_signature(pointer_type);
let void_thunk_sig = translator::void_thunk_signature(pointer_type, host_cc);
- let flags_logic_sig = translator::flags_logic_signature(pointer_type, host_cc);
- let flags_arith_sig = translator::flags_arith_signature(pointer_type, host_cc);
- let flags_ac_sig = translator::flags_ac_signature(pointer_type, host_cc);
let dmem_read_sig = translator::dmem_read_signature(pointer_type, host_cc);
let dmem_write_sig = translator::dmem_write_signature(pointer_type, host_cc);
let ar_unary_sig = translator::ar_unary_signature(pointer_type, host_cc);
let ar_binary_sig = translator::ar_binary_signature(pointer_type, host_cc);
let dynamic_shift_sig = translator::dynamic_shift_signature(pointer_type, host_cc);
- let cache_ext_ac_id = module
- .declare_function(syms.cache_ext_ac.0, Linkage::Import, &void_thunk_sig)
- .expect("declare cache_ext_ac thunk");
let loop_tail_id = module
.declare_function(syms.loop_tail.0, Linkage::Import, &void_thunk_sig)
.expect("declare loop_tail thunk");
- let update_flags_logic_id = module
- .declare_function(syms.update_flags_logic.0, Linkage::Import, &flags_logic_sig)
- .expect("declare update_flags_logic thunk");
- let update_flags_add_id = module
- .declare_function(syms.update_flags_add.0, Linkage::Import, &flags_arith_sig)
- .expect("declare update_flags_add thunk");
- let update_flags_sub_id = module
- .declare_function(syms.update_flags_sub.0, Linkage::Import, &flags_arith_sig)
- .expect("declare update_flags_sub thunk");
- let update_flags_ac_id = module
- .declare_function(syms.update_flags_ac.0, Linkage::Import, &flags_ac_sig)
- .expect("declare update_flags_ac thunk");
let read_dmem_id = module
.declare_function(syms.read_dmem.0, Linkage::Import, &dmem_read_sig)
.expect("declare read_dmem thunk");
@@ -262,9 +201,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
let read_imem_id = module
.declare_function(syms.read_imem.0, Linkage::Import, &dmem_read_sig)
.expect("declare read_imem thunk");
- let write_ac_mid_sxm_id = module
- .declare_function(syms.write_ac_mid_sxm.0, Linkage::Import, &dmem_write_sig)
- .expect("declare write_ac_mid_sxm thunk");
let call_stack_push_sig = {
let mut sig = cranelift_codegen::ir::Signature::new(host_cc);
@@ -301,12 +237,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
.expect("declare loop_setup thunk");
let extern_funcs = translator::ExternFuncs {
- cache_ext_ac: cache_ext_ac_id,
loop_tail: loop_tail_id,
- update_flags_logic: update_flags_logic_id,
- update_flags_add: update_flags_add_id,
- update_flags_sub: update_flags_sub_id,
- update_flags_ac: update_flags_ac_id,
read_dmem: read_dmem_id,
write_dmem: write_dmem_id,
inc_ar: inc_ar_id,
@@ -315,7 +246,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
decrease_ar_ix: decrease_ar_ix_id,
dynamic_shift: dynamic_shift_id,
read_imem: read_imem_id,
- write_ac_mid_sxm: write_ac_mid_sxm_id,
call_stack_push: call_stack_push_id,
call_stack_pop: call_stack_pop_id,
data_stack_pop: data_stack_pop_id,
@@ -348,6 +278,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
trampoline_fn,
block_seq: 0,
block_lookup_table_addr,
+ loop_end_table: unsafe { Box::<[u8; 0x10000]>::new_zeroed().assume_init() },
+ loop_end_table_built: false,
chain_depth_total: 0,
dispatcher_entries_total: 0,
chain_depth_histogram: [0; CHAIN_DEPTH_HISTOGRAM_LEN],
@@ -377,9 +309,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
self.chain_depth_histogram[bucket] = self.chain_depth_histogram[bucket].wrapping_add(1);
}
+ fn lookup_table_index(pc: u16) -> usize {
+ let pc_u32 = pc as u32;
+ ((pc_u32 & 0xFFF) | ((pc_u32 >> 3) & 0x1000)) as usize
+ }
+
fn register_in_lookup_table(&self, pc: u16, entry: usize) {
let pc_u32 = pc as u32;
- let idx = ((pc_u32 & 0xFFF) | ((pc_u32 >> 3) & 0x1000)) as usize;
+ let idx = Self::lookup_table_index(pc);
unsafe {
let table = self.block_lookup_table_addr as *mut DspBlockLookupSlot;
(*table.add(idx)).pc = pc_u32;
@@ -397,6 +334,51 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
}
}
+ fn rebuild_loop_end_table(&mut self, iram: &[u8], irom: &[u8]) {
+ self.loop_end_table.fill(0);
+
+ for (base, mem) in [(0x0000u32, iram), (0x8000u32, irom)] {
+ let words = mem.len() / 2;
+ for i in 0..words {
+ let w = u16::from_be_bytes([mem[i * 2], mem[i * 2 + 1]]);
+
+ if (w & 0xFFE0) == 0x0040 || (w & 0xFF00) == 0x1000 {
+ let end = (base + i as u32 + 1) as u16;
+ self.loop_end_table[end as usize] = 1;
+ }
+
+ if ((w & 0xFFE0) == 0x0060 || (w & 0xFF00) == 0x1100) && i + 1 < words {
+ let end = u16::from_be_bytes([mem[i * 2 + 2], mem[i * 2 + 3]]);
+ self.loop_end_table[end as usize] = 1;
+ }
+ }
+ }
+
+ self.loop_end_table_built = true;
+ }
+
+ pub fn lookup_block_fast(&mut self, pc: u16) -> Option<BlockEntry> {
+ let pc_u32 = pc as u32;
+ let idx = Self::lookup_table_index(pc);
+
+ let entry = unsafe {
+ let table = self.block_lookup_table_addr as *const DspBlockLookupSlot;
+ let slot = &*table.add(idx);
+ if slot.pc == pc_u32 && slot.entry != 0 {
+ Some(slot.entry)
+ } else {
+ None
+ }
+ };
+
+ #[cfg(feature = "jit-stats")]
+ if entry.is_some() {
+ *self.hits.entry(pc).or_insert(0) += 1;
+ }
+
+ entry
+ }
+
pub fn lookup_or_compile(&mut self, iram: &[u8], irom: &[u8], start_pc: u16) -> BlockEntry {
if let Some(&entry) = self.cache.get(&start_pc) {
#[cfg(feature = "jit-stats")]
@@ -405,6 +387,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
}
return entry;
}
+
+ if !self.loop_end_table_built {
+ self.rebuild_loop_end_table(iram, irom);
+ }
let spec = block::discover(iram, irom, start_pc);
let entry = self.compile(&spec);
self.cache.insert(start_pc, entry);
@@ -489,6 +475,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
#[cfg(feature = "jit-stats")]
pub fn dump_block_clif(&mut self, start_pc: u16, iram: &[u8], irom: &[u8]) {
+ if !self.loop_end_table_built {
+ self.rebuild_loop_end_table(iram, irom);
+ }
+
let spec = block::discover(iram, irom, start_pc);
self.ctx.clear();
@@ -501,6 +491,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
&spec,
self.block_lookup_table_addr as i64,
None,
+ &self.loop_end_table,
);
tracing::info!(
"CLIF for DSP block pc={:04X} (len={}, term={:?})",
@@ -531,6 +522,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
spec,
self.block_lookup_table_addr as i64,
entry_counter_addr,
+ &self.loop_end_table,
);
self.module
@@ -548,6 +540,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
pub fn flush(&mut self) {
self.cache.clear();
self.block_func_ids.clear();
+ self.loop_end_table_built = false;
self.clear_lookup_table();
}
diff --git a/crates/gecko/src/flipper/dsp/jit/block.rs b/crates/gecko/src/flipper/dsp/jit/block.rs
index 715adb2..0637865 100644
--- a/crates/gecko/src/flipper/dsp/jit/block.rs
+++ b/crates/gecko/src/flipper/dsp/jit/block.rs
@@ -35,16 +35,40 @@ fn read_imem_word(iram: &[u8], irom: &[u8], addr: u16) -> Option<u16> {
}
}
+pub fn is_terminator_word(primary: u16) -> bool {
+ self::classify(primary).is_some()
+}
+
fn classify(primary: u16) -> Option<TermKind> {
- let nibble = (primary >> 12) & 0xF;
- match nibble {
- 0x0 | 0x1 => {
- if primary == 0x0021 {
- Some(TermKind::Halt)
- } else {
- Some(TermKind::Jump)
- }
- }
+ match primary {
+ 0x0021 => Some(TermKind::Halt),
+
+ 0x0040..=0x007F | 0x1000..=0x11FF => Some(TermKind::LoopSetup),
+
+ 0x0270..=0x027F => Some(TermKind::IfCc),
+ 0x0290..=0x029F => Some(TermKind::Jump),
+ 0x02B0..=0x02BF => Some(TermKind::Call),
+ 0x02D0..=0x02DF | 0x02F0..=0x02FF => Some(TermKind::Ret),
+ _ if (primary & 0xFF10) == 0x1700 => Some(TermKind::Jump),
+ _ if (primary & 0xFF10) == 0x1710 => Some(TermKind::Call),
+
+ 0x0000 | 0x0004..=0x001F => None,
+ 0x0080..=0x009F | 0x00C0..=0x00FF => None,
+ 0x02CA | 0x02CB => None,
+ 0x0400..=0x0FFF => None,
+ 0x1200..=0x1207 | 0x1300..=0x1307 => None,
+ 0x1400..=0x16FF | 0x1800..=0x1FFF => None,
+ _ if (primary & 0xFEFF) == 0x0200 => None,
+ _ if (primary & 0xFEFF) == 0x0220 => None,
+ _ if (primary & 0xFEFF) == 0x0240 => None,
+ _ if (primary & 0xFEFF) == 0x0260 => None,
+ _ if (primary & 0xFEFF) == 0x0280 => None,
+ _ if (primary & 0xFEFF) == 0x02A0 => None,
+ _ if (primary & 0xFEFF) == 0x02C0 => None,
+ _ if (primary & 0xFEF0) == 0x0210 => None,
+
+ _ if (primary >> 12) <= 0x1 => Some(TermKind::Jump),
+
_ => None,
}
}
diff --git a/crates/gecko/src/flipper/dsp/jit/runtime.rs b/crates/gecko/src/flipper/dsp/jit/runtime.rs
index afb2f8d..478b754 100644
--- a/crates/gecko/src/flipper/dsp/jit/runtime.rs
+++ b/crates/gecko/src/flipper/dsp/jit/runtime.rs
@@ -44,62 +44,6 @@ pub extern "C" fn dsp_loop_tail_wii(sys: *mut core::ffi::c_void) {
loop_tail::<WII>(sys.cast());
}
-#[inline(always)]
-fn update_flags_logic<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, result16: u32, ac_full: i64) {
- let sys = unsafe { &mut *sys };
- sys.dsp.registers.update_flags_logic(result16 as u16, ac_full);
-}
-
-pub extern "C" fn dsp_update_flags_logic_gc(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {
- update_flags_logic::<GC>(sys.cast(), result, ac_full);
-}
-
-pub extern "C" fn dsp_update_flags_logic_wii(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {
- update_flags_logic::<WII>(sys.cast(), result, ac_full);
-}
-
-#[inline(always)]
-fn update_flags_add<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {
- let sys = unsafe { &mut *sys };
- sys.dsp.registers.update_flags_add(a, b, result);
-}
-
-pub extern "C" fn dsp_update_flags_add_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {
- update_flags_add::<GC>(sys.cast(), a, b, result);
-}
-
-pub extern "C" fn dsp_update_flags_add_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {
- update_flags_add::<WII>(sys.cast(), a, b, result);
-}
-
-#[inline(always)]
-fn update_flags_sub<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {
- let sys = unsafe { &mut *sys };
- sys.dsp.registers.update_flags_sub(a, b, result);
-}
-
-pub extern "C" fn dsp_update_flags_sub_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {
- update_flags_sub::<GC>(sys.cast(), a, b, result);
-}
-
-pub extern "C" fn dsp_update_flags_sub_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {
- update_flags_sub::<WII>(sys.cast(), a, b, result);
-}
-
-#[inline(always)]
-fn update_flags_ac<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ac_full: i64) {
- let sys = unsafe { &mut *sys };
- sys.dsp.registers.update_flags_ac(ac_full);
-}
-
-pub extern "C" fn dsp_update_flags_ac_gc(sys: *mut core::ffi::c_void, ac_full: i64) {
- update_flags_ac::<GC>(sys.cast(), ac_full);
-}
-
-pub extern "C" fn dsp_update_flags_ac_wii(sys: *mut core::ffi::c_void, ac_full: i64) {
- update_flags_ac::<WII>(sys.cast(), ac_full);
-}
-
pub extern "C" fn dsp_read_dmem_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {
let sys = unsafe { &mut *(sys as *mut System<GC>) };
dsp::read_dmem(sys, addr as u16) as u32
@@ -215,20 +159,6 @@ pub extern "C" fn dsp_read_imem_wii(sys: *mut core::ffi::c_void, addr: u32) -> u
read_imem::<WII>(sys.cast(), addr)
}
-#[inline(always)]
-fn write_ac_mid_sxm<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32, value: u32) {
- let sys = unsafe { &mut *sys };
- sys.dsp.registers.write::<true>(30 + idx as u8, value as u16);
-}
-
-pub extern "C" fn dsp_write_ac_mid_sxm_gc(sys: *mut core::ffi::c_void, idx: u32, value: u32) {
- write_ac_mid_sxm::<GC>(sys.cast(), idx, value);
-}
-
-pub extern "C" fn dsp_write_ac_mid_sxm_wii(sys: *mut core::ffi::c_void, idx: u32, value: u32) {
- write_ac_mid_sxm::<WII>(sys.cast(), idx, value);
-}
-
#[inline(always)]
fn call_stack_push<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, value: u32) {
let sys = unsafe { &mut *sys };
diff --git a/crates/gecko/src/flipper/dsp/jit/translate.rs b/crates/gecko/src/flipper/dsp/jit/translate.rs
index ce7331b..4a11363 100644
--- a/crates/gecko/src/flipper/dsp/jit/translate.rs
+++ b/crates/gecko/src/flipper/dsp/jit/translate.rs
@@ -446,33 +446,132 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {
self.store_u16(val, m_off);
}
+ fn store_status_bits(&mut self, bits: Value, clear_mask: i64) {
+ let off = abi::dsp_status_offset() as i32;
+ let sr = self.load_u16(off);
+ let cleared = self.builder.ins().band_imm(sr, clear_mask);
+ let new_sr = self.builder.ins().bor(cleared, bits);
+ self.store_u16(new_sr, off);
+ }
+
+ fn flags40_as32_pos(&mut self, r40: Value) -> Value {
+ use cranelift_codegen::ir::condcodes::IntCC;
+
+ let upper9_raw = self.builder.ins().ushr_imm(r40, 31);
+ let upper9 = self.builder.ins().band_imm(upper9_raw, 0x1FF);
+ let ne0 = self.builder.ins().icmp_imm(IntCC::NotEqual, upper9, 0);
+ let ne1ff = self.builder.ins().icmp_imm(IntCC::NotEqual, upper9, 0x1FF);
+ let as32 = self.builder.ins().band(ne0, ne1ff);
+ let as32_16 = self.builder.ins().uextend(types::I16, as32);
+ self.builder.ins().ishl_imm(as32_16, 4)
+ }
+
+ fn flags40_szat(&mut self, r40: Value) -> Value {
+ use cranelift_codegen::ir::condcodes::IntCC;
+
+ let sign64 = self.builder.ins().ushr_imm(r40, 39);
+ let sign16 = self.builder.ins().ireduce(types::I16, sign64);
+ let s_pos = self.builder.ins().ishl_imm(sign16, 3);
+
+ let z = self.builder.ins().icmp_imm(IntCC::Equal, r40, 0);
+ let z16 = self.builder.ins().uextend(types::I16, z);
+ let z_pos = self.builder.ins().ishl_imm(z16, 2);
+
+ let top2 = self.builder.ins().band_imm(r40, 0xC000_0000_i64);
+ let tb0 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0);
+ let tb3 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0xC000_0000_i64);
+ let tb = self.builder.ins().bor(tb0, tb3);
+ let tb16 = self.builder.ins().uextend(types::I16, tb);
+ let tb_pos = self.builder.ins().ishl_imm(tb16, 5);
+
+ let as32_pos = self.flags40_as32_pos(r40);
+
+ let sz = self.builder.ins().bor(s_pos, z_pos);
+ let at = self.builder.ins().bor(as32_pos, tb_pos);
+ self.builder.ins().bor(sz, at)
+ }
+
+ fn flags40_carry_overflow(&mut self, a: Value, b: Value, a40: Value, r40: Value, sub: bool) -> Value {
+ use cranelift_codegen::ir::condcodes::IntCC;
+
+ let carry = if sub {
+ self.builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, a40, r40)
+ } else {
+ self.builder.ins().icmp(IntCC::UnsignedLessThan, r40, a40)
+ };
+ let c_pos = self.builder.ins().uextend(types::I16, carry);
+
+ let a_shift = self.builder.ins().ushr_imm(a, 39);
+ let a_sign = self.builder.ins().band_imm(a_shift, 1);
+ let b_shift = self.builder.ins().ushr_imm(b, 39);
+ let b_sign = self.builder.ins().band_imm(b_shift, 1);
+ let r_sign = self.builder.ins().ushr_imm(r40, 39);
+
+ let ab = if sub {
+ self.builder.ins().icmp(IntCC::NotEqual, a_sign, b_sign)
+ } else {
+ self.builder.ins().icmp(IntCC::Equal, a_sign, b_sign)
+ };
+ let ra = self.builder.ins().icmp(IntCC::NotEqual, r_sign, a_sign);
+ let overflow = self.builder.ins().band(ab, ra);
+ let o16 = self.builder.ins().uextend(types::I16, overflow);
+ let o_pos = self.builder.ins().ishl_imm(o16, 1);
+ let os_pos = self.builder.ins().ishl_imm(o16, 7);
+
+ let co = self.builder.ins().bor(c_pos, o_pos);
+ self.builder.ins().bor(co, os_pos)
+ }
+
pub fn emit_update_flags_logic(&mut self, result16: Value, ac_full: Value) {
- let result32 = self.builder.ins().uextend(types::I32, result16);
- let f = self
- .module
- .declare_func_in_func(self.extern_funcs.update_flags_logic, self.builder.func);
- self.builder.ins().call(f, &[self.sys_ptr, result32, ac_full]);
+ use cranelift_codegen::ir::condcodes::IntCC;
+
+ let sign = self.builder.ins().ushr_imm(result16, 15);
+ let s_pos = self.builder.ins().ishl_imm(sign, 3);
+
+ let z = self.builder.ins().icmp_imm(IntCC::Equal, result16, 0);
+ let z16 = self.builder.ins().uextend(types::I16, z);
+ let z_pos = self.builder.ins().ishl_imm(z16, 2);
+
+ let top2 = self.builder.ins().ushr_imm(result16, 14);
+ let tb0 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 0);
+ let tb3 = self.builder.ins().icmp_imm(IntCC::Equal, top2, 3);
+ let tb = self.builder.ins().bor(tb0, tb3);
+ let tb16 = self.builder.ins().uextend(types::I16, tb);
+ let tb_pos = self.builder.ins().ishl_imm(tb16, 5);
+
+ let r40 = self.builder.ins().band_imm(ac_full, 0xFF_FFFF_FFFF_i64);
+ let as32_pos = self.flags40_as32_pos(r40);
+
+ let sz = self.builder.ins().bor(s_pos, z_pos);
+ let at = self.builder.ins().bor(as32_pos, tb_pos);
+ let bits = self.builder.ins().bor(sz, at);
+ self.store_status_bits(bits, !0x003C_i64);
}
pub fn emit_update_flags_add(&mut self, a: Value, b: Value, result: Value) {
- let f = self
- .module
- .declare_func_in_func(self.extern_funcs.update_flags_add, self.builder.func);
- self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);
+ let r40 = self.builder.ins().band_imm(result, 0xFF_FFFF_FFFF_i64);
+ let a40 = self.builder.ins().band_imm(a, 0xFF_FFFF_FFFF_i64);
+
+ let szat = self.flags40_szat(r40);
+ let co = self.flags40_carry_overflow(a, b, a40, r40, false);
+ let bits = self.builder.ins().bor(szat, co);
+ self.store_status_bits(bits, !0x003F_i64);
}
pub fn emit_update_flags_sub(&mut self, a: Value, b: Value, result: Value) {
- let f = self
- .module
- .declare_func_in_func(self.extern_funcs.update_flags_sub, self.builder.func);
- self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);
+ let r40 = self.builder.ins().band_imm(result, 0xFF_FFFF_FFFF_i64);
+ let a40 = self.builder.ins().band_imm(a, 0xFF_FFFF_FFFF_i64);
+
+ let szat = self.flags40_szat(r40);
+ let co = self.flags40_carry_overflow(a, b, a40, r40, true);
+ let bits = self.builder.ins().bor(szat, co);
+ self.store_status_bits(bits, !0x003F_i64);
}
pub fn emit_update_flags_ac(&mut self, ac_full: Value) {
- let f = self
- .module
- .declare_func_in_func(self.extern_funcs.update_flags_ac, self.builder.func);
- self.builder.ins().call(f, &[self.sys_ptr, ac_full]);
+ let r40 = self.builder.ins().band_imm(ac_full, 0xFF_FFFF_FFFF_i64);
+ let bits = self.flags40_szat(r40);
+ self.store_status_bits(bits, !0x003C_i64);
}
pub fn emit_read_dmem(&mut self, addr_u32: Value) -> Value {
diff --git a/crates/gecko/src/flipper/dsp/jit/translator.rs b/crates/gecko/src/flipper/dsp/jit/translator.rs
index 5611b24..3972116 100644
--- a/crates/gecko/src/flipper/dsp/jit/translator.rs
+++ b/crates/gecko/src/flipper/dsp/jit/translator.rs
@@ -5,15 +5,10 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::JITModule;
use cranelift_module::{FuncId, Module};
-use super::block::BlockSpec;
+use super::block::{BlockSpec, TermKind};
#[derive(Clone, Copy)]
pub struct ExternFuncs {
- pub cache_ext_ac: FuncId,
pub loop_tail: FuncId,
- pub update_flags_logic: FuncId,
- pub update_flags_add: FuncId,
- pub update_flags_sub: FuncId,
- pub update_flags_ac: FuncId,
pub read_dmem: FuncId,
pub write_dmem: FuncId,
pub inc_ar: FuncId,
@@ -22,7 +17,6 @@ pub struct ExternFuncs {
pub decrease_ar_ix: FuncId,
pub dynamic_shift: FuncId,
pub read_imem: FuncId,
- pub write_ac_mid_sxm: FuncId,
pub call_stack_push: FuncId,
pub call_stack_pop: FuncId,
pub data_stack_pop: FuncId,
@@ -39,6 +33,7 @@ pub fn translate(
spec: &BlockSpec,
block_lookup_table_addr: i64,
entry_counter_addr: Option<usize>,
+ loop_end_table: &[u8; 0x10000],
) {
let pointer_type = module.target_config().pointer_type();
@@ -65,7 +60,9 @@ pub fn translate(
builder.ins().store(MemFlagsData::trusted(), next, slot_v, 0);
}
- for entry in &spec.instrs {
+ let block_sig_ref = builder.import_signature(block_signature(pointer_type));
+
+ for (idx, entry) in spec.instrs.iter().enumerate() {
let natural_nia = entry.pc.wrapping_add(entry.size as u16);
let nia_v = builder.ins().iconst(types::I16, natural_nia as i64);
builder.ins().store(MemFlagsData::trusted(), nia_v, ctx_ptr, nia_offset);
@@ -104,6 +101,16 @@ pub fn translate(
jit_lut::dispatch_gc_dsp_ext(&mut tctx2, crate::flipper::dsp::instruction::GcDspExt(ext_byte as u8));
}
+ let is_last = idx + 1 == spec.instrs.len();
+ let dynamic_nia = is_last && spec.terminator != TermKind::LengthLimit;
+ let marked_loop_end = loop_end_table[entry.pc as usize] != 0;
+
+ if !dynamic_nia && !marked_loop_end {
+ let pc_v = builder.ins().iconst(types::I16, natural_nia as i64);
+ builder.ins().store(MemFlagsData::trusted(), pc_v, ctx_ptr, pc_offset);
+ continue;
+ }
+
let loop_ptr = builder
.ins()
.load(types::I8, MemFlagsData::trusted(), ctx_ptr, loop_addr_ptr_offset);
@@ -141,14 +148,18 @@ pub fn translate(
builder.switch_to_block(exit_block);
builder.seal_block(exit_block);
- let pc_u32 = builder.ins().uextend(types::I32, pc_after);
- builder.ins().return_(&[pc_u32]);
+ emit_block_tail_chain(
+ &mut builder,
+ ctx_ptr,
+ block_sig_ref,
+ block_lookup_table_addr,
+ idx as i64 + 1,
+ );
builder.switch_to_block(continue_block);
builder.seal_block(continue_block);
}
- let block_sig_ref = builder.import_signature(block_signature(pointer_type));
emit_block_tail_chain(
&mut builder,
ctx_ptr,
@@ -324,30 +335,6 @@ pub fn void_thunk_signature(pointer_type: cranelift_codegen::ir::Type, host_cc:
sig
}
-pub fn flags_logic_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {
- let mut sig = Signature::new(host_cc);
- sig.params.push(AbiParam::new(pointer_type));
- sig.params.push(AbiParam::new(types::I32));
- sig.params.push(AbiParam::new(types::I64));
- sig
-}
-
-pub fn flags_arith_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {
- let mut sig = Signature::new(host_cc);
- sig.params.push(AbiParam::new(pointer_type));
- sig.params.push(AbiParam::new(types::I64));
- sig.params.push(AbiParam::new(types::I64));
- sig.params.push(AbiParam::new(types::I64));
- sig
-}
-
-pub fn flags_ac_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {
- let mut sig = Signature::new(host_cc);
- sig.params.push(AbiParam::new(pointer_type));
- sig.params.push(AbiParam::new(types::I64));
- sig
-}
-
pub fn dmem_read_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {
let mut sig = Signature::new(host_cc);
sig.params.push(AbiParam::new(pointer_type));
diff --git a/crates/gecko/src/scheduler.rs b/crates/gecko/src/scheduler.rs
index 3253a0d..069f981 100644
--- a/crates/gecko/src/scheduler.rs
+++ b/crates/gecko/src/scheduler.rs
@@ -197,10 +197,7 @@ pub fn dsp_batch_handler<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {
return;
}
- let cpu_mail_quiet = !sys.dsp.mailbox_to_dsp_hi.busy();
- let dsp_mail_full = sys.dsp.mailbox_to_cpu_hi.busy();
- let (waits_cpu, waits_dsp) = sys.dsp.mailbox_wait_state();
- let in_idle_wait = (cpu_mail_quiet && waits_cpu) || (dsp_mail_full && waits_dsp);
+ let in_idle_wait = sys.dsp.parked_in_mailbox_wait();
let pending_interrupt = sys.dsp.csr.pi_interrupt() && sys.dsp.registers.status.external_interrupt_enable();
if in_idle_wait && !pending_interrupt {
diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rs
index 43a3cb1..db2242b 100644
--- a/crates/gecko/src/system.rs
+++ b/crates/gecko/src/system.rs
@@ -457,6 +457,39 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
let dsp_suspends = crate::flipper::dsp::DSP_SUSPEND_COUNT.load(Ordering::Relaxed);
let dsp_wakes = crate::flipper::dsp::DSP_WAKE_COUNT.load(Ordering::Relaxed);
+ let drain_calls = crate::flipper::dsp::DSP_DRAIN_CALLS.load(Ordering::Relaxed);
+ let drain_steps = crate::flipper::dsp::DSP_DRAIN_STEPS.load(Ordering::Relaxed);
+ let drain_blocks = crate::flipper::dsp::DSP_DRAIN_BLOCKS.load(Ordering::Relaxed);
+ let drain_nanos = crate::flipper::dsp::DSP_DRAIN_NANOS.load(Ordering::Relaxed);
+ let drain_halt = crate::flipper::dsp::DSP_DRAIN_EXIT_HALT.load(Ordering::Relaxed);
+ let drain_answered = crate::flipper::dsp::DSP_DRAIN_EXIT_ANSWERED.load(Ordering::Relaxed);
+ let drain_wait = crate::flipper::dsp::DSP_DRAIN_EXIT_WAIT.load(Ordering::Relaxed);
+ let drain_budget = crate::flipper::dsp::DSP_DRAIN_EXIT_BUDGET.load(Ordering::Relaxed);
+ let batch_calls = crate::flipper::dsp::DSP_BATCH_CALLS.load(Ordering::Relaxed);
+ let batch_steps = crate::flipper::dsp::DSP_BATCH_STEPS.load(Ordering::Relaxed);
+ let batch_nanos = crate::flipper::dsp::DSP_BATCH_NANOS.load(Ordering::Relaxed);
+
+ let drain_avg_steps = if drain_calls > 0 {
+ drain_steps as f64 / drain_calls as f64
+ } else {
+ 0.0
+ };
+ let drain_avg_block = if drain_blocks > 0 {
+ drain_steps as f64 / drain_blocks as f64
+ } else {
+ 0.0
+ };
+ let drain_ns_per_step = if drain_steps > 0 {
+ drain_nanos as f64 / drain_steps as f64
+ } else {
+ 0.0
+ };
+ let batch_ns_per_step = if batch_steps > 0 {
+ batch_nanos as f64 / batch_steps as f64
+ } else {
+ 0.0
+ };
+
let event_breakdown = self.event_breakdown_top_n(20);
let path = self.heatmap.out_dir.join("idle-skip.txt");
@@ -467,6 +500,33 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
self.vsync_count, calls, cycles, avg, dsp_suspends, dsp_wakes
)?;
+ writeln!(
+ f,
+ "dsp_drain_calls={}\ndsp_drain_steps={}\ndsp_drain_blocks={}\ndsp_drain_ms={:.1}\ndsp_drain_avg_steps={:.1}\ndsp_drain_avg_steps_per_block={:.1}\ndsp_drain_ns_per_step={:.1}",
+ drain_calls,
+ drain_steps,
+ drain_blocks,
+ drain_nanos as f64 / 1e6,
+ drain_avg_steps,
+ drain_avg_block,
+ drain_ns_per_step
+ )?;
+
+ writeln!(
+ f,
+ "dsp_drain_exit_halt={}\ndsp_drain_exit_answered={}\ndsp_drain_exit_wait={}\ndsp_drain_exit_budget={}",
+ drain_halt, drain_answered, drain_wait, drain_budget
+ )?;
+
+ writeln!(
+ f,
+ "dsp_batch_calls={}\ndsp_batch_steps={}\ndsp_batch_ms={:.1}\ndsp_batch_ns_per_step={:.1}",
+ batch_calls,
+ batch_steps,
+ batch_nanos as f64 / 1e6,
+ batch_ns_per_step
+ )?;
+
writeln!(f, "\n--- top scheduler events by fire count ---")?;
for (name, count) in &event_breakdown {