A GameCube/Wii emulator and debugger written in Rust.
merge dsp jit
Layleebb1b40b08fb
diff --git a/crates/gecko/build.rs b/crates/gecko/build.rsindex 2eb6c25..d8048ed 100644--- a/crates/gecko/build.rs+++ b/crates/gecko/build.rs@@ -5,4 +5,5 @@ fn main() {chipi_build::generate_bindings("wii_dsp.bindings.chipi").expect("chipi codegen failed (wii dsp)");chipi_build::generate_bindings("gekko_jit.bindings.chipi").expect("chipi codegen failed (gekko jit)");chipi_build::generate_bindings("wii_gekko_jit.bindings.chipi").expect("chipi codegen failed (wii gekko jit)");+ chipi_build::generate_bindings("dsp_jit.bindings.chipi").expect("chipi codegen failed (dsp jit)");}diff --git a/crates/gecko/dsp_jit.bindings.chipi b/crates/gecko/dsp_jit.bindings.chipinew file mode 100644index 0000000..459c8c7--- /dev/null+++ b/crates/gecko/dsp_jit.bindings.chipi@@ -0,0 +1,228 @@+include "../../submodules/chipi-spec/gamecube/dsp.chipi"++target rust {+ dispatch GcDsp {+ output "$OUT_DIR/dsp_jit_lut.rs"++ context crate::flipper::dsp::jit::translate::TranslatorCtx+ handlers crate::flipper::dsp::jit::translate+ handler_const crate::system::GC+ strategy fn_ptr_lut++ invalid_handler crate::flipper::dsp::jit::translate::invalid++ instruction_type crate::flipper::dsp::instruction::Instruction++ handler control {+ nop+ halt+ ifcc+ jcc+ callcc+ retcc+ rticc+ jrcc+ callrcc+ }++ handler loops {+ loop_reg+ bloop+ loopi+ bloopi+ }++ handler addr_reg {+ dar+ iar+ subarn+ addarn+ }++ handler load_store {+ lri+ lr+ sr+ mrr+ si+ lrr+ lrrd+ lrri+ lrrn+ srr+ srrd+ srri+ srrn+ lrs+ srsh+ srs+ ilrr+ ilrrd+ ilrri+ ilrrn+ }++ handler imm_alu {+ addi+ xori+ andi+ ori+ cmpi+ andf+ andcf+ addis+ cmpis+ lris+ }++ handler add_sub {+ addr+ addax+ add+ addp+ subr+ subax+ sub+ subp+ addaxl+ addpaxz+ }++ handler move_ops {+ movr+ movax+ mov+ movp+ movpz+ movnp+ }++ handler inc_dec {+ incm+ inc+ decm+ dec+ neg+ abs_ac+ }++ handler cmp_test {+ cmp+ cmpaxh+ tst+ tstprod+ tstaxh+ nx_0+ nx_1+ clr+ clrp+ clrl+ }++ handler logic {+ xorr+ andr+ orr+ andc+ orc+ xorc+ not_ac+ }++ handler shifts {+ lsl+ lsr+ asl+ asr+ lsl16+ lsr16+ asr16+ lsrn+ asrn+ lsrnrx+ asrnrx+ lsrnr+ asrnr+ }++ handler status {+ sbclr+ sbset+ m2+ m0+ clr15+ set15+ set16+ set40+ }++ handler mul {+ mul+ mulx+ mulc+ mulaxh+ mulmv+ mulmvz+ mulac+ mulxmv+ mulxmvz+ mulxac+ mulcmv+ mulcmvz+ mulcac+ madd+ msub+ maddx+ msubx+ maddc+ msubc+ }++ subdispatch GcDspExt {+ invalid_handler crate::flipper::dsp::jit::translate::invalid_ext++ instruction_type crate::flipper::dsp::instruction::GcDspExt++ handler ext_addr {+ ext_dr+ ext_ir+ ext_nr+ }++ handler ext_load {+ ext_l+ ext_ln+ }++ handler ext_store {+ ext_s+ ext_sn+ }++ handler ext_load_store {+ ext_ls+ ext_sl+ ext_lsn+ ext_sln+ ext_lsm+ ext_slm+ ext_lsnm+ ext_slnm+ }++ handler ext_ld {+ ext_ld_00+ ext_ldn_01+ ext_ldm_10+ ext_ldnm_11+ }++ handler ext_ldax {+ ext_ldax+ ext_ldaxn+ ext_ldaxm+ ext_ldaxnm+ }+ }+ }+}diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rsindex bde0642..6f3a00c 100644--- a/crates/gecko/src/flipper/dsp.rs+++ b/crates/gecko/src/flipper/dsp.rs@@ -5,6 +5,8 @@ pub mod core;#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]pub mod instruction;pub mod interpreter;+#[cfg(feature = "jit")]+pub mod jit;pub mod regs;#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]@@ -21,23 +23,63 @@ use crate::flipper::dsp::instruction::Instruction;use crate::mmio::Mmio;use crate::system::{System, SystemId};+#[cfg(feature = "jit")]+pub const DSP_JIT_CHAIN_BUDGET: u32 = 16;++#[cfg(feature = "jit")]+pub trait DspJitHandle {+ fn run_block(&mut self, ctx_ptr: *mut ::core::ffi::c_void, iram: &[u8], irom: &[u8], start_pc: u16) -> u16;+ fn record_chain_depth(&mut self, depth: u32);+ fn flush(&mut self);+ fn dump_hot_blocks(&self, top_k: usize);+ fn dump_top_clif(&mut self, top_k: usize, iram: &[u8], irom: &[u8]);+}++#[cfg(feature = "jit")]+impl<const SYSTEM: SystemId> DspJitHandle for jit::JitEngine<SYSTEM> {+ fn run_block(&mut self, ctx_ptr: *mut ::core::ffi::c_void, iram: &[u8], irom: &[u8], start_pc: u16) -> u16 {+ let entry = self.lookup_or_compile(iram, irom, start_pc);+ Self::run_block(self, ctx_ptr, entry)+ }+ fn record_chain_depth(&mut self, depth: u32) {+ Self::record_chain_depth(self, depth);+ }+ fn flush(&mut self) {+ Self::flush(self);+ }+ fn dump_hot_blocks(&self, _top_k: usize) {+ #[cfg(feature = "jit-stats")]+ Self::dump_hot_blocks(self, _top_k);+ #[cfg(not(feature = "jit-stats"))]+ eprintln!("[dsp-jit-stats] feature `jit-stats` is not enabled — rebuild with `--features jit-stats`");+ }+ fn dump_top_clif(&mut self, _top_k: usize, _iram: &[u8], _irom: &[u8]) {+ #[cfg(feature = "jit-stats")]+ {+ let mut pcs: Vec<(u16, u64)> = self.hits.iter().map(|(&pc, &n)| (pc, n)).collect();+ pcs.sort_by(|a, b| b.1.cmp(&a.1));+ for (pc, hits) in pcs.into_iter().take(_top_k) {+ eprintln!("\n[dsp-jit-stats] hits={hits} pc={pc:04X}");+ self.dump_block_clif(pc, _iram, _irom);+ }+ }+ #[cfg(not(feature = "jit-stats"))]+ eprintln!("[dsp-jit-stats] feature `jit-stats` is not enabled — rebuild with `--features jit-stats`");+ }+}+pub struct Dsp {- // Registerspub registers: core::Registers,- // IMEM = IRAM + IROM- pub iram: Box<[u8; 0x2000]>, // 0x0000 - 0x0FFF- pub irom: Box<[u8; 0x2000]>, // 0x8000 - 0x8FFF+ pub iram: Box<[u8; 0x2000]>,+ pub irom: Box<[u8; 0x2000]>,- // DMEM = DRAM + COEF + IFX- pub dram: Box<[u8; 0x2000]>, // 0x0000 - 0x0FFF (0x1000 words)- pub coef: Box<[u8; 0x2000]>, // 0x1000 - 0x1FFF (0x1000 words)- pub ifx: Box<[u8; 0x200]>, // 0xFF00 - 0xFFFF (0x100 words)+ pub dram: Box<[u8; 0x2000]>,+ pub coef: Box<[u8; 0x2000]>,+ pub ifx: Box<[u8; 0x200]>,- // Auxiliary RAM (16 MB)pub aram: Box<[u8; 16 * 1024 * 1024]>,- // I/O Registerspub csr: regs::ControlStatus,pub mailbox_to_dsp_hi: regs::MailboxToDspHi,pub mailbox_to_dsp_lo: regs::MailboxToDspLo,@@ -52,15 +94,24 @@ pub struct Dsp {pub audio_dma_start_addr: regs::AudioDmaStartAddr,pub audio_dma_control: regs::AudioDmaControl,- // IFX Registerspub dma_control: core::regs::DspDmaControl,pub dma_length: u16,pub dma_dsp_addr: u16,pub dma_ram_addr_hi: u16,pub dma_ram_addr_lo: u16,- // Audio sample accelerator (DSP IFX 0xFFD0..0xFFDF).pub accelerator: accelerator::Accelerator,++ #[cfg(feature = "jit")]+ pub jit: Option<Box<dyn DspJitHandle + Send>>,++ #[cfg(feature = "jit")]+ pub chain_budget: u32,++ #[cfg(feature = "jit")]+ pub instr_count: u32,++ pub poll_cache: std::cell::Cell<Option<(u16, bool, bool)>>,}impl Dsp {@@ -99,7 +150,97 @@ impl Dsp {dma_ram_addr_hi: 0,dma_ram_addr_lo: 0,accelerator: accelerator::Accelerator::new(),++ #[cfg(feature = "jit")]+ jit: None,+ #[cfg(feature = "jit")]+ chain_budget: 0,+ #[cfg(feature = "jit")]+ instr_count: 0,+ poll_cache: std::cell::Cell::new(None),+ }+ }++ #[inline]+ pub fn is_waiting_for_cpu_mail(&self) -> bool {+ let pc = self.registers.pc;+ if let Some((cached_pc, cpu, _dsp)) = self.poll_cache.get() {+ if cached_pc == pc {+ return cpu;+ }+ }++ let cpu = (0i16..=0)+ .chain(std::iter::once(-1))+ .chain(std::iter::once(-3))+ .any(|offset| self.matches_cpu_mail_wait(offset));+ let dsp = (0i16..=0)+ .chain(std::iter::once(-1))+ .chain(std::iter::once(-3))+ .any(|offset| self.matches_dsp_mail_wait(offset));++ self.poll_cache.set(Some((pc, cpu, dsp)));++ cpu+ }++ fn matches_cpu_mail_wait(&self, offset: i16) -> bool {+ let pc = self.registers.pc;+ let start = pc.wrapping_add_signed(offset);+ let words = [+ self.read_imem(start),+ self.read_imem(start.wrapping_add(1)),+ self.read_imem(start.wrapping_add(2)),+ self.read_imem(start.wrapping_add(3)),+ self.read_imem(start.wrapping_add(4)),+ ];++ let pattern_a = [0x26FE, 0x02C0, 0x8000, 0x029C, start];+ let pattern_b = [0x27FE, 0x03C0, 0x8000, 0x029C, start];+ let pattern_c = [0x26FE, 0x02A0, 0x8000, 0x029D, start];+ let pattern_d = [0x27FE, 0x03A0, 0x8000, 0x029D, start];+ words == pattern_a || words == pattern_b || words == pattern_c || words == pattern_d+ }++ #[inline]+ pub fn is_waiting_for_dsp_mail(&self) -> bool {+ let pc = self.registers.pc;+ if let Some((cached_pc, _cpu, dsp)) = self.poll_cache.get() {+ if cached_pc == pc {+ return dsp;+ }}++ let cpu = (0i16..=0)+ .chain(std::iter::once(-1))+ .chain(std::iter::once(-3))+ .any(|offset| self.matches_cpu_mail_wait(offset));+ let dsp = (0i16..=0)+ .chain(std::iter::once(-1))+ .chain(std::iter::once(-3))+ .any(|offset| self.matches_dsp_mail_wait(offset));++ self.poll_cache.set(Some((pc, cpu, dsp)));++ dsp+ }++ fn matches_dsp_mail_wait(&self, offset: i16) -> bool {+ let pc = self.registers.pc;+ let start = pc.wrapping_add_signed(offset);+ let words = [+ self.read_imem(start),+ self.read_imem(start.wrapping_add(1)),+ self.read_imem(start.wrapping_add(2)),+ self.read_imem(start.wrapping_add(3)),+ self.read_imem(start.wrapping_add(4)),+ ];++ let pattern_a = [0x26FC, 0x02C0, 0x8000, 0x029D, start];+ let pattern_b = [0x27FC, 0x03C0, 0x8000, 0x029D, start];+ let pattern_c = [0x26FC, 0x02A0, 0x8000, 0x029C, start];+ let pattern_d = [0x27FC, 0x03A0, 0x8000, 0x029C, start];+ words == pattern_a || words == pattern_b || words == pattern_c || words == pattern_d}pub fn process_aram_dma<const SYSTEM: SystemId>(&mut self, mmio: &mut Mmio<SYSTEM>) {@@ -148,11 +289,18 @@ impl Dsp {self.csr.set_dma_status(false);self.csr.set_dsp_interrupt(true);++ self.poll_cache.set(None);++ #[cfg(feature = "jit")]+ if let Some(jit) = self.jit.as_mut() {+ jit.flush();+ }}pub fn process_dsp_dma<const SYSTEM: SystemId>(&mut self, mmio: &mut Mmio<SYSTEM>) {let ram_addr = ((self.dma_ram_addr_hi as u32) << 16) | self.dma_ram_addr_lo as u32;- let dsp_addr = (self.dma_dsp_addr as usize) * 2; // word address -> byte offset+ let dsp_addr = (self.dma_dsp_addr as usize) * 2;let len = self.dma_length as usize;tracing::debug!(@@ -169,7 +317,9 @@ impl Dsp {core::regs::DspMemoryType::Instruction => &mut *self.iram,};- match self.dma_control.direction() {+ let mem_type = self.dma_control.memory_type();+ let direction = self.dma_control.direction();+ match direction {core::regs::DspDmaDirection::MainToDsp => {let src = mmio.virt_slice(ram_addr, len);memory[dsp_addr..dsp_addr + len].copy_from_slice(&src);@@ -181,20 +331,31 @@ impl Dsp {}}+ if matches!(+ (mem_type, direction),+ (+ core::regs::DspMemoryType::Instruction,+ core::regs::DspDmaDirection::MainToDsp+ )+ ) {+ self.poll_cache.set(None);+ #[cfg(feature = "jit")]+ if let Some(jit) = self.jit.as_mut() {+ jit.flush();+ }+ }+self.dma_length = 0;}}impl<const SYSTEM: SystemId> System<SYSTEM> {- /// Execute a single DSP instruction. Returns `false` if the DSP is halted or- /// in reset (no instruction was executed).#[inline(always)]pub fn step_dsp_instruction(&mut self) -> bool {if self.dsp.csr.reset() || self.dsp.csr.halt() {return false;}- // Check for external interrupt (CPU->DSP mailbox interrupt via CSR bit 1)if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);self.dsp.registers.call_stack.push(self.dsp.registers.pc);@@ -223,7 +384,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self::dispatch_gc_dsp_ext(self, instruction::GcDspExt(ext));}- // Check if we've reached the end of a loop.let at_loop_end =!self.dsp.registers.loop_addr.is_empty() && self.dsp.registers.nia == self.dsp.registers.loop_addr.top();if at_loop_end {@@ -242,7 +402,66 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {true}- /// Execute a batch of DSP instructions (scheduler hot path).+ #[cfg(feature = "jit")]+ pub fn execute_dsp_batch(&mut self) {+ if self.dsp.csr.reset() || self.dsp.csr.halt() {+ self::refresh_interrupts(self);+ return;+ }++ if self.dsp.jit.is_none() {+ self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));+ }++ let mut budget = crate::scheduler::DSP_BATCH_SIZE as u64;+ while budget > 0 {+ let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();+ let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();++ if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())+ || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())+ {+ break;+ }++ if self.dsp.csr.reset() || self.dsp.csr.halt() {+ break;+ }++ if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {+ self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);+ self.dsp.registers.call_stack.push(self.dsp.registers.pc);+ self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());+ self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);+ self.dsp.registers.pc = 0x000E;+ }++ let start_pc = self.dsp.registers.pc;+ self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;+ self.dsp.instr_count = 0;++ let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;+ let iram_ptr = self.dsp.iram.as_ptr();+ let irom_ptr = self.dsp.irom.as_ptr();+ let iram_len = self.dsp.iram.len();+ let irom_len = self.dsp.irom.len();+ let iram = unsafe { ::core::slice::from_raw_parts(iram_ptr, iram_len) };+ let irom = unsafe { ::core::slice::from_raw_parts(irom_ptr, irom_len) };++ let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);+ self.dsp.registers.pc = next_pc;++ let consumed = (self.dsp.instr_count as u64).max(1);+ budget = budget.saturating_sub(consumed);++ let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;+ self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);+ }++ self::refresh_interrupts(self);+ }++ #[cfg(not(feature = "jit"))]#[inline(always)]pub fn execute_dsp_batch(&mut self) {for _ in 0..crate::scheduler::DSP_BATCH_SIZE {@@ -253,11 +472,71 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self::refresh_interrupts(self);}- /// Run the DSP synchronously up to `max_steps` instructions or until- /// the DSP-to-CPU mailbox is set busy (i.e. the DSP has answered the- /// command). Used by AID DMA path to make the CPUs mailbox writes- /// behave as if the DSP processed them immediately, so AX has time to- /// render and DMA the buffer before AID consumes it.+ #[cfg(feature = "jit")]+ pub fn drain_dsp_synchronous(&mut self, max_steps: u32) {+ let already_busy = self.dsp.mailbox_to_cpu_hi.busy();++ if self.dsp.csr.reset() || self.dsp.csr.halt() {+ self::refresh_interrupts(self);+ return;+ }++ if self.dsp.jit.is_none() {+ self.dsp.jit = Some(Box::new(jit::JitEngine::<SYSTEM>::new()));+ }++ let mut budget = max_steps as u64;+ while budget > 0 {+ if self.dsp.csr.reset() || self.dsp.csr.halt() {+ break;+ }++ if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {+ break;+ }++ let cpu_mail_quiet = !self.dsp.mailbox_to_dsp_hi.busy();+ let dsp_mail_full = self.dsp.mailbox_to_cpu_hi.busy();+ if (cpu_mail_quiet && self.dsp.is_waiting_for_cpu_mail())+ || (dsp_mail_full && self.dsp.is_waiting_for_dsp_mail())+ {+ break;+ }++ if self.dsp.csr.pi_interrupt() && self.dsp.registers.status.external_interrupt_enable() {+ self.dsp.csr = self.dsp.csr.with_pi_interrupt(false);+ self.dsp.registers.call_stack.push(self.dsp.registers.pc);+ self.dsp.registers.data_stack.push(self.dsp.registers.status.raw());+ self.dsp.registers.status = self.dsp.registers.status.with_external_interrupt_enable(false);+ self.dsp.registers.pc = 0x000E;+ }++ let start_pc = self.dsp.registers.pc;+ self.dsp.chain_budget = DSP_JIT_CHAIN_BUDGET;+ self.dsp.instr_count = 0;++ let ctx_ptr = self as *mut crate::system::System<SYSTEM> as *mut ::core::ffi::c_void;+ let iram_ptr = self.dsp.iram.as_ptr();+ let irom_ptr = self.dsp.irom.as_ptr();+ let iram_len = self.dsp.iram.len();+ let irom_len = self.dsp.irom.len();+ let iram = unsafe { ::core::slice::from_raw_parts(iram_ptr, iram_len) };+ let irom = unsafe { ::core::slice::from_raw_parts(irom_ptr, irom_len) };++ let next_pc = self.dsp.jit.as_mut().unwrap().run_block(ctx_ptr, iram, irom, start_pc);+ self.dsp.registers.pc = next_pc;++ let consumed = (self.dsp.instr_count as u64).max(1);+ budget = budget.saturating_sub(consumed);++ let chain_depth = DSP_JIT_CHAIN_BUDGET - self.dsp.chain_budget;+ self.dsp.jit.as_mut().unwrap().record_chain_depth(chain_depth);+ }++ self::refresh_interrupts(self);+ }++ #[cfg(not(feature = "jit"))]pub fn drain_dsp_synchronous(&mut self, max_steps: u32) {let already_busy = self.dsp.mailbox_to_cpu_hi.busy();@@ -267,7 +546,6 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}if !already_busy && self.dsp.mailbox_to_cpu_hi.busy() {- // DSP responded; assume AX render is complete.break;}}@@ -298,7 +576,6 @@ crate::mmio_device_dispatch! {}impl Dsp {- /// Read a 16-bit word from instruction memory (IRAM 0x0000-0x0FFF, IROM 0x8000-0x8FFF).#[inline(always)]pub fn read_imem(&self, addr: u16) -> u16 {match addr {@@ -308,14 +585,12 @@ impl Dsp {}}- /// Load a binary file into IROM.pub fn load_irom(&mut self, data: &[u8]) {let len = data.len().min(self.irom.len());self.irom[..len].copy_from_slice(&data[..len]);tracing::info!(size = len, "loaded DSP IROM");}- /// Load a binary file into the coefficient ROM (DMEM 0x1000..0x1FFF).pub fn load_coef(&mut self, data: &[u8]) {let len = data.len().min(self.coef.len());self.coef[..len].copy_from_slice(&data[..len]);@@ -425,7 +700,7 @@ fn write_ifx<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, addr: u16, value:}// CMBH/CMBL are read-only from DSP sideaddr::IFX_CMBH | addr::IFX_CMBL => {}- // DSP DMA: writing DSBL triggers the transfer, run inline.+addr::IFX_DSBL => {sys.dsp.dma_length = value;sys.dsp.process_dsp_dma(&mut sys.mmio);diff --git a/crates/gecko/src/flipper/dsp/jit.rs b/crates/gecko/src/flipper/dsp/jit.rsnew file mode 100644index 0000000..f1d2b18--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit.rs@@ -0,0 +1,701 @@+pub mod abi;+pub mod block;+pub mod runtime;+pub mod translate;+pub mod translator;++#[allow(dead_code, unused_variables, non_upper_case_globals, clippy::all)]+pub mod jit_lut {+ include!(concat!(env!("OUT_DIR"), "/dsp_jit_lut.rs"));+}++use cranelift_codegen::Context;+use cranelift_codegen::ir::Signature;+use cranelift_frontend::FunctionBuilderContext;+use cranelift_jit::{JITBuilder, JITModule};+use cranelift_module::{FuncId, Linkage, Module, default_libcall_names};+use rustc_hash::FxHashMap;++use crate::system::{GC, SystemId, WII};++pub type BlockEntry = usize;++type TrampolineFn = unsafe extern "C" fn(*mut core::ffi::c_void, usize) -> u32;++#[repr(C)]+#[derive(Copy, Clone)]+pub struct DspBlockLookupSlot {+ pub pc: u32,+ pub _pad: u32,+ pub entry: usize,+}++pub const DSP_BLOCK_LOOKUP_TABLE_SIZE: usize = 8192;+pub const DSP_BLOCK_LOOKUP_TABLE_MASK: u32 = (DSP_BLOCK_LOOKUP_TABLE_SIZE as u32) - 1;++const CHAIN_DEPTH_HISTOGRAM_LEN: usize = super::DSP_JIT_CHAIN_BUDGET as usize + 2;++pub struct JitEngine<const SYSTEM: SystemId> {+ module: JITModule,+ ctx: Context,+ builder_ctx: FunctionBuilderContext,+ cache: FxHashMap<u16, BlockEntry>,+ block_func_ids: FxHashMap<u16, FuncId>,+ block_sig: Signature,+ extern_funcs: translator::ExternFuncs,+ trampoline_fn: TrampolineFn,+ block_seq: u64,+ block_lookup_table_addr: usize,+ chain_depth_total: u64,+ dispatcher_entries_total: u64,+ chain_depth_histogram: [u64; CHAIN_DEPTH_HISTOGRAM_LEN],+ #[cfg(feature = "jit-stats")]+ pub(crate) hits: FxHashMap<u16, u64>,+ #[cfg(feature = "jit-stats")]+ block_specs: FxHashMap<u16, block::BlockSpec>,+ #[cfg(feature = "jit-stats")]+ block_entry_counter_ptrs: FxHashMap<u16, usize>,+}++impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {+ pub fn new() -> Self {+ use cranelift_codegen::settings::{self, Configurable};++ let mut flag_builder = settings::builder();+ flag_builder.set("use_colocated_libcalls", "false").unwrap();+ flag_builder.set("is_pic", "false").unwrap();+ flag_builder.set("opt_level", "speed").unwrap();+ flag_builder.set("preserve_frame_pointers", "true").unwrap();+ flag_builder.set("enable_verifier", "false").unwrap();+ flag_builder.set("enable_probestack", "false").unwrap();+ flag_builder.set("unwind_info", "false").unwrap();+ flag_builder+ .set("enable_heap_access_spectre_mitigation", "false")+ .unwrap();+ flag_builder+ .set("enable_table_access_spectre_mitigation", "false")+ .unwrap();+ let isa_builder = cranelift_native::builder().expect("host ISA");+ let isa = isa_builder+ .finish(settings::Flags::new(flag_builder))+ .expect("ISA finish");++ let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());++ struct Syms {+ cache_ext_ac: (&'static str, *const u8),+ loop_tail: (&'static str, *const u8),+ update_flags_logic: (&'static str, *const u8),+ update_flags_add: (&'static str, *const u8),+ update_flags_sub: (&'static str, *const u8),+ update_flags_ac: (&'static str, *const u8),+ read_dmem: (&'static str, *const u8),+ write_dmem: (&'static str, *const u8),+ inc_ar: (&'static str, *const u8),+ dec_ar: (&'static str, *const u8),+ increase_ar: (&'static str, *const u8),+ decrease_ar_ix: (&'static str, *const u8),+ dynamic_shift: (&'static str, *const u8),+ read_imem: (&'static str, *const u8),+ write_ac_mid_sxm: (&'static str, *const u8),+ call_stack_push: (&'static str, *const u8),+ call_stack_pop: (&'static str, *const u8),+ data_stack_pop: (&'static str, *const u8),+ read_reg_full: (&'static str, *const u8),+ write_reg_full: (&'static str, *const u8),+ loop_setup: (&'static str, *const u8),+ }+ let syms: Syms = match SYSTEM {+ GC => Syms {+ cache_ext_ac: (+ "gecko_dsp_jit_cache_ext_ac_gc",+ runtime::dsp_cache_ext_ac_gc as *const u8,+ ),+ loop_tail: ("gecko_dsp_jit_loop_tail_gc", runtime::dsp_loop_tail_gc as *const u8),+ update_flags_logic: (+ "gecko_dsp_jit_uflog_gc",+ runtime::dsp_update_flags_logic_gc as *const u8,+ ),+ update_flags_add: ("gecko_dsp_jit_ufadd_gc", runtime::dsp_update_flags_add_gc as *const u8),+ update_flags_sub: ("gecko_dsp_jit_ufsub_gc", runtime::dsp_update_flags_sub_gc as *const u8),+ update_flags_ac: ("gecko_dsp_jit_ufac_gc", runtime::dsp_update_flags_ac_gc as *const u8),+ read_dmem: ("gecko_dsp_jit_rdmem_gc", runtime::dsp_read_dmem_gc as *const u8),+ write_dmem: ("gecko_dsp_jit_wdmem_gc", runtime::dsp_write_dmem_gc as *const u8),+ inc_ar: ("gecko_dsp_jit_incar_gc", runtime::dsp_increment_ar_gc as *const u8),+ dec_ar: ("gecko_dsp_jit_decar_gc", runtime::dsp_decrement_ar_gc as *const u8),+ increase_ar: ("gecko_dsp_jit_incrar_gc", runtime::dsp_increase_ar_gc as *const u8),+ decrease_ar_ix: ("gecko_dsp_jit_decarix_gc", runtime::dsp_decrease_ar_ix_gc as *const u8),+ dynamic_shift: ("gecko_dsp_jit_dynshift_gc", runtime::dsp_dynamic_shift_gc as *const u8),+ read_imem: ("gecko_dsp_jit_rimem_gc", runtime::dsp_read_imem_gc as *const u8),+ write_ac_mid_sxm: ("gecko_dsp_jit_wamsxm_gc", runtime::dsp_write_ac_mid_sxm_gc as *const u8),+ call_stack_push: ("gecko_dsp_jit_cspush_gc", runtime::dsp_call_stack_push_gc as *const u8),+ call_stack_pop: ("gecko_dsp_jit_cspop_gc", runtime::dsp_call_stack_pop_gc as *const u8),+ data_stack_pop: ("gecko_dsp_jit_dspop_gc", runtime::dsp_data_stack_pop_gc as *const u8),+ read_reg_full: ("gecko_dsp_jit_rdregf_gc", runtime::dsp_read_reg_full_gc as *const u8),+ write_reg_full: ("gecko_dsp_jit_wrregf_gc", runtime::dsp_write_reg_full_gc as *const u8),+ loop_setup: ("gecko_dsp_jit_loopsetup_gc", runtime::dsp_loop_setup_gc as *const u8),+ },+ WII => Syms {+ cache_ext_ac: (+ "gecko_dsp_jit_cache_ext_ac_wii",+ runtime::dsp_cache_ext_ac_wii as *const u8,+ ),+ loop_tail: ("gecko_dsp_jit_loop_tail_wii", runtime::dsp_loop_tail_wii as *const u8),+ update_flags_logic: (+ "gecko_dsp_jit_uflog_wii",+ runtime::dsp_update_flags_logic_wii as *const u8,+ ),+ update_flags_add: (+ "gecko_dsp_jit_ufadd_wii",+ runtime::dsp_update_flags_add_wii as *const u8,+ ),+ update_flags_sub: (+ "gecko_dsp_jit_ufsub_wii",+ runtime::dsp_update_flags_sub_wii as *const u8,+ ),+ update_flags_ac: ("gecko_dsp_jit_ufac_wii", runtime::dsp_update_flags_ac_wii as *const u8),+ read_dmem: ("gecko_dsp_jit_rdmem_wii", runtime::dsp_read_dmem_wii as *const u8),+ write_dmem: ("gecko_dsp_jit_wdmem_wii", runtime::dsp_write_dmem_wii as *const u8),+ inc_ar: ("gecko_dsp_jit_incar_wii", runtime::dsp_increment_ar_wii as *const u8),+ dec_ar: ("gecko_dsp_jit_decar_wii", runtime::dsp_decrement_ar_wii as *const u8),+ increase_ar: ("gecko_dsp_jit_incrar_wii", runtime::dsp_increase_ar_wii as *const u8),+ decrease_ar_ix: (+ "gecko_dsp_jit_decarix_wii",+ runtime::dsp_decrease_ar_ix_wii as *const u8,+ ),+ dynamic_shift: (+ "gecko_dsp_jit_dynshift_wii",+ runtime::dsp_dynamic_shift_wii as *const u8,+ ),+ read_imem: ("gecko_dsp_jit_rimem_wii", runtime::dsp_read_imem_wii as *const u8),+ write_ac_mid_sxm: (+ "gecko_dsp_jit_wamsxm_wii",+ runtime::dsp_write_ac_mid_sxm_wii as *const u8,+ ),+ call_stack_push: (+ "gecko_dsp_jit_cspush_wii",+ runtime::dsp_call_stack_push_wii as *const u8,+ ),+ call_stack_pop: ("gecko_dsp_jit_cspop_wii", runtime::dsp_call_stack_pop_wii as *const u8),+ data_stack_pop: ("gecko_dsp_jit_dspop_wii", runtime::dsp_data_stack_pop_wii as *const u8),+ read_reg_full: ("gecko_dsp_jit_rdregf_wii", runtime::dsp_read_reg_full_wii as *const u8),+ write_reg_full: ("gecko_dsp_jit_wrregf_wii", runtime::dsp_write_reg_full_wii as *const u8),+ loop_setup: ("gecko_dsp_jit_loopsetup_wii", runtime::dsp_loop_setup_wii as *const u8),+ },+ _ => unreachable!(),+ };+ jit_builder.symbol(syms.cache_ext_ac.0, syms.cache_ext_ac.1);+ jit_builder.symbol(syms.loop_tail.0, syms.loop_tail.1);+ jit_builder.symbol(syms.update_flags_logic.0, syms.update_flags_logic.1);+ jit_builder.symbol(syms.update_flags_add.0, syms.update_flags_add.1);+ jit_builder.symbol(syms.update_flags_sub.0, syms.update_flags_sub.1);+ jit_builder.symbol(syms.update_flags_ac.0, syms.update_flags_ac.1);+ jit_builder.symbol(syms.read_dmem.0, syms.read_dmem.1);+ jit_builder.symbol(syms.write_dmem.0, syms.write_dmem.1);+ jit_builder.symbol(syms.inc_ar.0, syms.inc_ar.1);+ jit_builder.symbol(syms.dec_ar.0, syms.dec_ar.1);+ jit_builder.symbol(syms.increase_ar.0, syms.increase_ar.1);+ jit_builder.symbol(syms.decrease_ar_ix.0, syms.decrease_ar_ix.1);+ jit_builder.symbol(syms.dynamic_shift.0, syms.dynamic_shift.1);+ jit_builder.symbol(syms.read_imem.0, syms.read_imem.1);+ jit_builder.symbol(syms.write_ac_mid_sxm.0, syms.write_ac_mid_sxm.1);+ jit_builder.symbol(syms.call_stack_push.0, syms.call_stack_push.1);+ jit_builder.symbol(syms.call_stack_pop.0, syms.call_stack_pop.1);+ jit_builder.symbol(syms.data_stack_pop.0, syms.data_stack_pop.1);+ jit_builder.symbol(syms.read_reg_full.0, syms.read_reg_full.1);+ jit_builder.symbol(syms.write_reg_full.0, syms.write_reg_full.1);+ jit_builder.symbol(syms.loop_setup.0, syms.loop_setup.1);++ let mut module = JITModule::new(jit_builder);++ let pointer_type = module.target_config().pointer_type();+ let host_cc = module.target_config().default_call_conv;++ let block_sig = translator::block_signature(pointer_type);+ let void_thunk_sig = translator::void_thunk_signature(pointer_type, host_cc);+ let flags_logic_sig = translator::flags_logic_signature(pointer_type, host_cc);+ let flags_arith_sig = translator::flags_arith_signature(pointer_type, host_cc);+ let flags_ac_sig = translator::flags_ac_signature(pointer_type, host_cc);+ let dmem_read_sig = translator::dmem_read_signature(pointer_type, host_cc);+ let dmem_write_sig = translator::dmem_write_signature(pointer_type, host_cc);+ let ar_unary_sig = translator::ar_unary_signature(pointer_type, host_cc);+ let ar_binary_sig = translator::ar_binary_signature(pointer_type, host_cc);+ let dynamic_shift_sig = translator::dynamic_shift_signature(pointer_type, host_cc);++ let cache_ext_ac_id = module+ .declare_function(syms.cache_ext_ac.0, Linkage::Import, &void_thunk_sig)+ .expect("declare cache_ext_ac thunk");+ let loop_tail_id = module+ .declare_function(syms.loop_tail.0, Linkage::Import, &void_thunk_sig)+ .expect("declare loop_tail thunk");+ let update_flags_logic_id = module+ .declare_function(syms.update_flags_logic.0, Linkage::Import, &flags_logic_sig)+ .expect("declare update_flags_logic thunk");+ let update_flags_add_id = module+ .declare_function(syms.update_flags_add.0, Linkage::Import, &flags_arith_sig)+ .expect("declare update_flags_add thunk");+ let update_flags_sub_id = module+ .declare_function(syms.update_flags_sub.0, Linkage::Import, &flags_arith_sig)+ .expect("declare update_flags_sub thunk");+ let update_flags_ac_id = module+ .declare_function(syms.update_flags_ac.0, Linkage::Import, &flags_ac_sig)+ .expect("declare update_flags_ac thunk");+ let read_dmem_id = module+ .declare_function(syms.read_dmem.0, Linkage::Import, &dmem_read_sig)+ .expect("declare read_dmem thunk");+ let write_dmem_id = module+ .declare_function(syms.write_dmem.0, Linkage::Import, &dmem_write_sig)+ .expect("declare write_dmem thunk");+ let inc_ar_id = module+ .declare_function(syms.inc_ar.0, Linkage::Import, &ar_unary_sig)+ .expect("declare inc_ar thunk");+ let dec_ar_id = module+ .declare_function(syms.dec_ar.0, Linkage::Import, &ar_unary_sig)+ .expect("declare dec_ar thunk");+ let increase_ar_id = module+ .declare_function(syms.increase_ar.0, Linkage::Import, &ar_binary_sig)+ .expect("declare increase_ar thunk");+ let decrease_ar_ix_id = module+ .declare_function(syms.decrease_ar_ix.0, Linkage::Import, &ar_binary_sig)+ .expect("declare decrease_ar_ix thunk");+ let dynamic_shift_id = module+ .declare_function(syms.dynamic_shift.0, Linkage::Import, &dynamic_shift_sig)+ .expect("declare dynamic_shift thunk");+ let read_imem_id = module+ .declare_function(syms.read_imem.0, Linkage::Import, &dmem_read_sig)+ .expect("declare read_imem thunk");+ let write_ac_mid_sxm_id = module+ .declare_function(syms.write_ac_mid_sxm.0, Linkage::Import, &dmem_write_sig)+ .expect("declare write_ac_mid_sxm thunk");++ let call_stack_push_sig = {+ let mut sig = cranelift_codegen::ir::Signature::new(host_cc);+ sig.params.push(cranelift_codegen::ir::AbiParam::new(pointer_type));+ sig.params+ .push(cranelift_codegen::ir::AbiParam::new(cranelift_codegen::ir::types::I32));+ sig+ };+ let call_stack_push_id = module+ .declare_function(syms.call_stack_push.0, Linkage::Import, &call_stack_push_sig)+ .expect("declare call_stack_push thunk");++ let stack_pop_sig = translator::stack_pop_signature(pointer_type, host_cc);+ let call_stack_pop_id = module+ .declare_function(syms.call_stack_pop.0, Linkage::Import, &stack_pop_sig)+ .expect("declare call_stack_pop thunk");+ let data_stack_pop_id = module+ .declare_function(syms.data_stack_pop.0, Linkage::Import, &stack_pop_sig)+ .expect("declare data_stack_pop thunk");++ let read_reg_full_id = module+ .declare_function(syms.read_reg_full.0, Linkage::Import, &dmem_read_sig)+ .expect("declare read_reg_full thunk");++ let write_reg_full_id = module+ .declare_function(syms.write_reg_full.0, Linkage::Import, &dmem_write_sig)+ .expect("declare write_reg_full thunk");+ let loop_setup_id = module+ .declare_function(+ syms.loop_setup.0,+ Linkage::Import,+ &translator::loop_setup_signature(pointer_type, host_cc),+ )+ .expect("declare loop_setup thunk");++ let extern_funcs = translator::ExternFuncs {+ cache_ext_ac: cache_ext_ac_id,+ loop_tail: loop_tail_id,+ update_flags_logic: update_flags_logic_id,+ update_flags_add: update_flags_add_id,+ update_flags_sub: update_flags_sub_id,+ update_flags_ac: update_flags_ac_id,+ read_dmem: read_dmem_id,+ write_dmem: write_dmem_id,+ inc_ar: inc_ar_id,+ dec_ar: dec_ar_id,+ increase_ar: increase_ar_id,+ decrease_ar_ix: decrease_ar_ix_id,+ dynamic_shift: dynamic_shift_id,+ read_imem: read_imem_id,+ write_ac_mid_sxm: write_ac_mid_sxm_id,+ call_stack_push: call_stack_push_id,+ call_stack_pop: call_stack_pop_id,+ data_stack_pop: data_stack_pop_id,+ read_reg_full: read_reg_full_id,+ write_reg_full: write_reg_full_id,+ loop_setup: loop_setup_id,+ };++ let trampoline_fn = build_trampoline(&mut module, host_cc, pointer_type, &block_sig);++ let table = vec![+ DspBlockLookupSlot {+ pc: 0,+ _pad: 0,+ entry: 0+ };+ DSP_BLOCK_LOOKUP_TABLE_SIZE+ ]+ .into_boxed_slice();+ let block_lookup_table_addr = Box::leak(table).as_mut_ptr() as usize;++ Self {+ module,+ ctx: Context::new(),+ builder_ctx: FunctionBuilderContext::new(),+ cache: FxHashMap::default(),+ block_func_ids: FxHashMap::default(),+ block_sig,+ extern_funcs,+ trampoline_fn,+ block_seq: 0,+ block_lookup_table_addr,+ chain_depth_total: 0,+ dispatcher_entries_total: 0,+ chain_depth_histogram: [0; CHAIN_DEPTH_HISTOGRAM_LEN],+ #[cfg(feature = "jit-stats")]+ hits: FxHashMap::default(),+ #[cfg(feature = "jit-stats")]+ block_specs: FxHashMap::default(),+ #[cfg(feature = "jit-stats")]+ block_entry_counter_ptrs: FxHashMap::default(),+ }+ }++ #[cfg(feature = "jit-stats")]+ fn block_entry_counter_slot(&mut self, pc: u16) -> usize {+ if let Some(&addr) = self.block_entry_counter_ptrs.get(&pc) {+ return addr;+ }+ let slot: &'static mut u64 = Box::leak(Box::new(0u64));+ let addr = slot as *mut u64 as usize;+ self.block_entry_counter_ptrs.insert(pc, addr);+ addr+ }++ pub fn record_chain_depth(&mut self, depth: u32) {+ self.chain_depth_total = self.chain_depth_total.wrapping_add(depth as u64);+ self.dispatcher_entries_total = self.dispatcher_entries_total.wrapping_add(1);+ let bucket = (depth as usize).min(self.chain_depth_histogram.len() - 1);+ self.chain_depth_histogram[bucket] = self.chain_depth_histogram[bucket].wrapping_add(1);+ }++ fn register_in_lookup_table(&self, pc: u16, entry: usize) {+ let pc_u32 = pc as u32;+ let idx = ((pc_u32 & 0xFFF) | ((pc_u32 >> 3) & 0x1000)) as usize;+ unsafe {+ let table = self.block_lookup_table_addr as *mut DspBlockLookupSlot;+ (*table.add(idx)).pc = pc_u32;+ (*table.add(idx)).entry = entry;+ }+ }++ fn clear_lookup_table(&self) {+ unsafe {+ let table = self.block_lookup_table_addr as *mut DspBlockLookupSlot;+ for i in 0..DSP_BLOCK_LOOKUP_TABLE_SIZE {+ (*table.add(i)).pc = 0;+ (*table.add(i)).entry = 0;+ }+ }+ }++ pub fn lookup_or_compile(&mut self, iram: &[u8], irom: &[u8], start_pc: u16) -> BlockEntry {+ if let Some(&entry) = self.cache.get(&start_pc) {+ #[cfg(feature = "jit-stats")]+ {+ *self.hits.entry(start_pc).or_insert(0) += 1;+ }+ return entry;+ }+ let spec = block::discover(iram, irom, start_pc);+ let entry = self.compile(&spec);+ self.cache.insert(start_pc, entry);+ #[cfg(feature = "jit-stats")]+ {+ *self.hits.entry(start_pc).or_insert(0) += 1;+ self.block_specs.insert(start_pc, spec);+ }+ entry+ }++ fn func_id_for(&mut self, pc: u16) -> FuncId {+ if let Some(&id) = self.block_func_ids.get(&pc) {+ return id;+ }+ self.block_seq = self.block_seq.wrapping_add(1);+ let name = format!("gecko_dsp_block_{:04x}_{}", pc, self.block_seq);+ let id = self+ .module+ .declare_function(&name, Linkage::Local, &self.block_sig)+ .expect("declare dsp block");+ self.block_func_ids.insert(pc, id);+ id+ }++ #[cfg(feature = "jit-stats")]+ pub fn dump_block_clif(&mut self, start_pc: u16, iram: &[u8], irom: &[u8]) {+ let spec = block::discover(iram, irom, start_pc);++ self.ctx.clear();+ self.ctx.func.signature = self.block_sig.clone();+ translator::translate(+ &mut self.ctx,+ &mut self.builder_ctx,+ &mut self.module,+ &self.extern_funcs,+ &spec,+ self.block_lookup_table_addr as i64,+ None,+ );+ tracing::info!(+ "CLIF for DSP block pc={:04X} (len={}, term={:?})",+ spec.start_pc,+ spec.instrs.len(),+ spec.terminator,+ );+ tracing::info!("{}", self.ctx.func.display());+ }++ fn compile(&mut self, spec: &block::BlockSpec) -> BlockEntry {+ let func_id = self.func_id_for(spec.start_pc);++ self.ctx.clear();+ self.ctx.func.signature = self.block_sig.clone();++ #[cfg(feature = "jit-stats")]+ let entry_counter_addr = Some(self.block_entry_counter_slot(spec.start_pc));+ #[cfg(not(feature = "jit-stats"))]+ let entry_counter_addr: Option<usize> = None;++ translator::translate(+ &mut self.ctx,+ &mut self.builder_ctx,+ &mut self.module,+ &self.extern_funcs,+ spec,+ self.block_lookup_table_addr as i64,+ entry_counter_addr,+ );++ self.module+ .define_function(func_id, &mut self.ctx)+ .expect("define dsp block");+ self.module.finalize_definitions().expect("finalize dsp jit");++ let entry = self.module.get_finalized_function(func_id) as usize;++ self.register_in_lookup_table(spec.start_pc, entry);++ entry+ }++ pub fn flush(&mut self) {+ self.cache.clear();+ self.block_func_ids.clear();++ self.clear_lookup_table();+ }++ pub fn run_block(&mut self, ctx_ptr: *mut core::ffi::c_void, entry: BlockEntry) -> u16 {+ let next_pc_u32 = unsafe { (self.trampoline_fn)(ctx_ptr, entry) };+ next_pc_u32 as u16+ }++ #[cfg(feature = "jit-stats")]+ pub fn dump_hot_blocks(&self, top_k: usize) {+ let mut entries: Vec<(u16, u64)> = self.hits.iter().map(|(&pc, &n)| (pc, n)).collect();+ entries.sort_by(|a, b| b.1.cmp(&a.1));++ if entries.is_empty() {+ tracing::info!("no dispatcher-entry hits recorded");+ return;+ }++ let total_entries: u64 = entries.iter().map(|e| e.1).sum();+ let total_instrs: u64 = entries+ .iter()+ .map(|(pc, n)| {+ let len = self.block_specs.get(pc).map(|s| s.instrs.len() as u64).unwrap_or(0);+ n.saturating_mul(len)+ })+ .sum();++ tracing::info!(+ "DSP JIT hot blocks (top {} of {}, {} dispatcher entries, {} instr executions)",+ top_k.min(entries.len()),+ entries.len(),+ total_entries,+ total_instrs,+ );++ if self.dispatcher_entries_total > 0 {+ let avg_depth = self.chain_depth_total as f64 / self.dispatcher_entries_total as f64;+ let budget_bucket = super::DSP_JIT_CHAIN_BUDGET as usize;+ let saturated = self.chain_depth_histogram.get(budget_bucket).copied().unwrap_or(0);+ let zero_depth = self.chain_depth_histogram[0];+ let saturated_pct = (saturated as f64) * 100.0 / (self.dispatcher_entries_total as f64);+ let zero_pct = (zero_depth as f64) * 100.0 / (self.dispatcher_entries_total as f64);+ tracing::info!(+ " chain link: avg depth {:.2} (over {} run_block entries, {} chained tail-calls)",+ avg_depth,+ self.dispatcher_entries_total,+ self.chain_depth_total,+ );+ tracing::info!(+ " chain link: {:.1}% returned without chaining (depth=0), {:.1}% saturated at the budget",+ zero_pct,+ saturated_pct,+ );++ let buckets: Vec<String> = self+ .chain_depth_histogram+ .iter()+ .enumerate()+ .filter(|(_, n)| **n > 0)+ .map(|(i, n)| format!("{}:{}", i, n))+ .collect();+ tracing::info!(" chain depth histogram (non-zero): {}", buckets.join(" "));+ } else {+ tracing::info!(" chain link: no run_block entries observed yet");+ }++ let mut per_mnem: rustc_hash::FxHashMap<String, u64> = rustc_hash::FxHashMap::default();+ for (pc, hits) in entries.iter() {+ if let Some(spec) = self.block_specs.get(pc) {+ for entry in &spec.instrs {+ let mnem = disasm_mnemonic(entry.raw);+ *per_mnem.entry(mnem).or_insert(0) += hits;+ }+ }+ }++ for (rank, (pc, hits)) in entries.iter().take(top_k).enumerate() {+ let pct = (*hits as f64) * 100.0 / (total_entries as f64);+ let spec = self.block_specs.get(pc);+ let len = spec.map(|s| s.instrs.len()).unwrap_or(0);+ let term = spec.map(|s| format!("{:?}", s.terminator)).unwrap_or_default();++ let total_block_entries = self+ .block_entry_counter_ptrs+ .get(pc)+ .map(|&addr| unsafe { *(addr as *const u64) })+ .unwrap_or(0);+ let chain_arrivals = total_block_entries.saturating_sub(*hits);+ let chain_rate = if total_block_entries > 0 {+ (chain_arrivals as f64) * 100.0 / (total_block_entries as f64)+ } else {+ 0.0+ };+ tracing::info!(+ " #{:>3} pc={:04X} hits={} ({:.1}%) total={} chain={:.1}% len={} term={}",+ rank,+ pc,+ hits,+ pct,+ total_block_entries,+ chain_rate,+ len,+ term,+ );+ if let Some(spec) = spec {+ for entry in &spec.instrs {+ let bytes = [+ ((entry.raw >> 8) & 0xFF) as u8,+ (entry.raw & 0xFF) as u8,+ ((entry.raw >> 24) & 0xFF) as u8,+ ((entry.raw >> 16) & 0xFF) as u8,+ ];+ let asm = disasm::dsp::GcDspInstruction::decode(&bytes)+ .map(|(ins, _)| format!("{ins}"))+ .unwrap_or_else(|| "<unknown>".to_string());+ tracing::info!(" {:04X} {:08X} {}", entry.pc, entry.raw, asm,);+ }+ }+ }++ let mut mnem_entries: Vec<(String, u64)> = per_mnem.into_iter().collect();+ mnem_entries.sort_by(|a, b| b.1.cmp(&a.1));+ let mnem_total: u64 = mnem_entries.iter().map(|e| e.1).sum();+ tracing::info!(+ "DSP JIT hot mnemonics (top {} of {}, {} total executions)",+ top_k.min(mnem_entries.len()),+ mnem_entries.len(),+ mnem_total,+ );+ for (rank, (mnem, n)) in mnem_entries.iter().take(top_k).enumerate() {+ let pct = (*n as f64) * 100.0 / (mnem_total.max(1) as f64);+ tracing::info!(" #{:>3} {:6} {:>10} ({:.1}%)", rank, mnem, n, pct);+ }+ }+}++#[cfg(feature = "jit-stats")]+fn disasm_mnemonic(raw: u32) -> String {+ let bytes = [+ ((raw >> 8) & 0xFF) as u8,+ (raw & 0xFF) as u8,+ ((raw >> 24) & 0xFF) as u8,+ ((raw >> 16) & 0xFF) as u8,+ ];++ let Some((ins, _)) = disasm::dsp::GcDspInstruction::decode(&bytes) else {+ return "<unk>".to_string();+ };++ let s = format!("{ins}");+ s.split_whitespace().next().map(|m| m.to_string()).unwrap_or_else(|| s)+}++impl<const SYSTEM: SystemId> Default for JitEngine<SYSTEM> {+ fn default() -> Self {+ Self::new()+ }+}++fn build_trampoline(+ module: &mut JITModule,+ host_cc: cranelift_codegen::isa::CallConv,+ pointer_type: cranelift_codegen::ir::Type,+ block_sig: &Signature,+) -> TrampolineFn {+ use cranelift_codegen::Context;+ use cranelift_codegen::ir::{AbiParam, InstBuilder, types};+ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};++ let mut tramp_sig = Signature::new(host_cc);+ tramp_sig.params.push(AbiParam::new(pointer_type));+ tramp_sig.params.push(AbiParam::new(pointer_type));+ tramp_sig.returns.push(AbiParam::new(types::I32));++ let id = module+ .declare_function("gecko_dsp_jit_trampoline", Linkage::Local, &tramp_sig)+ .expect("declare dsp trampoline");++ let mut ctx = Context::new();+ ctx.func.signature = tramp_sig;++ let mut bctx = FunctionBuilderContext::new();+ {+ let mut builder = FunctionBuilder::new(&mut ctx.func, &mut bctx);+ let entry = builder.create_block();+ builder.append_block_params_for_function_params(entry);+ builder.switch_to_block(entry);+ builder.seal_block(entry);++ let ctx_ptr = builder.block_params(entry)[0];+ let block_ptr = builder.block_params(entry)[1];++ let sig_ref = builder.import_signature(block_sig.clone());+ let call = builder.ins().call_indirect(sig_ref, block_ptr, &[ctx_ptr]);+ let ret = builder.inst_results(call)[0];+ builder.ins().return_(&[ret]);+ builder.finalize();+ }++ module.define_function(id, &mut ctx).expect("define dsp trampoline");+ module.finalize_definitions().expect("finalize dsp trampoline");++ let raw = module.get_finalized_function(id);+ unsafe { core::mem::transmute::<*const u8, TrampolineFn>(raw) }+}diff --git a/crates/gecko/src/flipper/dsp/jit/abi.rs b/crates/gecko/src/flipper/dsp/jit/abi.rsnew file mode 100644index 0000000..a2c7d13--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit/abi.rs@@ -0,0 +1,134 @@+use core::mem::offset_of;++use crate::flipper::dsp::Dsp;+use crate::flipper::dsp::core::Registers;+use crate::flipper::dsp::regs::ControlStatus;+use crate::system::{GC, System, SystemId, WII};++macro_rules! make_dsp_offset {+ ($name:ident, $reg_field:ident, $($rest:tt)*) => {+ #[inline(always)]+ pub const fn $name() -> usize {+ const fn off<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, dsp)+ + offset_of!(Dsp, registers)+ + offset_of!(Registers, $reg_field)+ $($rest)*+ }+ const G: usize = off::<GC>();+ const W: usize = off::<WII>();+ const _: () = {+ if G != W {+ panic!(concat!(+ stringify!($name),+ " offset differs between GC and Wii layouts"+ ));+ }+ };+ G+ }+ };+}++macro_rules! make_dsp_field_offset {+ ($name:ident, $field:ident) => {+ #[inline(always)]+ pub const fn $name() -> usize {+ const fn off<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, dsp) + offset_of!(Dsp, $field)+ }+ const G: usize = off::<GC>();+ const W: usize = off::<WII>();+ const _: () = {+ if G != W {+ panic!(concat!(+ stringify!($name),+ " offset differs between GC and Wii layouts"+ ));+ }+ };+ G+ }+ };+}++make_dsp_offset!(dsp_pc_offset_max, pc,);+make_dsp_offset!(dsp_cia_offset_max, cia,);+make_dsp_offset!(dsp_nia_offset_max, nia,);+make_dsp_offset!(dsp_ac0_high_offset, ac0_high,);+make_dsp_offset!(dsp_ac1_high_offset, ac1_high,);+make_dsp_offset!(dsp_ac0_mid_offset, ac0_mid,);+make_dsp_offset!(dsp_ac1_mid_offset, ac1_mid,);+make_dsp_offset!(dsp_ac0_low_offset, ac0_low,);+make_dsp_offset!(dsp_ac1_low_offset, ac1_low,);+make_dsp_offset!(dsp_config_offset, config,);+make_dsp_offset!(dsp_product_low_offset, product_low,);+make_dsp_offset!(dsp_product_mid1_offset, product_mid1,);+make_dsp_offset!(dsp_product_high_offset, product_high,);+make_dsp_offset!(dsp_product_mid2_offset, product_mid2,);+make_dsp_offset!(dsp_ar_base_offset, ar,);+make_dsp_offset!(dsp_ix_base_offset, ix,);+make_dsp_offset!(dsp_wr_base_offset, wr,);+make_dsp_offset!(dsp_ax_base_offset, ax,);+make_dsp_offset!(dsp_axh_base_offset, axh,);+make_dsp_offset!(dsp_status_offset, status,);+make_dsp_offset!(dsp_ext_ac_cache_base_offset, ext_ac_cache,);++#[inline(always)]+pub const fn dsp_loop_addr_ptr_offset_max() -> usize {+ const fn off<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, dsp)+ + offset_of!(Dsp, registers)+ + offset_of!(Registers, loop_addr)+ + core::mem::size_of::<[u16; 32]>()+ }+ const G: usize = off::<GC>();+ const W: usize = off::<WII>();+ const _: () = {+ if G != W {+ panic!("dsp_loop_addr_ptr_offset differs between GC and Wii");+ }+ };+ G+}++make_dsp_field_offset!(dsp_csr_offset, csr);+make_dsp_field_offset!(dsp_chain_budget_offset, chain_budget);+make_dsp_field_offset!(dsp_instr_count_offset, instr_count);++const _: () = {+ let _ = dsp_pc_offset_max();+ let _ = dsp_cia_offset_max();+ let _ = dsp_nia_offset_max();+ let _ = dsp_ac0_high_offset();+ let _ = dsp_ac1_high_offset();+ let _ = dsp_ac0_mid_offset();+ let _ = dsp_ac1_mid_offset();+ let _ = dsp_ac0_low_offset();+ let _ = dsp_ac1_low_offset();+ let _ = dsp_config_offset();+ let _ = dsp_product_low_offset();+ let _ = dsp_product_mid1_offset();+ let _ = dsp_product_high_offset();+ let _ = dsp_product_mid2_offset();+ let _ = dsp_ar_base_offset();+ let _ = dsp_ix_base_offset();+ let _ = dsp_wr_base_offset();+ let _ = dsp_ax_base_offset();+ let _ = dsp_axh_base_offset();+ let _ = dsp_status_offset();+ let _ = dsp_ext_ac_cache_base_offset();+ let _ = dsp_csr_offset();+ let _ = dsp_loop_addr_ptr_offset_max();+ let _ = dsp_chain_budget_offset();+ let _ = dsp_instr_count_offset();+};++const _: () = {+ if core::mem::size_of::<ControlStatus>() != 2 {+ panic!("ControlStatus is no longer 2 bytes");+ }+ if core::mem::align_of::<ControlStatus>() != 2 {+ panic!("ControlStatus alignment changed");+ }+};diff --git a/crates/gecko/src/flipper/dsp/jit/block.rs b/crates/gecko/src/flipper/dsp/jit/block.rsnew file mode 100644index 0000000..715adb2--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit/block.rs@@ -0,0 +1,93 @@+pub const MAX_BLOCK_INSTRS: usize = 64;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TermKind {+ Halt,+ Jump,+ Call,+ Ret,+ IfCc,+ LoopSetup,+ LengthLimit,+}++#[derive(Debug, Clone)]+pub struct InstrEntry {+ pub pc: u16,+ pub raw: u32,+ pub size: u8,+}++#[derive(Debug, Clone)]+pub struct BlockSpec {+ pub start_pc: u16,+ pub instrs: Vec<InstrEntry>,+ pub terminator: TermKind,+ pub fallthrough_pc: u16,+}++fn read_imem_word(iram: &[u8], irom: &[u8], addr: u16) -> Option<u16> {+ let off = (addr & 0x0FFF) as usize * 2;+ match addr & 0xF000 {+ 0x0000 => Some(u16::from_be_bytes([iram[off], iram[off + 1]])),+ 0x8000 => Some(u16::from_be_bytes([irom[off], irom[off + 1]])),+ _ => None,+ }+}++fn classify(primary: u16) -> Option<TermKind> {+ let nibble = (primary >> 12) & 0xF;+ match nibble {+ 0x0 | 0x1 => {+ if primary == 0x0021 {+ Some(TermKind::Halt)+ } else {+ Some(TermKind::Jump)+ }+ }+ _ => None,+ }+}++fn approx_size(primary: u16) -> u8 {+ use crate::flipper::dsp::instruction::Instruction;+ use crate::flipper::dsp::lut;+ lut::instr_size(Instruction(primary as u32)) as u8+}++pub fn discover(iram: &[u8], irom: &[u8], start_pc: u16) -> BlockSpec {+ let mut instrs = Vec::with_capacity(8);+ let mut pc = start_pc;+ let mut term = TermKind::LengthLimit;++ while instrs.len() < MAX_BLOCK_INSTRS {+ let Some(w0) = read_imem_word(iram, irom, pc) else {+ term = TermKind::LengthLimit;+ break;+ };++ let size = approx_size(w0);+ let raw = if size == 2 {+ let w1 = read_imem_word(iram, irom, pc.wrapping_add(1)).unwrap_or(0);+ (w0 as u32) | ((w1 as u32) << 16)+ } else {+ w0 as u32+ };+ instrs.push(InstrEntry { pc, raw, size });++ if let Some(kind) = classify(w0) {+ term = kind;+ pc = pc.wrapping_add(size as u16);+ break;+ }++ pc = pc.wrapping_add(size as u16);+ }++ BlockSpec {+ start_pc,+ instrs,+ terminator: term,+ fallthrough_pc: pc,+ }+}diff --git a/crates/gecko/src/flipper/dsp/jit/runtime.rs b/crates/gecko/src/flipper/dsp/jit/runtime.rsnew file mode 100644index 0000000..afb2f8d--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit/runtime.rs@@ -0,0 +1,322 @@+use crate::flipper::dsp::{self};+use crate::system::{GC, System, SystemId, WII};++#[inline(always)]+fn cache_ext_ac<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.cache_ext_ac();+}++pub extern "C" fn dsp_cache_ext_ac_gc(sys: *mut core::ffi::c_void) {+ cache_ext_ac::<GC>(sys.cast());+}++pub extern "C" fn dsp_cache_ext_ac_wii(sys: *mut core::ffi::c_void) {+ cache_ext_ac::<WII>(sys.cast());+}++#[inline(always)]+fn loop_tail<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) {+ let sys = unsafe { &mut *sys };+ let dsp = &mut sys.dsp;+ let at_loop_end = !dsp.registers.loop_addr.is_empty() && dsp.registers.nia == dsp.registers.loop_addr.top();++ if at_loop_end {+ let counter = dsp.registers.loop_counter.top().wrapping_sub(1);+ if counter != 0 {+ dsp.registers.loop_counter.set_top(counter);+ dsp.registers.nia = dsp.registers.call_stack.top();+ } else {+ dsp.registers.loop_counter.pop();+ dsp.registers.loop_addr.pop();+ dsp.registers.call_stack.pop();+ }+ }++ dsp.registers.pc = dsp.registers.nia;+}++pub extern "C" fn dsp_loop_tail_gc(sys: *mut core::ffi::c_void) {+ loop_tail::<GC>(sys.cast());+}++pub extern "C" fn dsp_loop_tail_wii(sys: *mut core::ffi::c_void) {+ loop_tail::<WII>(sys.cast());+}++#[inline(always)]+fn update_flags_logic<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, result16: u32, ac_full: i64) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.update_flags_logic(result16 as u16, ac_full);+}++pub extern "C" fn dsp_update_flags_logic_gc(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {+ update_flags_logic::<GC>(sys.cast(), result, ac_full);+}++pub extern "C" fn dsp_update_flags_logic_wii(sys: *mut core::ffi::c_void, result: u32, ac_full: i64) {+ update_flags_logic::<WII>(sys.cast(), result, ac_full);+}++#[inline(always)]+fn update_flags_add<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.update_flags_add(a, b, result);+}++pub extern "C" fn dsp_update_flags_add_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {+ update_flags_add::<GC>(sys.cast(), a, b, result);+}++pub extern "C" fn dsp_update_flags_add_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {+ update_flags_add::<WII>(sys.cast(), a, b, result);+}++#[inline(always)]+fn update_flags_sub<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, a: i64, b: i64, result: i64) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.update_flags_sub(a, b, result);+}++pub extern "C" fn dsp_update_flags_sub_gc(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {+ update_flags_sub::<GC>(sys.cast(), a, b, result);+}++pub extern "C" fn dsp_update_flags_sub_wii(sys: *mut core::ffi::c_void, a: i64, b: i64, result: i64) {+ update_flags_sub::<WII>(sys.cast(), a, b, result);+}++#[inline(always)]+fn update_flags_ac<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ac_full: i64) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.update_flags_ac(ac_full);+}++pub extern "C" fn dsp_update_flags_ac_gc(sys: *mut core::ffi::c_void, ac_full: i64) {+ update_flags_ac::<GC>(sys.cast(), ac_full);+}++pub extern "C" fn dsp_update_flags_ac_wii(sys: *mut core::ffi::c_void, ac_full: i64) {+ update_flags_ac::<WII>(sys.cast(), ac_full);+}++pub extern "C" fn dsp_read_dmem_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ dsp::read_dmem(sys, addr as u16) as u32+}++pub extern "C" fn dsp_read_dmem_wii(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ dsp::read_dmem(sys, addr as u16) as u32+}++pub extern "C" fn dsp_write_dmem_gc(sys: *mut core::ffi::c_void, addr: u32, value: u32) {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ dsp::write_dmem(sys, addr as u16, value as u16);+}++pub extern "C" fn dsp_write_dmem_wii(sys: *mut core::ffi::c_void, addr: u32, value: u32) {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ dsp::write_dmem(sys, addr as u16, value as u16);+}++pub extern "C" fn dsp_increment_ar_gc(sys: *mut core::ffi::c_void, reg: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ sys.dsp.registers.increment_ar(reg as usize) as u32+}++pub extern "C" fn dsp_increment_ar_wii(sys: *mut core::ffi::c_void, reg: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ sys.dsp.registers.increment_ar(reg as usize) as u32+}++pub extern "C" fn dsp_decrement_ar_gc(sys: *mut core::ffi::c_void, reg: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ sys.dsp.registers.decrement_ar(reg as usize) as u32+}++pub extern "C" fn dsp_decrement_ar_wii(sys: *mut core::ffi::c_void, reg: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ sys.dsp.registers.decrement_ar(reg as usize) as u32+}++pub extern "C" fn dsp_increase_ar_gc(sys: *mut core::ffi::c_void, reg: u32, ix: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ sys.dsp.registers.increase_ar(reg as usize, ix as i16) as u32+}++pub extern "C" fn dsp_increase_ar_wii(sys: *mut core::ffi::c_void, reg: u32, ix: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ sys.dsp.registers.increase_ar(reg as usize, ix as i16) as u32+}++pub extern "C" fn dsp_decrease_ar_ix_gc(sys: *mut core::ffi::c_void, reg: u32, ix: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<GC>) };+ sys.dsp.registers.decrease_ar_ix(reg as usize, ix as i16) as u32+}++pub extern "C" fn dsp_decrease_ar_ix_wii(sys: *mut core::ffi::c_void, reg: u32, ix: u32) -> u32 {+ let sys = unsafe { &mut *(sys as *mut System<WII>) };+ sys.dsp.registers.decrease_ar_ix(reg as usize, ix as i16) as u32+}++#[inline(always)]+fn dynamic_shift_impl<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, d: u32, shift_val: u32, mode: u32) {+ let logical = mode & 1 != 0;+ let reversed = mode & 2 != 0;++ let sys = unsafe { &mut *sys };++ let regs = &mut sys.dsp.registers;+ let shift_val = shift_val as i16;+ let low6 = (shift_val & 63) as u32;+ let bit6 = shift_val & 64 != 0;+ let amount = if bit6 { (64 - low6) % 64 } else { low6 };+ let shift_left = if reversed { !bit6 } else { bit6 };+ let dn = d as u8;++ if shift_left {+ let ac = ((regs.ac(dn) as u64 & 0xFF_FFFF_FFFF) << amount) as i64;+ regs.set_ac(dn, ac);+ } else if amount != 0 {+ let ac = if logical {+ ((regs.ac(dn) as u64 & 0xFF_FFFF_FFFF) >> amount) as i64+ } else {+ regs.ac(dn) >> amount+ };+ regs.set_ac(dn, ac);+ }++ let ac = regs.ac(dn);+ regs.update_flags_ac(ac);+ regs.status.set_o(false);+ regs.status.set_c(false);+}++pub extern "C" fn dsp_dynamic_shift_gc(sys: *mut core::ffi::c_void, d: u32, shift_val: u32, mode: u32) {+ dynamic_shift_impl::<GC>(sys.cast(), d, shift_val, mode);+}++pub extern "C" fn dsp_dynamic_shift_wii(sys: *mut core::ffi::c_void, d: u32, shift_val: u32, mode: u32) {+ dynamic_shift_impl::<WII>(sys.cast(), d, shift_val, mode);+}++#[inline(always)]+fn read_imem<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, addr: u32) -> u32 {+ let sys = unsafe { &mut *sys };+ sys.dsp.read_imem(addr as u16) as u32+}++pub extern "C" fn dsp_read_imem_gc(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ read_imem::<GC>(sys.cast(), addr)+}++pub extern "C" fn dsp_read_imem_wii(sys: *mut core::ffi::c_void, addr: u32) -> u32 {+ read_imem::<WII>(sys.cast(), addr)+}++#[inline(always)]+fn write_ac_mid_sxm<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, idx: u32, value: u32) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.write::<true>(30 + idx as u8, value as u16);+}++pub extern "C" fn dsp_write_ac_mid_sxm_gc(sys: *mut core::ffi::c_void, idx: u32, value: u32) {+ write_ac_mid_sxm::<GC>(sys.cast(), idx, value);+}++pub extern "C" fn dsp_write_ac_mid_sxm_wii(sys: *mut core::ffi::c_void, idx: u32, value: u32) {+ write_ac_mid_sxm::<WII>(sys.cast(), idx, value);+}++#[inline(always)]+fn call_stack_push<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, value: u32) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.call_stack.push(value as u16);+}++pub extern "C" fn dsp_call_stack_push_gc(sys: *mut core::ffi::c_void, value: u32) {+ call_stack_push::<GC>(sys.cast(), value);+}++pub extern "C" fn dsp_call_stack_push_wii(sys: *mut core::ffi::c_void, value: u32) {+ call_stack_push::<WII>(sys.cast(), value);+}++#[inline(always)]+fn call_stack_pop<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) -> u32 {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.call_stack.pop() as u32+}++#[inline(always)]+fn data_stack_pop<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>) -> u32 {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.data_stack.pop() as u32+}++pub extern "C" fn dsp_call_stack_pop_gc(sys: *mut core::ffi::c_void) -> u32 {+ call_stack_pop::<GC>(sys.cast())+}++pub extern "C" fn dsp_call_stack_pop_wii(sys: *mut core::ffi::c_void) -> u32 {+ call_stack_pop::<WII>(sys.cast())+}++pub extern "C" fn dsp_data_stack_pop_gc(sys: *mut core::ffi::c_void) -> u32 {+ data_stack_pop::<GC>(sys.cast())+}++pub extern "C" fn dsp_data_stack_pop_wii(sys: *mut core::ffi::c_void) -> u32 {+ data_stack_pop::<WII>(sys.cast())+}++#[inline(always)]+fn read_reg_full<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, slot: u32) -> u32 {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.read::<true>(slot as u8) as u32+}++pub extern "C" fn dsp_read_reg_full_gc(sys: *mut core::ffi::c_void, slot: u32) -> u32 {+ read_reg_full::<GC>(sys.cast(), slot)+}++pub extern "C" fn dsp_read_reg_full_wii(sys: *mut core::ffi::c_void, slot: u32) -> u32 {+ read_reg_full::<WII>(sys.cast(), slot)+}++#[inline(always)]+fn write_reg_full<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, slot: u32, value: u32) {+ let sys = unsafe { &mut *sys };+ sys.dsp.registers.write::<true>(slot as u8, value as u16);+}++pub extern "C" fn dsp_write_reg_full_gc(sys: *mut core::ffi::c_void, slot: u32, value: u32) {+ write_reg_full::<GC>(sys.cast(), slot, value);+}++pub extern "C" fn dsp_write_reg_full_wii(sys: *mut core::ffi::c_void, slot: u32, value: u32) {+ write_reg_full::<WII>(sys.cast(), slot, value);+}++#[inline(always)]+fn loop_setup<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, end_addr_p1: u32, counter: u32, call_stack_val: u32) {+ let sys = unsafe { &mut *sys };+ let regs = &mut sys.dsp.registers;++ if counter != 0 {+ regs.call_stack.push(call_stack_val as u16);+ regs.loop_addr.push(end_addr_p1 as u16);+ regs.loop_counter.push(counter as u16);+ } else {+ regs.nia = end_addr_p1 as u16;+ }+}++pub extern "C" fn dsp_loop_setup_gc(sys: *mut core::ffi::c_void, end_addr_p1: u32, counter: u32, call_stack_val: u32) {+ loop_setup::<GC>(sys.cast(), end_addr_p1, counter, call_stack_val);+}++pub extern "C" fn dsp_loop_setup_wii(sys: *mut core::ffi::c_void, end_addr_p1: u32, counter: u32, call_stack_val: u32) {+ loop_setup::<WII>(sys.cast(), end_addr_p1, counter, call_stack_val);+}diff --git a/crates/gecko/src/flipper/dsp/jit/translate.rs b/crates/gecko/src/flipper/dsp/jit/translate.rsnew file mode 100644index 0000000..7c77a4b--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit/translate.rs@@ -0,0 +1,2386 @@+use cranelift_codegen::ir::{InstBuilder, MemFlags, Value, types};+use cranelift_frontend::FunctionBuilder;+use cranelift_module::Module;++use crate::flipper::dsp::instruction::{GcDspExt, Instruction};+use crate::flipper::dsp::lut::*;+use crate::system::SystemId;++use super::abi;+use super::translator::ExternFuncs;++pub struct TranslatorCtx<'a, 'b> {+ pub builder: &'a mut FunctionBuilder<'b>,+ pub module: &'a mut cranelift_jit::JITModule,+ pub extern_funcs: ExternFuncs,+ pub sys_ptr: Value,+ pub pc: u16,+ pub size: u8,+}++impl<'a, 'b> TranslatorCtx<'a, 'b> {+ #[inline]+ fn iconst16(&mut self, v: u16) -> Value {+ self.builder.ins().iconst(types::I16, v as i64)+ }++ #[inline]+ fn load_u16(&mut self, offset: i32) -> Value {+ self.builder+ .ins()+ .load(types::I16, MemFlags::trusted(), self.sys_ptr, offset)+ }++ #[inline]+ fn store_u16(&mut self, val: Value, offset: i32) {+ self.builder.ins().store(MemFlags::trusted(), val, self.sys_ptr, offset);+ }++ fn simple_reg_offset(slot: u8) -> Option<i32> {+ Some(+ (match slot {+ 0..=3 => abi::dsp_ar_base_offset() + (slot as usize) * 2,+ 4..=7 => abi::dsp_ix_base_offset() + (slot as usize - 4) * 2,+ 8..=11 => abi::dsp_wr_base_offset() + (slot as usize - 8) * 2,+ 12..=15 => return None,+ 16 => abi::dsp_ac0_high_offset(),+ 17 => abi::dsp_ac1_high_offset(),+ 18 => abi::dsp_config_offset(),+ 19 => abi::dsp_status_offset(),+ 20 => abi::dsp_product_low_offset(),+ 21 => abi::dsp_product_mid1_offset(),+ 22 => abi::dsp_product_high_offset(),+ 23 => abi::dsp_product_mid2_offset(),+ 24 | 25 => abi::dsp_ax_base_offset() + (slot as usize - 24) * 2,+ 26 | 27 => abi::dsp_axh_base_offset() + (slot as usize - 26) * 2,+ 28 => abi::dsp_ac0_low_offset(),+ 29 => abi::dsp_ac1_low_offset(),+ 30..=31 => return None,+ _ => return None,+ }) as i32,+ )+ }++ pub fn try_emit_reg_read(&mut self, slot: u8) -> Option<Value> {+ let off = Self::simple_reg_offset(slot)?;+ let raw = self.load_u16(off);+ let v = match slot {+ 19 => self.builder.ins().band_imm(raw, !0x0100i64 & 0xFFFF),+ 22 => self.builder.ins().band_imm(raw, 0xFF),+ _ => raw,+ };++ Some(v)+ }++ pub fn try_emit_reg_write(&mut self, slot: u8, value: Value) -> bool {+ let Some(off) = Self::simple_reg_offset(slot) else {+ return false;+ };++ let to_store = match slot {+ 16 | 17 => {+ let trunc = self.builder.ins().ireduce(types::I8, value);+ self.builder.ins().sextend(types::I16, trunc)+ }+ _ => value,+ };+ self.store_u16(to_store, off);++ true+ }++ fn load_csr(&mut self) -> Value {+ let off = abi::dsp_csr_offset() as i32;+ self.load_u16(off)+ }++ fn store_csr(&mut self, val: Value) {+ let off = abi::dsp_csr_offset() as i32;+ self.store_u16(val, off);+ }++ #[allow(dead_code)]+ const SR_BIT_C: u8 = 0;+ #[allow(dead_code)]+ const SR_BIT_O: u8 = 1;+ #[allow(dead_code)]+ const SR_BIT_Z: u8 = 2;+ #[allow(dead_code)]+ const SR_BIT_S: u8 = 3;+ #[allow(dead_code)]+ const SR_BIT_AS32: u8 = 4;+ #[allow(dead_code)]+ const SR_BIT_TB: u8 = 5;+ #[allow(dead_code)]+ const SR_BIT_LZ: u8 = 6;+ #[allow(dead_code)]+ const SR_BIT_OS: u8 = 7;++ fn load_sr(&mut self) -> Value {+ let off = abi::dsp_status_offset() as i32;+ self.load_u16(off)+ }++ fn sr_bit(&mut self, sr: Value, bit: u8) -> Value {+ let shifted = self.builder.ins().ushr_imm(sr, bit as i64);+ self.builder.ins().band_imm(shifted, 1)+ }++ fn ac_offsets(d: u8) -> (i32, i32, i32) {+ if d == 0 {+ (+ abi::dsp_ac0_high_offset() as i32,+ abi::dsp_ac0_mid_offset() as i32,+ abi::dsp_ac0_low_offset() as i32,+ )+ } else {+ (+ abi::dsp_ac1_high_offset() as i32,+ abi::dsp_ac1_mid_offset() as i32,+ abi::dsp_ac1_low_offset() as i32,+ )+ }+ }++ pub fn read_ac_i64(&mut self, d: u8) -> Value {+ let (h_off, m_off, l_off) = Self::ac_offsets(d);+ let high = self.load_u16(h_off);+ let mid = self.load_u16(m_off);+ let low = self.load_u16(l_off);++ let high64 = self.builder.ins().sextend(types::I64, high);+ let high_byte = self.builder.ins().band_imm(high64, 0xFFi64);+ let mid64 = self.builder.ins().uextend(types::I64, mid);+ let low64 = self.builder.ins().uextend(types::I64, low);+ let h_shl = self.builder.ins().ishl_imm(high_byte, 32);+ let m_shl = self.builder.ins().ishl_imm(mid64, 16);+ let hm = self.builder.ins().bor(h_shl, m_shl);+ let merged = self.builder.ins().bor(hm, low64);++ let shl = self.builder.ins().ishl_imm(merged, 24);+ self.builder.ins().sshr_imm(shl, 24)+ }++ pub fn write_ac_i64(&mut self, d: u8, val: Value) {+ let (h_off, m_off, l_off) = Self::ac_offsets(d);+ let mid_shift = self.builder.ins().sshr_imm(val, 16);+ let mid = self.builder.ins().ireduce(types::I16, mid_shift);+ let low = self.builder.ins().ireduce(types::I16, val);+ let high_shift = self.builder.ins().sshr_imm(val, 32);++ let high_byte = self.builder.ins().ireduce(types::I8, high_shift);+ let high = self.builder.ins().sextend(types::I16, high_byte);+ self.store_u16(high, h_off);+ self.store_u16(mid, m_off);+ self.store_u16(low, l_off);+ }++ pub fn read_product_i64(&mut self) -> Value {+ let ph = self.load_u16(abi::dsp_product_high_offset() as i32);+ let pm1 = self.load_u16(abi::dsp_product_mid1_offset() as i32);+ let pm2 = self.load_u16(abi::dsp_product_mid2_offset() as i32);+ let pl = self.load_u16(abi::dsp_product_low_offset() as i32);++ let ph_u8 = self.builder.ins().band_imm(ph, 0xFF);+ let ph_shl = self.builder.ins().ishl_imm(ph_u8, 8);+ let ph_i16 = self.builder.ins().sshr_imm(ph_shl, 8);+ let ph_i64 = self.builder.ins().sextend(types::I64, ph_i16);+ let ph_shifted = self.builder.ins().ishl_imm(ph_i64, 32);++ let pm1_64 = self.builder.ins().uextend(types::I64, pm1);+ let pm2_64 = self.builder.ins().uextend(types::I64, pm2);+ let mid_sum = self.builder.ins().iadd(pm1_64, pm2_64);+ let mid_shifted = self.builder.ins().ishl_imm(mid_sum, 16);++ let pl_64 = self.builder.ins().uextend(types::I64, pl);++ let part1 = self.builder.ins().iadd(ph_shifted, mid_shifted);+ self.builder.ins().iadd(part1, pl_64)+ }++ pub fn write_product_i64(&mut self, val: Value) {+ let high_shift = self.builder.ins().sshr_imm(val, 32);+ let high_i8 = self.builder.ins().ireduce(types::I8, high_shift);+ let high = self.builder.ins().sextend(types::I16, high_i8);+ self.store_u16(high, abi::dsp_product_high_offset() as i32);++ let pm1_shift = self.builder.ins().sshr_imm(val, 16);+ let pm1 = self.builder.ins().ireduce(types::I16, pm1_shift);+ self.store_u16(pm1, abi::dsp_product_mid1_offset() as i32);++ let pl = self.builder.ins().ireduce(types::I16, val);+ self.store_u16(pl, abi::dsp_product_low_offset() as i32);++ let zero = self.iconst16(0);+ self.store_u16(zero, abi::dsp_product_mid2_offset() as i32);+ }++ pub fn multiply_i64(&mut self, a_i16: Value, b_i16: Value) -> Value {+ let a32 = self.builder.ins().sextend(types::I32, a_i16);+ let b32 = self.builder.ins().sextend(types::I32, b_i16);+ let mul32 = self.builder.ins().imul(a32, b32);+ let mul64 = self.builder.ins().sextend(types::I64, mul32);++ let status = self.load_u16(abi::dsp_status_offset() as i32);+ let am_shift = self.builder.ins().ushr_imm(status, 13);+ let am = self.builder.ins().band_imm(am_shift, 1);++ use cranelift_codegen::ir::condcodes::IntCC;++ let am_set = self.builder.ins().icmp_imm(IntCC::NotEqual, am, 0);+ let shifted = self.builder.ins().ishl_imm(mul64, 1);+ self.builder.ins().select(am_set, mul64, shifted)+ }++ pub fn multiply_accumulate_i64(&mut self, a_i16: Value, b_i16: Value, add: bool) {+ let mul = self.multiply_i64(a_i16, b_i16);+ let prod = self.read_product_i64();+ let result = if add {+ self.builder.ins().iadd(prod, mul)+ } else {+ self.builder.ins().isub(prod, mul)+ };++ self.write_product_i64(result);+ }++ pub fn mulx_operands_u16(&mut self, s: u8, t: u8) -> (Value, Value) {+ let a_off = if s != 0 {+ abi::dsp_axh_base_offset()+ } else {+ abi::dsp_ax_base_offset()+ } as i32;++ let b_off = if t != 0 {+ (abi::dsp_axh_base_offset() + 2) as i32+ } else {+ (abi::dsp_ax_base_offset() + 2) as i32+ };++ (self.load_u16(a_off), self.load_u16(b_off))+ }++ pub fn multiply_mulx(&mut self, s: u8, t: u8) {+ use cranelift_codegen::ir::condcodes::IntCC;++ let (a, b) = self.mulx_operands_u16(s, t);+ let a_sext = self.builder.ins().sextend(types::I64, a);+ let b_sext = self.builder.ins().sextend(types::I64, b);+ let a_zext = self.builder.ins().uextend(types::I64, a);+ let b_zext = self.builder.ins().uextend(types::I64, b);++ let signed_mul = self.builder.ins().imul(a_sext, b_sext);+ let unsigned_mul = match (s, t) {+ (0, 0) => self.builder.ins().imul(a_zext, b_zext),+ (0, 1) => self.builder.ins().imul(a_zext, b_sext),+ (1, 0) => self.builder.ins().imul(a_sext, b_zext),+ _ => signed_mul,+ };++ let status = self.load_u16(abi::dsp_status_offset() as i32);+ let su_shift = self.builder.ins().ushr_imm(status, 15);+ let su = self.builder.ins().band_imm(su_shift, 1);+ let su_set = self.builder.ins().icmp_imm(IntCC::NotEqual, su, 0);+ let mul = self.builder.ins().select(su_set, unsigned_mul, signed_mul);++ let am_shift = self.builder.ins().ushr_imm(status, 13);+ let am = self.builder.ins().band_imm(am_shift, 1);+ let am_set = self.builder.ins().icmp_imm(IntCC::NotEqual, am, 0);+ let shifted = self.builder.ins().ishl_imm(mul, 1);+ let result = self.builder.ins().select(am_set, mul, shifted);++ self.write_product_i64(result);+ }++ pub fn mulc_operands_i16(&mut self, s: u8, t: u8) -> (Value, Value) {+ let mid_off = if s != 0 {+ abi::dsp_ac1_mid_offset()+ } else {+ abi::dsp_ac0_mid_offset()+ } as i32;++ let axh_off = (abi::dsp_axh_base_offset() + (t as usize) * 2) as i32;+ (self.load_u16(mid_off), self.load_u16(axh_off))+ }++ pub fn read_status_bit(&mut self, bit: u32) -> Value {+ let status = self.load_u16(abi::dsp_status_offset() as i32);+ let shifted = self.builder.ins().ushr_imm(status, bit as i64);+ self.builder.ins().band_imm(shifted, 1)+ }++ pub fn set_status_bit(&mut self, bit: u32, value: Value) {+ let off = abi::dsp_status_offset() as i32;+ let status = self.load_u16(off);+ let mask = !(1u16 << bit);+ let cleared = self.builder.ins().band_imm(status, mask as i64);+ let v_clean = self.builder.ins().band_imm(value, 1);+ let v_shifted = self.builder.ins().ishl_imm(v_clean, bit as i64);+ let new_status = self.builder.ins().bor(cleared, v_shifted);+ self.store_u16(new_status, off);+ }++ pub fn set_status_bit_const(&mut self, bit: u32, value: bool) {+ let off = abi::dsp_status_offset() as i32;+ let status = self.load_u16(off);+ let new_status = if value {+ self.builder.ins().bor_imm(status, 1i64 << bit)+ } else {+ self.builder.ins().band_imm(status, !(1i64 << bit))+ };+ self.store_u16(new_status, off);+ }++ pub fn or_status_bit(&mut self, bit: u32, value: Value) {+ let off = abi::dsp_status_offset() as i32;+ let status = self.load_u16(off);+ let v_clean = self.builder.ins().band_imm(value, 1);+ let v_shifted = self.builder.ins().ishl_imm(v_clean, bit as i64);+ let new_status = self.builder.ins().bor(status, v_shifted);+ self.store_u16(new_status, off);+ }++ pub fn product_flags_i16(&mut self) -> (Value, Value) {+ use cranelift_codegen::ir::condcodes::IntCC;++ let pm1 = self.load_u16(abi::dsp_product_mid1_offset() as i32);+ let pm2 = self.load_u16(abi::dsp_product_mid2_offset() as i32);+ let ph = self.load_u16(abi::dsp_product_high_offset() as i32);++ let pm1_32 = self.builder.ins().uextend(types::I32, pm1);+ let pm2_32 = self.builder.ins().uextend(types::I32, pm2);+ let sum_32 = self.builder.ins().iadd(pm1_32, pm2_32);+ let mid_carry_32 = self.builder.ins().ushr_imm(sum_32, 16);+ let mid_carry = self.builder.ins().ireduce(types::I16, mid_carry_32);++ let ph_u8 = self.builder.ins().band_imm(ph, 0xFF);++ let ph_32 = self.builder.ins().uextend(types::I32, ph_u8);+ let mc_32 = self.builder.ins().uextend(types::I32, mid_carry);+ let total_32 = self.builder.ins().iadd(ph_32, mc_32);+ let carry_32 = self.builder.ins().icmp_imm(IntCC::UnsignedGreaterThan, total_32, 0xFF);+ let carry = self.builder.ins().uextend(types::I16, carry_32);++ let ph_eq_7f = self.builder.ins().icmp_imm(IntCC::Equal, ph_u8, 0x7F);+ let mc_nonzero = self.builder.ins().icmp_imm(IntCC::NotEqual, mid_carry, 0);+ let overflow_b8 = self.builder.ins().band(ph_eq_7f, mc_nonzero);+ let overflow = self.builder.ins().uextend(types::I16, overflow_b8);++ (carry, overflow)+ }++ pub fn apply_product_oc(&mut self, carry: Value, overflow: Value) {+ self.set_status_bit(1, overflow);+ self.or_status_bit(7, overflow);+ self.set_status_bit(0, carry);+ }++ pub fn move_prod_to_ac(&mut self, r: u8) {+ let (carry, overflow) = self.product_flags_i16();+ let prod = self.read_product_i64();+ self.write_ac_i64(r, prod);+ self.emit_update_flags_ac(prod);+ self.apply_product_oc(carry, overflow);+ }++ pub fn round_half_to_even(&mut self, val: Value) -> (Value, Value) {+ use cranelift_codegen::ir::condcodes::IntCC;++ let val40 = self.builder.ins().band_imm(val, 0xFF_FFFF_FFFF_i64);+ let lower = self.builder.ins().band_imm(val40, 0xFFFF);+ let bit16 = self.builder.ins().band_imm(val40, 0x10000);+ let bit16_set = self.builder.ins().icmp_imm(IntCC::NotEqual, bit16, 0);++ let lower_gt = self.builder.ins().icmp_imm(IntCC::UnsignedGreaterThan, lower, 0x8000);+ let lower_eq = self.builder.ins().icmp_imm(IntCC::Equal, lower, 0x8000);+ let lower_eq_and_bit16 = self.builder.ins().band(lower_eq, bit16_set);+ let round_up = self.builder.ins().bor(lower_gt, lower_eq_and_bit16);++ let truncated = self.builder.ins().band_imm(val40, !0xFFFF_i64);+ let sum = self.builder.ins().iadd_imm(val40, 0x10000);+ let rounded = self.builder.ins().band_imm(sum, !0xFFFF_i64);+ let sum_overflow = self+ .builder+ .ins()+ .icmp_imm(IntCC::UnsignedGreaterThan, sum, 0xFF_FFFF_FFFF_i64);+ let sum_overflow_i16 = self.builder.ins().uextend(types::I16, sum_overflow);+ let zero_i16 = self.iconst16(0);++ let result = self.builder.ins().select(round_up, rounded, truncated);+ let carry_out = self.builder.ins().select(round_up, sum_overflow_i16, zero_i16);++ (result, carry_out)+ }++ pub fn move_prod_to_ac_zero(&mut self, r: u8) {+ let (carry, overflow) = self.product_flags_i16();+ let raw = self.read_product_i64();+ let (prod, rounding_carry) = self.round_half_to_even(raw);+ self.write_ac_i64(r, prod);+ self.emit_update_flags_ac(prod);++ let combined_carry = self.builder.ins().bor(carry, rounding_carry);+ self.apply_product_oc(combined_carry, overflow);+ }++ pub fn add_prod_to_ac(&mut self, r: u8) {+ let a = self.read_ac_i64(r);+ let (pc, po) = self.product_flags_i16();+ let b = self.read_product_i64();+ let result = self.builder.ins().iadd(a, b);+ let os_before = self.read_status_bit(7);+ self.write_ac_i64(r, result);+ self.emit_update_flags_add(a, b, result);+ self.set_status_bit(7, os_before);++ let c_now = self.read_status_bit(0);+ let o_now = self.read_status_bit(1);+ let c_xor = self.builder.ins().bxor(c_now, pc);+ let o_xor = self.builder.ins().bxor(o_now, po);+ self.set_status_bit(0, c_xor);+ self.set_status_bit(1, o_xor);+ self.or_status_bit(7, o_xor);+ }++ pub fn read_ac_mid(&mut self, d: u8) -> Value {+ let (_, m_off, _) = Self::ac_offsets(d);+ self.load_u16(m_off)+ }++ pub fn write_ac_mid(&mut self, d: u8, val: Value) {+ let (_, m_off, _) = Self::ac_offsets(d);+ self.store_u16(val, m_off);+ }++ pub fn emit_update_flags_logic(&mut self, result16: Value, ac_full: Value) {+ let result32 = self.builder.ins().uextend(types::I32, result16);+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.update_flags_logic, self.builder.func);+ self.builder.ins().call(f, &[self.sys_ptr, result32, ac_full]);+ }++ pub fn emit_update_flags_add(&mut self, a: Value, b: Value, result: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.update_flags_add, self.builder.func);+ self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);+ }++ pub fn emit_update_flags_sub(&mut self, a: Value, b: Value, result: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.update_flags_sub, self.builder.func);+ self.builder.ins().call(f, &[self.sys_ptr, a, b, result]);+ }++ pub fn emit_update_flags_ac(&mut self, ac_full: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.update_flags_ac, self.builder.func);+ self.builder.ins().call(f, &[self.sys_ptr, ac_full]);+ }++ pub fn emit_read_dmem(&mut self, addr_u32: Value) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.read_dmem, self.builder.func);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, addr_u32]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_write_dmem(&mut self, addr_u32: Value, value_u32: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.write_dmem, self.builder.func);+ self.builder.ins().call(f, &[self.sys_ptr, addr_u32, value_u32]);+ }++ pub fn emit_inc_ar(&mut self, reg: u32) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.inc_ar, self.builder.func);+ let reg_v = self.builder.ins().iconst(types::I32, reg as i64);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, reg_v]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_dec_ar(&mut self, reg: u32) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.dec_ar, self.builder.func);+ let reg_v = self.builder.ins().iconst(types::I32, reg as i64);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, reg_v]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_increase_ar(&mut self, reg: u32, ix_value: Value) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.increase_ar, self.builder.func);+ let reg_v = self.builder.ins().iconst(types::I32, reg as i64);+ let ix_u32 = self.builder.ins().uextend(types::I32, ix_value);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, reg_v, ix_u32]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_decrease_ar_ix(&mut self, reg: u32, ix_value: Value) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.decrease_ar_ix, self.builder.func);+ let reg_v = self.builder.ins().iconst(types::I32, reg as i64);+ let ix_u32 = self.builder.ins().uextend(types::I32, ix_value);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, reg_v, ix_u32]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_dynamic_shift(&mut self, d_const: u32, shift_val: Value, mode_const: u32) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.dynamic_shift, self.builder.func);+ let d_v = self.builder.ins().iconst(types::I32, d_const as i64);+ let shift_u32 = self.builder.ins().uextend(types::I32, shift_val);+ let mode_v = self.builder.ins().iconst(types::I32, mode_const as i64);+ self.builder.ins().call(f, &[self.sys_ptr, d_v, shift_u32, mode_v]);+ }++ pub fn emit_read_imem(&mut self, addr_u32: Value) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.read_imem, self.builder.func);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, addr_u32]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_write_ac_mid_sxm(&mut self, idx: u32, value: Value) {+ use cranelift_codegen::ir::condcodes::IntCC;+ let (mid_off, high_off, low_off) = if idx == 0 {+ (+ abi::dsp_ac0_mid_offset() as i32,+ abi::dsp_ac0_high_offset() as i32,+ abi::dsp_ac0_low_offset() as i32,+ )+ } else {+ (+ abi::dsp_ac1_mid_offset() as i32,+ abi::dsp_ac1_high_offset() as i32,+ abi::dsp_ac1_low_offset() as i32,+ )+ };++ self.store_u16(value, mid_off);++ let status_off = abi::dsp_status_offset() as i32;+ let status = self.load_u16(status_off);+ let sxm_shifted = self.builder.ins().ushr_imm(status, 14);+ let sxm = self.builder.ins().band_imm(sxm_shifted, 1);+ let sxm_set = self.builder.ins().icmp_imm(IntCC::NotEqual, sxm, 0);++ let sxm_block = self.builder.create_block();+ let join_block = self.builder.create_block();+ self.builder.ins().brif(sxm_set, sxm_block, &[], join_block, &[]);++ self.builder.switch_to_block(sxm_block);+ self.builder.seal_block(sxm_block);++ let sign_ext_high = self.builder.ins().sshr_imm(value, 15);+ self.store_u16(sign_ext_high, high_off);+ let zero = self.iconst16(0);+ self.store_u16(zero, low_off);+ self.builder.ins().jump(join_block, &[]);++ self.builder.switch_to_block(join_block);+ self.builder.seal_block(join_block);+ }++ pub fn emit_call_stack_push(&mut self, value: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.call_stack_push, self.builder.func);+ let v_u32 = self.builder.ins().uextend(types::I32, value);+ self.builder.ins().call(f, &[self.sys_ptr, v_u32]);+ }++ pub fn emit_call_stack_pop(&mut self) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.call_stack_pop, self.builder.func);+ let inst = self.builder.ins().call(f, &[self.sys_ptr]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_data_stack_pop(&mut self) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.data_stack_pop, self.builder.func);+ let inst = self.builder.ins().call(f, &[self.sys_ptr]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_read_reg_full(&mut self, slot: u32) -> Value {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.read_reg_full, self.builder.func);+ let slot_v = self.builder.ins().iconst(types::I32, slot as i64);+ let inst = self.builder.ins().call(f, &[self.sys_ptr, slot_v]);+ self.builder.inst_results(inst)[0]+ }++ pub fn emit_write_reg_full(&mut self, slot: u32, value: Value) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.write_reg_full, self.builder.func);+ let slot_v = self.builder.ins().iconst(types::I32, slot as i64);+ let value_u32 = self.builder.ins().uextend(types::I32, value);+ self.builder.ins().call(f, &[self.sys_ptr, slot_v, value_u32]);+ }++ pub fn emit_read_reg(&mut self, slot: u8) -> Value {+ if let Some(v) = self.try_emit_reg_read(slot) {+ return v;+ }++ let v32 = self.emit_read_reg_full(slot as u32);+ self.builder.ins().ireduce(types::I16, v32)+ }++ pub fn emit_write_reg(&mut self, slot: u8, value: Value) {+ if !self.try_emit_reg_write(slot, value) {+ self.emit_write_reg_full(slot as u32, value);+ }+ }++ pub fn emit_loop_setup(&mut self, end_addr_p1: u16, counter: Value, call_stack_val: u16) {+ let f = self+ .module+ .declare_func_in_func(self.extern_funcs.loop_setup, self.builder.func);+ let end_v = self.builder.ins().iconst(types::I32, end_addr_p1 as i64);+ let counter_u32 = self.builder.ins().uextend(types::I32, counter);+ let csv_v = self.builder.ins().iconst(types::I32, call_stack_val as i64);+ self.builder.ins().call(f, &[self.sys_ptr, end_v, counter_u32, csv_v]);+ }++ pub fn emit_clear_oc(&mut self) {+ let sr_off = abi::dsp_status_offset() as i32;+ let sr = self.load_u16(sr_off);+ let mask = !((1i64 << Self::SR_BIT_O) | (1i64 << Self::SR_BIT_C)) & 0xFFFF;+ let cleared = self.builder.ins().band_imm(sr, mask);+ self.store_u16(cleared, sr_off);+ }++ pub fn emit_set_lz(&mut self, lz: Value) {+ let sr_off = abi::dsp_status_offset() as i32;+ let sr = self.load_u16(sr_off);+ let bit_pos = Self::SR_BIT_LZ;+ let mask = !(1i64 << bit_pos) & 0xFFFF;+ let cleared = self.builder.ins().band_imm(sr, mask);+ let lz1 = self.builder.ins().band_imm(lz, 1);+ let shifted = self.builder.ins().ishl_imm(lz1, bit_pos as i64);+ let updated = self.builder.ins().bor(cleared, shifted);+ self.store_u16(updated, sr_off);+ }++ fn emit_eval_condition(&mut self, cond: u8) -> Value {+ let sr = self.load_sr();+ let c = self.sr_bit(sr, Self::SR_BIT_C);+ let o = self.sr_bit(sr, Self::SR_BIT_O);+ let z = self.sr_bit(sr, Self::SR_BIT_Z);+ let s = self.sr_bit(sr, Self::SR_BIT_S);+ let as32 = self.sr_bit(sr, Self::SR_BIT_AS32);+ let tb = self.sr_bit(sr, Self::SR_BIT_TB);+ let lz = self.sr_bit(sr, Self::SR_BIT_LZ);++ let and = |b: &mut FunctionBuilder, x: Value, y: Value| b.ins().band(x, y);+ let or = |b: &mut FunctionBuilder, x: Value, y: Value| b.ins().bor(x, y);+ let not = |b: &mut FunctionBuilder, x: Value| b.ins().bxor_imm(x, 1);+ let eqb = |b: &mut FunctionBuilder, x: Value, y: Value| {+ let x = b.ins().bxor(x, y);+ b.ins().bxor_imm(x, 1)+ };+ let neb = |b: &mut FunctionBuilder, x: Value, y: Value| b.ins().bxor(x, y);++ match cond {+ 0b0000 => eqb(self.builder, o, s),+ 0b0001 => neb(self.builder, o, s),+ 0b0010 => {+ let ge = eqb(self.builder, o, s);+ let nz = not(self.builder, z);+ and(self.builder, ge, nz)+ }+ 0b0011 => {+ let lt = neb(self.builder, o, s);+ or(self.builder, lt, z)+ }+ 0b0100 => not(self.builder, z),+ 0b0101 => z,+ 0b0110 => not(self.builder, c),+ 0b0111 => c,+ 0b1000 => not(self.builder, as32),+ 0b1001 => as32,+ 0b1010 => {+ let either = or(self.builder, as32, tb);+ let nz = not(self.builder, z);+ and(self.builder, either, nz)+ }+ 0b1011 => {+ let nas = not(self.builder, as32);+ let ntb = not(self.builder, tb);+ let neither = and(self.builder, nas, ntb);+ or(self.builder, neither, z)+ }+ 0b1100 => not(self.builder, lz),+ 0b1101 => lz,+ 0b1110 => o,+ _ => self.builder.ins().iconst(types::I16, 1),+ }+ }+}++#[cold]+#[inline(never)]+pub fn invalid(_ctx: &mut TranslatorCtx, instr: Instruction) {+ panic!("DSP JIT translate: opcode {:#06x} hit invalid handler", instr.0);+}++#[cold]+#[inline(never)]+pub fn invalid_ext(_ctx: &mut TranslatorCtx, instr: GcDspExt) {+ panic!("DSP JIT translate: ext opcode {:#04x} hit invalid_ext handler", instr.0);+}++#[inline(always)]+pub fn ext_mv<const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let d = instr.d_4_5();+ let s = instr.s_6_7();+ let cache_off = (abi::dsp_ext_ac_cache_base_offset() + (s as usize) * 2) as i32;+ let value = ctx.load_u16(cache_off);+ let dst = 24u8 + d;+ ctx.try_emit_reg_write(dst, value);+}++#[inline(always)]+pub fn ext_nop<const SYSTEM: SystemId>(_ctx: &mut TranslatorCtx, _instr: GcDspExt) {}++#[inline(always)]+pub fn control<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_NOP => {}+ OP_HALT => {+ let csr = ctx.load_csr();+ let updated = ctx.builder.ins().bor_imm(csr, 1i64 << 2);+ ctx.store_csr(updated);+ }+ OP_IFCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let skip_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, end_block, &[], skip_block, &[]);++ ctx.builder.switch_to_block(skip_block);+ ctx.builder.seal_block(skip_block);+ let nia_off = abi::dsp_nia_offset_max() as i32;+ let nia = ctx.load_u16(nia_off);+ let new_nia = ctx.builder.ins().iadd_imm(nia, 1);+ ctx.store_u16(new_nia, nia_off);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ OP_JCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);+ let target = ctx.iconst16(instr.addr());+ let nia_off = abi::dsp_nia_offset_max() as i32;+ ctx.store_u16(target, nia_off);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ OP_CALLCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);++ let nia = ctx.load_u16(abi::dsp_nia_offset_max() as i32);+ ctx.emit_call_stack_push(nia);++ let target = ctx.iconst16(instr.addr());+ ctx.store_u16(target, abi::dsp_nia_offset_max() as i32);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }++ OP_RETCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);+ let popped = ctx.emit_call_stack_pop();+ let popped16 = ctx.builder.ins().ireduce(types::I16, popped);+ ctx.store_u16(popped16, abi::dsp_nia_offset_max() as i32);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ OP_RTICC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);+ let new_status = ctx.emit_data_stack_pop();+ let new_status16 = ctx.builder.ins().ireduce(types::I16, new_status);+ ctx.store_u16(new_status16, abi::dsp_status_offset() as i32);++ let popped = ctx.emit_call_stack_pop();+ let popped16 = ctx.builder.ins().ireduce(types::I16, popped);+ ctx.store_u16(popped16, abi::dsp_nia_offset_max() as i32);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ OP_JRCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);+ let slot = instr.reg_8_10();+ let val = if let Some(v) = ctx.try_emit_reg_read(slot) {+ v+ } else {+ let v32 = ctx.emit_read_reg_full(slot as u32);+ ctx.builder.ins().ireduce(types::I16, v32)+ };+ ctx.store_u16(val, abi::dsp_nia_offset_max() as i32);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ OP_CALLRCC => {+ let cond_v = ctx.emit_eval_condition(instr.cond());+ let take_block = ctx.builder.create_block();+ let end_block = ctx.builder.create_block();+ ctx.builder.ins().brif(cond_v, take_block, &[], end_block, &[]);++ ctx.builder.switch_to_block(take_block);+ ctx.builder.seal_block(take_block);++ let slot = instr.reg_8_10();+ let target = if let Some(v) = ctx.try_emit_reg_read(slot) {+ v+ } else {+ let v32 = ctx.emit_read_reg_full(slot as u32);+ ctx.builder.ins().ireduce(types::I16, v32)+ };+ let nia = ctx.load_u16(abi::dsp_nia_offset_max() as i32);+ ctx.emit_call_stack_push(nia);+ ctx.store_u16(target, abi::dsp_nia_offset_max() as i32);+ ctx.builder.ins().jump(end_block, &[]);++ ctx.builder.switch_to_block(end_block);+ ctx.builder.seal_block(end_block);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn loops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ let natural_nia = ctx.pc.wrapping_add(ctx.size as u16);+ let (end_addr_p1, call_stack_val) = match OP {+ OP_LOOP_REG | OP_LOOPI => (natural_nia.wrapping_add(1), natural_nia),+ OP_BLOOP | OP_BLOOPI => (instr.addr().wrapping_add(1), natural_nia),+ _ => unreachable!("DSP JIT: unexpected loop OP"),+ };++ let counter = match OP {+ OP_LOOPI | OP_BLOOPI => ctx.iconst16(instr.imm_8_15_u8() as u16),+ OP_LOOP_REG | OP_BLOOP => {+ let reg = instr.reg_11_15();+ match ctx.try_emit_reg_read(reg) {+ Some(v) => v,+ None => {+ let v32 = ctx.emit_read_reg_full(reg as u32);+ ctx.builder.ins().ireduce(types::I16, v32)+ }+ }+ }+ _ => unreachable!(),+ };++ ctx.emit_loop_setup(end_addr_p1, counter, call_stack_val);+}++#[inline(always)]+pub fn addr_reg<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_DAR => {+ let d = instr.d_14_15();+ let new_ar = ctx.emit_dec_ar(d as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ let ar_off = (abi::dsp_ar_base_offset() + (d as usize) * 2) as i32;+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_IAR => {+ let d = instr.d_14_15();+ let new_ar = ctx.emit_inc_ar(d as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ let ar_off = (abi::dsp_ar_base_offset() + (d as usize) * 2) as i32;+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_SUBARN => {+ let d = instr.d_14_15();+ let ix_off = (abi::dsp_ix_base_offset() + (d as usize) * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ let new_ar = ctx.emit_decrease_ar_ix(d as u32, ix);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ let ar_off = (abi::dsp_ar_base_offset() + (d as usize) * 2) as i32;+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_ADDARN => {+ let s = instr.s_12_13();+ let d = instr.d_14_15();+ let ix_off = (abi::dsp_ix_base_offset() + (s as usize) * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ let new_ar = ctx.emit_increase_ar(d as u32, ix);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ let ar_off = (abi::dsp_ar_base_offset() + (d as usize) * 2) as i32;+ ctx.store_u16(new_ar16, ar_off);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn load_store<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_LRI => {+ let dst = instr.d_11_15();+ let imm = instr.imm_16_31();+ let v = ctx.iconst16(imm);++ if dst == 30 || dst == 31 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, v);+ } else {+ ctx.emit_write_reg(dst, v);+ }+ }+ OP_MRR => {+ let src = instr.src();+ let dst = instr.dst();+ let v = ctx.emit_read_reg(src);++ if dst == 30 || dst == 31 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, v);+ } else {+ ctx.emit_write_reg(dst, v);+ }+ }+ OP_LR => {+ let dst = instr.d_11_15();+ let imm = instr.imm_16_31();+ let addr = ctx.builder.ins().iconst(types::I32, imm as i64);+ let val_u32 = ctx.emit_read_dmem(addr);+ let val = ctx.builder.ins().ireduce(types::I16, val_u32);++ if dst == 30 || dst == 31 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, val);+ } else {+ ctx.emit_write_reg(dst, val);+ }+ }+ OP_SR => {+ let src = instr.d_11_15();+ let v = ctx.emit_read_reg(src);+ let imm = instr.imm_16_31();+ let addr = ctx.builder.ins().iconst(types::I32, imm as i64);+ let v_u32 = ctx.builder.ins().uextend(types::I32, v);+ ctx.emit_write_dmem(addr, v_u32);+ }+ OP_SI => {+ let addr_low = instr.mem_8_15_u16();+ let addr = (0xFF00u32 | (addr_low as u32)) as i64;+ let addr_v = ctx.builder.ins().iconst(types::I32, addr);+ let val = ctx.builder.ins().iconst(types::I32, instr.imm_16_31() as i64);+ ctx.emit_write_dmem(addr_v, val);+ }+ OP_LRR | OP_LRRD | OP_LRRI | OP_LRRN => {+ let s = instr.s_9_10();+ let dst = instr.d_11_15();+ let ar_off = (abi::dsp_ar_base_offset() + (s as usize) * 2) as i32;+ let ar_u16 = ctx.load_u16(ar_off);+ let addr = ctx.builder.ins().uextend(types::I32, ar_u16);+ let new_ar_i32: Option<Value> = match OP {+ OP_LRRD => Some(ctx.emit_dec_ar(s as u32)),+ OP_LRRI => Some(ctx.emit_inc_ar(s as u32)),+ OP_LRRN => {+ let ix_off = (abi::dsp_ix_base_offset() + (s as usize) * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ Some(ctx.emit_increase_ar(s as u32, ix))+ }+ _ => None,+ };++ let val_u32 = ctx.emit_read_dmem(addr);+ let val = ctx.builder.ins().ireduce(types::I16, val_u32);+ if dst == 30 || dst == 31 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, val);+ } else {+ ctx.emit_write_reg(dst, val);+ }++ if let Some(ar_v) = new_ar_i32 {+ if dst as usize != s as usize {+ let ar_v16 = ctx.builder.ins().ireduce(types::I16, ar_v);+ ctx.store_u16(ar_v16, ar_off);+ }+ }+ }+ OP_SRR | OP_SRRD | OP_SRRI | OP_SRRN => {+ let ar_idx = instr.s_9_10();+ let src = instr.d_11_15();+ let v = ctx.emit_read_reg(src);+ let new_ar_i32: Option<Value> = match OP {+ OP_SRRD => Some(ctx.emit_dec_ar(ar_idx as u32)),+ OP_SRRI => Some(ctx.emit_inc_ar(ar_idx as u32)),+ OP_SRRN => {+ let ix_off = (abi::dsp_ix_base_offset() + (ar_idx as usize) * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ Some(ctx.emit_increase_ar(ar_idx as u32, ix))+ }+ _ => None,+ };++ let ar_off = (abi::dsp_ar_base_offset() + (ar_idx as usize) * 2) as i32;+ let ar_u16 = ctx.load_u16(ar_off);+ let addr = ctx.builder.ins().uextend(types::I32, ar_u16);+ let v_u32 = ctx.builder.ins().uextend(types::I32, v);+ ctx.emit_write_dmem(addr, v_u32);++ if let Some(ar_v) = new_ar_i32 {+ let ar_v16 = ctx.builder.ins().ireduce(types::I16, ar_v);+ ctx.store_u16(ar_v16, ar_off);+ }+ }+ OP_LRS => {+ let dst = 24u8 + instr.reg_5_7();+ let mem_low = instr.mem_8_15_u16();+ let cfg = ctx.load_u16(abi::dsp_config_offset() as i32);+ let cfg_shl = ctx.builder.ins().ishl_imm(cfg, 8);+ let addr_u16 = ctx.builder.ins().bor_imm(cfg_shl, mem_low as i64);+ let addr = ctx.builder.ins().uextend(types::I32, addr_u16);+ let val_u32 = ctx.emit_read_dmem(addr);+ let val = ctx.builder.ins().ireduce(types::I16, val_u32);++ if dst >= 30 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, val);+ } else {+ ctx.try_emit_reg_write(dst, val);+ }+ }+ OP_SRS => {+ let src = 28u8 + instr.reg_6_7();+ let v = ctx.emit_read_reg(src);+ let mem_low = instr.mem_8_15_u16();++ let cfg = ctx.load_u16(abi::dsp_config_offset() as i32);+ let cfg_shl = ctx.builder.ins().ishl_imm(cfg, 8);+ let addr_u16 = ctx.builder.ins().bor_imm(cfg_shl, mem_low as i64);+ let addr = ctx.builder.ins().uextend(types::I32, addr_u16);+ let v_u32 = ctx.builder.ins().uextend(types::I32, v);+ ctx.emit_write_dmem(addr, v_u32);+ }+ OP_SRSH => {+ let src = if instr.s_7_7() != 0 { 17u8 } else { 16u8 };+ let v = ctx.try_emit_reg_read(src).expect("AC*H is simple");+ let mem_low = instr.mem_8_15_u16();++ let cfg = ctx.load_u16(abi::dsp_config_offset() as i32);+ let cfg_shl = ctx.builder.ins().ishl_imm(cfg, 8);+ let addr_u16 = ctx.builder.ins().bor_imm(cfg_shl, mem_low as i64);+ let addr = ctx.builder.ins().uextend(types::I32, addr_u16);+ let v_u32 = ctx.builder.ins().uextend(types::I32, v);+ ctx.emit_write_dmem(addr, v_u32);+ }+ OP_ILRR | OP_ILRRD | OP_ILRRI | OP_ILRRN => {+ let src = instr.s_14_15() as usize;+ let d = instr.d_7_7();+ let ar_off = (abi::dsp_ar_base_offset() + src * 2) as i32;+ let ar = ctx.load_u16(ar_off);+ let addr = ctx.builder.ins().uextend(types::I32, ar);+ let new_ar: Option<Value> = match OP {+ OP_ILRRD => Some(ctx.emit_dec_ar(src as u32)),+ OP_ILRRI => Some(ctx.emit_inc_ar(src as u32)),+ OP_ILRRN => {+ let ix_off = (abi::dsp_ix_base_offset() + src * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ Some(ctx.emit_increase_ar(src as u32, ix))+ }+ _ => None,+ };++ let val_u32 = ctx.emit_read_imem(addr);+ let val = ctx.builder.ins().ireduce(types::I16, val_u32);+ ctx.emit_write_ac_mid_sxm(d as u32, val);++ if let Some(ar_v) = new_ar {+ let ar_v16 = ctx.builder.ins().ireduce(types::I16, ar_v);+ ctx.store_u16(ar_v16, ar_off);+ }+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn imm_alu<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_LRIS => {+ let dst = 24u8 + (instr.reg_5_7() as u8);+ let imm_signed = instr.imm_8_15_i16();+ let v = ctx.iconst16(imm_signed as u16);+ ctx.emit_write_reg(dst, v);+ }+ OP_ANDI => {+ let d = instr.d_7_7();+ let imm = instr.imm_16_31();+ let mid = ctx.read_ac_mid(d);+ let result = ctx.builder.ins().band_imm(mid, imm as i64);+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }+ OP_ORI => {+ let d = instr.d_7_7();+ let imm = instr.imm_16_31();+ let mid = ctx.read_ac_mid(d);+ let result = ctx.builder.ins().bor_imm(mid, imm as i64);+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }+ OP_XORI => {+ let d = instr.d_7_7();+ let imm = instr.imm_16_31();+ let mid = ctx.read_ac_mid(d);+ let result = ctx.builder.ins().bxor_imm(mid, imm as i64);+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }+ OP_ANDF => {+ let d = instr.d_7_7();+ let imm = instr.imm_16_31();+ let mid = ctx.read_ac_mid(d);+ let masked = ctx.builder.ins().band_imm(mid, imm as i64);+ let lz = ctx+ .builder+ .ins()+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, masked, 0);+ let lz_i16 = ctx.builder.ins().uextend(types::I16, lz);+ ctx.emit_set_lz(lz_i16);+ }+ OP_ANDCF => {+ let d = instr.d_7_7();+ let imm = instr.imm_16_31();+ let mid = ctx.read_ac_mid(d);+ let masked = ctx.builder.ins().band_imm(mid, imm as i64);+ let imm_v = ctx.iconst16(imm);+ let lz = ctx+ .builder+ .ins()+ .icmp(cranelift_codegen::ir::condcodes::IntCC::Equal, masked, imm_v);+ let lz_i16 = ctx.builder.ins().uextend(types::I16, lz);+ ctx.emit_set_lz(lz_i16);+ }+ OP_ADDI => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);++ let b = (instr.imm_16_31() as i16 as i64) << 16;+ let b_v = ctx.builder.ins().iconst(types::I64, b);+ let result = ctx.builder.ins().iadd(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b_v, new_ac);+ }+ OP_ADDIS => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = (instr.imm_8_15_i16() as i64) << 16;+ let b_v = ctx.builder.ins().iconst(types::I64, b);+ let result = ctx.builder.ins().iadd(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b_v, new_ac);+ }+ OP_CMPI => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = (instr.imm_16_31() as i16 as i64) << 16;+ let b_v = ctx.builder.ins().iconst(types::I64, b);+ let result = ctx.builder.ins().isub(a, b_v);+ ctx.emit_update_flags_sub(a, b_v, result);+ }+ OP_CMPIS => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = (instr.imm_8_15_i16() as i64) << 16;+ let b_v = ctx.builder.ins().iconst(types::I64, b);+ let result = ctx.builder.ins().isub(a, b_v);+ ctx.emit_update_flags_sub(a, b_v, result);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn add_sub<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ fn read_simple_signed_i64(ctx: &mut TranslatorCtx, slot: u8) -> Value {+ let raw = ctx.try_emit_reg_read(slot).expect("simple slot");+ ctx.builder.ins().sextend(types::I64, raw)+ }++ fn read_axh_ax_combined(ctx: &mut TranslatorCtx, s: u8) -> Value {+ let axh_off = (abi::dsp_axh_base_offset() + (s as usize) * 2) as i32;+ let ax_off = (abi::dsp_ax_base_offset() + (s as usize) * 2) as i32;+ let axh = ctx.load_u16(axh_off);+ let ax = ctx.load_u16(ax_off);+ let axh_i64 = ctx.builder.ins().sextend(types::I64, axh);+ let axh_shl = ctx.builder.ins().ishl_imm(axh_i64, 16);+ let ax_i64 = ctx.builder.ins().uextend(types::I64, ax);+ ctx.builder.ins().bor(axh_shl, ax_i64)+ }++ match OP {+ OP_ADDR => {+ let ss = instr.ss();+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let src = read_simple_signed_i64(ctx, 24 + ss);+ let b = ctx.builder.ins().ishl_imm(src, 16);+ let result = ctx.builder.ins().iadd(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b, new_ac);+ }+ OP_ADD => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = ctx.read_ac_i64(1 - d);+ let result = ctx.builder.ins().iadd(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b, new_ac);+ }+ OP_ADDAX => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = read_axh_ax_combined(ctx, s);+ let result = ctx.builder.ins().iadd(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b, new_ac);+ }+ OP_ADDAXL => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let ax_off = (abi::dsp_ax_base_offset() + (s as usize) * 2) as i32;+ let ax = ctx.load_u16(ax_off);+ let b = ctx.builder.ins().uextend(types::I64, ax);+ let result = ctx.builder.ins().iadd(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b, new_ac);+ }+ OP_SUBR => {+ let ss = instr.ss();+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let src = read_simple_signed_i64(ctx, 24 + ss);+ let b = ctx.builder.ins().ishl_imm(src, 16);+ let result = ctx.builder.ins().isub(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(a, b, new_ac);+ }+ OP_SUB => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = ctx.read_ac_i64(1 - d);+ let result = ctx.builder.ins().isub(a, b);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(a, b, new_ac);+ }+ OP_SUBAX => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = read_axh_ax_combined(ctx, s);+ let result = ctx.builder.ins().isub(a, b);+ ctx.write_ac_i64(d, result);+ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(a, b, new_ac);+ }+ OP_ADDP => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let (pc, po) = ctx.product_flags_i16();+ let b = ctx.read_product_i64();+ let result = ctx.builder.ins().iadd(a, b);+ let os_before = ctx.read_status_bit(7);+ ctx.write_ac_i64(d, result);+ ctx.emit_update_flags_add(a, b, result);+ ctx.set_status_bit(7, os_before);++ let c_now = ctx.read_status_bit(0);+ let o_now = ctx.read_status_bit(1);+ let c_xor = ctx.builder.ins().bxor(c_now, pc);+ let o_xor = ctx.builder.ins().bxor(o_now, po);+ ctx.set_status_bit(0, c_xor);+ ctx.set_status_bit(1, o_xor);+ ctx.or_status_bit(7, o_xor);+ }+ OP_SUBP => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let (pc, po) = ctx.product_flags_i16();+ let b = ctx.read_product_i64();+ let result = ctx.builder.ins().isub(a, b);+ let os_before = ctx.read_status_bit(7);+ ctx.write_ac_i64(d, result);+ ctx.emit_update_flags_sub(a, b, result);+ ctx.set_status_bit(7, os_before);++ let c_now = ctx.read_status_bit(0);+ let o_now = ctx.read_status_bit(1);++ let one = ctx.iconst16(1);+ let not_pc = ctx.builder.ins().isub(one, pc);+ let c_xor = ctx.builder.ins().bxor(c_now, not_pc);+ let o_xor = ctx.builder.ins().bxor(o_now, po);+ ctx.set_status_bit(0, c_xor);+ ctx.set_status_bit(1, o_xor);+ ctx.or_status_bit(7, o_xor);+ }+ OP_ADDPAXZ => {+ use cranelift_codegen::ir::condcodes::IntCC;++ let s = instr.s_6_6() as usize;+ let d = instr.d_7_7();+ let (pc, po) = ctx.product_flags_i16();+ let a = ctx.read_product_i64();+ let axh_off = (abi::dsp_axh_base_offset() + s * 2) as i32;+ let axh16 = ctx.load_u16(axh_off);+ let axh_i64 = ctx.builder.ins().sextend(types::I64, axh16);+ let b = ctx.builder.ins().ishl_imm(axh_i64, 16);++ let val40 = ctx.builder.ins().band_imm(a, 0xFF_FFFF_FFFF_i64);+ let lower = ctx.builder.ins().band_imm(val40, 0xFFFF);+ let sum_pre = ctx.builder.ins().iadd(val40, b);+ let sum40 = ctx.builder.ins().band_imm(sum_pre, 0xFF_FFFF_FFFF_i64);+ let bit16 = ctx.builder.ins().band_imm(sum40, 0x10000);+ let bit16_set = ctx.builder.ins().icmp_imm(IntCC::NotEqual, bit16, 0);+ let lower_gt = ctx.builder.ins().icmp_imm(IntCC::UnsignedGreaterThan, lower, 0x8000);+ let lower_eq = ctx.builder.ins().icmp_imm(IntCC::Equal, lower, 0x8000);+ let lower_eq_and_bit16 = ctx.builder.ins().band(lower_eq, bit16_set);+ let round_up = ctx.builder.ins().bor(lower_gt, lower_eq_and_bit16);++ let truncated = ctx.builder.ins().band_imm(val40, !0xFFFF_i64);+ let plus = ctx.builder.ins().iadd_imm(val40, 0x10000);+ let rounded_path = ctx.builder.ins().band_imm(plus, !0xFFFF_i64);+ let plus_overflow = ctx+ .builder+ .ins()+ .icmp_imm(IntCC::UnsignedGreaterThan, plus, 0xFF_FFFF_FFFF_i64);+ let plus_overflow_i16 = ctx.builder.ins().uextend(types::I16, plus_overflow);+ let zero_i16 = ctx.iconst16(0);++ let rounded = ctx.builder.ins().select(round_up, rounded_path, truncated);+ let rounding_carry = ctx.builder.ins().select(round_up, plus_overflow_i16, zero_i16);++ let result_pre = ctx.builder.ins().iadd(rounded, b);+ let result = ctx.builder.ins().band_imm(result_pre, !0xFFFF_i64);++ let os_before = ctx.read_status_bit(7);+ ctx.write_ac_i64(d, result);+ ctx.emit_update_flags_add(rounded, b, result);++ let c_in = ctx.builder.ins().bor(pc, rounding_carry);+ ctx.set_status_bit(7, os_before);++ let c_now = ctx.read_status_bit(0);+ let o_now = ctx.read_status_bit(1);+ let c_xor = ctx.builder.ins().bxor(c_now, c_in);+ let o_xor = ctx.builder.ins().bxor(o_now, po);+ ctx.set_status_bit(0, c_xor);+ ctx.set_status_bit(1, o_xor);+ ctx.or_status_bit(7, o_xor);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn move_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_MOVR => {+ let ss = instr.ss();+ let d = instr.d_7_7();++ let raw = ctx.try_emit_reg_read(24 + ss).expect("simple slot");+ let signed = ctx.builder.ins().sextend(types::I64, raw);+ let val = ctx.builder.ins().ishl_imm(signed, 16);+ ctx.write_ac_i64(d, val);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_ac(new_ac);+ ctx.emit_clear_oc();+ }+ OP_MOVAX => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let axh_off = (abi::dsp_axh_base_offset() + (s as usize) * 2) as i32;+ let ax_off = (abi::dsp_ax_base_offset() + (s as usize) * 2) as i32;+ let axh = ctx.load_u16(axh_off);+ let ax = ctx.load_u16(ax_off);+ let axh_i64 = ctx.builder.ins().sextend(types::I64, axh);+ let axh_shl = ctx.builder.ins().ishl_imm(axh_i64, 16);+ let ax_i64 = ctx.builder.ins().uextend(types::I64, ax);+ let val = ctx.builder.ins().bor(axh_shl, ax_i64);+ ctx.write_ac_i64(d, val);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_ac(new_ac);+ ctx.emit_clear_oc();+ }+ OP_MOV => {+ let d = instr.d_7_7();+ let val = ctx.read_ac_i64(1 - d);+ ctx.write_ac_i64(d, val);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_ac(new_ac);+ ctx.emit_clear_oc();+ }+ OP_MOVP => {+ let d = instr.d_7_7();+ ctx.move_prod_to_ac(d);+ }+ OP_MOVPZ => {+ let d = instr.d_7_7();+ ctx.move_prod_to_ac_zero(d);+ }+ OP_MOVNP => {+ use cranelift_codegen::ir::condcodes::IntCC;++ let d = instr.d_7_7();+ let prod = ctx.read_product_i64();+ let (carry, overflow) = ctx.product_flags_i16();+ let val = ctx.builder.ins().ineg(prod);+ ctx.write_ac_i64(d, val);+ ctx.emit_update_flags_ac(val);++ ctx.set_status_bit(1, overflow);+ ctx.or_status_bit(7, overflow);++ let prod40 = ctx.builder.ins().band_imm(prod, 0xFF_FFFF_FFFF_i64);+ let prod_zero = ctx.builder.ins().icmp_imm(IntCC::Equal, prod40, 0);+ let prod_zero_i16 = ctx.builder.ins().uextend(types::I16, prod_zero);+ let one = ctx.iconst16(1);+ let not_carry = ctx.builder.ins().isub(one, carry);+ let c_val = ctx.builder.ins().bxor(not_carry, prod_zero_i16);+ ctx.set_status_bit(0, c_val);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn inc_dec<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_INCM => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b = 0x10000i64;+ let b_v = ctx.builder.ins().iconst(types::I64, b);+ let result = ctx.builder.ins().iadd(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b_v, new_ac);+ }+ OP_INC => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b_v = ctx.builder.ins().iconst(types::I64, 1);+ let result = ctx.builder.ins().iadd(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_add(a, b_v, new_ac);+ }+ OP_DECM => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b_v = ctx.builder.ins().iconst(types::I64, 0x10000);+ let result = ctx.builder.ins().isub(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(a, b_v, new_ac);+ }+ OP_DEC => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let b_v = ctx.builder.ins().iconst(types::I64, 1);+ let result = ctx.builder.ins().isub(a, b_v);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(a, b_v, new_ac);+ }+ OP_NEG => {+ let d = instr.d_7_7();+ let a = ctx.read_ac_i64(d);+ let zero = ctx.builder.ins().iconst(types::I64, 0);+ let result = ctx.builder.ins().isub(zero, a);+ ctx.write_ac_i64(d, result);++ let new_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_sub(zero, a, new_ac);+ }+ OP_ABS_AC => {+ use cranelift_codegen::ir::condcodes::IntCC;++ let d = instr.d_4_4();+ let a = ctx.read_ac_i64(d);+ let was_negative = ctx.builder.ins().icmp_imm(IntCC::SignedLessThan, a, 0);+ let neg = ctx.builder.ins().ineg(a);++ let new_ac = ctx.builder.ins().select(was_negative, neg, a);+ ctx.write_ac_i64(d, new_ac);++ let stored_ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_ac(stored_ac);++ let still_neg = ctx.builder.ins().icmp_imm(IntCC::SignedLessThan, stored_ac, 0);+ let o_b8 = ctx.builder.ins().band(was_negative, still_neg);+ let o_i16 = ctx.builder.ins().uextend(types::I16, o_b8);+ ctx.set_status_bit(1, o_i16);+ ctx.or_status_bit(7, o_i16);+ ctx.set_status_bit_const(0, false);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn cmp_test<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_CMP => {+ let a = ctx.read_ac_i64(0);+ let b = ctx.read_ac_i64(1);+ let result = ctx.builder.ins().isub(a, b);+ ctx.emit_update_flags_sub(a, b, result);+ }+ OP_CMPAXH => {+ let s = instr.s_4_4();+ let r = instr.s_3_3();+ let a = ctx.read_ac_i64(s);+ let axh_off = (abi::dsp_axh_base_offset() + (r as usize) * 2) as i32;+ let axh = ctx.load_u16(axh_off);+ let axh_signed = ctx.builder.ins().sextend(types::I64, axh);+ let b = ctx.builder.ins().ishl_imm(axh_signed, 16);+ let result = ctx.builder.ins().isub(a, b);+ ctx.emit_update_flags_sub(a, b, result);+ }+ OP_TST => {+ let r = instr.r_4_4();+ let ac = ctx.read_ac_i64(r);+ ctx.emit_update_flags_ac(ac);+ ctx.emit_clear_oc();+ }+ OP_TSTAXH => {+ let r = instr.r_7_7();+ let axh_off = (abi::dsp_axh_base_offset() + (r as usize) * 2) as i32;+ let axh = ctx.load_u16(axh_off);+ let axh_signed = ctx.builder.ins().sextend(types::I64, axh);+ let val = ctx.builder.ins().ishl_imm(axh_signed, 16);+ ctx.emit_update_flags_ac(val);++ let sr_off = abi::dsp_status_offset() as i32;+ let sr = ctx.load_u16(sr_off);+ let mask = !((1i64 << 4) | (1 << 1) | 1) & 0xFFFF;+ let cleared = ctx.builder.ins().band_imm(sr, mask);+ ctx.store_u16(cleared, sr_off);+ }+ OP_NX_0 | OP_NX_1 => {}+ OP_CLR => {+ let r = instr.r_4_4();+ let zero64 = ctx.builder.ins().iconst(types::I64, 0);+ ctx.write_ac_i64(r, zero64);++ let sr_off = abi::dsp_status_offset() as i32;+ let sr = ctx.load_u16(sr_off);+ let mask = !((1i64 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5)) & 0xFFFF;+ let cleared = ctx.builder.ins().band_imm(sr, mask);+ let updated = ctx.builder.ins().bor_imm(cleared, (1i64 << 5) | (1 << 2));+ ctx.store_u16(updated, sr_off);+ }+ OP_CLRP => {+ let pl = ctx.iconst16(0x0000);+ let pm1 = ctx.iconst16(0xFFF0);+ let ph = ctx.iconst16(0x00FF);+ let pm2 = ctx.iconst16(0x0010);+ ctx.store_u16(pl, abi::dsp_product_low_offset() as i32);+ ctx.store_u16(pm1, abi::dsp_product_mid1_offset() as i32);+ ctx.store_u16(ph, abi::dsp_product_high_offset() as i32);+ ctx.store_u16(pm2, abi::dsp_product_mid2_offset() as i32);+ }+ OP_TSTPROD => {+ let (pc, po) = ctx.product_flags_i16();+ let prod = ctx.read_product_i64();+ ctx.emit_update_flags_ac(prod);+ ctx.set_status_bit(1, po);+ ctx.or_status_bit(7, po);+ ctx.set_status_bit(0, pc);+ }+ OP_CLRL => {+ let r = instr.r_7_7();+ let ac = ctx.read_ac_i64(r);+ let (rounded, carry) = ctx.round_half_to_even(ac);+ ctx.write_ac_i64(r, rounded);+ ctx.emit_update_flags_ac(rounded);+ ctx.set_status_bit_const(1, false);+ ctx.set_status_bit(0, carry);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn logic<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ fn emit_logic_with_axh(ctx: &mut TranslatorCtx, d: u8, s: u8, kind: LogicKind) {+ let mid = ctx.read_ac_mid(d);+ let axh_off = (abi::dsp_axh_base_offset() + (s as usize) * 2) as i32;+ let other = ctx.load_u16(axh_off);+ let result = match kind {+ LogicKind::And => ctx.builder.ins().band(mid, other),+ LogicKind::Or => ctx.builder.ins().bor(mid, other),+ LogicKind::Xor => ctx.builder.ins().bxor(mid, other),+ };+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }++ fn emit_logic_with_other_ac_mid(ctx: &mut TranslatorCtx, d: u8, kind: LogicKind) {+ let mid = ctx.read_ac_mid(d);+ let other_d = 1 - d;+ let other = ctx.read_ac_mid(other_d);++ let result = match kind {+ LogicKind::And => ctx.builder.ins().band(mid, other),+ LogicKind::Or => ctx.builder.ins().bor(mid, other),+ LogicKind::Xor => ctx.builder.ins().bxor(mid, other),+ };+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }++ match OP {+ OP_XORR => emit_logic_with_axh(ctx, instr.d_7_7(), instr.s_6_6(), LogicKind::Xor),+ OP_ANDR => emit_logic_with_axh(ctx, instr.d_7_7(), instr.s_6_6(), LogicKind::And),+ OP_ORR => emit_logic_with_axh(ctx, instr.d_7_7(), instr.s_6_6(), LogicKind::Or),+ OP_ANDC => emit_logic_with_other_ac_mid(ctx, instr.d_7_7(), LogicKind::And),+ OP_ORC => emit_logic_with_other_ac_mid(ctx, instr.d_7_7(), LogicKind::Or),+ OP_XORC => emit_logic_with_other_ac_mid(ctx, instr.d_7_7(), LogicKind::Xor),+ OP_NOT_AC => {+ let d = instr.d_7_7();+ let mid = ctx.read_ac_mid(d);+ let result = ctx.builder.ins().bnot(mid);+ ctx.write_ac_mid(d, result);++ let ac_full = ctx.read_ac_i64(d);+ ctx.emit_update_flags_logic(result, ac_full);+ ctx.emit_clear_oc();+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[derive(Clone, Copy)]+enum LogicKind {+ And,+ Or,+ Xor,+}++#[inline(always)]+pub fn shifts<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ fn finish(ctx: &mut TranslatorCtx, d: u8, val: Value) {+ ctx.write_ac_i64(d, val);+ let ac = ctx.read_ac_i64(d);+ ctx.emit_update_flags_ac(ac);+ ctx.emit_clear_oc();+ }++ match OP {+ OP_LSL | OP_ASL => {+ let r = instr.r_7_7();+ let i = instr.n() as i64;+ let ac = ctx.read_ac_i64(r);+ let masked = ctx.builder.ins().band_imm(ac, 0xFF_FFFF_FFFFi64);+ let shifted = ctx.builder.ins().ishl_imm(masked, i);+ finish(ctx, r, shifted);+ }+ OP_LSR => {+ let r = instr.r_7_7();+ let i = instr.n() as i64;++ if i != 0 {+ let ac = ctx.read_ac_i64(r);+ let masked = ctx.builder.ins().band_imm(ac, 0xFF_FFFF_FFFFi64);+ let shifted = ctx.builder.ins().ushr_imm(masked, 64 - i);+ finish(ctx, r, shifted);+ } else {+ let ac = ctx.read_ac_i64(r);+ ctx.emit_update_flags_ac(ac);+ ctx.emit_clear_oc();+ }+ }+ OP_ASR => {+ let r = instr.r_7_7();+ let i = instr.n() as i64;++ if i != 0 {+ let ac = ctx.read_ac_i64(r);+ let shifted = ctx.builder.ins().sshr_imm(ac, 64 - i);+ finish(ctx, r, shifted);+ } else {+ let ac = ctx.read_ac_i64(r);+ ctx.emit_update_flags_ac(ac);+ ctx.emit_clear_oc();+ }+ }+ OP_LSL16 => {+ let r = instr.r_7_7();+ let ac = ctx.read_ac_i64(r);+ let masked = ctx.builder.ins().band_imm(ac, 0xFF_FFFF_FFFFi64);+ let shifted = ctx.builder.ins().ishl_imm(masked, 16);+ finish(ctx, r, shifted);+ }+ OP_LSR16 => {+ let r = instr.r_7_7();+ let ac = ctx.read_ac_i64(r);+ let masked = ctx.builder.ins().band_imm(ac, 0xFF_FFFF_FFFFi64);+ let shifted = ctx.builder.ins().ushr_imm(masked, 16);+ finish(ctx, r, shifted);+ }+ OP_ASR16 => {+ let r = instr.r_4_4();+ let ac = ctx.read_ac_i64(r);+ let shifted = ctx.builder.ins().sshr_imm(ac, 16);+ finish(ctx, r, shifted);+ }+ OP_LSRN => {+ let shift = ctx.load_u16(abi::dsp_ac1_mid_offset() as i32);+ ctx.emit_dynamic_shift(0, shift, 0b01);+ }+ OP_ASRN => {+ let shift = ctx.load_u16(abi::dsp_ac1_mid_offset() as i32);+ ctx.emit_dynamic_shift(0, shift, 0b00);+ }+ OP_LSRNRX => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let axh_off = (abi::dsp_axh_base_offset() + (s as usize) * 2) as i32;+ let shift = ctx.load_u16(axh_off);+ ctx.emit_dynamic_shift(d as u32, shift, 0b11);+ }+ OP_ASRNRX => {+ let s = instr.s_6_6();+ let d = instr.d_7_7();+ let axh_off = (abi::dsp_axh_base_offset() + (s as usize) * 2) as i32;+ let shift = ctx.load_u16(axh_off);+ ctx.emit_dynamic_shift(d as u32, shift, 0b10);+ }+ OP_LSRNR => {+ let d = instr.d_7_7();+ let other = ctx.read_ac_mid(1 - d);+ ctx.emit_dynamic_shift(d as u32, other, 0b11);+ }+ OP_ASRNR => {+ let d = instr.d_7_7();+ let other = ctx.read_ac_mid(1 - d);+ ctx.emit_dynamic_shift(d as u32, other, 0b10);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn status<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ let sr_off = abi::dsp_status_offset() as i32;++ match OP {+ OP_SBCLR => {+ let idx = 6 + instr.bit();+ if idx == 13 {+ return;+ }++ let sr = ctx.load_u16(sr_off);+ let mask = !(1u16 << idx);+ let new_sr = ctx.builder.ins().band_imm(sr, mask as i64);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_SBSET => {+ let idx = 6 + instr.bit();+ if idx == 13 || idx == 8 {+ return;+ }++ let sr = ctx.load_u16(sr_off);+ let bit = 1i64 << idx;+ let new_sr = ctx.builder.ins().bor_imm(sr, bit);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_M2 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().band_imm(sr, !(1i64 << 13) & 0xFFFF);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_M0 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().bor_imm(sr, 1i64 << 13);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_CLR15 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().band_imm(sr, !(1i64 << 15) & 0xFFFF);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_SET15 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().bor_imm(sr, 1i64 << 15);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_SET16 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().band_imm(sr, !(1i64 << 14) & 0xFFFF);+ ctx.store_u16(new_sr, sr_off);+ }+ OP_SET40 => {+ let sr = ctx.load_u16(sr_off);+ let new_sr = ctx.builder.ins().bor_imm(sr, 1i64 << 14);+ ctx.store_u16(new_sr, sr_off);+ }+ _ => unreachable!("DSP JIT: unexpected OP routed to native translator"),+ }+}++#[inline(always)]+pub fn mul<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: Instruction) {+ match OP {+ OP_MUL => {+ let s = (instr.r_4_4() as usize) * 2;+ let ax_off = (abi::dsp_ax_base_offset() + s) as i32;+ let axh_off = (abi::dsp_axh_base_offset() + s) as i32;+ let a = ctx.load_u16(ax_off);+ let b = ctx.load_u16(axh_off);+ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MULAXH => {+ let axh_off = abi::dsp_axh_base_offset() as i32;+ let a = ctx.load_u16(axh_off);+ let p = ctx.multiply_i64(a, a);+ ctx.write_product_i64(p);+ }+ OP_MULX => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ ctx.multiply_mulx(s, t);+ }+ OP_MULC => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let (a, b) = ctx.mulc_operands_i16(s, t);+ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MADD => {+ let s = (instr.s_7_7() as usize) * 2;+ let ax_off = (abi::dsp_ax_base_offset() + s) as i32;+ let axh_off = (abi::dsp_axh_base_offset() + s) as i32;+ let a = ctx.load_u16(ax_off);+ let b = ctx.load_u16(axh_off);+ ctx.multiply_accumulate_i64(a, b, true);+ }+ OP_MSUB => {+ let s = (instr.s_7_7() as usize) * 2;+ let ax_off = (abi::dsp_ax_base_offset() + s) as i32;+ let axh_off = (abi::dsp_axh_base_offset() + s) as i32;+ let a = ctx.load_u16(ax_off);+ let b = ctx.load_u16(axh_off);+ ctx.multiply_accumulate_i64(a, b, false);+ }+ OP_MADDC | OP_MSUBC => {+ let s = instr.s_6_6();+ let t = instr.t_7_7();+ let (a, b) = ctx.mulc_operands_i16(s, t);+ ctx.multiply_accumulate_i64(a, b, OP == OP_MADDC);+ }+ OP_MULMV => {+ let s = (instr.r_4_4() as usize) * 2;+ let r = instr.r_7_7();+ ctx.move_prod_to_ac(r);++ let a = ctx.load_u16((abi::dsp_ax_base_offset() + s) as i32);+ let b = ctx.load_u16((abi::dsp_axh_base_offset() + s) as i32);+ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MULAC => {+ let s = (instr.r_4_4() as usize) * 2;+ let r = instr.r_7_7();+ ctx.add_prod_to_ac(r);++ let a = ctx.load_u16((abi::dsp_ax_base_offset() + s) as i32);+ let b = ctx.load_u16((abi::dsp_axh_base_offset() + s) as i32);+ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MULXMV => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ ctx.move_prod_to_ac(r);+ ctx.multiply_mulx(s, t);+ }+ OP_MULXAC => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ ctx.add_prod_to_ac(r);+ ctx.multiply_mulx(s, t);+ }+ OP_MULCMV => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ let (a, b) = ctx.mulc_operands_i16(s, t);+ ctx.move_prod_to_ac(r);++ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MULCAC => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ let (a, b) = ctx.mulc_operands_i16(s, t);+ ctx.add_prod_to_ac(r);++ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MADDX | OP_MSUBX => {+ let s = instr.s_6_6();+ let t = instr.t_7_7();+ let (a, b) = ctx.mulx_operands_u16(s, t);+ ctx.multiply_accumulate_i64(a, b, OP == OP_MADDX);+ }+ OP_MULMVZ => {+ let s = (instr.r_4_4() as usize) * 2;+ let r = instr.r_7_7();+ ctx.move_prod_to_ac_zero(r);++ let a = ctx.load_u16((abi::dsp_ax_base_offset() + s) as i32);+ let b = ctx.load_u16((abi::dsp_axh_base_offset() + s) as i32);+ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ OP_MULXMVZ => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ ctx.move_prod_to_ac_zero(r);+ ctx.multiply_mulx(s, t);+ }+ OP_MULCMVZ => {+ let s = instr.s_3_3();+ let t = instr.t_4_4();+ let r = instr.r_7_7();+ let (a, b) = ctx.mulc_operands_i16(s, t);+ ctx.move_prod_to_ac_zero(r);++ let p = ctx.multiply_i64(a, b);+ ctx.write_product_i64(p);+ }+ _ => unreachable!("DSP JIT: unexpected mul OP"),+ }+}++#[inline(always)]+pub fn ext_addr<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let r = instr.r_6_7() as usize;+ let ar_off = (abi::dsp_ar_base_offset() + r * 2) as i32;++ match OP {+ OP_EXT_DR => {+ let new_ar = ctx.emit_dec_ar(r as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_EXT_IR => {+ let new_ar = ctx.emit_inc_ar(r as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_EXT_NR => {+ let ix_off = (abi::dsp_ix_base_offset() + r * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ let new_ar = ctx.emit_increase_ar(r as u32, ix);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ _ => unreachable!("DSP JIT: unexpected ext OP routed to native translator"),+ }+}++#[inline(always)]+pub fn ext_load<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let d = instr.d_2_4();+ let s = instr.s_6_7() as usize;+ let dst = 24u8 + d;++ let ar_off = (abi::dsp_ar_base_offset() + s * 2) as i32;+ let ar = ctx.load_u16(ar_off);+ let addr = ctx.builder.ins().uextend(types::I32, ar);+ let val_u32 = ctx.emit_read_dmem(addr);+ let val = ctx.builder.ins().ireduce(types::I16, val_u32);+ if dst >= 30 {+ ctx.emit_write_ac_mid_sxm((dst - 30) as u32, val);+ } else {+ ctx.try_emit_reg_write(dst, val);+ }++ match OP {+ OP_EXT_L => {+ let new_ar = ctx.emit_inc_ar(s as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_EXT_LN => {+ let ix_off = (abi::dsp_ix_base_offset() + s * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ let new_ar = ctx.emit_increase_ar(s as u32, ix);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ _ => {}+ }+}++#[inline(always)]+pub fn ext_store<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let d = instr.d_6_7() as usize;+ let s = instr.s_3_4() as usize;+ let cache_off = (abi::dsp_ext_ac_cache_base_offset() + s * 2) as i32;+ let value = ctx.load_u16(cache_off);+ let v_u32 = ctx.builder.ins().uextend(types::I32, value);+ let ar_off = (abi::dsp_ar_base_offset() + d * 2) as i32;+ let ar = ctx.load_u16(ar_off);+ let addr = ctx.builder.ins().uextend(types::I32, ar);+ ctx.emit_write_dmem(addr, v_u32);++ match OP {+ OP_EXT_S => {+ let new_ar = ctx.emit_inc_ar(d as u32);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ OP_EXT_SN => {+ let ix_off = (abi::dsp_ix_base_offset() + d * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ let new_ar = ctx.emit_increase_ar(d as u32, ix);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar_off);+ }+ _ => {}+ }+}++#[inline(always)]+pub fn ext_load_store<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let s = instr.s_7_7() as usize;+ let d = instr.d_2_3();+ let dst = 24u8 + d;++ let ar0_off = abi::dsp_ar_base_offset() as i32;+ let ar3_off = (abi::dsp_ar_base_offset() + 3 * 2) as i32;+ let ix0_off = abi::dsp_ix_base_offset() as i32;+ let ix3_off = (abi::dsp_ix_base_offset() + 3 * 2) as i32;+ let cache_off = (abi::dsp_ext_ac_cache_base_offset() + (4 + s) * 2) as i32;++ let is_ls = matches!(OP, OP_EXT_LS | OP_EXT_LSM | OP_EXT_LSN | OP_EXT_LSNM);+ if is_ls {+ let ar0 = ctx.load_u16(ar0_off);+ let load_addr = ctx.builder.ins().uextend(types::I32, ar0);+ let load_val_u32 = ctx.emit_read_dmem(load_addr);+ let load_val = ctx.builder.ins().ireduce(types::I16, load_val_u32);+ ctx.try_emit_reg_write(dst, load_val);++ let store_val = ctx.load_u16(cache_off);+ let store_val_u32 = ctx.builder.ins().uextend(types::I32, store_val);+ let ar3 = ctx.load_u16(ar3_off);+ let store_addr = ctx.builder.ins().uextend(types::I32, ar3);+ ctx.emit_write_dmem(store_addr, store_val_u32);+ } else {+ let store_val = ctx.load_u16(cache_off);+ let store_val_u32 = ctx.builder.ins().uextend(types::I32, store_val);+ let ar0 = ctx.load_u16(ar0_off);+ let store_addr = ctx.builder.ins().uextend(types::I32, ar0);+ ctx.emit_write_dmem(store_addr, store_val_u32);++ let ar3 = ctx.load_u16(ar3_off);+ let load_addr = ctx.builder.ins().uextend(types::I32, ar3);+ let load_val_u32 = ctx.emit_read_dmem(load_addr);+ let load_val = ctx.builder.ins().ireduce(types::I16, load_val_u32);+ ctx.try_emit_reg_write(dst, load_val);+ }++ let (a0_inc, a3_inc) = match OP {+ OP_EXT_LS | OP_EXT_SL => (true, true),+ OP_EXT_LSM | OP_EXT_SLM => (true, false),+ OP_EXT_LSN | OP_EXT_SLN => (false, true),+ OP_EXT_LSNM | OP_EXT_SLNM => (false, false),+ _ => unreachable!(),+ };++ if a0_inc {+ let new_ar = ctx.emit_inc_ar(0);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar0_off);+ } else {+ let ix0 = ctx.load_u16(ix0_off);+ let new_ar = ctx.emit_increase_ar(0, ix0);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar0_off);+ }++ if a3_inc {+ let new_ar = ctx.emit_inc_ar(3);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar3_off);+ } else {+ let ix3 = ctx.load_u16(ix3_off);+ let new_ar = ctx.emit_increase_ar(3, ix3);+ let new_ar16 = ctx.builder.ins().ireduce(types::I16, new_ar);+ ctx.store_u16(new_ar16, ar3_off);+ }+}++#[inline(always)]+pub fn ext_ld<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let d = instr.d_2_2();+ let r = instr.r_3_3();+ let s = instr.s_6_7() as usize;++ let ar_s_off = (abi::dsp_ar_base_offset() + s * 2) as i32;+ let ar_3_off = (abi::dsp_ar_base_offset() + 3 * 2) as i32;+ let ar_s = ctx.load_u16(ar_s_off);+ let ar_3 = ctx.load_u16(ar_3_off);++ let s_bank = ctx.builder.ins().ushr_imm(ar_s, 10);+ let three_bank = ctx.builder.ins().ushr_imm(ar_3, 10);+ let same_bank_i8 = ctx+ .builder+ .ins()+ .icmp(cranelift_codegen::ir::condcodes::IntCC::Equal, s_bank, three_bank);++ let addr0 = ctx.builder.ins().uextend(types::I32, ar_s);+ let value0_u32 = ctx.emit_read_dmem(addr0);+ let value0 = ctx.builder.ins().ireduce(types::I16, value0_u32);+ let d_reg = if d != 0 { 26u8 } else { 24u8 };+ ctx.try_emit_reg_write(d_reg, value0);++ let value1 = {+ let same_block = ctx.builder.create_block();+ let diff_block = ctx.builder.create_block();+ let join_block = ctx.builder.create_block();+ ctx.builder.append_block_param(join_block, types::I16);+ ctx.builder.ins().brif(same_bank_i8, same_block, &[], diff_block, &[]);++ ctx.builder.switch_to_block(same_block);+ ctx.builder.seal_block(same_block);+ ctx.builder.ins().jump(join_block, &[value0.into()]);++ ctx.builder.switch_to_block(diff_block);+ ctx.builder.seal_block(diff_block);+ let addr1 = ctx.builder.ins().uextend(types::I32, ar_3);+ let v1_u32 = ctx.emit_read_dmem(addr1);+ let v1 = ctx.builder.ins().ireduce(types::I16, v1_u32);+ ctx.builder.ins().jump(join_block, &[v1.into()]);++ ctx.builder.switch_to_block(join_block);+ ctx.builder.seal_block(join_block);+ ctx.builder.block_params(join_block)[0]+ };++ let r_reg = if r != 0 { 27u8 } else { 25u8 };+ ctx.try_emit_reg_write(r_reg, value1);++ let (s_inc, three_inc) = match OP {+ OP_EXT_LD_00 => (true, true),+ OP_EXT_LDN_01 => (false, true),+ OP_EXT_LDM_10 => (true, false),+ OP_EXT_LDNM_11 => (false, false),+ _ => unreachable!(),+ };++ let new_s = if s_inc {+ ctx.emit_inc_ar(s as u32)+ } else {+ let ix_off = (abi::dsp_ix_base_offset() + s * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ ctx.emit_increase_ar(s as u32, ix)+ };+ let new_s16 = ctx.builder.ins().ireduce(types::I16, new_s);+ ctx.store_u16(new_s16, ar_s_off);++ let new_3 = if three_inc {+ ctx.emit_inc_ar(3)+ } else {+ let ix_off = (abi::dsp_ix_base_offset() + 3 * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ ctx.emit_increase_ar(3, ix)+ };+ let new_316 = ctx.builder.ins().ireduce(types::I16, new_3);+ ctx.store_u16(new_316, ar_3_off);+}++#[inline(always)]+pub fn ext_ldax<const OP: u32, const SYSTEM: SystemId>(ctx: &mut TranslatorCtx, instr: GcDspExt) {+ let s = instr.d_2_2() as usize;+ let r = instr.r_3_3() as usize;++ let ar_s_off = (abi::dsp_ar_base_offset() + s * 2) as i32;+ let ar_3_off = (abi::dsp_ar_base_offset() + 3 * 2) as i32;+ let ar_s = ctx.load_u16(ar_s_off);+ let ar_3 = ctx.load_u16(ar_3_off);++ let s_bank = ctx.builder.ins().ushr_imm(ar_s, 10);+ let three_bank = ctx.builder.ins().ushr_imm(ar_3, 10);+ let same_bank_i8 = ctx+ .builder+ .ins()+ .icmp(cranelift_codegen::ir::condcodes::IntCC::Equal, s_bank, three_bank);++ let addr_s = ctx.builder.ins().uextend(types::I32, ar_s);+ let high_u32 = ctx.emit_read_dmem(addr_s);+ let high = ctx.builder.ins().ireduce(types::I16, high_u32);++ let low = {+ let same_block = ctx.builder.create_block();+ let diff_block = ctx.builder.create_block();+ let join_block = ctx.builder.create_block();+ ctx.builder.append_block_param(join_block, types::I16);+ ctx.builder.ins().brif(same_bank_i8, same_block, &[], diff_block, &[]);+ ctx.builder.switch_to_block(same_block);+ ctx.builder.seal_block(same_block);+ ctx.builder.ins().jump(join_block, &[high.into()]);+ ctx.builder.switch_to_block(diff_block);+ ctx.builder.seal_block(diff_block);+ let addr_3 = ctx.builder.ins().uextend(types::I32, ar_3);+ let lo_u32 = ctx.emit_read_dmem(addr_3);+ let lo = ctx.builder.ins().ireduce(types::I16, lo_u32);+ ctx.builder.ins().jump(join_block, &[lo.into()]);+ ctx.builder.switch_to_block(join_block);+ ctx.builder.seal_block(join_block);+ ctx.builder.block_params(join_block)[0]+ };++ let axh_off = (abi::dsp_axh_base_offset() + r * 2) as i32;+ let ax_off = (abi::dsp_ax_base_offset() + r * 2) as i32;+ ctx.store_u16(high, axh_off);+ ctx.store_u16(low, ax_off);++ let (s_inc, three_inc) = match OP {+ OP_EXT_LDAX => (true, true),+ OP_EXT_LDAXN => (false, true),+ OP_EXT_LDAXM => (true, false),+ OP_EXT_LDAXNM => (false, false),+ _ => unreachable!(),+ };++ let new_s = if s_inc {+ ctx.emit_inc_ar(s as u32)+ } else {+ let ix_off = (abi::dsp_ix_base_offset() + s * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ ctx.emit_increase_ar(s as u32, ix)+ };+ let new_s16 = ctx.builder.ins().ireduce(types::I16, new_s);+ ctx.store_u16(new_s16, ar_s_off);++ let new_3 = if three_inc {+ ctx.emit_inc_ar(3)+ } else {+ let ix_off = (abi::dsp_ix_base_offset() + 3 * 2) as i32;+ let ix = ctx.load_u16(ix_off);+ ctx.emit_increase_ar(3, ix)+ };+ let new_316 = ctx.builder.ins().ireduce(types::I16, new_3);+ ctx.store_u16(new_316, ar_3_off);+}diff --git a/crates/gecko/src/flipper/dsp/jit/translator.rs b/crates/gecko/src/flipper/dsp/jit/translator.rsnew file mode 100644index 0000000..b2c571b--- /dev/null+++ b/crates/gecko/src/flipper/dsp/jit/translator.rs@@ -0,0 +1,390 @@+use cranelift_codegen::Context;+use cranelift_codegen::ir::{AbiParam, InstBuilder, Signature, Value, types};+use cranelift_codegen::isa::CallConv;+use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};+use cranelift_jit::JITModule;+use cranelift_module::{FuncId, Module};++use super::block::BlockSpec;+#[derive(Clone, Copy)]+pub struct ExternFuncs {+ pub cache_ext_ac: FuncId,+ pub loop_tail: FuncId,+ pub update_flags_logic: FuncId,+ pub update_flags_add: FuncId,+ pub update_flags_sub: FuncId,+ pub update_flags_ac: FuncId,+ pub read_dmem: FuncId,+ pub write_dmem: FuncId,+ pub inc_ar: FuncId,+ pub dec_ar: FuncId,+ pub increase_ar: FuncId,+ pub decrease_ar_ix: FuncId,+ pub dynamic_shift: FuncId,+ pub read_imem: FuncId,+ pub write_ac_mid_sxm: FuncId,+ pub call_stack_push: FuncId,+ pub call_stack_pop: FuncId,+ pub data_stack_pop: FuncId,+ pub read_reg_full: FuncId,+ pub write_reg_full: FuncId,+ pub loop_setup: FuncId,+}++pub fn translate(+ ctx: &mut Context,+ builder_ctx: &mut FunctionBuilderContext,+ module: &mut JITModule,+ extern_funcs: &ExternFuncs,+ spec: &BlockSpec,+ block_lookup_table_addr: i64,+ entry_counter_addr: Option<usize>,+) {+ let pointer_type = module.target_config().pointer_type();++ let mut builder = FunctionBuilder::new(&mut ctx.func, builder_ctx);+ let entry = builder.create_block();+ builder.append_block_params_for_function_params(entry);+ builder.switch_to_block(entry);+ builder.seal_block(entry);++ let ctx_ptr: Value = builder.block_params(entry)[0];++ use crate::flipper::dsp::instruction::Instruction;+ use crate::flipper::dsp::jit::{jit_lut, translate as t};+ use cranelift_codegen::ir::MemFlags;+ let nia_offset = super::abi::dsp_nia_offset_max() as i32;+ let pc_offset = super::abi::dsp_pc_offset_max() as i32;+ let loop_addr_ptr_offset = super::abi::dsp_loop_addr_ptr_offset_max() as i32;+ let loop_tail_ref = module.declare_func_in_func(extern_funcs.loop_tail, builder.func);++ if let Some(addr) = entry_counter_addr {+ let slot_v = builder.ins().iconst(types::I64, addr as i64);+ let cur = builder.ins().load(types::I64, MemFlags::trusted(), slot_v, 0);+ let next = builder.ins().iadd_imm(cur, 1);+ builder.ins().store(MemFlags::trusted(), next, slot_v, 0);+ }++ for entry in &spec.instrs {+ let natural_nia = entry.pc.wrapping_add(entry.size as u16);+ let nia_v = builder.ins().iconst(types::I16, natural_nia as i64);+ builder.ins().store(MemFlags::trusted(), nia_v, ctx_ptr, nia_offset);++ let primary = (entry.raw & 0xFFFF) as u16;+ let has_ext = ((primary >> 12) & 0xF) >= 3;+ if has_ext {+ emit_cache_ext_ac_inline(&mut builder, ctx_ptr);+ }++ let mut tctx = t::TranslatorCtx {+ builder: &mut builder,+ module,+ extern_funcs: *extern_funcs,+ sys_ptr: ctx_ptr,+ pc: entry.pc,+ size: entry.size,+ };+ jit_lut::dispatch(&mut tctx, Instruction(entry.raw));++ if has_ext {+ let ext_byte = if ((primary >> 12) & 0xF) == 3 {+ primary & 0x7F+ } else {+ primary & 0xFF+ };++ let mut tctx2 = t::TranslatorCtx {+ builder: &mut builder,+ module,+ extern_funcs: *extern_funcs,+ sys_ptr: ctx_ptr,+ pc: entry.pc,+ size: entry.size,+ };+ jit_lut::dispatch_gc_dsp_ext(&mut tctx2, crate::flipper::dsp::instruction::GcDspExt(ext_byte as u8));+ }++ let loop_ptr = builder+ .ins()+ .load(types::I8, MemFlags::trusted(), ctx_ptr, loop_addr_ptr_offset);+ let in_loop = builder+ .ins()+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::NotEqual, loop_ptr, 0);+ let slow_block = builder.create_block();+ let fast_block = builder.create_block();+ let join_block = builder.create_block();+ builder.ins().brif(in_loop, slow_block, &[], fast_block, &[]);++ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);+ let nia_v = builder.ins().load(types::I16, MemFlags::trusted(), ctx_ptr, nia_offset);+ builder.ins().store(MemFlags::trusted(), nia_v, ctx_ptr, pc_offset);+ builder.ins().jump(join_block, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.ins().call(loop_tail_ref, &[ctx_ptr]);+ builder.ins().jump(join_block, &[]);++ builder.switch_to_block(join_block);+ builder.seal_block(join_block);+ }++ let block_sig_ref = builder.import_signature(block_signature(pointer_type));+ emit_block_tail_chain(+ &mut builder,+ ctx_ptr,+ block_sig_ref,+ block_lookup_table_addr,+ spec.instrs.len() as i64,+ );++ builder.finalize();+}++fn emit_block_tail_chain(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ block_sig_ref: cranelift_codegen::ir::SigRef,+ block_lookup_table_addr: i64,+ block_instr_count: i64,+) {+ use cranelift_codegen::ir::MemFlags;+ use cranelift_codegen::ir::condcodes::IntCC;++ let pc_offset = super::abi::dsp_pc_offset_max() as i32;+ let chain_budget_offset = super::abi::dsp_chain_budget_offset() as i32;+ let instr_count_offset = super::abi::dsp_instr_count_offset() as i32;++ let instr_count = builder+ .ins()+ .load(types::I32, MemFlags::trusted(), ctx_ptr, instr_count_offset);+ let new_instr_count = builder.ins().iadd_imm(instr_count, block_instr_count);+ builder+ .ins()+ .store(MemFlags::trusted(), new_instr_count, ctx_ptr, instr_count_offset);++ let pc_u16 = builder.ins().load(types::I16, MemFlags::trusted(), ctx_ptr, pc_offset);+ let pc_u32 = builder.ins().uextend(types::I32, pc_u16);++ let budget = builder+ .ins()+ .load(types::I32, MemFlags::trusted(), ctx_ptr, chain_budget_offset);++ let pc_low12 = builder.ins().band_imm(pc_u32, 0xFFF);+ let pc_shifted = builder.ins().ushr_imm(pc_u32, 3);+ let pc_high_bit = builder.ins().band_imm(pc_shifted, 0x1000);+ let idx = builder.ins().bor(pc_low12, pc_high_bit);+ let idx64 = builder.ins().uextend(types::I64, idx);++ let entry_size = std::mem::size_of::<super::DspBlockLookupSlot>() as i64;+ let off64 = builder.ins().imul_imm(idx64, entry_size);+ let table_base = builder.ins().iconst(types::I64, block_lookup_table_addr);+ let slot_addr = builder.ins().iadd(table_base, off64);++ let slot_pc = builder.ins().load(types::I32, MemFlags::trusted(), slot_addr, 0);+ let slot_entry = builder.ins().load(types::I64, MemFlags::trusted(), slot_addr, 8);++ let pc_match = builder.ins().icmp(IntCC::Equal, slot_pc, pc_u32);+ let entry_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, slot_entry, 0);+ let budget_nonzero = builder.ins().icmp_imm(IntCC::NotEqual, budget, 0);+ let pc_and_entry = builder.ins().band(pc_match, entry_nonzero);+ let ok = builder.ins().band(pc_and_entry, budget_nonzero);++ let chain_block = builder.create_block();+ let return_block = builder.create_block();+ builder.ins().brif(ok, chain_block, &[], return_block, &[]);++ builder.switch_to_block(return_block);+ builder.seal_block(return_block);+ builder.ins().return_(&[pc_u32]);++ builder.switch_to_block(chain_block);+ builder.seal_block(chain_block);+ let new_budget = builder.ins().iadd_imm(budget, -1);+ builder+ .ins()+ .store(MemFlags::trusted(), new_budget, ctx_ptr, chain_budget_offset);+ builder+ .ins()+ .return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr]);+}++fn emit_cache_ext_ac_inline(builder: &mut FunctionBuilder, ctx_ptr: Value) {+ use cranelift_codegen::ir::MemFlags;+ use cranelift_codegen::ir::condcodes::IntCC;++ let ac0_low_off = super::abi::dsp_ac0_low_offset() as i32;+ let ac1_low_off = super::abi::dsp_ac1_low_offset() as i32;+ let ac0_mid_off = super::abi::dsp_ac0_mid_offset() as i32;+ let ac1_mid_off = super::abi::dsp_ac1_mid_offset() as i32;+ let ac0_hi_off = super::abi::dsp_ac0_high_offset() as i32;+ let ac1_hi_off = super::abi::dsp_ac1_high_offset() as i32;+ let status_off = super::abi::dsp_status_offset() as i32;+ let cache_base = super::abi::dsp_ext_ac_cache_base_offset() as i32;++ let ac0_low = builder+ .ins()+ .load(types::I16, MemFlags::trusted(), ctx_ptr, ac0_low_off);+ let ac1_low = builder+ .ins()+ .load(types::I16, MemFlags::trusted(), ctx_ptr, ac1_low_off);+ let ac0_mid = builder+ .ins()+ .load(types::I16, MemFlags::trusted(), ctx_ptr, ac0_mid_off);+ let ac1_mid = builder+ .ins()+ .load(types::I16, MemFlags::trusted(), ctx_ptr, ac1_mid_off);+ let ac0_hi = builder.ins().load(types::I16, MemFlags::trusted(), ctx_ptr, ac0_hi_off);+ let ac1_hi = builder.ins().load(types::I16, MemFlags::trusted(), ctx_ptr, ac1_hi_off);++ builder.ins().store(MemFlags::trusted(), ac0_low, ctx_ptr, cache_base);+ builder+ .ins()+ .store(MemFlags::trusted(), ac1_low, ctx_ptr, cache_base + 2);+ builder+ .ins()+ .store(MemFlags::trusted(), ac0_mid, ctx_ptr, cache_base + 8);+ builder+ .ins()+ .store(MemFlags::trusted(), ac1_mid, ctx_ptr, cache_base + 10);++ let status = builder.ins().load(types::I16, MemFlags::trusted(), ctx_ptr, status_off);+ let sxm_shifted = builder.ins().ushr_imm(status, 14);+ let sxm = builder.ins().band_imm(sxm_shifted, 1);+ let sxm_set = builder.ins().icmp_imm(IntCC::NotEqual, sxm, 0);++ let sat0 = emit_saturate_ac_mid(builder, ac0_hi, ac0_mid);+ let sat1 = emit_saturate_ac_mid(builder, ac1_hi, ac1_mid);++ let sel0 = builder.ins().select(sxm_set, sat0, ac0_mid);+ let sel1 = builder.ins().select(sxm_set, sat1, ac1_mid);+ builder.ins().store(MemFlags::trusted(), sel0, ctx_ptr, cache_base + 4);+ builder.ins().store(MemFlags::trusted(), sel1, ctx_ptr, cache_base + 6);+}++fn emit_saturate_ac_mid(builder: &mut FunctionBuilder, high: Value, mid: Value) -> Value {+ use cranelift_codegen::ir::condcodes::IntCC;++ let sign_ext = builder.ins().sshr_imm(mid, 15);++ let high_eq_signext = builder.ins().icmp(IntCC::Equal, high, sign_ext);++ let high_neg = builder.ins().band_imm(high, 0x80);+ let high_neg_set = builder.ins().icmp_imm(IntCC::NotEqual, high_neg, 0);+ let neg_max = builder.ins().iconst(types::I16, 0x8000_u16 as i64);+ let pos_max = builder.ins().iconst(types::I16, 0x7FFF);+ let sat = builder.ins().select(high_neg_set, neg_max, pos_max);++ builder.ins().select(high_eq_signext, mid, sat)+}++pub fn block_signature(pointer_type: cranelift_codegen::ir::Type) -> Signature {+ let mut sig = Signature::new(CallConv::Tail);+ sig.params.push(AbiParam::new(pointer_type));+ sig.returns.push(AbiParam::new(types::I32));+ sig+}++pub fn step_thunk_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig+}++pub fn void_thunk_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig+}++pub fn flags_logic_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I64));+ sig+}++pub fn flags_arith_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I64));+ sig.params.push(AbiParam::new(types::I64));+ sig.params.push(AbiParam::new(types::I64));+ sig+}++pub fn flags_ac_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I64));+ sig+}++pub fn dmem_read_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.returns.push(AbiParam::new(types::I32));+ sig+}++pub fn dmem_write_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+}++pub fn ar_unary_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.returns.push(AbiParam::new(types::I32));+ sig+}++pub fn ar_binary_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.returns.push(AbiParam::new(types::I32));+ sig+}++pub fn dynamic_shift_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+}++pub fn kind_dispatch_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+}++pub fn stack_pop_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.returns.push(AbiParam::new(types::I32));+ sig+}++pub fn loop_setup_signature(pointer_type: cranelift_codegen::ir::Type, host_cc: CallConv) -> Signature {+ let mut sig = Signature::new(host_cc);+ sig.params.push(AbiParam::new(pointer_type));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig.params.push(AbiParam::new(types::I32));+ sig+}