A GameCube/Wii emulator and debugger written in Rust.
icache invalidation
Laylea7e8fcdf4097
diff --git a/Cargo.lock b/Cargo.lockindex 916d178..44b86e1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1802,6 +1802,7 @@ dependencies = ["image 0.1.0","multiversion","rustc-hash 2.1.2",+ "smallvec","tracing","twox-hash","windows-sys 0.61.2",diff --git a/crates/gecko/Cargo.toml b/crates/gecko/Cargo.tomlindex a5694d9..445c59d 100644--- a/crates/gecko/Cargo.toml+++ b/crates/gecko/Cargo.toml@@ -42,6 +42,7 @@ crossbeam-channel = "0.5.15"hound = { version = "3.5.1", optional = true }multiversion = "0.8.0"rustc-hash = "2.1.2"+smallvec = "1.15.1"crc32fast = "1.5.0"zerocopy = { version = "0.8.48", features = ["derive"] }cranelift-codegen = { version = "0.131.1", optional = true }diff --git a/crates/gecko/spec/gekko_jit.bindings.chipi b/crates/gecko/spec/gekko_jit.bindings.chipiindex c57bd55..0500ef8 100644--- a/crates/gecko/spec/gekko_jit.bindings.chipi+++ b/crates/gecko/spec/gekko_jit.bindings.chipi@@ -162,7 +162,6 @@ target rust {dcbadcbzdcbz_l- icbitlbietlbiatlbsync@@ -170,6 +169,10 @@ target rust {tlbli}+ handler icbi {+ icbi+ }+handler logical {andxorxdiff --git a/crates/gecko/spec/wii_gekko_jit.bindings.chipi b/crates/gecko/spec/wii_gekko_jit.bindings.chipiindex 64e3a1a..9bb86e8 100644--- a/crates/gecko/spec/wii_gekko_jit.bindings.chipi+++ b/crates/gecko/spec/wii_gekko_jit.bindings.chipi@@ -162,7 +162,6 @@ target rust {dcbadcbzdcbz_l- icbitlbietlbiatlbsync@@ -170,6 +169,10 @@ target rust {tlbli}+ handler icbi {+ icbi+ }+handler logical {andxorxdiff --git a/crates/gecko/src/dvd.rs b/crates/gecko/src/dvd.rsindex a47827e..e6680f5 100644--- a/crates/gecko/src/dvd.rs+++ b/crates/gecko/src/dvd.rs@@ -147,6 +147,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Colet dst = sys.di.dma_address.address();let buffer = sys.mmio.phys_slice_mut(dst, 0x20);buffer.copy_from_slice(&[0x69; 0x20]); // TODO: Drive Info?+ #[cfg(feature = "jit")]+ sys.mmio.queue_icbi_for_range(dst, 0x20);}Command::ReadSectorData => {let src = sys.di.cmdbuf1 << 2;@@ -156,6 +158,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Colet buffer = sys.mmio.phys_slice_mut(dst, len);dvd.read_disc_into(src as usize, buffer);+ #[cfg(feature = "jit")]+ sys.mmio.queue_icbi_for_range(dst, len as u32);tracing::debug!(src = format!("{:08X}", src),@@ -171,6 +175,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Colet buffer = sys.mmio.phys_slice_mut(dst, len);dvd.read_disc_into(0, buffer);+ #[cfg(feature = "jit")]+ sys.mmio.queue_icbi_for_range(dst, len as u32);tracing::debug!(src = format!("{:08X}", src),diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rsindex 108918c..edc9914 100644--- a/crates/gecko/src/flipper/dsp.rs+++ b/crates/gecko/src/flipper/dsp.rs@@ -195,6 +195,8 @@ impl Dsp {let src = &self.aram[aram_addr..aram_addr + count];let dst = mmio.virt_slice_mut(ram_addr as u32, count);dst.copy_from_slice(src);+ #[cfg(feature = "jit")]+ mmio.queue_icbi_for_range(crate::mmio::virt_to_phys(ram_addr as u32), count as u32);}regs::DmaDirection::RamToAram if within_bounds => {let src = mmio.virt_slice(ram_addr as u32, count);@@ -261,6 +263,8 @@ impl Dsp {let src = &memory[dsp_addr..dsp_addr + len];let dst = mmio.virt_slice_mut(ram_addr, len);dst.copy_from_slice(src);+ #[cfg(feature = "jit")]+ mmio.queue_icbi_for_range(crate::mmio::virt_to_phys(ram_addr), len as u32);}}diff --git a/crates/gecko/src/flipper/exi.rs b/crates/gecko/src/flipper/exi.rsindex 93aeb96..65ab10c 100644--- a/crates/gecko/src/flipper/exi.rs+++ b/crates/gecko/src/flipper/exi.rs@@ -264,7 +264,11 @@ pub fn run_dma<const CHANNEL: usize, const SYSTEM: SystemId>(sys: &mut System<SYif let Some(device) = &mut sys.exi.devices[CHANNEL][slot] {match transfer_type {- TransferType::Read => device.dma_read(sys.mmio.phys_slice_mut(address, length as usize)),+ TransferType::Read => {+ device.dma_read(sys.mmio.phys_slice_mut(address, length as usize));+ #[cfg(feature = "jit")]+ sys.mmio.queue_icbi_for_range(address, length);+ }TransferType::Write => device.dma_write(sys.mmio.phys_slice(address, length as usize)),TransferType::ReadAndWrite | TransferType::Reserved => {tracing::error!(diff --git a/crates/gecko/src/gekko/interpreter/system.rs b/crates/gecko/src/gekko/interpreter/system.rsindex 2a59005..ef4209c 100644--- a/crates/gecko/src/gekko/interpreter/system.rs+++ b/crates/gecko/src/gekko/interpreter/system.rs@@ -44,8 +44,15 @@ pub fn spr<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, inst923 => {ctx.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);if ctx.gekko.spr.dmal.trigger() {- ctx.mmio- .process_locked_cache_dma(&ctx.gekko.spr.dmau, &ctx.gekko.spr.dmal);+ let dmau = ctx.gekko.spr.dmau;+ let dmal = ctx.gekko.spr.dmal;+ let written = ctx.mmio.process_locked_cache_dma(&dmau, &dmal);+ #[cfg(feature = "jit")]+ if let Some((phys, len)) = written {+ ctx.mmio.queue_icbi_for_range(phys, len);+ }+ #[cfg(not(feature = "jit"))]+ let _ = written;}}_ => ctx.gekko.spr.write(spr_num, val),diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rsindex 92e31a4..1736ccc 100644--- a/crates/gecko/src/gekko/jit.rs+++ b/crates/gecko/src/gekko/jit.rs@@ -74,6 +74,8 @@ pub struct ExternFuncs {pub do_psq_load: FuncId,pub do_psq_store: FuncId,pub read_timebase: FuncId,+ pub cause_icbi: FuncId,+ pub cause_smc_write: FuncId,}pub struct JitEngine<const SYSTEM: SystemId> {@@ -82,7 +84,9 @@ pub struct JitEngine<const SYSTEM: SystemId> {builder_ctx: FunctionBuilderContext,cache: FxHashMap<u32, BlockEntry>,block_func_ids: FxHashMap<u32, FuncId>,- pending_chain_slots: FxHashMap<u32, Vec<usize>>,+ target_slots: std::cell::RefCell<FxHashMap<u32, usize>>,+ blocks_by_line: FxHashMap<u32, smallvec::SmallVec<[u32; 2]>>,+ pub(crate) drain_scratch: Vec<u32>,#[cfg(feature = "jit-stats")]hits: FxHashMap<u32, u64>,pub(crate) block_specs: FxHashMap<u32, block::BlockSpec>,@@ -150,6 +154,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_load: (&'static str, *const u8),do_psq_store: (&'static str, *const u8),read_timebase: (&'static str, *const u8),+ cause_icbi: (&'static str, *const u8),+ cause_smc_write: (&'static str, *const u8),}let syms: SymTable = match SYSTEM {GC => SymTable {@@ -201,6 +207,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_load: ("gecko_jit_do_psq_load_gc", runtime::do_psq_load_gc as *const u8),do_psq_store: ("gecko_jit_do_psq_store_gc", runtime::do_psq_store_gc as *const u8),read_timebase: ("gecko_jit_read_timebase_gc", runtime::read_timebase_gc as *const u8),+ cause_icbi: ("gecko_jit_cause_icbi_gc", runtime::cause_icbi_gc as *const u8),+ cause_smc_write: ("gecko_jit_cause_smc_write_gc", runtime::cause_smc_write_gc as *const u8),},WII => SymTable {cause_invalid_opcode: (@@ -254,6 +262,11 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_load: ("gecko_jit_do_psq_load_wii", runtime::do_psq_load_wii as *const u8),do_psq_store: ("gecko_jit_do_psq_store_wii", runtime::do_psq_store_wii as *const u8),read_timebase: ("gecko_jit_read_timebase_wii", runtime::read_timebase_wii as *const u8),+ cause_icbi: ("gecko_jit_cause_icbi_wii", runtime::cause_icbi_wii as *const u8),+ cause_smc_write: (+ "gecko_jit_cause_smc_write_wii",+ runtime::cause_smc_write_wii as *const u8,+ ),},_ => unreachable!(),};@@ -289,6 +302,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {syms.do_psq_load,syms.do_psq_store,syms.read_timebase,+ syms.cause_icbi,+ syms.cause_smc_write,] {jit_builder.symbol(name, addr);}@@ -456,6 +471,12 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {read_timebase: module.declare_function(syms.read_timebase.0, Linkage::Import, &mem_read_sig).expect("declare read_timebase"),+ cause_icbi: module+ .declare_function(syms.cause_icbi.0, Linkage::Import, &ctx_u32_sig)+ .expect("declare cause_icbi"),+ cause_smc_write: module+ .declare_function(syms.cause_smc_write.0, Linkage::Import, &mem_write_sig)+ .expect("declare cause_smc_write"),};let trampoline_fn = build_trampoline(&mut module, call_conv, pointer_type, &block_sig);@@ -477,7 +498,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {builder_ctx: FunctionBuilderContext::new(),cache: FxHashMap::default(),block_func_ids: FxHashMap::default(),- pending_chain_slots: FxHashMap::default(),+ target_slots: std::cell::RefCell::new(FxHashMap::default()),+ blocks_by_line: FxHashMap::default(),+ drain_scratch: Vec::new(),#[cfg(feature = "jit-stats")]hits: FxHashMap::default(),block_specs: FxHashMap::default(),@@ -494,6 +517,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}}+ #[inline]+ fn target_slot_addr(&self, target_pc: u32) -> usize {+ *self.target_slots.borrow_mut().entry(target_pc).or_insert_with(|| {+ let slot: &'static mut usize = Box::leak(Box::new(0usize));+ slot as *mut usize as usize+ })+ }+pub fn block_lookup_table_addr(&self) -> usize {self.block_lookup_table_addr}@@ -592,10 +623,77 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let entry = self.compile(&spec, &gprs_snapshot);self.cache.insert(pc, entry);self.block_specs.insert(pc, spec);+ self.register_block(&mut sys.mmio, pc);entry}+ fn register_block(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, pc: u32) {+ let Some(spec) = self.block_specs.get(&pc) else { return };+ let mut last = u32::MAX;+ for &vpc in &spec.pcs {+ let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;+ if line == last {+ continue;+ }++ last = line;+ self.blocks_by_line.entry(line).or_default().push(pc);+ mmio.mark_code(line, crate::mmio::CODE_LINE_BYTES);+ }+ }++ fn unregister_block(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, pc: u32) {+ let Some(spec) = self.block_specs.remove(&pc) else {+ return;+ };+ self.cache.remove(&pc);+ self.block_func_ids.remove(&pc);++ if let Some(&slot) = self.target_slots.borrow().get(&pc) {+ unsafe {+ *(slot as *mut usize) = 0;+ }+ }++ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;+ unsafe {+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;+ if (*table.add(idx)).pc == pc {+ (*table.add(idx)).entry = 0;+ }+ }++ let mut last = u32::MAX;+ for &vpc in &spec.pcs {+ let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;+ if line == last {+ continue;+ }++ last = line;+ mmio.unmark_code(line, crate::mmio::CODE_LINE_BYTES);+ if let Some(v) = self.blocks_by_line.get_mut(&line) {+ v.retain(|p| *p != pc);+ if v.is_empty() {+ self.blocks_by_line.remove(&line);+ }+ }+ }+ }++ pub fn invalidate_line(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, line: u32) -> bool {+ let Some(pcs) = self.blocks_by_line.remove(&line) else {+ return false;+ };++ for pc in pcs.into_iter() {+ self.unregister_block(mmio, pc);+ }++ true+ }+#[cfg(feature = "jit-stats")]fn block_entry_counter_slot(&mut self, pc: u32) -> usize {if let Some(&addr) = self.block_entry_counter_ptrs.get(&pc) {@@ -796,7 +894,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {pub fn flush(&mut self) {self.cache.clear();self.block_func_ids.clear();- self.pending_chain_slots.clear();+ self.block_specs.clear();+ self.blocks_by_line.clear();++ for &slot_addr in self.target_slots.borrow().values() {+ unsafe {+ *(slot_addr as *mut usize) = 0;+ }+ }unsafe {let table = self.block_lookup_table_addr as *mut BlockLookupSlot;@@ -807,6 +912,13 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}}+ pub fn flush_with_refcount(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>) {+ self.flush();+ mmio.clear_code_refcount();+ mmio.pending_icbi.clear();+ mmio.jit_dirty = 0;+ }+#[cfg_attr(feature = "hotpath", hotpath::measure(label = "ppc_jit_compile"))]fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {let func_id = self.func_id_for(spec.start_pc);@@ -814,24 +926,18 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.ctx.clear();self.ctx.func.signature = self.block_sig.clone();- let self_slot: &'static mut usize = Box::leak(Box::new(0usize));- let self_slot_addr = self_slot as *mut usize as i64;-- let mut pending_for_this_block: Vec<(u32, usize)> = Vec::new();+ let _ = self.target_slot_addr(spec.start_pc);#[cfg(feature = "jit-stats")]let entry_counter_addr = Some(self.block_entry_counter_slot(spec.start_pc));#[cfg(not(feature = "jit-stats"))]let entry_counter_addr: Option<usize> = None;+ let block_lookup_table_addr = self.block_lookup_table_addr as i64;let chain = translator::ChainContext {self_pc: spec.start_pc,- self_func_id: func_id,- self_slot_addr,- compiled_func_ids: &self.block_func_ids,- compiled_cache: &self.cache,- pending: std::cell::RefCell::new(&mut pending_for_this_block),- block_lookup_table_addr: self.block_lookup_table_addr as i64,+ target_slots: &self.target_slots,+ block_lookup_table_addr,};translator::translate::<SYSTEM>(@@ -846,9 +952,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {);drop(chain);- for (target_pc, slot_addr) in pending_for_this_block.drain(..) {- self.pending_chain_slots.entry(target_pc).or_default().push(slot_addr);- }let want_dump = self.dump_pc == Some(spec.start_pc);if want_dump {@@ -901,16 +1004,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.module.finalize_definitions().expect("finalize");let entry = self.module.get_finalized_function(func_id) as usize;+ let slot_addr = self.target_slot_addr(spec.start_pc);unsafe {- *(self_slot_addr as *mut usize) = entry;- }-- if let Some(slots) = self.pending_chain_slots.remove(&spec.start_pc) {- for slot_addr in slots {- unsafe {- *(slot_addr as *mut usize) = entry;- }- }+ *(slot_addr as *mut usize) = entry;}self.register_in_lookup_table(spec.start_pc, entry);diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rsindex c4f8d81..f268f93 100644--- a/crates/gecko/src/gekko/jit/abi.rs+++ b/crates/gecko/src/gekko/jit/abi.rs@@ -98,6 +98,16 @@ pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)}+#[inline(always)]+pub const fn code_refcount_ptr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, code_refcount_ptr)+}++#[inline(always)]+pub const fn jit_dirty_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, jit_dirty)+}+#[inline(always)]pub const fn gqr_offset<const SYSTEM: SystemId>(i: u8) -> usize {let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);diff --git a/crates/gecko/src/gekko/jit/handlers.rs b/crates/gecko/src/gekko/jit/handlers.rsindex 2249350..eba0950 100644--- a/crates/gecko/src/gekko/jit/handlers.rs+++ b/crates/gecko/src/gekko/jit/handlers.rs@@ -488,20 +488,60 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,lut::OP_LHA => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),lut::OP_LHAU => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),- lut::OP_STW => {- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)- }- lut::OP_STWU => {- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)- }- lut::OP_STB => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false),- lut::OP_STBU => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true),- lut::OP_STH => {- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)- }- lut::OP_STHU => {- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)- }+ lut::OP_STW => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STWU => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ true,+ ),+ lut::OP_STB => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U8,+ local.write_u8,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STBU => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U8,+ local.write_u8,+ local.cause_smc_write,+ true,+ ),+ lut::OP_STH => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U16,+ local.write_u16,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STHU => translator::emit_store::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U16,+ local.write_u16,+ local.cause_smc_write,+ true,+ ),lut::OP_LMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, false),lut::OP_STMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, true),@@ -527,24 +567,60 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,lut::OP_LHAX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),lut::OP_LHAUX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),- lut::OP_STWX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)- }- lut::OP_STWUX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)- }- lut::OP_STBX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false)- }- lut::OP_STBUX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true)- }- lut::OP_STHX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)- }- lut::OP_STHUX => {- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)- }+ lut::OP_STWX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STWUX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ true,+ ),+ lut::OP_STBX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U8,+ local.write_u8,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STBUX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U8,+ local.write_u8,+ local.cause_smc_write,+ true,+ ),+ lut::OP_STHX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U16,+ local.write_u16,+ local.cause_smc_write,+ false,+ ),+ lut::OP_STHUX => translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U16,+ local.write_u16,+ local.cause_smc_write,+ true,+ ),lut::OP_LWBRX => {translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32)@@ -552,12 +628,22 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,lut::OP_LHBRX => {translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16)}- lut::OP_STWBRX => {- translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32)- }- lut::OP_STHBRX => {- translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16)- }+ lut::OP_STWBRX => translator::emit_store_xform_brx::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ ),+ lut::OP_STHBRX => translator::emit_store_xform_brx::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U16,+ local.write_u16,+ local.cause_smc_write,+ ),_ => return,}@@ -577,9 +663,18 @@ pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslatolut::OP_LFD => translator::emit_lfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),lut::OP_LFDU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),- lut::OP_STFS => translator::emit_stfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.write_u32),+ lut::OP_STFS => translator::emit_stfs::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ local.write_f32,+ local.write_u32,+ local.cause_smc_write,+ ),lut::OP_STFSU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),- lut::OP_STFD => translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),+ lut::OP_STFD => {+ translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64, local.cause_smc_write)+ }lut::OP_STFDU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),lut::OP_LFSX => translator::emit_lfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32),@@ -587,12 +682,18 @@ pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslatolut::OP_LFDX => translator::emit_lfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),lut::OP_LFDUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),- lut::OP_STFSX => translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32),+ lut::OP_STFSX => {+ translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.cause_smc_write)+ }lut::OP_STFSUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),- lut::OP_STFDX => translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),+ lut::OP_STFDX => {+ translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64, local.cause_smc_write)+ }lut::OP_STFDUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),- lut::OP_STFIWX => translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32),+ lut::OP_STFIWX => {+ translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32, local.cause_smc_write)+ }_ => return,}@@ -644,7 +745,6 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:| lut::OP_DCBA| lut::OP_DCBZ| lut::OP_DCBZ_L- | lut::OP_ICBI| lut::OP_TLBIE| lut::OP_TLBIA| lut::OP_TLBSYNC@@ -657,6 +757,38 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:let _ = builder;}+#[inline(always)]+pub fn icbi<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {+ let _ = OP;+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };++ let ra = instr.ra();+ let rb = instr.rb();+ let ra_v = if ra == 0 {+ builder.ins().iconst(types::I32, 0)+ } else {+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, ra)+ };+ let rb_v = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, rb);+ let ea = builder.ins().iadd(ra_v, rb_v);++ builder.ins().call(local.cause_icbi, &[t.ctx_ptr, ea]);++ let dirty_off = abi::jit_dirty_offset::<SYSTEM>() as i32;+ let dirty = builder+ .ins()+ .load(types::I8, translator::vmctx_flags(), t.ctx_ptr, dirty_off);+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, dirty, 0);++ let continue_block = builder.create_block();+ let nia = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);+ builder.ins().brif(nz, t.exit_block, &[nia.into()], continue_block, &[]);+ builder.switch_to_block(continue_block);+ builder.seal_block(continue_block);++ t.handled_natively = true;+}+#[inline(always)]pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {if !t.is_terminator {@@ -931,7 +1063,15 @@ pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false);+ translator::emit_store_xform::<SYSTEM>(+ builder,+ t.ctx_ptr,+ instr,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ false,+ );t.handled_natively = true;}diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rsindex 63715ef..f2172ce 100644--- a/crates/gecko/src/gekko/jit/runtime.rs+++ b/crates/gecko/src/gekko/jit/runtime.rs@@ -207,8 +207,11 @@ fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u3923 => {sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);if sys.gekko.spr.dmal.trigger() {- sys.mmio- .process_locked_cache_dma(&sys.gekko.spr.dmau, &sys.gekko.spr.dmal);+ let dmau = sys.gekko.spr.dmau;+ let dmal = sys.gekko.spr.dmal;+ if let Some((phys, len)) = sys.mmio.process_locked_cache_dma(&dmau, &dmal) {+ sys.mmio.queue_icbi_for_range(phys, len);+ }}}_ => sys.gekko.spr.write(num, val),@@ -550,3 +553,37 @@ pub extern "C" fn do_rfi_gc(sys: *mut core::ffi::c_void) {pub extern "C" fn do_rfi_wii(sys: *mut core::ffi::c_void) {do_rfi::<WII>(sys.cast());}++#[inline(always)]+fn cause_icbi<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) {+ let sys = unsafe { &mut *sys };+ let phys = crate::mmio::virt_to_phys(ea);+ if !sys.mmio.is_code_chunk(phys) {+ return;+ }++ let line = phys & crate::mmio::CODE_LINE_MASK;+ sys.mmio.pending_icbi.insert(line);+ sys.mmio.jit_dirty = 1;+}++pub extern "C" fn cause_icbi_gc(sys: *mut core::ffi::c_void, ea: u32) {+ cause_icbi::<GC>(sys.cast(), ea);+}+pub extern "C" fn cause_icbi_wii(sys: *mut core::ffi::c_void, ea: u32) {+ cause_icbi::<WII>(sys.cast(), ea);+}++#[inline(always)]+fn cause_smc_write<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, size: u32) {+ let sys = unsafe { &mut *sys };+ let phys = crate::mmio::virt_to_phys(ea);+ sys.mmio.queue_icbi_for_range(phys, size);+}++pub extern "C" fn cause_smc_write_gc(sys: *mut core::ffi::c_void, ea: u32, size: u32) {+ cause_smc_write::<GC>(sys.cast(), ea, size);+}+pub extern "C" fn cause_smc_write_wii(sys: *mut core::ffi::c_void, ea: u32, size: u32) {+ cause_smc_write::<WII>(sys.cast(), ea, size);+}diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rsindex 4b2906a..1dc8609 100644--- a/crates/gecko/src/gekko/jit/translator.rs+++ b/crates/gecko/src/gekko/jit/translator.rs@@ -4,7 +4,7 @@ use cranelift_codegen::ir::{AliasRegion, Block, FuncRef, InstBuilder, MemFlags,use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};use cranelift_jit::JITModule;-use cranelift_module::{FuncId, Module};+use cranelift_module::Module;use rustc_hash::FxHashMap;use crate::gekko::instruction::Instruction;@@ -32,29 +32,18 @@ pub(crate) fn heap_flags() -> MemFlags {pub struct ChainContext<'a> {pub self_pc: u32,- pub self_func_id: FuncId,- pub self_slot_addr: i64,- pub compiled_func_ids: &'a FxHashMap<u32, FuncId>,- pub compiled_cache: &'a FxHashMap<u32, super::BlockEntry>,- pub pending: std::cell::RefCell<&'a mut Vec<(u32, usize)>>,+ pub target_slots: &'a std::cell::RefCell<FxHashMap<u32, usize>>,pub block_lookup_table_addr: i64,}impl<'a> ChainContext<'a> {fn target_for(&self, target_pc: u32) -> Option<i64> {- if target_pc == self.self_pc {- return Some(self.self_slot_addr);- }-- if let Some(&entry) = self.compiled_cache.get(&target_pc) {- let slot: &'static mut usize = Box::leak(Box::new(entry));- return Some(slot as *mut usize as i64);- }-- let slot: &'static mut usize = Box::leak(Box::new(0usize));- let slot_addr_usize = slot as *mut usize as usize;- self.pending.borrow_mut().push((target_pc, slot_addr_usize));- Some(slot_addr_usize as i64)+ let mut slots = self.target_slots.borrow_mut();+ let addr = *slots.entry(target_pc).or_insert_with(|| {+ let slot: &'static mut usize = Box::leak(Box::new(0usize));+ slot as *mut usize as usize+ });+ Some(addr as i64)}}@@ -92,6 +81,8 @@ pub(crate) struct LocalFuncs {pub(crate) do_psq_load: FuncRef,pub(crate) do_psq_store: FuncRef,pub(crate) read_timebase: FuncRef,+ pub(crate) cause_icbi: FuncRef,+ pub(crate) cause_smc_write: FuncRef,pub(crate) fp_guard_emitted: std::cell::Cell<bool>,}@@ -152,6 +143,8 @@ pub fn translate<const SYSTEM: SystemId>(do_psq_load: module.declare_func_in_func(extern_funcs.do_psq_load, &mut ctx.func),do_psq_store: module.declare_func_in_func(extern_funcs.do_psq_store, &mut ctx.func),read_timebase: module.declare_func_in_func(extern_funcs.read_timebase, &mut ctx.func),+ cause_icbi: module.declare_func_in_func(extern_funcs.cause_icbi, &mut ctx.func),+ cause_smc_write: module.declare_func_in_func(extern_funcs.cause_smc_write, &mut ctx.func),fp_guard_emitted: std::cell::Cell::new(false),};@@ -186,6 +179,15 @@ pub fn translate<const SYSTEM: SystemId>(let exit_block = builder.create_block();builder.append_block_param(exit_block, types::I32);+ let dirty_off = abi::jit_dirty_offset::<SYSTEM>() as i32;+ let dirty = builder.ins().load(types::I8, vmctx_flags(), ctx_ptr, dirty_off);+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, dirty, 0);+ let alive = builder.create_block();+ let start_nia = builder.ins().iconst(types::I32, spec.start_pc as i64);+ builder.ins().brif(nz, exit_block, &[start_nia.into()], alive, &[]);+ builder.switch_to_block(alive);+ builder.seal_block(alive);+if let IdleClass::PointerIterLoop { gpr, stride } = idle_class {let rb_off = self::gpr_offset::<SYSTEM>(gpr);let ctr_off = abi::ctr_offset::<SYSTEM>() as i32;@@ -2026,6 +2028,59 @@ pub(crate) fn emit_x_form_ea<const SYSTEM: SystemId>(builder.ins().iadd(base, b)}+pub(crate) fn emit_smc_check<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ size_bytes: u32,+ cause_smc_write: FuncRef,+) {+ let phys = builder.ins().band_imm(ea, crate::mmio::PHYS_MASK as i64);+ let in_mem1 = builder+ .ins()+ .icmp_imm(IntCC::UnsignedLessThan, phys, crate::mmio::constants::RAM_SIZE as i64);++ let check_block = builder.create_block();+ let post_block = builder.create_block();+ builder.ins().brif(in_mem1, check_block, &[], post_block, &[]);++ builder.switch_to_block(check_block);+ builder.seal_block(check_block);++ let rc_off = abi::code_refcount_ptr_offset::<SYSTEM>() as i32;+ let rc_base = builder.ins().load(types::I64, vmctx_readonly_flags(), ctx_ptr, rc_off);++ let lo_idx_u32 = builder.ins().ushr_imm(phys, crate::mmio::CODE_LINE_SHIFT as i64);+ let lo_idx = builder.ins().uextend(types::I64, lo_idx_u32);+ let lo_addr = builder.ins().iadd(rc_base, lo_idx);+ let rc_lo = builder.ins().load(types::I8, vmctx_flags(), lo_addr, 0);++ let combined = if size_bytes <= 1 {+ rc_lo+ } else {+ let hi_phys = builder.ins().iadd_imm(phys, (size_bytes as i64) - 1);+ let hi_idx_u32 = builder.ins().ushr_imm(hi_phys, crate::mmio::CODE_LINE_SHIFT as i64);+ let hi_idx = builder.ins().uextend(types::I64, hi_idx_u32);+ let hi_addr = builder.ins().iadd(rc_base, hi_idx);+ let rc_hi = builder.ins().load(types::I8, vmctx_flags(), hi_addr, 0);+ builder.ins().bor(rc_lo, rc_hi)+ };+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, combined, 0);++ let slow_block = builder.create_block();+ builder.set_cold_block(slow_block);+ builder.ins().brif(nz, slow_block, &[], post_block, &[]);++ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ let size_v = builder.ins().iconst(types::I32, size_bytes as i64);+ builder.ins().call(cause_smc_write, &[ctx_ptr, ea, size_v]);+ builder.ins().jump(post_block, &[]);++ builder.switch_to_block(post_block);+ builder.seal_block(post_block);+}+pub(crate) fn emit_fastmem_lookup<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,@@ -2135,9 +2190,9 @@ pub(crate) fn emit_stfx_update<const SYSTEM: SystemId>() {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32, local.cause_smc_write);} else {- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64, local.cause_smc_write);}gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);}@@ -2147,6 +2202,7 @@ pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,slow_u32: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);@@ -2154,7 +2210,7 @@ pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(let u64v = builder.ins().bitcast(types::I64, MemFlags::new(), f);let u32v = builder.ins().ireduce(types::I32, u64v);- emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32);+ emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32, cause_smc_write);}pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(@@ -2163,6 +2219,7 @@ pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(ea: Value,val: Value,slow: FuncRef,+ cause_smc_write: FuncRef,) {let fast_block = builder.create_block();let slow_block = builder.create_block();@@ -2181,6 +2238,7 @@ pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(let addr = builder.block_params(fast_block)[0];let bswapped = builder.ins().bswap(val);builder.ins().store(heap_flags(), bswapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -2215,11 +2273,11 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(builder.seal_block(fast_block);if store {let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);if !instr.w_21_21() {let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rd());let ea1 = builder.ins().iadd_imm(ea, 4);- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);}} else {let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);@@ -2324,6 +2382,7 @@ pub(crate) fn emit_stwcx_dot<const SYSTEM: SystemId>(0,MemSize::U32,local.write_u32,+ local.cause_smc_write,false,);builder.ins().jump(cr_block, &[matched.into()]);@@ -2360,7 +2419,17 @@ pub(crate) fn emit_lmw_stmw<const SYSTEM: SystemId>(builder.ins().iadd_imm(ea_base, off as i64)};if store {- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.write_u32, false);+ emit_store_at_ea::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ r,+ 0,+ MemSize::U32,+ local.write_u32,+ local.cause_smc_write,+ false,+ );} else {emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.read_u32, false);}@@ -2474,6 +2543,7 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(instr: Instruction,size: MemSize,slow: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());@@ -2500,6 +2570,12 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(MemSize::U8 => builder.ins().ireduce(types::I8, val32),};builder.ins().store(heap_flags(), truncated, addr, 0);+ let size_bytes = match size {+ MemSize::U8 => 1,+ MemSize::U16 => 2,+ MemSize::U32 => 4,+ };+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -2631,9 +2707,9 @@ pub(crate) fn emit_stf_d_form_update<const SYSTEM: SystemId>() {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32, local.cause_smc_write);} else {- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64, local.cause_smc_write);}gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);}@@ -2729,6 +2805,7 @@ pub(crate) fn emit_stfs<const SYSTEM: SystemId>(instr: Instruction,slow_f32: FuncRef,_slow_u32: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());@@ -2752,6 +2829,7 @@ pub(crate) fn emit_stfs<const SYSTEM: SystemId>(let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);let swapped = builder.ins().bswap(bits);builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -2768,6 +2846,7 @@ pub(crate) fn emit_stfd<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,slow_f64: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());@@ -2790,6 +2869,7 @@ pub(crate) fn emit_stfd<const SYSTEM: SystemId>(let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);let swapped = builder.ins().bswap(bits);builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -2826,9 +2906,10 @@ pub(crate) fn emit_stfsx<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,slow_f32: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32);+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32, cause_smc_write);}pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(@@ -2836,9 +2917,10 @@ pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,slow_f64: FuncRef,+ cause_smc_write: FuncRef,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64);+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64, cause_smc_write);}pub(crate) fn emit_lfs_at_ea<const SYSTEM: SystemId>(@@ -2928,6 +3010,7 @@ pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(rs: u8,ea: Value,slow_f32: FuncRef,+ cause_smc_write: FuncRef,) {let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);@@ -2950,6 +3033,7 @@ pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);let swapped = builder.ins().bswap(bits);builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -2967,6 +3051,7 @@ pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(rs: u8,ea: Value,slow_f64: FuncRef,+ cause_smc_write: FuncRef,) {let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);@@ -2988,6 +3073,7 @@ pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);let swapped = builder.ins().bswap(bits);builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -3078,11 +3164,11 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(builder.switch_to_block(fast_block);builder.seal_block(fast_block);let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);if !instr.w_16_16() {let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rs());let ea1 = builder.ins().iadd_imm(ea, 4);- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);}builder.ins().jump(merge, &[]);@@ -3415,6 +3501,8 @@ pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(builder.ins().store(heap_flags(), ps1_be, addr, 4);}+ let psq_size = if w { 4 } else { 8 };+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, psq_size, local.cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_f32_block);@@ -3475,6 +3563,7 @@ pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(ea: Value,f64v: Value,slow_f32: FuncRef,+ cause_smc_write: FuncRef,) {let fast_block = builder.create_block();let slow_block = builder.create_block();@@ -3495,6 +3584,7 @@ pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);let swapped = builder.ins().bswap(bits);builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -4278,10 +4368,21 @@ pub(crate) fn emit_store<const SYSTEM: SystemId>(instr: Instruction,size: MemSize,slow: FuncRef,+ cause_smc_write: FuncRef,update: bool,) {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);+ emit_store_at_ea::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rs(),+ instr.ra(),+ size,+ slow,+ cause_smc_write,+ update,+ );}pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(@@ -4290,10 +4391,21 @@ pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(instr: Instruction,size: MemSize,slow: FuncRef,+ cause_smc_write: FuncRef,update: bool,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);+ emit_store_at_ea::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ instr.rs(),+ instr.ra(),+ size,+ slow,+ cause_smc_write,+ update,+ );}pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(@@ -4304,6 +4416,7 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(ra: u8,size: MemSize,slow: FuncRef,+ cause_smc_write: FuncRef,update: bool,) {let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);@@ -4334,6 +4447,12 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),};builder.ins().store(flags, val_to_store, addr, 0);+ let size_bytes = match size {+ MemSize::U8 => 1,+ MemSize::U16 => 2,+ MemSize::U32 => 4,+ };+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);diff --git a/crates/gecko/src/hollywood/ipc/di.rs b/crates/gecko/src/hollywood/ipc/di.rsindex c0edf92..a6ec5d7 100644--- a/crates/gecko/src/hollywood/ipc/di.rs+++ b/crates/gecko/src/hollywood/ipc/di.rs@@ -98,6 +98,8 @@ impl DiskInterface {let dst = ctx.mmio.phys_slice_mut(out_ptr, length as usize);dvd.read_raw_disc(pos_bytes as usize, dst);+ #[cfg(feature = "jit")]+ ctx.mmio.queue_icbi_for_range(out_ptr, length);self.last_error = DI_ERROR_OK;tracing::debug!(@@ -183,6 +185,8 @@ impl DiskInterface {let dst = ctx.mmio.phys_slice_mut(out_ptr, size as usize);dvd.read_disc_into(pos_bytes as usize, dst);+ #[cfg(feature = "jit")]+ ctx.mmio.queue_icbi_for_range(out_ptr, size);self.last_error = DI_ERROR_OK;tracing::debug!(diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rsindex 37fafbc..c032704 100644--- a/crates/gecko/src/mmio.rs+++ b/crates/gecko/src/mmio.rs@@ -6,12 +6,22 @@ pub mod traits;use crate::system::{SystemId, WII};use constants::*;+use rustc_hash::FxHashSet;pub const FASTMEM_LUT_PAGES: usize = 1 << 15;pub const FASTMEM_PAGE_BYTES: usize = 1 << 17;pub const FASTMEM_PAGE_MASK: u32 = (FASTMEM_PAGE_BYTES as u32) - 1;pub const FASTMEM_PAGE_SHIFT: u32 = 17;+pub const CODE_LINE_BYTES: u32 = 32;+pub const CODE_LINE_SHIFT: u32 = 5;+pub const CODE_LINE_MASK: u32 = !(CODE_LINE_BYTES - 1);++pub const PHYS_MASK: u32 = 0x3FFF_FFFF;++pub const MEM1_LINES: usize = RAM_SIZE >> CODE_LINE_SHIFT as usize;+pub const MEM2_LINES: usize = MEM2_SIZE >> CODE_LINE_SHIFT as usize;+pub struct Mmio<const SYSTEM: SystemId> {pub ram: Vec<u8>,pub efb: Vec<u8>,@@ -22,6 +32,12 @@ pub struct Mmio<const SYSTEM: SystemId> {pub ram_ptr: usize,pub fastmem_lut: Vec<usize>,pub fastmem_lut_ptr: usize,+ pub code_refcount: Box<[u8]>,+ pub code_refcount_ptr: usize,+ #[cfg(feature = "jit")]+ pub pending_icbi: FxHashSet<u32>,+ #[cfg(feature = "jit")]+ pub jit_dirty: u8,}/// Read-only view over MEM1 plus (on Wii) MEM2, addressed by physical@@ -122,6 +138,14 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {let fastmem_lut_ptr = fastmem_lut.as_ptr() as usize;+ let refcount_len = if SYSTEM == WII {+ MEM1_LINES + MEM2_LINES+ } else {+ MEM1_LINES+ };+ let code_refcount: Box<[u8]> = vec![0u8; refcount_len].into_boxed_slice();+ let code_refcount_ptr = code_refcount.as_ptr() as usize;+Mmio {ram,efb: vec![0; EFB_SIZE],@@ -132,6 +156,12 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {ram_ptr,fastmem_lut,fastmem_lut_ptr,+ code_refcount,+ code_refcount_ptr,+ #[cfg(feature = "jit")]+ pending_icbi: FxHashSet::default(),+ #[cfg(feature = "jit")]+ jit_dirty: 0,}}@@ -284,6 +314,79 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {&mut slice[offset..offset + len]}+ #[inline(always)]+ fn code_refcount_index(phys: u32) -> Option<usize> {+ match phys {+ RAM_BASE..=RAM_END => Some((phys >> CODE_LINE_SHIFT) as usize),+ MEM2_BASE..=MEM2_END if SYSTEM == WII => {+ Some(MEM1_LINES + ((phys - MEM2_BASE) >> CODE_LINE_SHIFT) as usize)+ }+ _ => None,+ }+ }++ #[inline(always)]+ fn for_each_code_line(start: u32, len: u32, mut f: impl FnMut(u32)) {+ if len == 0 {+ return;+ }++ let mut p = start & CODE_LINE_MASK;+ let end = start.wrapping_add(len - 1) & CODE_LINE_MASK;+ loop {+ f(p);++ if p == end {+ break;+ }++ p = p.wrapping_add(CODE_LINE_BYTES);+ }+ }++ pub fn mark_code(&mut self, start: u32, len: u32) {+ Self::for_each_code_line(start, len, |line| {+ if let Some(i) = Self::code_refcount_index(line) {+ let v = &mut self.code_refcount[i];+ *v = v.saturating_add(1);+ }+ });+ }++ pub fn unmark_code(&mut self, start: u32, len: u32) {+ Self::for_each_code_line(start, len, |line| {+ if let Some(i) = Self::code_refcount_index(line) {+ let v = &mut self.code_refcount[i];+ if *v != u8::MAX {+ *v = v.saturating_sub(1);+ }+ }+ });+ }++ #[inline(always)]+ pub fn is_code_chunk(&self, phys: u32) -> bool {+ match Self::code_refcount_index(phys) {+ Some(i) => self.code_refcount[i] != 0,+ None => false,+ }+ }++ pub fn clear_code_refcount(&mut self) {+ self.code_refcount.fill(0);+ }++ #[cfg(feature = "jit")]+ #[inline(always)]+ pub fn queue_icbi_for_range(&mut self, phys: u32, len: u32) {+ Self::for_each_code_line(phys, len, |line| {+ if self.is_code_chunk(line) {+ self.pending_icbi.insert(line);+ self.jit_dirty = 1;+ }+ });+ }+/// Read-only view spanning MEM1 and (on Wii) MEM2. Used by the GP path/// so texture / vertex / TLUT reads can resolve addresses in either bank.#[inline(always)]@@ -352,8 +455,11 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {}}- /// Process a locked cache DMA transfer triggered by writing to SPR DMAL (923).- pub fn process_locked_cache_dma(&mut self, dmau: &crate::gekko::spr::DmaUpper, dmal: &crate::gekko::spr::DmaLower) {+ pub fn process_locked_cache_dma(+ &mut self,+ dmau: &crate::gekko::spr::DmaUpper,+ dmal: &crate::gekko::spr::DmaLower,+ ) -> Option<(u32, u32)> {let ram_addr = dmau.ram_addr() << 5;let lcache_vaddr = dmal.lcache_addr() << 5;let lcache_paddr = (lcache_vaddr - LCACHE_BASE) as usize;@@ -372,9 +478,11 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {if dmal.load() {let src = self.virt_slice(ram_addr, length).to_vec();self.lcache[lcache_paddr..lcache_paddr + length].copy_from_slice(&src);+ None} else {let src = self.lcache[lcache_paddr..lcache_paddr + length].to_vec();self.virt_slice_mut(ram_addr, length).copy_from_slice(&src);+ Some((virt_to_phys(ram_addr), length as u32))}}diff --git a/crates/gecko/src/mmio/bus.rs b/crates/gecko/src/mmio/bus.rsindex 7f4c659..7637180 100644--- a/crates/gecko/src/mmio/bus.rs+++ b/crates/gecko/src/mmio/bus.rs@@ -194,6 +194,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {bus_write_hooks!(self, addr, phys, 1, val, {if phys <= RAM_END {self.mmio.ram_write_u8(phys, val);+ #[cfg(feature = "jit")]+ self.mmio.queue_icbi_for_range(phys, 1);} else {self.write_u8_mmio(phys, addr, val);}@@ -206,6 +208,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {bus_write_hooks!(self, addr, phys, 2, val, {if phys <= RAM_END - 1 {self.mmio.ram_write_u16(phys, val);+ #[cfg(feature = "jit")]+ self.mmio.queue_icbi_for_range(phys, 2);} else {self.write_u16_mmio(phys, addr, val);}@@ -218,6 +222,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {bus_write_hooks!(self, addr, phys, 4, val, {if phys <= RAM_END - 3 {self.mmio.ram_write_u32(phys, val);+ #[cfg(feature = "jit")]+ self.mmio.queue_icbi_for_range(phys, 4);} else {self.write_u32_mmio(phys, addr, val);}diff --git a/crates/gecko/src/mmio/fp.rs b/crates/gecko/src/mmio/fp.rsindex dfbe63f..5059e75 100644--- a/crates/gecko/src/mmio/fp.rs+++ b/crates/gecko/src/mmio/fp.rs@@ -22,6 +22,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let bits = val.to_bits();if phys <= RAM_END - 7 {self.mmio.ram_write_u64(phys, bits);+ #[cfg(feature = "jit")]+ self.mmio.queue_icbi_for_range(phys, 8);return;}diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rsindex 3c8ba36..d3d39b5 100644--- a/crates/gecko/src/system.rs+++ b/crates/gecko/src/system.rs@@ -545,6 +545,14 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}jit.run_block(self);++ if self.mmio.jit_dirty != 0 {+ jit.drain_scratch.extend(self.mmio.pending_icbi.drain());+ while let Some(line) = jit.drain_scratch.pop() {+ jit.invalidate_line(&mut self.mmio, line);+ }+ self.mmio.jit_dirty = 0;+ }}self.jit = Some(jit);