Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

icache invalidation

Laylea7e8fcdf4097
diff --git a/Cargo.lock b/Cargo.lock
index 916d178..44b86e1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1802,6 +1802,7 @@ dependencies = [
"image 0.1.0",
"multiversion",
"rustc-hash 2.1.2",
+ "smallvec",
"tracing",
"twox-hash",
"windows-sys 0.61.2",
diff --git a/crates/gecko/Cargo.toml b/crates/gecko/Cargo.toml
index a5694d9..445c59d 100644
--- a/crates/gecko/Cargo.toml
+++ b/crates/gecko/Cargo.toml
@@ -42,6 +42,7 @@ crossbeam-channel = "0.5.15"
hound = { version = "3.5.1", optional = true }
multiversion = "0.8.0"
rustc-hash = "2.1.2"
+smallvec = "1.15.1"
crc32fast = "1.5.0"
zerocopy = { version = "0.8.48", features = ["derive"] }
cranelift-codegen = { version = "0.131.1", optional = true }
diff --git a/crates/gecko/spec/gekko_jit.bindings.chipi b/crates/gecko/spec/gekko_jit.bindings.chipi
index c57bd55..0500ef8 100644
--- a/crates/gecko/spec/gekko_jit.bindings.chipi
+++ b/crates/gecko/spec/gekko_jit.bindings.chipi
@@ -162,7 +162,6 @@ target rust {
dcba
dcbz
dcbz_l
- icbi
tlbie
tlbia
tlbsync
@@ -170,6 +169,10 @@ target rust {
tlbli
}
+ handler icbi {
+ icbi
+ }
+
handler logical {
andx
orx
diff --git a/crates/gecko/spec/wii_gekko_jit.bindings.chipi b/crates/gecko/spec/wii_gekko_jit.bindings.chipi
index 64e3a1a..9bb86e8 100644
--- a/crates/gecko/spec/wii_gekko_jit.bindings.chipi
+++ b/crates/gecko/spec/wii_gekko_jit.bindings.chipi
@@ -162,7 +162,6 @@ target rust {
dcba
dcbz
dcbz_l
- icbi
tlbie
tlbia
tlbsync
@@ -170,6 +169,10 @@ target rust {
tlbli
}
+ handler icbi {
+ icbi
+ }
+
handler logical {
andx
orx
diff --git a/crates/gecko/src/dvd.rs b/crates/gecko/src/dvd.rs
index a47827e..e6680f5 100644
--- a/crates/gecko/src/dvd.rs
+++ b/crates/gecko/src/dvd.rs
@@ -147,6 +147,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Co
let dst = sys.di.dma_address.address();
let buffer = sys.mmio.phys_slice_mut(dst, 0x20);
buffer.copy_from_slice(&[0x69; 0x20]); // TODO: Drive Info?
+ #[cfg(feature = "jit")]
+ sys.mmio.queue_icbi_for_range(dst, 0x20);
}
Command::ReadSectorData => {
let src = sys.di.cmdbuf1 << 2;
@@ -156,6 +158,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Co
let buffer = sys.mmio.phys_slice_mut(dst, len);
dvd.read_disc_into(src as usize, buffer);
+ #[cfg(feature = "jit")]
+ sys.mmio.queue_icbi_for_range(dst, len as u32);
tracing::debug!(
src = format!("{:08X}", src),
@@ -171,6 +175,8 @@ fn process_dvd_command<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>, cmd: Co
let buffer = sys.mmio.phys_slice_mut(dst, len);
dvd.read_disc_into(0, buffer);
+ #[cfg(feature = "jit")]
+ sys.mmio.queue_icbi_for_range(dst, len as u32);
tracing::debug!(
src = format!("{:08X}", src),
diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rs
index 108918c..edc9914 100644
--- a/crates/gecko/src/flipper/dsp.rs
+++ b/crates/gecko/src/flipper/dsp.rs
@@ -195,6 +195,8 @@ impl Dsp {
let src = &self.aram[aram_addr..aram_addr + count];
let dst = mmio.virt_slice_mut(ram_addr as u32, count);
dst.copy_from_slice(src);
+ #[cfg(feature = "jit")]
+ mmio.queue_icbi_for_range(crate::mmio::virt_to_phys(ram_addr as u32), count as u32);
}
regs::DmaDirection::RamToAram if within_bounds => {
let src = mmio.virt_slice(ram_addr as u32, count);
@@ -261,6 +263,8 @@ impl Dsp {
let src = &memory[dsp_addr..dsp_addr + len];
let dst = mmio.virt_slice_mut(ram_addr, len);
dst.copy_from_slice(src);
+ #[cfg(feature = "jit")]
+ mmio.queue_icbi_for_range(crate::mmio::virt_to_phys(ram_addr), len as u32);
}
}
diff --git a/crates/gecko/src/flipper/exi.rs b/crates/gecko/src/flipper/exi.rs
index 93aeb96..65ab10c 100644
--- a/crates/gecko/src/flipper/exi.rs
+++ b/crates/gecko/src/flipper/exi.rs
@@ -264,7 +264,11 @@ pub fn run_dma<const CHANNEL: usize, const SYSTEM: SystemId>(sys: &mut System<SY
if let Some(device) = &mut sys.exi.devices[CHANNEL][slot] {
match transfer_type {
- TransferType::Read => device.dma_read(sys.mmio.phys_slice_mut(address, length as usize)),
+ TransferType::Read => {
+ device.dma_read(sys.mmio.phys_slice_mut(address, length as usize));
+ #[cfg(feature = "jit")]
+ sys.mmio.queue_icbi_for_range(address, length);
+ }
TransferType::Write => device.dma_write(sys.mmio.phys_slice(address, length as usize)),
TransferType::ReadAndWrite | TransferType::Reserved => {
tracing::error!(
diff --git a/crates/gecko/src/gekko/interpreter/system.rs b/crates/gecko/src/gekko/interpreter/system.rs
index 2a59005..ef4209c 100644
--- a/crates/gecko/src/gekko/interpreter/system.rs
+++ b/crates/gecko/src/gekko/interpreter/system.rs
@@ -44,8 +44,15 @@ pub fn spr<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, inst
923 => {
ctx.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
if ctx.gekko.spr.dmal.trigger() {
- ctx.mmio
- .process_locked_cache_dma(&ctx.gekko.spr.dmau, &ctx.gekko.spr.dmal);
+ let dmau = ctx.gekko.spr.dmau;
+ let dmal = ctx.gekko.spr.dmal;
+ let written = ctx.mmio.process_locked_cache_dma(&dmau, &dmal);
+ #[cfg(feature = "jit")]
+ if let Some((phys, len)) = written {
+ ctx.mmio.queue_icbi_for_range(phys, len);
+ }
+ #[cfg(not(feature = "jit"))]
+ let _ = written;
}
}
_ => ctx.gekko.spr.write(spr_num, val),
diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rs
index 92e31a4..1736ccc 100644
--- a/crates/gecko/src/gekko/jit.rs
+++ b/crates/gecko/src/gekko/jit.rs
@@ -74,6 +74,8 @@ pub struct ExternFuncs {
pub do_psq_load: FuncId,
pub do_psq_store: FuncId,
pub read_timebase: FuncId,
+ pub cause_icbi: FuncId,
+ pub cause_smc_write: FuncId,
}
pub struct JitEngine<const SYSTEM: SystemId> {
@@ -82,7 +84,9 @@ pub struct JitEngine<const SYSTEM: SystemId> {
builder_ctx: FunctionBuilderContext,
cache: FxHashMap<u32, BlockEntry>,
block_func_ids: FxHashMap<u32, FuncId>,
- pending_chain_slots: FxHashMap<u32, Vec<usize>>,
+ target_slots: std::cell::RefCell<FxHashMap<u32, usize>>,
+ blocks_by_line: FxHashMap<u32, smallvec::SmallVec<[u32; 2]>>,
+ pub(crate) drain_scratch: Vec<u32>,
#[cfg(feature = "jit-stats")]
hits: FxHashMap<u32, u64>,
pub(crate) block_specs: FxHashMap<u32, block::BlockSpec>,
@@ -150,6 +154,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
do_psq_load: (&'static str, *const u8),
do_psq_store: (&'static str, *const u8),
read_timebase: (&'static str, *const u8),
+ cause_icbi: (&'static str, *const u8),
+ cause_smc_write: (&'static str, *const u8),
}
let syms: SymTable = match SYSTEM {
GC => SymTable {
@@ -201,6 +207,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
do_psq_load: ("gecko_jit_do_psq_load_gc", runtime::do_psq_load_gc as *const u8),
do_psq_store: ("gecko_jit_do_psq_store_gc", runtime::do_psq_store_gc as *const u8),
read_timebase: ("gecko_jit_read_timebase_gc", runtime::read_timebase_gc as *const u8),
+ cause_icbi: ("gecko_jit_cause_icbi_gc", runtime::cause_icbi_gc as *const u8),
+ cause_smc_write: ("gecko_jit_cause_smc_write_gc", runtime::cause_smc_write_gc as *const u8),
},
WII => SymTable {
cause_invalid_opcode: (
@@ -254,6 +262,11 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
do_psq_load: ("gecko_jit_do_psq_load_wii", runtime::do_psq_load_wii as *const u8),
do_psq_store: ("gecko_jit_do_psq_store_wii", runtime::do_psq_store_wii as *const u8),
read_timebase: ("gecko_jit_read_timebase_wii", runtime::read_timebase_wii as *const u8),
+ cause_icbi: ("gecko_jit_cause_icbi_wii", runtime::cause_icbi_wii as *const u8),
+ cause_smc_write: (
+ "gecko_jit_cause_smc_write_wii",
+ runtime::cause_smc_write_wii as *const u8,
+ ),
},
_ => unreachable!(),
};
@@ -289,6 +302,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
syms.do_psq_load,
syms.do_psq_store,
syms.read_timebase,
+ syms.cause_icbi,
+ syms.cause_smc_write,
] {
jit_builder.symbol(name, addr);
}
@@ -456,6 +471,12 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
read_timebase: module
.declare_function(syms.read_timebase.0, Linkage::Import, &mem_read_sig)
.expect("declare read_timebase"),
+ cause_icbi: module
+ .declare_function(syms.cause_icbi.0, Linkage::Import, &ctx_u32_sig)
+ .expect("declare cause_icbi"),
+ cause_smc_write: module
+ .declare_function(syms.cause_smc_write.0, Linkage::Import, &mem_write_sig)
+ .expect("declare cause_smc_write"),
};
let trampoline_fn = build_trampoline(&mut module, call_conv, pointer_type, &block_sig);
@@ -477,7 +498,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
builder_ctx: FunctionBuilderContext::new(),
cache: FxHashMap::default(),
block_func_ids: FxHashMap::default(),
- pending_chain_slots: FxHashMap::default(),
+ target_slots: std::cell::RefCell::new(FxHashMap::default()),
+ blocks_by_line: FxHashMap::default(),
+ drain_scratch: Vec::new(),
#[cfg(feature = "jit-stats")]
hits: FxHashMap::default(),
block_specs: FxHashMap::default(),
@@ -494,6 +517,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
}
}
+ #[inline]
+ fn target_slot_addr(&self, target_pc: u32) -> usize {
+ *self.target_slots.borrow_mut().entry(target_pc).or_insert_with(|| {
+ let slot: &'static mut usize = Box::leak(Box::new(0usize));
+ slot as *mut usize as usize
+ })
+ }
+
pub fn block_lookup_table_addr(&self) -> usize {
self.block_lookup_table_addr
}
@@ -592,10 +623,77 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
let entry = self.compile(&spec, &gprs_snapshot);
self.cache.insert(pc, entry);
self.block_specs.insert(pc, spec);
+ self.register_block(&mut sys.mmio, pc);
entry
}
+ fn register_block(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, pc: u32) {
+ let Some(spec) = self.block_specs.get(&pc) else { return };
+ let mut last = u32::MAX;
+ for &vpc in &spec.pcs {
+ let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;
+ if line == last {
+ continue;
+ }
+
+ last = line;
+ self.blocks_by_line.entry(line).or_default().push(pc);
+ mmio.mark_code(line, crate::mmio::CODE_LINE_BYTES);
+ }
+ }
+
+ fn unregister_block(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, pc: u32) {
+ let Some(spec) = self.block_specs.remove(&pc) else {
+ return;
+ };
+ self.cache.remove(&pc);
+ self.block_func_ids.remove(&pc);
+
+ if let Some(&slot) = self.target_slots.borrow().get(&pc) {
+ unsafe {
+ *(slot as *mut usize) = 0;
+ }
+ }
+
+ let idx = ((pc >> 2) & BLOCK_LOOKUP_TABLE_MASK) as usize;
+ unsafe {
+ let table = self.block_lookup_table_addr as *mut BlockLookupSlot;
+ if (*table.add(idx)).pc == pc {
+ (*table.add(idx)).entry = 0;
+ }
+ }
+
+ let mut last = u32::MAX;
+ for &vpc in &spec.pcs {
+ let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;
+ if line == last {
+ continue;
+ }
+
+ last = line;
+ mmio.unmark_code(line, crate::mmio::CODE_LINE_BYTES);
+ if let Some(v) = self.blocks_by_line.get_mut(&line) {
+ v.retain(|p| *p != pc);
+ if v.is_empty() {
+ self.blocks_by_line.remove(&line);
+ }
+ }
+ }
+ }
+
+ pub fn invalidate_line(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, line: u32) -> bool {
+ let Some(pcs) = self.blocks_by_line.remove(&line) else {
+ return false;
+ };
+
+ for pc in pcs.into_iter() {
+ self.unregister_block(mmio, pc);
+ }
+
+ true
+ }
+
#[cfg(feature = "jit-stats")]
fn block_entry_counter_slot(&mut self, pc: u32) -> usize {
if let Some(&addr) = self.block_entry_counter_ptrs.get(&pc) {
@@ -796,7 +894,14 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
pub fn flush(&mut self) {
self.cache.clear();
self.block_func_ids.clear();
- self.pending_chain_slots.clear();
+ self.block_specs.clear();
+ self.blocks_by_line.clear();
+
+ for &slot_addr in self.target_slots.borrow().values() {
+ unsafe {
+ *(slot_addr as *mut usize) = 0;
+ }
+ }
unsafe {
let table = self.block_lookup_table_addr as *mut BlockLookupSlot;
@@ -807,6 +912,13 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
}
}
+ pub fn flush_with_refcount(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>) {
+ self.flush();
+ mmio.clear_code_refcount();
+ mmio.pending_icbi.clear();
+ mmio.jit_dirty = 0;
+ }
+
#[cfg_attr(feature = "hotpath", hotpath::measure(label = "ppc_jit_compile"))]
fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {
let func_id = self.func_id_for(spec.start_pc);
@@ -814,24 +926,18 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
self.ctx.clear();
self.ctx.func.signature = self.block_sig.clone();
- let self_slot: &'static mut usize = Box::leak(Box::new(0usize));
- let self_slot_addr = self_slot as *mut usize as i64;
-
- let mut pending_for_this_block: Vec<(u32, usize)> = Vec::new();
+ let _ = self.target_slot_addr(spec.start_pc);
#[cfg(feature = "jit-stats")]
let entry_counter_addr = Some(self.block_entry_counter_slot(spec.start_pc));
#[cfg(not(feature = "jit-stats"))]
let entry_counter_addr: Option<usize> = None;
+ let block_lookup_table_addr = self.block_lookup_table_addr as i64;
let chain = translator::ChainContext {
self_pc: spec.start_pc,
- self_func_id: func_id,
- self_slot_addr,
- compiled_func_ids: &self.block_func_ids,
- compiled_cache: &self.cache,
- pending: std::cell::RefCell::new(&mut pending_for_this_block),
- block_lookup_table_addr: self.block_lookup_table_addr as i64,
+ target_slots: &self.target_slots,
+ block_lookup_table_addr,
};
translator::translate::<SYSTEM>(
@@ -846,9 +952,6 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
);
drop(chain);
- for (target_pc, slot_addr) in pending_for_this_block.drain(..) {
- self.pending_chain_slots.entry(target_pc).or_default().push(slot_addr);
- }
let want_dump = self.dump_pc == Some(spec.start_pc);
if want_dump {
@@ -901,16 +1004,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
self.module.finalize_definitions().expect("finalize");
let entry = self.module.get_finalized_function(func_id) as usize;
+ let slot_addr = self.target_slot_addr(spec.start_pc);
unsafe {
- *(self_slot_addr as *mut usize) = entry;
- }
-
- if let Some(slots) = self.pending_chain_slots.remove(&spec.start_pc) {
- for slot_addr in slots {
- unsafe {
- *(slot_addr as *mut usize) = entry;
- }
- }
+ *(slot_addr as *mut usize) = entry;
}
self.register_in_lookup_table(spec.start_pc, entry);
diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rs
index c4f8d81..f268f93 100644
--- a/crates/gecko/src/gekko/jit/abi.rs
+++ b/crates/gecko/src/gekko/jit/abi.rs
@@ -98,6 +98,16 @@ pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {
offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)
}
+#[inline(always)]
+pub const fn code_refcount_ptr_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, code_refcount_ptr)
+}
+
+#[inline(always)]
+pub const fn jit_dirty_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, jit_dirty)
+}
+
#[inline(always)]
pub const fn gqr_offset<const SYSTEM: SystemId>(i: u8) -> usize {
let spr_off = gekko_offset::<SYSTEM>() + offset_of!(Gekko, spr);
diff --git a/crates/gecko/src/gekko/jit/handlers.rs b/crates/gecko/src/gekko/jit/handlers.rs
index 2249350..eba0950 100644
--- a/crates/gecko/src/gekko/jit/handlers.rs
+++ b/crates/gecko/src/gekko/jit/handlers.rs
@@ -488,20 +488,60 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,
lut::OP_LHA => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),
lut::OP_LHAU => translator::emit_lha_d_form::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),
- lut::OP_STW => {
- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)
- }
- lut::OP_STWU => {
- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)
- }
- lut::OP_STB => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false),
- lut::OP_STBU => translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true),
- lut::OP_STH => {
- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)
- }
- lut::OP_STHU => {
- translator::emit_store::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)
- }
+ lut::OP_STW => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STWU => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ true,
+ ),
+ lut::OP_STB => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U8,
+ local.write_u8,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STBU => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U8,
+ local.write_u8,
+ local.cause_smc_write,
+ true,
+ ),
+ lut::OP_STH => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U16,
+ local.write_u16,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STHU => translator::emit_store::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U16,
+ local.write_u16,
+ local.cause_smc_write,
+ true,
+ ),
lut::OP_LMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, false),
lut::OP_STMW => translator::emit_lmw_stmw::<SYSTEM>(builder, t.ctx_ptr, instr, local, true),
@@ -527,24 +567,60 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,
lut::OP_LHAX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, false),
lut::OP_LHAUX => translator::emit_lha_xform::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_u16, true),
- lut::OP_STWX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false)
- }
- lut::OP_STWUX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, true)
- }
- lut::OP_STBX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, false)
- }
- lut::OP_STBUX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U8, local.write_u8, true)
- }
- lut::OP_STHX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, false)
- }
- lut::OP_STHUX => {
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16, true)
- }
+ lut::OP_STWX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STWUX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ true,
+ ),
+ lut::OP_STBX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U8,
+ local.write_u8,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STBUX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U8,
+ local.write_u8,
+ local.cause_smc_write,
+ true,
+ ),
+ lut::OP_STHX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U16,
+ local.write_u16,
+ local.cause_smc_write,
+ false,
+ ),
+ lut::OP_STHUX => translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U16,
+ local.write_u16,
+ local.cause_smc_write,
+ true,
+ ),
lut::OP_LWBRX => {
translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32)
@@ -552,12 +628,22 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,
lut::OP_LHBRX => {
translator::emit_load_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.read_u16)
}
- lut::OP_STWBRX => {
- translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32)
- }
- lut::OP_STHBRX => {
- translator::emit_store_xform_brx::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U16, local.write_u16)
- }
+ lut::OP_STWBRX => translator::emit_store_xform_brx::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ ),
+ lut::OP_STHBRX => translator::emit_store_xform_brx::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U16,
+ local.write_u16,
+ local.cause_smc_write,
+ ),
_ => return,
}
@@ -577,9 +663,18 @@ pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslato
lut::OP_LFD => translator::emit_lfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),
lut::OP_LFDU => translator::emit_lf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
- lut::OP_STFS => translator::emit_stfs::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.write_u32),
+ lut::OP_STFS => translator::emit_stfs::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ local.write_f32,
+ local.write_u32,
+ local.cause_smc_write,
+ ),
lut::OP_STFSU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
- lut::OP_STFD => translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),
+ lut::OP_STFD => {
+ translator::emit_stfd::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64, local.cause_smc_write)
+ }
lut::OP_STFDU => translator::emit_stf_d_form_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
lut::OP_LFSX => translator::emit_lfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f32),
@@ -587,12 +682,18 @@ pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslato
lut::OP_LFDX => translator::emit_lfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.read_f64),
lut::OP_LFDUX => translator::emit_lfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
- lut::OP_STFSX => translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32),
+ lut::OP_STFSX => {
+ translator::emit_stfsx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f32, local.cause_smc_write)
+ }
lut::OP_STFSUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, true, local),
- lut::OP_STFDX => translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64),
+ lut::OP_STFDX => {
+ translator::emit_stfdx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_f64, local.cause_smc_write)
+ }
lut::OP_STFDUX => translator::emit_stfx_update::<SYSTEM>(builder, t.ctx_ptr, instr, false, local),
- lut::OP_STFIWX => translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32),
+ lut::OP_STFIWX => {
+ translator::emit_stfiwx::<SYSTEM>(builder, t.ctx_ptr, instr, local.write_u32, local.cause_smc_write)
+ }
_ => return,
}
@@ -644,7 +745,6 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:
| lut::OP_DCBA
| lut::OP_DCBZ
| lut::OP_DCBZ_L
- | lut::OP_ICBI
| lut::OP_TLBIE
| lut::OP_TLBIA
| lut::OP_TLBSYNC
@@ -657,6 +757,38 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:
let _ = builder;
}
+#[inline(always)]
+pub fn icbi<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
+ let _ = OP;
+ let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
+
+ let ra = instr.ra();
+ let rb = instr.rb();
+ let ra_v = if ra == 0 {
+ builder.ins().iconst(types::I32, 0)
+ } else {
+ translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, ra)
+ };
+ let rb_v = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, rb);
+ let ea = builder.ins().iadd(ra_v, rb_v);
+
+ builder.ins().call(local.cause_icbi, &[t.ctx_ptr, ea]);
+
+ let dirty_off = abi::jit_dirty_offset::<SYSTEM>() as i32;
+ let dirty = builder
+ .ins()
+ .load(types::I8, translator::vmctx_flags(), t.ctx_ptr, dirty_off);
+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, dirty, 0);
+
+ let continue_block = builder.create_block();
+ let nia = builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64);
+ builder.ins().brif(nz, t.exit_block, &[nia.into()], continue_block, &[]);
+ builder.switch_to_block(continue_block);
+ builder.seal_block(continue_block);
+
+ t.handled_natively = true;
+}
+
#[inline(always)]
pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
if !t.is_terminator {
@@ -931,7 +1063,15 @@ pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)
pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {
let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };
- translator::emit_store_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.write_u32, false);
+ translator::emit_store_xform::<SYSTEM>(
+ builder,
+ t.ctx_ptr,
+ instr,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ false,
+ );
t.handled_natively = true;
}
diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rs
index 63715ef..f2172ce 100644
--- a/crates/gecko/src/gekko/jit/runtime.rs
+++ b/crates/gecko/src/gekko/jit/runtime.rs
@@ -207,8 +207,11 @@ fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u3
923 => {
sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
if sys.gekko.spr.dmal.trigger() {
- sys.mmio
- .process_locked_cache_dma(&sys.gekko.spr.dmau, &sys.gekko.spr.dmal);
+ let dmau = sys.gekko.spr.dmau;
+ let dmal = sys.gekko.spr.dmal;
+ if let Some((phys, len)) = sys.mmio.process_locked_cache_dma(&dmau, &dmal) {
+ sys.mmio.queue_icbi_for_range(phys, len);
+ }
}
}
_ => sys.gekko.spr.write(num, val),
@@ -550,3 +553,37 @@ pub extern "C" fn do_rfi_gc(sys: *mut core::ffi::c_void) {
pub extern "C" fn do_rfi_wii(sys: *mut core::ffi::c_void) {
do_rfi::<WII>(sys.cast());
}
+
+#[inline(always)]
+fn cause_icbi<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) {
+ let sys = unsafe { &mut *sys };
+ let phys = crate::mmio::virt_to_phys(ea);
+ if !sys.mmio.is_code_chunk(phys) {
+ return;
+ }
+
+ let line = phys & crate::mmio::CODE_LINE_MASK;
+ sys.mmio.pending_icbi.insert(line);
+ sys.mmio.jit_dirty = 1;
+}
+
+pub extern "C" fn cause_icbi_gc(sys: *mut core::ffi::c_void, ea: u32) {
+ cause_icbi::<GC>(sys.cast(), ea);
+}
+pub extern "C" fn cause_icbi_wii(sys: *mut core::ffi::c_void, ea: u32) {
+ cause_icbi::<WII>(sys.cast(), ea);
+}
+
+#[inline(always)]
+fn cause_smc_write<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32, size: u32) {
+ let sys = unsafe { &mut *sys };
+ let phys = crate::mmio::virt_to_phys(ea);
+ sys.mmio.queue_icbi_for_range(phys, size);
+}
+
+pub extern "C" fn cause_smc_write_gc(sys: *mut core::ffi::c_void, ea: u32, size: u32) {
+ cause_smc_write::<GC>(sys.cast(), ea, size);
+}
+pub extern "C" fn cause_smc_write_wii(sys: *mut core::ffi::c_void, ea: u32, size: u32) {
+ cause_smc_write::<WII>(sys.cast(), ea, size);
+}
diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rs
index 4b2906a..1dc8609 100644
--- a/crates/gecko/src/gekko/jit/translator.rs
+++ b/crates/gecko/src/gekko/jit/translator.rs
@@ -4,7 +4,7 @@ use cranelift_codegen::ir::{AliasRegion, Block, FuncRef, InstBuilder, MemFlags,
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::JITModule;
-use cranelift_module::{FuncId, Module};
+use cranelift_module::Module;
use rustc_hash::FxHashMap;
use crate::gekko::instruction::Instruction;
@@ -32,29 +32,18 @@ pub(crate) fn heap_flags() -> MemFlags {
pub struct ChainContext<'a> {
pub self_pc: u32,
- pub self_func_id: FuncId,
- pub self_slot_addr: i64,
- pub compiled_func_ids: &'a FxHashMap<u32, FuncId>,
- pub compiled_cache: &'a FxHashMap<u32, super::BlockEntry>,
- pub pending: std::cell::RefCell<&'a mut Vec<(u32, usize)>>,
+ pub target_slots: &'a std::cell::RefCell<FxHashMap<u32, usize>>,
pub block_lookup_table_addr: i64,
}
impl<'a> ChainContext<'a> {
fn target_for(&self, target_pc: u32) -> Option<i64> {
- if target_pc == self.self_pc {
- return Some(self.self_slot_addr);
- }
-
- if let Some(&entry) = self.compiled_cache.get(&target_pc) {
- let slot: &'static mut usize = Box::leak(Box::new(entry));
- return Some(slot as *mut usize as i64);
- }
-
- let slot: &'static mut usize = Box::leak(Box::new(0usize));
- let slot_addr_usize = slot as *mut usize as usize;
- self.pending.borrow_mut().push((target_pc, slot_addr_usize));
- Some(slot_addr_usize as i64)
+ let mut slots = self.target_slots.borrow_mut();
+ let addr = *slots.entry(target_pc).or_insert_with(|| {
+ let slot: &'static mut usize = Box::leak(Box::new(0usize));
+ slot as *mut usize as usize
+ });
+ Some(addr as i64)
}
}
@@ -92,6 +81,8 @@ pub(crate) struct LocalFuncs {
pub(crate) do_psq_load: FuncRef,
pub(crate) do_psq_store: FuncRef,
pub(crate) read_timebase: FuncRef,
+ pub(crate) cause_icbi: FuncRef,
+ pub(crate) cause_smc_write: FuncRef,
pub(crate) fp_guard_emitted: std::cell::Cell<bool>,
}
@@ -152,6 +143,8 @@ pub fn translate<const SYSTEM: SystemId>(
do_psq_load: module.declare_func_in_func(extern_funcs.do_psq_load, &mut ctx.func),
do_psq_store: module.declare_func_in_func(extern_funcs.do_psq_store, &mut ctx.func),
read_timebase: module.declare_func_in_func(extern_funcs.read_timebase, &mut ctx.func),
+ cause_icbi: module.declare_func_in_func(extern_funcs.cause_icbi, &mut ctx.func),
+ cause_smc_write: module.declare_func_in_func(extern_funcs.cause_smc_write, &mut ctx.func),
fp_guard_emitted: std::cell::Cell::new(false),
};
@@ -186,6 +179,15 @@ pub fn translate<const SYSTEM: SystemId>(
let exit_block = builder.create_block();
builder.append_block_param(exit_block, types::I32);
+ let dirty_off = abi::jit_dirty_offset::<SYSTEM>() as i32;
+ let dirty = builder.ins().load(types::I8, vmctx_flags(), ctx_ptr, dirty_off);
+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, dirty, 0);
+ let alive = builder.create_block();
+ let start_nia = builder.ins().iconst(types::I32, spec.start_pc as i64);
+ builder.ins().brif(nz, exit_block, &[start_nia.into()], alive, &[]);
+ builder.switch_to_block(alive);
+ builder.seal_block(alive);
+
if let IdleClass::PointerIterLoop { gpr, stride } = idle_class {
let rb_off = self::gpr_offset::<SYSTEM>(gpr);
let ctr_off = abi::ctr_offset::<SYSTEM>() as i32;
@@ -2026,6 +2028,59 @@ pub(crate) fn emit_x_form_ea<const SYSTEM: SystemId>(
builder.ins().iadd(base, b)
}
+pub(crate) fn emit_smc_check<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ size_bytes: u32,
+ cause_smc_write: FuncRef,
+) {
+ let phys = builder.ins().band_imm(ea, crate::mmio::PHYS_MASK as i64);
+ let in_mem1 = builder
+ .ins()
+ .icmp_imm(IntCC::UnsignedLessThan, phys, crate::mmio::constants::RAM_SIZE as i64);
+
+ let check_block = builder.create_block();
+ let post_block = builder.create_block();
+ builder.ins().brif(in_mem1, check_block, &[], post_block, &[]);
+
+ builder.switch_to_block(check_block);
+ builder.seal_block(check_block);
+
+ let rc_off = abi::code_refcount_ptr_offset::<SYSTEM>() as i32;
+ let rc_base = builder.ins().load(types::I64, vmctx_readonly_flags(), ctx_ptr, rc_off);
+
+ let lo_idx_u32 = builder.ins().ushr_imm(phys, crate::mmio::CODE_LINE_SHIFT as i64);
+ let lo_idx = builder.ins().uextend(types::I64, lo_idx_u32);
+ let lo_addr = builder.ins().iadd(rc_base, lo_idx);
+ let rc_lo = builder.ins().load(types::I8, vmctx_flags(), lo_addr, 0);
+
+ let combined = if size_bytes <= 1 {
+ rc_lo
+ } else {
+ let hi_phys = builder.ins().iadd_imm(phys, (size_bytes as i64) - 1);
+ let hi_idx_u32 = builder.ins().ushr_imm(hi_phys, crate::mmio::CODE_LINE_SHIFT as i64);
+ let hi_idx = builder.ins().uextend(types::I64, hi_idx_u32);
+ let hi_addr = builder.ins().iadd(rc_base, hi_idx);
+ let rc_hi = builder.ins().load(types::I8, vmctx_flags(), hi_addr, 0);
+ builder.ins().bor(rc_lo, rc_hi)
+ };
+ let nz = builder.ins().icmp_imm(IntCC::NotEqual, combined, 0);
+
+ let slow_block = builder.create_block();
+ builder.set_cold_block(slow_block);
+ builder.ins().brif(nz, slow_block, &[], post_block, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let size_v = builder.ins().iconst(types::I32, size_bytes as i64);
+ builder.ins().call(cause_smc_write, &[ctx_ptr, ea, size_v]);
+ builder.ins().jump(post_block, &[]);
+
+ builder.switch_to_block(post_block);
+ builder.seal_block(post_block);
+}
+
pub(crate) fn emit_fastmem_lookup<const SYSTEM: SystemId>(
builder: &mut FunctionBuilder,
ctx_ptr: Value,
@@ -2135,9 +2190,9 @@ pub(crate) fn emit_stfx_update<const SYSTEM: SystemId>(
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
if single {
- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32, local.cause_smc_write);
} else {
- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64, local.cause_smc_write);
}
gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
}
@@ -2147,6 +2202,7 @@ pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(
ctx_ptr: Value,
instr: Instruction,
slow_u32: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
@@ -2154,7 +2210,7 @@ pub(crate) fn emit_stfiwx<const SYSTEM: SystemId>(
let u64v = builder.ins().bitcast(types::I64, MemFlags::new(), f);
let u32v = builder.ins().ireduce(types::I32, u64v);
- emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32);
+ emit_u32_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, u32v, slow_u32, cause_smc_write);
}
pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(
@@ -2163,6 +2219,7 @@ pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(
ea: Value,
val: Value,
slow: FuncRef,
+ cause_smc_write: FuncRef,
) {
let fast_block = builder.create_block();
let slow_block = builder.create_block();
@@ -2181,6 +2238,7 @@ pub(crate) fn emit_u32_store_at_ea<const SYSTEM: SystemId>(
let addr = builder.block_params(fast_block)[0];
let bswapped = builder.ins().bswap(val);
builder.ins().store(heap_flags(), bswapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -2215,11 +2273,11 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(
builder.seal_block(fast_block);
if store {
let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
if !instr.w_21_21() {
let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);
}
} else {
let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
@@ -2324,6 +2382,7 @@ pub(crate) fn emit_stwcx_dot<const SYSTEM: SystemId>(
0,
MemSize::U32,
local.write_u32,
+ local.cause_smc_write,
false,
);
builder.ins().jump(cr_block, &[matched.into()]);
@@ -2360,7 +2419,17 @@ pub(crate) fn emit_lmw_stmw<const SYSTEM: SystemId>(
builder.ins().iadd_imm(ea_base, off as i64)
};
if store {
- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.write_u32, false);
+ emit_store_at_ea::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ r,
+ 0,
+ MemSize::U32,
+ local.write_u32,
+ local.cause_smc_write,
+ false,
+ );
} else {
emit_load_at_ea::<SYSTEM>(builder, ctx_ptr, ea, r, 0, MemSize::U32, local.read_u32, false);
}
@@ -2474,6 +2543,7 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(
instr: Instruction,
size: MemSize,
slow: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
@@ -2500,6 +2570,12 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(
MemSize::U8 => builder.ins().ireduce(types::I8, val32),
};
builder.ins().store(heap_flags(), truncated, addr, 0);
+ let size_bytes = match size {
+ MemSize::U8 => 1,
+ MemSize::U16 => 2,
+ MemSize::U32 => 4,
+ };
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -2631,9 +2707,9 @@ pub(crate) fn emit_stf_d_form_update<const SYSTEM: SystemId>(
) {
let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
if single {
- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32);
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f32, local.cause_smc_write);
} else {
- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64);
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, local.write_f64, local.cause_smc_write);
}
gpr_store::<SYSTEM>(builder, ctx_ptr, instr.ra(), ea);
}
@@ -2729,6 +2805,7 @@ pub(crate) fn emit_stfs<const SYSTEM: SystemId>(
instr: Instruction,
slow_f32: FuncRef,
_slow_u32: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
@@ -2752,6 +2829,7 @@ pub(crate) fn emit_stfs<const SYSTEM: SystemId>(
let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
let swapped = builder.ins().bswap(bits);
builder.ins().store(heap_flags(), swapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -2768,6 +2846,7 @@ pub(crate) fn emit_stfd<const SYSTEM: SystemId>(
ctx_ptr: Value,
instr: Instruction,
slow_f64: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
@@ -2790,6 +2869,7 @@ pub(crate) fn emit_stfd<const SYSTEM: SystemId>(
let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);
let swapped = builder.ins().bswap(bits);
builder.ins().store(heap_flags(), swapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -2826,9 +2906,10 @@ pub(crate) fn emit_stfsx<const SYSTEM: SystemId>(
ctx_ptr: Value,
instr: Instruction,
slow_f32: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
- emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32);
+ emit_stfs_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f32, cause_smc_write);
}
pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(
@@ -2836,9 +2917,10 @@ pub(crate) fn emit_stfdx<const SYSTEM: SystemId>(
ctx_ptr: Value,
instr: Instruction,
slow_f64: FuncRef,
+ cause_smc_write: FuncRef,
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
- emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64);
+ emit_stfd_at_ea::<SYSTEM>(builder, ctx_ptr, instr.rs(), ea, slow_f64, cause_smc_write);
}
pub(crate) fn emit_lfs_at_ea<const SYSTEM: SystemId>(
@@ -2928,6 +3010,7 @@ pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(
rs: u8,
ea: Value,
slow_f32: FuncRef,
+ cause_smc_write: FuncRef,
) {
let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);
@@ -2950,6 +3033,7 @@ pub(crate) fn emit_stfs_at_ea<const SYSTEM: SystemId>(
let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
let swapped = builder.ins().bswap(bits);
builder.ins().store(heap_flags(), swapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -2967,6 +3051,7 @@ pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(
rs: u8,
ea: Value,
slow_f64: FuncRef,
+ cause_smc_write: FuncRef,
) {
let f64v = fpr_load::<SYSTEM>(builder, ctx_ptr, rs);
@@ -2988,6 +3073,7 @@ pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(
let bits = builder.ins().bitcast(types::I64, MemFlags::new(), f64v);
let swapped = builder.ins().bswap(bits);
builder.ins().store(heap_flags(), swapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -3078,11 +3164,11 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
if !instr.w_16_16() {
let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32);
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);
}
builder.ins().jump(merge, &[]);
@@ -3415,6 +3501,8 @@ pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(
builder.ins().store(heap_flags(), ps1_be, addr, 4);
}
+ let psq_size = if w { 4 } else { 8 };
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, psq_size, local.cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_f32_block);
@@ -3475,6 +3563,7 @@ pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(
ea: Value,
f64v: Value,
slow_f32: FuncRef,
+ cause_smc_write: FuncRef,
) {
let fast_block = builder.create_block();
let slow_block = builder.create_block();
@@ -3495,6 +3584,7 @@ pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(
let bits = builder.ins().bitcast(types::I32, MemFlags::new(), f32v);
let swapped = builder.ins().bswap(bits);
builder.ins().store(heap_flags(), swapped, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -4278,10 +4368,21 @@ pub(crate) fn emit_store<const SYSTEM: SystemId>(
instr: Instruction,
size: MemSize,
slow: FuncRef,
+ cause_smc_write: FuncRef,
update: bool,
) {
let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);
+ emit_store_at_ea::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rs(),
+ instr.ra(),
+ size,
+ slow,
+ cause_smc_write,
+ update,
+ );
}
pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(
@@ -4290,10 +4391,21 @@ pub(crate) fn emit_store_xform<const SYSTEM: SystemId>(
instr: Instruction,
size: MemSize,
slow: FuncRef,
+ cause_smc_write: FuncRef,
update: bool,
) {
let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);
- emit_store_at_ea::<SYSTEM>(builder, ctx_ptr, ea, instr.rs(), instr.ra(), size, slow, update);
+ emit_store_at_ea::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ instr.rs(),
+ instr.ra(),
+ size,
+ slow,
+ cause_smc_write,
+ update,
+ );
}
pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
@@ -4304,6 +4416,7 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
ra: u8,
size: MemSize,
slow: FuncRef,
+ cause_smc_write: FuncRef,
update: bool,
) {
let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);
@@ -4334,6 +4447,12 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),
};
builder.ins().store(flags, val_to_store, addr, 0);
+ let size_bytes = match size {
+ MemSize::U8 => 1,
+ MemSize::U16 => 2,
+ MemSize::U32 => 4,
+ };
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
diff --git a/crates/gecko/src/hollywood/ipc/di.rs b/crates/gecko/src/hollywood/ipc/di.rs
index c0edf92..a6ec5d7 100644
--- a/crates/gecko/src/hollywood/ipc/di.rs
+++ b/crates/gecko/src/hollywood/ipc/di.rs
@@ -98,6 +98,8 @@ impl DiskInterface {
let dst = ctx.mmio.phys_slice_mut(out_ptr, length as usize);
dvd.read_raw_disc(pos_bytes as usize, dst);
+ #[cfg(feature = "jit")]
+ ctx.mmio.queue_icbi_for_range(out_ptr, length);
self.last_error = DI_ERROR_OK;
tracing::debug!(
@@ -183,6 +185,8 @@ impl DiskInterface {
let dst = ctx.mmio.phys_slice_mut(out_ptr, size as usize);
dvd.read_disc_into(pos_bytes as usize, dst);
+ #[cfg(feature = "jit")]
+ ctx.mmio.queue_icbi_for_range(out_ptr, size);
self.last_error = DI_ERROR_OK;
tracing::debug!(
diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rs
index 37fafbc..c032704 100644
--- a/crates/gecko/src/mmio.rs
+++ b/crates/gecko/src/mmio.rs
@@ -6,12 +6,22 @@ pub mod traits;
use crate::system::{SystemId, WII};
use constants::*;
+use rustc_hash::FxHashSet;
pub const FASTMEM_LUT_PAGES: usize = 1 << 15;
pub const FASTMEM_PAGE_BYTES: usize = 1 << 17;
pub const FASTMEM_PAGE_MASK: u32 = (FASTMEM_PAGE_BYTES as u32) - 1;
pub const FASTMEM_PAGE_SHIFT: u32 = 17;
+pub const CODE_LINE_BYTES: u32 = 32;
+pub const CODE_LINE_SHIFT: u32 = 5;
+pub const CODE_LINE_MASK: u32 = !(CODE_LINE_BYTES - 1);
+
+pub const PHYS_MASK: u32 = 0x3FFF_FFFF;
+
+pub const MEM1_LINES: usize = RAM_SIZE >> CODE_LINE_SHIFT as usize;
+pub const MEM2_LINES: usize = MEM2_SIZE >> CODE_LINE_SHIFT as usize;
+
pub struct Mmio<const SYSTEM: SystemId> {
pub ram: Vec<u8>,
pub efb: Vec<u8>,
@@ -22,6 +32,12 @@ pub struct Mmio<const SYSTEM: SystemId> {
pub ram_ptr: usize,
pub fastmem_lut: Vec<usize>,
pub fastmem_lut_ptr: usize,
+ pub code_refcount: Box<[u8]>,
+ pub code_refcount_ptr: usize,
+ #[cfg(feature = "jit")]
+ pub pending_icbi: FxHashSet<u32>,
+ #[cfg(feature = "jit")]
+ pub jit_dirty: u8,
}
/// Read-only view over MEM1 plus (on Wii) MEM2, addressed by physical
@@ -122,6 +138,14 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
let fastmem_lut_ptr = fastmem_lut.as_ptr() as usize;
+ let refcount_len = if SYSTEM == WII {
+ MEM1_LINES + MEM2_LINES
+ } else {
+ MEM1_LINES
+ };
+ let code_refcount: Box<[u8]> = vec![0u8; refcount_len].into_boxed_slice();
+ let code_refcount_ptr = code_refcount.as_ptr() as usize;
+
Mmio {
ram,
efb: vec![0; EFB_SIZE],
@@ -132,6 +156,12 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
ram_ptr,
fastmem_lut,
fastmem_lut_ptr,
+ code_refcount,
+ code_refcount_ptr,
+ #[cfg(feature = "jit")]
+ pending_icbi: FxHashSet::default(),
+ #[cfg(feature = "jit")]
+ jit_dirty: 0,
}
}
@@ -284,6 +314,79 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
&mut slice[offset..offset + len]
}
+ #[inline(always)]
+ fn code_refcount_index(phys: u32) -> Option<usize> {
+ match phys {
+ RAM_BASE..=RAM_END => Some((phys >> CODE_LINE_SHIFT) as usize),
+ MEM2_BASE..=MEM2_END if SYSTEM == WII => {
+ Some(MEM1_LINES + ((phys - MEM2_BASE) >> CODE_LINE_SHIFT) as usize)
+ }
+ _ => None,
+ }
+ }
+
+ #[inline(always)]
+ fn for_each_code_line(start: u32, len: u32, mut f: impl FnMut(u32)) {
+ if len == 0 {
+ return;
+ }
+
+ let mut p = start & CODE_LINE_MASK;
+ let end = start.wrapping_add(len - 1) & CODE_LINE_MASK;
+ loop {
+ f(p);
+
+ if p == end {
+ break;
+ }
+
+ p = p.wrapping_add(CODE_LINE_BYTES);
+ }
+ }
+
+ pub fn mark_code(&mut self, start: u32, len: u32) {
+ Self::for_each_code_line(start, len, |line| {
+ if let Some(i) = Self::code_refcount_index(line) {
+ let v = &mut self.code_refcount[i];
+ *v = v.saturating_add(1);
+ }
+ });
+ }
+
+ pub fn unmark_code(&mut self, start: u32, len: u32) {
+ Self::for_each_code_line(start, len, |line| {
+ if let Some(i) = Self::code_refcount_index(line) {
+ let v = &mut self.code_refcount[i];
+ if *v != u8::MAX {
+ *v = v.saturating_sub(1);
+ }
+ }
+ });
+ }
+
+ #[inline(always)]
+ pub fn is_code_chunk(&self, phys: u32) -> bool {
+ match Self::code_refcount_index(phys) {
+ Some(i) => self.code_refcount[i] != 0,
+ None => false,
+ }
+ }
+
+ pub fn clear_code_refcount(&mut self) {
+ self.code_refcount.fill(0);
+ }
+
+ #[cfg(feature = "jit")]
+ #[inline(always)]
+ pub fn queue_icbi_for_range(&mut self, phys: u32, len: u32) {
+ Self::for_each_code_line(phys, len, |line| {
+ if self.is_code_chunk(line) {
+ self.pending_icbi.insert(line);
+ self.jit_dirty = 1;
+ }
+ });
+ }
+
/// Read-only view spanning MEM1 and (on Wii) MEM2. Used by the GP path
/// so texture / vertex / TLUT reads can resolve addresses in either bank.
#[inline(always)]
@@ -352,8 +455,11 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
}
}
- /// Process a locked cache DMA transfer triggered by writing to SPR DMAL (923).
- pub fn process_locked_cache_dma(&mut self, dmau: &crate::gekko::spr::DmaUpper, dmal: &crate::gekko::spr::DmaLower) {
+ pub fn process_locked_cache_dma(
+ &mut self,
+ dmau: &crate::gekko::spr::DmaUpper,
+ dmal: &crate::gekko::spr::DmaLower,
+ ) -> Option<(u32, u32)> {
let ram_addr = dmau.ram_addr() << 5;
let lcache_vaddr = dmal.lcache_addr() << 5;
let lcache_paddr = (lcache_vaddr - LCACHE_BASE) as usize;
@@ -372,9 +478,11 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
if dmal.load() {
let src = self.virt_slice(ram_addr, length).to_vec();
self.lcache[lcache_paddr..lcache_paddr + length].copy_from_slice(&src);
+ None
} else {
let src = self.lcache[lcache_paddr..lcache_paddr + length].to_vec();
self.virt_slice_mut(ram_addr, length).copy_from_slice(&src);
+ Some((virt_to_phys(ram_addr), length as u32))
}
}
diff --git a/crates/gecko/src/mmio/bus.rs b/crates/gecko/src/mmio/bus.rs
index 7f4c659..7637180 100644
--- a/crates/gecko/src/mmio/bus.rs
+++ b/crates/gecko/src/mmio/bus.rs
@@ -194,6 +194,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
bus_write_hooks!(self, addr, phys, 1, val, {
if phys <= RAM_END {
self.mmio.ram_write_u8(phys, val);
+ #[cfg(feature = "jit")]
+ self.mmio.queue_icbi_for_range(phys, 1);
} else {
self.write_u8_mmio(phys, addr, val);
}
@@ -206,6 +208,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
bus_write_hooks!(self, addr, phys, 2, val, {
if phys <= RAM_END - 1 {
self.mmio.ram_write_u16(phys, val);
+ #[cfg(feature = "jit")]
+ self.mmio.queue_icbi_for_range(phys, 2);
} else {
self.write_u16_mmio(phys, addr, val);
}
@@ -218,6 +222,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
bus_write_hooks!(self, addr, phys, 4, val, {
if phys <= RAM_END - 3 {
self.mmio.ram_write_u32(phys, val);
+ #[cfg(feature = "jit")]
+ self.mmio.queue_icbi_for_range(phys, 4);
} else {
self.write_u32_mmio(phys, addr, val);
}
diff --git a/crates/gecko/src/mmio/fp.rs b/crates/gecko/src/mmio/fp.rs
index dfbe63f..5059e75 100644
--- a/crates/gecko/src/mmio/fp.rs
+++ b/crates/gecko/src/mmio/fp.rs
@@ -22,6 +22,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
let bits = val.to_bits();
if phys <= RAM_END - 7 {
self.mmio.ram_write_u64(phys, bits);
+ #[cfg(feature = "jit")]
+ self.mmio.queue_icbi_for_range(phys, 8);
return;
}
diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rs
index 3c8ba36..d3d39b5 100644
--- a/crates/gecko/src/system.rs
+++ b/crates/gecko/src/system.rs
@@ -545,6 +545,14 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
}
jit.run_block(self);
+
+ if self.mmio.jit_dirty != 0 {
+ jit.drain_scratch.extend(self.mmio.pending_icbi.drain());
+ while let Some(line) = jit.drain_scratch.pop() {
+ jit.invalidate_line(&mut self.mmio, line);
+ }
+ self.mmio.jit_dirty = 0;
+ }
}
self.jit = Some(jit);