A GameCube/Wii emulator and debugger written in Rust.
optimize psq loads/stores, absolutely necessary for mario kart wii menu scenes with FMVs, they crippled performance to 50%. should be hitting around 100% now
Layle85fb63f883e8
diff --git a/crates/gecko/src/gekko/interpreter.rs b/crates/gecko/src/gekko/interpreter.rsindex 8c9816a..78bc481 100644--- a/crates/gecko/src/gekko/interpreter.rs+++ b/crates/gecko/src/gekko/interpreter.rs@@ -4,7 +4,7 @@ mod compare;mod cr_ops;mod fp_ops;mod ps_ops;-mod psq;+pub(crate) mod psq;mod rotate;mod store_load;mod system;diff --git a/crates/gecko/src/gekko/interpreter/psq.rs b/crates/gecko/src/gekko/interpreter/psq.rsindex cff172c..716fc7c 100644--- a/crates/gecko/src/gekko/interpreter/psq.rs+++ b/crates/gecko/src/gekko/interpreter/psq.rs@@ -34,13 +34,13 @@ const QUANT_TABLE: [f32; 64] = {table};-fn gqr_ld_type(gqr: u32) -> u8 {+pub(crate) fn gqr_ld_type(gqr: u32) -> u8 {((gqr >> 16) & 0x7) as u8}fn gqr_ld_scale(gqr: u32) -> u8 {((gqr >> 24) & 0x3f) as u8}-fn gqr_st_type(gqr: u32) -> u8 {+pub(crate) fn gqr_st_type(gqr: u32) -> u8 {(gqr & 0x7) as u8}fn gqr_st_scale(gqr: u32) -> u8 {diff --git a/crates/gecko/src/gekko/interpreter/system.rs b/crates/gecko/src/gekko/interpreter/system.rsindex 12a876b..e39ea31 100644--- a/crates/gecko/src/gekko/interpreter/system.rs+++ b/crates/gecko/src/gekko/interpreter/system.rs@@ -1,5 +1,6 @@use crate::gekko::instruction::Instruction;use crate::gekko::lut::*;+use crate::gekko::spr::Spr;use crate::gekko::sr::Sr;use crate::system::{System, SystemId};@@ -30,40 +31,7 @@ pub fn spr<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instOP_MTSPR => {let spr_num = instr.spr_swapped();let val = ctx.gekko.read_gpr(instr.rs());- match spr_num {- 22 => {- ctx.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);- ctx.gekko.dec.write(ctx.scheduler.cycles, val);- ctx.gekko.spr.dec = val;- ctx.scheduler.schedule_in(- crate::gekko::dec::cycles_until_underflow(val),- crate::scheduler::Handler::DecUnderflow,- );- tracing::debug!(cycles = ctx.scheduler.cycles, value = val, "decrementer set");- }- 284 => ctx.scheduler.set_timebase_lower(val),- 285 => ctx.scheduler.set_timebase_upper(val),- 921 => {- ctx.gekko.spr.wpar = val & !1;- ctx.cp.gather_pos = 0;- }- 923 => {- ctx.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);- if ctx.gekko.spr.dmal.trigger() {- let dmau = ctx.gekko.spr.dmau;- let dmal = ctx.gekko.spr.dmal;- let written = ctx.mmio.process_locked_cache_dma(&dmau, &dmal);- #[cfg(feature = "jit")]- if let Some((phys, len)) = written {- ctx.mmio.queue_icbi_for_range(phys, len);- }- #[cfg(not(feature = "jit"))]- let _ = written;- ctx.gekko.spr.dmal.set_trigger(false);- }- }- _ => ctx.gekko.spr.write(spr_num, val),- }+ ctx.write_spr(spr_num, val);}OP_MFSPR => {let spr_num = instr.spr_swapped();@@ -187,3 +155,46 @@ pub fn sc<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, _instr: Instruction)ctx.scheduler.cycles += crate::gekko::cycles::cycles_for_op(OP_SC) as u64;ctx.cause_syscall_interrupt();}++impl<const SYSTEM: SystemId> System<SYSTEM> {+ pub(crate) fn write_spr(&mut self, num: u32, val: u32) {+ match num {+ 22 => {+ self.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);+ self.gekko.dec.write(self.scheduler.cycles, val);+ self.gekko.spr.dec = val;+ self.scheduler.schedule_in(+ crate::gekko::dec::cycles_until_underflow(val),+ crate::scheduler::Handler::DecUnderflow,+ );+ tracing::debug!(cycles = self.scheduler.cycles, value = val, "decrementer set");+ }+ 284 => self.scheduler.set_timebase_lower(val),+ 285 => self.scheduler.set_timebase_upper(val),+ _ if Spr::is_dbat(num) => {+ self.gekko.spr.write(num, val);+ self.refresh_lcache_fastmem();+ }+ 921 => {+ self.gekko.spr.wpar = val & !1;+ self.cp.gather_pos = 0;+ }+ 923 => {+ self.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);+ if self.gekko.spr.dmal.trigger() {+ let dmau = self.gekko.spr.dmau;+ let dmal = self.gekko.spr.dmal;+ let written = self.mmio.process_locked_cache_dma(&dmau, &dmal);+ #[cfg(feature = "jit")]+ if let Some((phys, len)) = written {+ self.mmio.queue_icbi_for_range(phys, len);+ }+ #[cfg(not(feature = "jit"))]+ let _ = written;+ self.gekko.spr.dmal.set_trigger(false);+ }+ }+ _ => self.gekko.spr.write(num, val),+ }+ }+}diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rsindex ea16942..8023295 100644--- a/crates/gecko/src/gekko/jit.rs+++ b/crates/gecko/src/gekko/jit.rs@@ -28,6 +28,8 @@ use rustc_hash::FxHashMap;use crate::system::{GC, System, SystemId, WII};+use self::translator::GqrAssumptions;+pub type BlockEntry = usize;type TrampolineFn = unsafe extern "C" fn(*mut core::ffi::c_void, usize) -> u32;@@ -697,7 +699,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.hle_code_sizes.insert(pc, DIV2I_SIZE);self.compile_div2i(pc)} else {- self.compile(&spec, &gprs_snapshot)+ let gqrs_snapshot = core::array::from_fn(|i| sys.gekko.spr.read_gqr(i as u8));+ self.compile(&spec, &gprs_snapshot, &gqrs_snapshot)};self.cache.insert(pc, entry);@@ -816,17 +819,20 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {return id;}- self.block_seq = self.block_seq.wrapping_add(1);- let name = format!("gecko_block_{:08x}_{}", pc, self.block_seq);- let id = self- .module- .declare_function(&name, Linkage::Local, &self.block_sig)- .expect("declare block");+ let id = self.declare_block_fn("gecko_block", pc);self.block_func_ids.insert(pc, id);id}+ fn declare_block_fn(&mut self, prefix: &str, pc: u32) -> FuncId {+ self.block_seq = self.block_seq.wrapping_add(1);+ let name = format!("{prefix}_{pc:08x}_{}", self.block_seq);+ self.module+ .declare_function(&name, Linkage::Local, &self.block_sig)+ .expect("declare block")+ }+#[cfg_attr(feature = "hotpath", hotpath::measure(label = "ppc_jit_run_block"))]pub fn run_block(&mut self, sys: &mut System<SYSTEM>) {let pc = sys.gekko.pc;@@ -1158,7 +1164,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {entry}- fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {+ fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32], gqrs: &[u32; 8]) -> BlockEntry {let func_id = self.func_id_for(spec.start_pc);let _ = self.target_slot_addr(spec.start_pc);@@ -1168,7 +1174,13 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {#[cfg(not(feature = "jit-stats"))]let entry_counter_addr: Option<usize> = None;- let entry = self.compile_function(spec, gprs, func_id, entry_counter_addr);+ let assumptions = GqrAssumptions::for_block(spec, gqrs);+ let specialization = assumptions.as_ref().map(|assumptions| {+ let fallback_id = self.declare_block_fn("gecko_generic", spec.start_pc);+ let fallback = self.compile_function(spec, gprs, fallback_id, entry_counter_addr, None);+ (assumptions, fallback)+ });+ let entry = self.compile_function(spec, gprs, func_id, entry_counter_addr, specialization);let slot_addr = self.target_slot_addr(spec.start_pc);unsafe {@@ -1186,6 +1198,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {gprs: &[u32; 32],func_id: FuncId,entry_counter_addr: Option<usize>,+ specialization: Option<(&GqrAssumptions, BlockEntry)>,) -> BlockEntry {self.ctx.clear();self.ctx.func.signature = self.block_sig.clone();@@ -1206,6 +1219,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {gprs,&chain,entry_counter_addr,+ specialization,);drop(chain);diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rsindex e879e5e..da27f92 100644--- a/crates/gecko/src/gekko/jit/abi.rs+++ b/crates/gecko/src/gekko/jit/abi.rs@@ -120,6 +120,10 @@ pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)}+pub const fn lcache_fastmem_ptr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, lcache_fastmem_ptr)+}+#[inline(always)]pub const fn code_refcount_ptr_offset<const SYSTEM: SystemId>() -> usize {offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, code_refcount_ptr)diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rsindex b4e0776..40e8dd2 100644--- a/crates/gecko/src/gekko/jit/runtime.rs+++ b/crates/gecko/src/gekko/jit/runtime.rs@@ -189,45 +189,11 @@ pub extern "C" fn read_spr_wii(sys: *mut core::ffi::c_void, num: u32) -> u32 {read_spr::<WII>(sys.cast(), num)}-#[inline(always)]-fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u32) {- let sys = unsafe { &mut *sys };- match num {- 22 => {- sys.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);- sys.gekko.dec.write(sys.scheduler.cycles, val);- sys.gekko.spr.dec = val;- sys.scheduler.schedule_in(- crate::gekko::dec::cycles_until_underflow(val),- crate::scheduler::Handler::DecUnderflow,- );- }- 284 => sys.scheduler.set_timebase_lower(val),- 285 => sys.scheduler.set_timebase_upper(val),- 921 => {- sys.gekko.spr.wpar = val & !1;- sys.cp.gather_pos = 0;- }- 923 => {- sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);- if sys.gekko.spr.dmal.trigger() {- let dmau = sys.gekko.spr.dmau;- let dmal = sys.gekko.spr.dmal;- if let Some((phys, len)) = sys.mmio.process_locked_cache_dma(&dmau, &dmal) {- sys.mmio.queue_icbi_for_range(phys, len);- }- sys.gekko.spr.dmal.set_trigger(false);- }- }- _ => sys.gekko.spr.write(num, val),- }-}-pub extern "C" fn write_spr_gc(sys: *mut core::ffi::c_void, num: u32, val: u32) {- write_spr::<GC>(sys.cast(), num, val);+ unsafe { (*sys.cast::<System<GC>>()).write_spr(num, val) };}pub extern "C" fn write_spr_wii(sys: *mut core::ffi::c_void, num: u32, val: u32) {- write_spr::<WII>(sys.cast(), num, val);+ unsafe { (*sys.cast::<System<WII>>()).write_spr(num, val) };}#[inline(always)]diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rsindex b634890..976301e 100644--- a/crates/gecko/src/gekko/jit/translator.rs+++ b/crates/gecko/src/gekko/jit/translator.rs@@ -10,9 +10,15 @@ use rustc_hash::FxHashMap;use crate::gekko::instruction::Instruction;use crate::gekko::jit::block::{BlockSpec, TermKind};use crate::gekko::jit::idle::{self, IdleClass};-use crate::gekko::jit::{ExternFuncs, abi};+use crate::gekko::jit::{BlockEntry, ExternFuncs, abi};+use crate::gekko::spr::Spr;use crate::system::SystemId;+mod gqr;+mod quantized;++pub(crate) use gqr::GqrAssumptions;+struct GqrCache {values: [Option<Value>; 8],}@@ -153,7 +159,7 @@ pub struct JitTranslator {pub op_cycles: i64,}-pub fn translate<const SYSTEM: SystemId>(+pub(crate) fn translate<const SYSTEM: SystemId>(ctx: &mut Context,builder_ctx: &mut FunctionBuilderContext,module: &mut JITModule,@@ -162,6 +168,7 @@ pub fn translate<const SYSTEM: SystemId>(gprs: &[u32; 32],chain: &ChainContext<'_>,entry_counter_addr: Option<usize>,+ specialization: Option<(&GqrAssumptions, BlockEntry)>,) {let mut local = BlockEmitContext {cause_invalid_opcode: module.declare_func_in_func(extern_funcs.cause_invalid_opcode, &mut ctx.func),@@ -225,6 +232,17 @@ pub fn translate<const SYSTEM: SystemId>(let ctx_ptr: Value = builder.block_params(entry)[0];let host_return_pc: Value = builder.block_params(entry)[1];+ if let Some((assumptions, fallback)) = specialization {+ assumptions.emit_guard::<SYSTEM>(+ &mut builder,+ ctx_ptr,+ host_return_pc,+ block_sig_ref,+ &mut local,+ fallback,+ );+ }+let is_self_loop = should_use_native_self_loop(spec, idle_class);if let Some(addr) = entry_counter_addr {@@ -1049,7 +1067,8 @@ pub(crate) fn emit_mtspr<const SYSTEM: SystemId>() -> bool {let spr_num = instr.spr_swapped() as u16;let val = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());- if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {+ let direct = abi::spr_field_offset::<SYSTEM>(spr_num).filter(|_| !Spr::is_dbat(spr_num as u32));+ if let Some(off) = direct.map(|off| off as i32) {builder.ins().store(vmctx_flags(), val, ctx_ptr, off);if matches!(spr_num, 912..=919) {@@ -2379,6 +2398,21 @@ impl MemSize {MemSize::U32 => types::I32,}}++ const fn bytes(self) -> u32 {+ match self {+ MemSize::U8 => 1,+ MemSize::U16 => 2,+ MemSize::U32 => 4,+ }+ }++ fn swap(self, builder: &mut FunctionBuilder, raw: Value) -> Value {+ match self {+ MemSize::U8 => raw,+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw),+ }+ }}pub(crate) fn emit_d_form_ea<const SYSTEM: SystemId>(@@ -2637,7 +2671,6 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, local, instr.i_22_24(), store);let fast_block = builder.create_block();let slow_block = builder.create_block();- builder.set_cold_block(slow_block);let merge = builder.create_block();builder.ins().brif(easy, fast_block, &[], slow_block, &[]);@@ -2940,12 +2973,7 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(MemSize::U8 => builder.ins().ireduce(types::I8, val32),};builder.ins().store(heap_flags(), truncated, addr, 0);- let size_bytes = match size {- MemSize::U8 => 1,- MemSize::U16 => 2,- MemSize::U32 => 4,- };- emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size.bytes(), cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -3500,7 +3528,6 @@ pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(let fast_block = builder.create_block();let slow_block = builder.create_block();- builder.set_cold_block(slow_block);let merge = builder.create_block();builder.ins().brif(easy, fast_block, &[], slow_block, &[]);@@ -3556,7 +3583,6 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(let fast_block = builder.create_block();let slow_block = builder.create_block();- builder.set_cold_block(slow_block);let merge = builder.create_block();builder.ins().brif(easy, fast_block, &[], slow_block, &[]);@@ -3607,14 +3633,13 @@ pub(crate) fn emit_gqr_type_zero_check<const SYSTEM: SystemId>(builder.ins().icmp_imm(IntCC::Equal, bits, 0)}-pub(crate) fn emit_psq_dequant_element(+pub(crate) fn emit_psq_dequant_element<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,ea: Value,type_v: Value,scale_f32: Value,- read_u8_fn: FuncRef,- read_u16_fn: FuncRef,+ local: &BlockEmitContext,) -> Value {let half_bit = builder.ins().band_imm(type_v, 1);let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);@@ -3630,8 +3655,7 @@ pub(crate) fn emit_psq_dequant_element(builder.switch_to_block(byte_block);builder.seal_block(byte_block);- let call = builder.ins().call(read_u8_fn, &[ctx_ptr, ea]);- let raw_byte = builder.inst_results(call)[0];+ let raw_byte = quantized::load::<SYSTEM>(builder, ctx_ptr, ea, MemSize::U8, local);let trunc = builder.ins().ireduce(types::I8, raw_byte);let sext = builder.ins().sextend(types::I32, trunc);let val_i32 = builder.ins().select(is_signed, sext, raw_byte);@@ -3641,8 +3665,7 @@ pub(crate) fn emit_psq_dequant_element(builder.switch_to_block(half_block);builder.seal_block(half_block);- let call = builder.ins().call(read_u16_fn, &[ctx_ptr, ea]);- let raw_half = builder.inst_results(call)[0];+ let raw_half = quantized::load::<SYSTEM>(builder, ctx_ptr, ea, MemSize::U16, local);let trunc = builder.ins().ireduce(types::I16, raw_half);let sext = builder.ins().sextend(types::I32, trunc);let val_i32 = builder.ins().select(is_signed, sext, raw_half);@@ -3656,15 +3679,14 @@ pub(crate) fn emit_psq_dequant_element(builder.ins().fpromote(types::F64, scaled_f32)}-pub(crate) fn emit_psq_quant_element(+pub(crate) fn emit_psq_quant_element<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,ea: Value,value_f64: Value,type_v: Value,scale_f32: Value,- write_u8_fn: FuncRef,- write_u16_fn: FuncRef,+ local: &BlockEmitContext,) {let val_f32 = builder.ins().fdemote(types::F32, value_f64);let scaled_f32 = builder.ins().fmul(val_f32, scale_f32);@@ -3674,24 +3696,21 @@ pub(crate) fn emit_psq_quant_element(let signed_bit = builder.ins().band_imm(type_v, 2);let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);- let zero_f = builder.ins().f32const(0.0_f32);- let u8_max = builder.ins().f32const(255.0_f32);- let u16_max = builder.ins().f32const(65535.0_f32);- let i8_min = builder.ins().f32const(-128.0_f32);- let i8_max = builder.ins().f32const(127.0_f32);- let i16_min = builder.ins().f32const(-32768.0_f32);- let i16_max = builder.ins().f32const(32767.0_f32);-+ let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, scaled_f32);+ let zero = builder.ins().iconst(types::I32, 0);+ let u8_max = builder.ins().iconst(types::I32, 255);+ let u16_max = builder.ins().iconst(types::I32, 65535);+ let i8_min = builder.ins().iconst(types::I32, -128);+ let i8_max = builder.ins().iconst(types::I32, 127);+ let i16_min = builder.ins().iconst(types::I32, -32768);+ let i16_max = builder.ins().iconst(types::I32, 32767);let signed_min = builder.ins().select(is_half, i16_min, i8_min);- let min_f32 = builder.ins().select(is_signed, signed_min, zero_f);+ let min = builder.ins().select(is_signed, signed_min, zero);let signed_max = builder.ins().select(is_half, i16_max, i8_max);let unsigned_max = builder.ins().select(is_half, u16_max, u8_max);- let max_f32 = builder.ins().select(is_signed, signed_max, unsigned_max);-- let lo_clamped = builder.ins().fmax(scaled_f32, min_f32);- let clamped = builder.ins().fmin(lo_clamped, max_f32);-- let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, clamped);+ let max = builder.ins().select(is_signed, signed_max, unsigned_max);+ let clamped = builder.ins().smax(val_i32, min);+ let val_i32 = builder.ins().smin(clamped, max);let mask_byte = builder.ins().iconst(types::I32, 0xFFi64);let mask_half = builder.ins().iconst(types::I32, 0xFFFFi64);@@ -3705,12 +3724,12 @@ pub(crate) fn emit_psq_quant_element(builder.switch_to_block(byte_block);builder.seal_block(byte_block);- builder.ins().call(write_u8_fn, &[ctx_ptr, ea, val_to_write]);+ quantized::store::<SYSTEM>(builder, ctx_ptr, ea, val_to_write, MemSize::U8, local);builder.ins().jump(merge, &[]);builder.switch_to_block(half_block);builder.seal_block(half_block);- builder.ins().call(write_u16_fn, &[ctx_ptr, ea, val_to_write]);+ quantized::store::<SYSTEM>(builder, ctx_ptr, ea, val_to_write, MemSize::U16, local);builder.ins().jump(merge, &[]);builder.switch_to_block(merge);@@ -3757,13 +3776,13 @@ pub(crate) fn emit_psq_load_quantized<const SYSTEM: SystemId>(let scale_addr = builder.ins().iadd(table_addr, scale_offset);let scale_f32 = builder.ins().load(types::F32, vmctx_flags(), scale_addr, 0);- let ps0 = emit_psq_dequant_element(builder, ctx_ptr, ea, type_v, scale_f32, local.read_u8, local.read_u16);+ let ps0 = emit_psq_dequant_element::<SYSTEM>(builder, ctx_ptr, ea, type_v, scale_f32, local);let ps1 = if w {builder.ins().f64const(1.0)} else {let elem_size = emit_psq_elem_size(builder, type_v);let ea1 = builder.ins().iadd(ea, elem_size);- emit_psq_dequant_element(builder, ctx_ptr, ea1, type_v, scale_f32, local.read_u8, local.read_u16)+ emit_psq_dequant_element::<SYSTEM>(builder, ctx_ptr, ea1, type_v, scale_f32, local)};fpr_store::<SYSTEM>(builder, ctx_ptr, fd, ps0);ps1_store::<SYSTEM>(builder, ctx_ptr, fd, ps1);@@ -3842,30 +3861,12 @@ pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(let scale_f32 = builder.ins().load(types::F32, vmctx_flags(), scale_addr, 0);let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, fs);- emit_psq_quant_element(- builder,- ctx_ptr,- ea,- ps0,- type_v,- scale_f32,- local.write_u8,- local.write_u16,- );+ emit_psq_quant_element::<SYSTEM>(builder, ctx_ptr, ea, ps0, type_v, scale_f32, local);if !w {let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, fs);let elem_size = emit_psq_elem_size(builder, type_v);let ea1 = builder.ins().iadd(ea, elem_size);- emit_psq_quant_element(- builder,- ctx_ptr,- ea1,- ps1,- type_v,- scale_f32,- local.write_u8,- local.write_u16,- );+ emit_psq_quant_element::<SYSTEM>(builder, ctx_ptr, ea1, ps1, type_v, scale_f32, local);}builder.ins().jump(merge, &[]);@@ -4722,28 +4723,36 @@ pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(slow: FuncRef,update: bool,) {+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ let val = emit_fastmem_load(builder, ctx_ptr, ea, host_ptr, host_addr, size, slow);+ gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);++ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);+ }+}++pub(crate) fn emit_fastmem_load(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ host_ptr: Value,+ host_addr: Value,+ size: MemSize,+ slow: FuncRef,+) -> Value {let fast_block = builder.create_block();let slow_block = builder.create_block();builder.set_cold_block(slow_block);let merge = builder.create_block();builder.append_block_param(merge, types::I32);-- let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);- let host_addr = builder.ins().iadd(host_ptr, page_off);- builder- .ins()- .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);- builder.append_block_param(fast_block, types::I64);+ builder.ins().brif(host_ptr, fast_block, &[], slow_block, &[]);builder.switch_to_block(fast_block);builder.seal_block(fast_block);- let addr = builder.block_params(fast_block)[0];- let flags = heap_flags();- let raw_val = builder.ins().load(size.ir_type(), flags, addr, 0);- let val_swapped = match size {- MemSize::U8 => raw_val,- MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw_val),- };+ let raw_val = builder.ins().load(size.ir_type(), heap_flags(), host_addr, 0);+ let val_swapped = size.swap(builder, raw_val);let val32 = match size {MemSize::U32 => val_swapped,MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, val_swapped),@@ -4758,12 +4767,7 @@ pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(builder.switch_to_block(merge);builder.seal_block(merge);- let val = builder.block_params(merge)[0];- gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);-- if update {- gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);- }+ builder.block_params(merge)[0]}pub(crate) fn emit_store<const SYSTEM: SystemId>(@@ -4824,39 +4828,51 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(update: bool,) {let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let host_addr = builder.ins().iadd(host_ptr, page_off);+ emit_fastmem_store::<SYSTEM>(+ builder,+ ctx_ptr,+ ea,+ host_ptr,+ host_addr,+ val32,+ size,+ slow,+ cause_smc_write,+ );+ if update {+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);+ }+}++pub(crate) fn emit_fastmem_store<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ host_ptr: Value,+ host_addr: Value,+ val32: Value,+ size: MemSize,+ slow: FuncRef,+ cause_smc_write: FuncRef,+) {let fast_block = builder.create_block();let slow_block = builder.create_block();builder.set_cold_block(slow_block);let merge = builder.create_block();-- let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);- let host_addr = builder.ins().iadd(host_ptr, page_off);- builder- .ins()- .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);- builder.append_block_param(fast_block, types::I64);+ builder.ins().brif(host_ptr, fast_block, &[], slow_block, &[]);builder.switch_to_block(fast_block);builder.seal_block(fast_block);- let addr = builder.block_params(fast_block)[0];- let flags = heap_flags();let truncated = match size {MemSize::U32 => val32,- MemSize::U16 => builder.ins().ireduce(types::I16, val32),- MemSize::U8 => builder.ins().ireduce(types::I8, val32),+ MemSize::U16 | MemSize::U8 => builder.ins().ireduce(size.ir_type(), val32),};- let val_to_store = match size {- MemSize::U8 => truncated,- MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),- };- builder.ins().store(flags, val_to_store, addr, 0);- let size_bytes = match size {- MemSize::U8 => 1,- MemSize::U16 => 2,- MemSize::U32 => 4,- };- emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);+ let val_to_store = size.swap(builder, truncated);+ builder.ins().store(heap_flags(), val_to_store, host_addr, 0);+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size.bytes(), cause_smc_write);builder.ins().jump(merge, &[]);builder.switch_to_block(slow_block);@@ -4866,10 +4882,6 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(builder.switch_to_block(merge);builder.seal_block(merge);-- if update {- gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);- }}pub(crate) fn emit_addi<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {diff --git a/crates/gecko/src/gekko/jit/translator/gqr.rs b/crates/gecko/src/gekko/jit/translator/gqr.rsnew file mode 100644index 0000000..8f068cd--- /dev/null+++ b/crates/gecko/src/gekko/jit/translator/gqr.rs@@ -0,0 +1,88 @@+use cranelift_codegen::ir::condcodes::IntCC;+use cranelift_codegen::ir::{InstBuilder, SigRef, Value, types};+use cranelift_frontend::FunctionBuilder;++use crate::gekko::instruction::Instruction;+use crate::gekko::interpreter::psq::{gqr_ld_type, gqr_st_type};+use crate::gekko::jit::BlockEntry;+use crate::gekko::jit::block::BlockSpec;+use crate::system::SystemId;++use super::{BlockEmitContext, gqr_cache_store, gqr_load};++const MIN_QUANTIZED_ACCESSES: usize = 4;++pub(crate) struct GqrAssumptions {+ expected: [Option<u32>; 8],+}++impl GqrAssumptions {+ pub(crate) fn for_block(spec: &BlockSpec, gqrs: &[u32; 8]) -> Option<Self> {+ if cfg!(feature = "hooks") {+ return None;+ }++ let mut values = [None; 8];+ let mut count = 0;+ for &raw in &spec.instrs {+ let instr = Instruction(raw);+ let primary = instr.primary_opcode();+ if primary == 31 && instr.xo10() == 467 && matches!(instr.spr_swapped(), 912..=919) {+ return None;+ }++ let (index, store) = match primary {+ 56 | 57 => (instr.psq_i(), false),+ 60 | 61 => (instr.psq_i(), true),+ 4 => match instr.xo6() {+ 6 | 38 => (instr.psq_ix(), false),+ 7 | 39 => (instr.psq_ix(), true),+ _ => continue,+ },+ _ => continue,+ };+ let gqr = gqrs[index as usize];+ let qtype = if store { gqr_st_type(gqr) } else { gqr_ld_type(gqr) };+ if qtype >= 4 {+ values[index as usize] = Some(gqr);+ count += 1;+ }+ }++ (count >= MIN_QUANTIZED_ACCESSES).then_some(Self { expected: values })+ }++ pub(super) fn emit_guard<const SYSTEM: SystemId>(+ &self,+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ host_return_pc: Value,+ block_sig_ref: SigRef,+ local: &mut BlockEmitContext,+ fallback_entry: BlockEntry,+ ) {+ let mut matches = builder.ins().iconst(types::I8, 1);+ for (index, expected) in self.expected.iter().enumerate() {+ let Some(expected) = *expected else { continue };+ let actual = gqr_load::<SYSTEM>(builder, ctx_ptr, local, index as u8);+ let same = builder.ins().icmp_imm(IntCC::Equal, actual, expected as i64);+ matches = builder.ins().band(matches, same);+ let constant = builder.ins().iconst(types::I32, expected as i64);+ gqr_cache_store(local, index as u8, constant);+ }++ let specialized = builder.create_block();+ let fallback = builder.create_block();+ builder.ins().brif(matches, specialized, &[], fallback, &[]);++ builder.switch_to_block(fallback);+ builder.seal_block(fallback);+ let target = builder.ins().iconst(types::I64, fallback_entry as i64);+ builder+ .ins()+ .return_call_indirect(block_sig_ref, target, &[ctx_ptr, host_return_pc]);++ builder.switch_to_block(specialized);+ builder.seal_block(specialized);+ }+}diff --git a/crates/gecko/src/gekko/jit/translator/quantized.rs b/crates/gecko/src/gekko/jit/translator/quantized.rsnew file mode 100644index 0000000..6d93543--- /dev/null+++ b/crates/gecko/src/gekko/jit/translator/quantized.rs@@ -0,0 +1,92 @@+use cranelift_codegen::ir::condcodes::IntCC;+use cranelift_codegen::ir::{InstBuilder, Value, types};+use cranelift_frontend::FunctionBuilder;++use crate::gekko::jit::abi;+use crate::mmio::FASTMEM_PAGE_BYTES;+use crate::mmio::constants::{LCACHE_BASE, LCACHE_SIZE};+use crate::system::SystemId;++use super::{BlockEmitContext, MemSize, emit_fastmem_load, emit_fastmem_lookup, emit_fastmem_store, vmctx_flags};++fn pointer<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, ea: Value, size: MemSize) -> Value {+ let bytes = size.bytes() as usize;++ let (base, offset) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);+ let top = builder.ins().ushr_imm(ea, 28);+ let cached = builder.ins().icmp_imm(IntCC::Equal, top, 8);+ let uncached = builder.ins().icmp_imm(IntCC::Equal, top, 12);+ let alias = builder.ins().bor(cached, uncached);+ let mapped = builder.ins().icmp_imm(IntCC::NotEqual, base, 0);+ let in_page = builder.ins().icmp_imm(+ IntCC::UnsignedLessThanOrEqual,+ offset,+ (FASTMEM_PAGE_BYTES - bytes) as i64,+ );+ let valid = builder.ins().band(mapped, alias);+ let valid = builder.ins().band(valid, in_page);+ let addr = builder.ins().iadd(base, offset);+ let zero = builder.ins().iconst(types::I64, 0);+ let ram = builder.ins().select(valid, addr, zero);++ let lc_offset = builder.ins().iadd_imm(ea, -(LCACHE_BASE as i64));+ let in_cache = builder+ .ins()+ .icmp_imm(IntCC::UnsignedLessThanOrEqual, lc_offset, (LCACHE_SIZE - bytes) as i64);+ let lc_base = builder.ins().load(+ types::I64,+ vmctx_flags(),+ ctx_ptr,+ abi::lcache_fastmem_ptr_offset::<SYSTEM>() as i32,+ );+ let lc_mapped = builder.ins().icmp_imm(IntCC::NotEqual, lc_base, 0);+ let lc_valid = builder.ins().band(in_cache, lc_mapped);+ let lc_offset = builder.ins().uextend(types::I64, lc_offset);+ let lc_addr = builder.ins().iadd(lc_base, lc_offset);+ builder.ins().select(lc_valid, lc_addr, ram)+}++pub(super) fn load<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ size: MemSize,+ local: &BlockEmitContext,+) -> Value {+ let slow = match size {+ MemSize::U8 => local.read_u8,+ MemSize::U16 => local.read_u16,+ MemSize::U32 => local.read_u32,+ };++ if cfg!(feature = "hooks") {+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);+ return builder.inst_results(call)[0];+ }++ let ptr = self::pointer::<SYSTEM>(builder, ctx_ptr, ea, size);+ emit_fastmem_load(builder, ctx_ptr, ea, ptr, ptr, size, slow)+}++pub(super) fn store<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ ea: Value,+ value: Value,+ size: MemSize,+ local: &BlockEmitContext,+) {+ let slow = match size {+ MemSize::U8 => local.write_u8,+ MemSize::U16 => local.write_u16,+ MemSize::U32 => local.write_u32,+ };++ if cfg!(feature = "hooks") {+ builder.ins().call(slow, &[ctx_ptr, ea, value]);+ return;+ }++ let ptr = self::pointer::<SYSTEM>(builder, ctx_ptr, ea, size);+ emit_fastmem_store::<SYSTEM>(builder, ctx_ptr, ea, ptr, ptr, value, size, slow, local.cause_smc_write);+}diff --git a/crates/gecko/src/gekko/spr.rs b/crates/gecko/src/gekko/spr.rsindex 6ce678d..7eb4b5e 100644--- a/crates/gecko/src/gekko/spr.rs+++ b/crates/gecko/src/gekko/spr.rs@@ -137,6 +137,10 @@ pub struct Spr {}impl Spr {+ pub const fn is_dbat(spr_num: u32) -> bool {+ matches!(spr_num, 536..=543 | 568..=575)+ }+pub fn read_gqr(&self, index: u8) -> u32 {match index {0 => self.gqr0,diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rsindex 7bd7607..c0e422d 100644--- a/crates/gecko/src/mmio.rs+++ b/crates/gecko/src/mmio.rs@@ -37,6 +37,7 @@ pub struct Mmio<const SYSTEM: SystemId> {pub ram_ptr: usize,pub fastmem_lut: Vec<usize>,pub fastmem_lut_ptr: usize,+ pub(crate) lcache_fastmem_ptr: usize,pub code_refcount: Box<[u8]>,pub code_refcount_ptr: usize,memory_write_generation: u64,@@ -193,6 +194,7 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {ram_ptr,fastmem_lut,fastmem_lut_ptr,+ lcache_fastmem_ptr: 0,code_refcount,code_refcount_ptr,memory_write_generation: 0,diff --git a/crates/gecko/src/mmio/bus.rs b/crates/gecko/src/mmio/bus.rsindex 727e12a..adf67be 100644--- a/crates/gecko/src/mmio/bus.rs+++ b/crates/gecko/src/mmio/bus.rs@@ -80,6 +80,15 @@ macro_rules! bus_write_hooks {}impl<const SYSTEM: SystemId> System<SYSTEM> {+ pub(crate) fn refresh_lcache_fastmem(&mut self) {+ let base = crate::mmio::constants::LCACHE_BASE;+ self.mmio.lcache_fastmem_ptr = if self.bat_translate(base) == base {+ self.mmio.lcache.as_ptr() as usize+ } else {+ 0+ };+ }+#[cold]pub(crate) fn sync_deferred_efb_writeback(&mut self, phys: u32, len: usize) {if !self.mmio.efb_writeback_needed(phys, len) {diff --git a/crates/gecko/src/savestate.rs b/crates/gecko/src/savestate.rsindex 4d67053..8548c80 100644--- a/crates/gecko/src/savestate.rs+++ b/crates/gecko/src/savestate.rs@@ -293,6 +293,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.gekko = r.pod()?;self.mmio.load_state(&mut r)?;+ self.refresh_lcache_fastmem();self.vi = r.pod()?;self.pe = r.pod()?;