Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

optimize psq loads/stores, absolutely necessary for mario kart wii menu scenes with FMVs, they crippled performance to 50%. should be hitting around 100% now

Layle85fb63f883e8
diff --git a/crates/gecko/src/gekko/interpreter.rs b/crates/gecko/src/gekko/interpreter.rs
index 8c9816a..78bc481 100644
--- a/crates/gecko/src/gekko/interpreter.rs
+++ b/crates/gecko/src/gekko/interpreter.rs
@@ -4,7 +4,7 @@ mod compare;
mod cr_ops;
mod fp_ops;
mod ps_ops;
-mod psq;
+pub(crate) mod psq;
mod rotate;
mod store_load;
mod system;
diff --git a/crates/gecko/src/gekko/interpreter/psq.rs b/crates/gecko/src/gekko/interpreter/psq.rs
index cff172c..716fc7c 100644
--- a/crates/gecko/src/gekko/interpreter/psq.rs
+++ b/crates/gecko/src/gekko/interpreter/psq.rs
@@ -34,13 +34,13 @@ const QUANT_TABLE: [f32; 64] = {
table
};
-fn gqr_ld_type(gqr: u32) -> u8 {
+pub(crate) fn gqr_ld_type(gqr: u32) -> u8 {
((gqr >> 16) & 0x7) as u8
}
fn gqr_ld_scale(gqr: u32) -> u8 {
((gqr >> 24) & 0x3f) as u8
}
-fn gqr_st_type(gqr: u32) -> u8 {
+pub(crate) fn gqr_st_type(gqr: u32) -> u8 {
(gqr & 0x7) as u8
}
fn gqr_st_scale(gqr: u32) -> u8 {
diff --git a/crates/gecko/src/gekko/interpreter/system.rs b/crates/gecko/src/gekko/interpreter/system.rs
index 12a876b..e39ea31 100644
--- a/crates/gecko/src/gekko/interpreter/system.rs
+++ b/crates/gecko/src/gekko/interpreter/system.rs
@@ -1,5 +1,6 @@
use crate::gekko::instruction::Instruction;
use crate::gekko::lut::*;
+use crate::gekko::spr::Spr;
use crate::gekko::sr::Sr;
use crate::system::{System, SystemId};
@@ -30,40 +31,7 @@ pub fn spr<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, inst
OP_MTSPR => {
let spr_num = instr.spr_swapped();
let val = ctx.gekko.read_gpr(instr.rs());
- match spr_num {
- 22 => {
- ctx.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);
- ctx.gekko.dec.write(ctx.scheduler.cycles, val);
- ctx.gekko.spr.dec = val;
- ctx.scheduler.schedule_in(
- crate::gekko::dec::cycles_until_underflow(val),
- crate::scheduler::Handler::DecUnderflow,
- );
- tracing::debug!(cycles = ctx.scheduler.cycles, value = val, "decrementer set");
- }
- 284 => ctx.scheduler.set_timebase_lower(val),
- 285 => ctx.scheduler.set_timebase_upper(val),
- 921 => {
- ctx.gekko.spr.wpar = val & !1;
- ctx.cp.gather_pos = 0;
- }
- 923 => {
- ctx.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
- if ctx.gekko.spr.dmal.trigger() {
- let dmau = ctx.gekko.spr.dmau;
- let dmal = ctx.gekko.spr.dmal;
- let written = ctx.mmio.process_locked_cache_dma(&dmau, &dmal);
- #[cfg(feature = "jit")]
- if let Some((phys, len)) = written {
- ctx.mmio.queue_icbi_for_range(phys, len);
- }
- #[cfg(not(feature = "jit"))]
- let _ = written;
- ctx.gekko.spr.dmal.set_trigger(false);
- }
- }
- _ => ctx.gekko.spr.write(spr_num, val),
- }
+ ctx.write_spr(spr_num, val);
}
OP_MFSPR => {
let spr_num = instr.spr_swapped();
@@ -187,3 +155,46 @@ pub fn sc<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, _instr: Instruction)
ctx.scheduler.cycles += crate::gekko::cycles::cycles_for_op(OP_SC) as u64;
ctx.cause_syscall_interrupt();
}
+
+impl<const SYSTEM: SystemId> System<SYSTEM> {
+ pub(crate) fn write_spr(&mut self, num: u32, val: u32) {
+ match num {
+ 22 => {
+ self.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);
+ self.gekko.dec.write(self.scheduler.cycles, val);
+ self.gekko.spr.dec = val;
+ self.scheduler.schedule_in(
+ crate::gekko::dec::cycles_until_underflow(val),
+ crate::scheduler::Handler::DecUnderflow,
+ );
+ tracing::debug!(cycles = self.scheduler.cycles, value = val, "decrementer set");
+ }
+ 284 => self.scheduler.set_timebase_lower(val),
+ 285 => self.scheduler.set_timebase_upper(val),
+ _ if Spr::is_dbat(num) => {
+ self.gekko.spr.write(num, val);
+ self.refresh_lcache_fastmem();
+ }
+ 921 => {
+ self.gekko.spr.wpar = val & !1;
+ self.cp.gather_pos = 0;
+ }
+ 923 => {
+ self.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
+ if self.gekko.spr.dmal.trigger() {
+ let dmau = self.gekko.spr.dmau;
+ let dmal = self.gekko.spr.dmal;
+ let written = self.mmio.process_locked_cache_dma(&dmau, &dmal);
+ #[cfg(feature = "jit")]
+ if let Some((phys, len)) = written {
+ self.mmio.queue_icbi_for_range(phys, len);
+ }
+ #[cfg(not(feature = "jit"))]
+ let _ = written;
+ self.gekko.spr.dmal.set_trigger(false);
+ }
+ }
+ _ => self.gekko.spr.write(num, val),
+ }
+ }
+}
diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rs
index ea16942..8023295 100644
--- a/crates/gecko/src/gekko/jit.rs
+++ b/crates/gecko/src/gekko/jit.rs
@@ -28,6 +28,8 @@ use rustc_hash::FxHashMap;
use crate::system::{GC, System, SystemId, WII};
+use self::translator::GqrAssumptions;
+
pub type BlockEntry = usize;
type TrampolineFn = unsafe extern "C" fn(*mut core::ffi::c_void, usize) -> u32;
@@ -697,7 +699,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
self.hle_code_sizes.insert(pc, DIV2I_SIZE);
self.compile_div2i(pc)
} else {
- self.compile(&spec, &gprs_snapshot)
+ let gqrs_snapshot = core::array::from_fn(|i| sys.gekko.spr.read_gqr(i as u8));
+ self.compile(&spec, &gprs_snapshot, &gqrs_snapshot)
};
self.cache.insert(pc, entry);
@@ -816,17 +819,20 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
return id;
}
- self.block_seq = self.block_seq.wrapping_add(1);
- let name = format!("gecko_block_{:08x}_{}", pc, self.block_seq);
- let id = self
- .module
- .declare_function(&name, Linkage::Local, &self.block_sig)
- .expect("declare block");
+ let id = self.declare_block_fn("gecko_block", pc);
self.block_func_ids.insert(pc, id);
id
}
+ fn declare_block_fn(&mut self, prefix: &str, pc: u32) -> FuncId {
+ self.block_seq = self.block_seq.wrapping_add(1);
+ let name = format!("{prefix}_{pc:08x}_{}", self.block_seq);
+ self.module
+ .declare_function(&name, Linkage::Local, &self.block_sig)
+ .expect("declare block")
+ }
+
#[cfg_attr(feature = "hotpath", hotpath::measure(label = "ppc_jit_run_block"))]
pub fn run_block(&mut self, sys: &mut System<SYSTEM>) {
let pc = sys.gekko.pc;
@@ -1158,7 +1164,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
entry
}
- fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {
+ fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32], gqrs: &[u32; 8]) -> BlockEntry {
let func_id = self.func_id_for(spec.start_pc);
let _ = self.target_slot_addr(spec.start_pc);
@@ -1168,7 +1174,13 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
#[cfg(not(feature = "jit-stats"))]
let entry_counter_addr: Option<usize> = None;
- let entry = self.compile_function(spec, gprs, func_id, entry_counter_addr);
+ let assumptions = GqrAssumptions::for_block(spec, gqrs);
+ let specialization = assumptions.as_ref().map(|assumptions| {
+ let fallback_id = self.declare_block_fn("gecko_generic", spec.start_pc);
+ let fallback = self.compile_function(spec, gprs, fallback_id, entry_counter_addr, None);
+ (assumptions, fallback)
+ });
+ let entry = self.compile_function(spec, gprs, func_id, entry_counter_addr, specialization);
let slot_addr = self.target_slot_addr(spec.start_pc);
unsafe {
@@ -1186,6 +1198,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
gprs: &[u32; 32],
func_id: FuncId,
entry_counter_addr: Option<usize>,
+ specialization: Option<(&GqrAssumptions, BlockEntry)>,
) -> BlockEntry {
self.ctx.clear();
self.ctx.func.signature = self.block_sig.clone();
@@ -1206,6 +1219,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {
gprs,
&chain,
entry_counter_addr,
+ specialization,
);
drop(chain);
diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rs
index e879e5e..da27f92 100644
--- a/crates/gecko/src/gekko/jit/abi.rs
+++ b/crates/gecko/src/gekko/jit/abi.rs
@@ -120,6 +120,10 @@ pub const fn fastmem_lut_ptr_offset<const SYSTEM: SystemId>() -> usize {
offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, fastmem_lut_ptr)
}
+pub const fn lcache_fastmem_ptr_offset<const SYSTEM: SystemId>() -> usize {
+ offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, lcache_fastmem_ptr)
+}
+
#[inline(always)]
pub const fn code_refcount_ptr_offset<const SYSTEM: SystemId>() -> usize {
offset_of!(System<SYSTEM>, mmio) + offset_of!(Mmio<SYSTEM>, code_refcount_ptr)
diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rs
index b4e0776..40e8dd2 100644
--- a/crates/gecko/src/gekko/jit/runtime.rs
+++ b/crates/gecko/src/gekko/jit/runtime.rs
@@ -189,45 +189,11 @@ pub extern "C" fn read_spr_wii(sys: *mut core::ffi::c_void, num: u32) -> u32 {
read_spr::<WII>(sys.cast(), num)
}
-#[inline(always)]
-fn write_spr<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, num: u32, val: u32) {
- let sys = unsafe { &mut *sys };
- match num {
- 22 => {
- sys.scheduler.cancel(crate::scheduler::Handler::DecUnderflow);
- sys.gekko.dec.write(sys.scheduler.cycles, val);
- sys.gekko.spr.dec = val;
- sys.scheduler.schedule_in(
- crate::gekko::dec::cycles_until_underflow(val),
- crate::scheduler::Handler::DecUnderflow,
- );
- }
- 284 => sys.scheduler.set_timebase_lower(val),
- 285 => sys.scheduler.set_timebase_upper(val),
- 921 => {
- sys.gekko.spr.wpar = val & !1;
- sys.cp.gather_pos = 0;
- }
- 923 => {
- sys.gekko.spr.dmal = crate::gekko::spr::DmaLower::from_raw(val);
- if sys.gekko.spr.dmal.trigger() {
- let dmau = sys.gekko.spr.dmau;
- let dmal = sys.gekko.spr.dmal;
- if let Some((phys, len)) = sys.mmio.process_locked_cache_dma(&dmau, &dmal) {
- sys.mmio.queue_icbi_for_range(phys, len);
- }
- sys.gekko.spr.dmal.set_trigger(false);
- }
- }
- _ => sys.gekko.spr.write(num, val),
- }
-}
-
pub extern "C" fn write_spr_gc(sys: *mut core::ffi::c_void, num: u32, val: u32) {
- write_spr::<GC>(sys.cast(), num, val);
+ unsafe { (*sys.cast::<System<GC>>()).write_spr(num, val) };
}
pub extern "C" fn write_spr_wii(sys: *mut core::ffi::c_void, num: u32, val: u32) {
- write_spr::<WII>(sys.cast(), num, val);
+ unsafe { (*sys.cast::<System<WII>>()).write_spr(num, val) };
}
#[inline(always)]
diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rs
index b634890..976301e 100644
--- a/crates/gecko/src/gekko/jit/translator.rs
+++ b/crates/gecko/src/gekko/jit/translator.rs
@@ -10,9 +10,15 @@ use rustc_hash::FxHashMap;
use crate::gekko::instruction::Instruction;
use crate::gekko::jit::block::{BlockSpec, TermKind};
use crate::gekko::jit::idle::{self, IdleClass};
-use crate::gekko::jit::{ExternFuncs, abi};
+use crate::gekko::jit::{BlockEntry, ExternFuncs, abi};
+use crate::gekko::spr::Spr;
use crate::system::SystemId;
+mod gqr;
+mod quantized;
+
+pub(crate) use gqr::GqrAssumptions;
+
struct GqrCache {
values: [Option<Value>; 8],
}
@@ -153,7 +159,7 @@ pub struct JitTranslator {
pub op_cycles: i64,
}
-pub fn translate<const SYSTEM: SystemId>(
+pub(crate) fn translate<const SYSTEM: SystemId>(
ctx: &mut Context,
builder_ctx: &mut FunctionBuilderContext,
module: &mut JITModule,
@@ -162,6 +168,7 @@ pub fn translate<const SYSTEM: SystemId>(
gprs: &[u32; 32],
chain: &ChainContext<'_>,
entry_counter_addr: Option<usize>,
+ specialization: Option<(&GqrAssumptions, BlockEntry)>,
) {
let mut local = BlockEmitContext {
cause_invalid_opcode: module.declare_func_in_func(extern_funcs.cause_invalid_opcode, &mut ctx.func),
@@ -225,6 +232,17 @@ pub fn translate<const SYSTEM: SystemId>(
let ctx_ptr: Value = builder.block_params(entry)[0];
let host_return_pc: Value = builder.block_params(entry)[1];
+ if let Some((assumptions, fallback)) = specialization {
+ assumptions.emit_guard::<SYSTEM>(
+ &mut builder,
+ ctx_ptr,
+ host_return_pc,
+ block_sig_ref,
+ &mut local,
+ fallback,
+ );
+ }
+
let is_self_loop = should_use_native_self_loop(spec, idle_class);
if let Some(addr) = entry_counter_addr {
@@ -1049,7 +1067,8 @@ pub(crate) fn emit_mtspr<const SYSTEM: SystemId>(
) -> bool {
let spr_num = instr.spr_swapped() as u16;
let val = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
- if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {
+ let direct = abi::spr_field_offset::<SYSTEM>(spr_num).filter(|_| !Spr::is_dbat(spr_num as u32));
+ if let Some(off) = direct.map(|off| off as i32) {
builder.ins().store(vmctx_flags(), val, ctx_ptr, off);
if matches!(spr_num, 912..=919) {
@@ -2379,6 +2398,21 @@ impl MemSize {
MemSize::U32 => types::I32,
}
}
+
+ const fn bytes(self) -> u32 {
+ match self {
+ MemSize::U8 => 1,
+ MemSize::U16 => 2,
+ MemSize::U32 => 4,
+ }
+ }
+
+ fn swap(self, builder: &mut FunctionBuilder, raw: Value) -> Value {
+ match self {
+ MemSize::U8 => raw,
+ MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw),
+ }
+ }
}
pub(crate) fn emit_d_form_ea<const SYSTEM: SystemId>(
@@ -2637,7 +2671,6 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(
let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, local, instr.i_22_24(), store);
let fast_block = builder.create_block();
let slow_block = builder.create_block();
- builder.set_cold_block(slow_block);
let merge = builder.create_block();
builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
@@ -2940,12 +2973,7 @@ pub(crate) fn emit_store_xform_brx<const SYSTEM: SystemId>(
MemSize::U8 => builder.ins().ireduce(types::I8, val32),
};
builder.ins().store(heap_flags(), truncated, addr, 0);
- let size_bytes = match size {
- MemSize::U8 => 1,
- MemSize::U16 => 2,
- MemSize::U32 => 4,
- };
- emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size.bytes(), cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -3500,7 +3528,6 @@ pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(
let fast_block = builder.create_block();
let slow_block = builder.create_block();
- builder.set_cold_block(slow_block);
let merge = builder.create_block();
builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
@@ -3556,7 +3583,6 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(
let fast_block = builder.create_block();
let slow_block = builder.create_block();
- builder.set_cold_block(slow_block);
let merge = builder.create_block();
builder.ins().brif(easy, fast_block, &[], slow_block, &[]);
@@ -3607,14 +3633,13 @@ pub(crate) fn emit_gqr_type_zero_check<const SYSTEM: SystemId>(
builder.ins().icmp_imm(IntCC::Equal, bits, 0)
}
-pub(crate) fn emit_psq_dequant_element(
+pub(crate) fn emit_psq_dequant_element<const SYSTEM: SystemId>(
builder: &mut FunctionBuilder,
ctx_ptr: Value,
ea: Value,
type_v: Value,
scale_f32: Value,
- read_u8_fn: FuncRef,
- read_u16_fn: FuncRef,
+ local: &BlockEmitContext,
) -> Value {
let half_bit = builder.ins().band_imm(type_v, 1);
let is_half = builder.ins().icmp_imm(IntCC::NotEqual, half_bit, 0);
@@ -3630,8 +3655,7 @@ pub(crate) fn emit_psq_dequant_element(
builder.switch_to_block(byte_block);
builder.seal_block(byte_block);
- let call = builder.ins().call(read_u8_fn, &[ctx_ptr, ea]);
- let raw_byte = builder.inst_results(call)[0];
+ let raw_byte = quantized::load::<SYSTEM>(builder, ctx_ptr, ea, MemSize::U8, local);
let trunc = builder.ins().ireduce(types::I8, raw_byte);
let sext = builder.ins().sextend(types::I32, trunc);
let val_i32 = builder.ins().select(is_signed, sext, raw_byte);
@@ -3641,8 +3665,7 @@ pub(crate) fn emit_psq_dequant_element(
builder.switch_to_block(half_block);
builder.seal_block(half_block);
- let call = builder.ins().call(read_u16_fn, &[ctx_ptr, ea]);
- let raw_half = builder.inst_results(call)[0];
+ let raw_half = quantized::load::<SYSTEM>(builder, ctx_ptr, ea, MemSize::U16, local);
let trunc = builder.ins().ireduce(types::I16, raw_half);
let sext = builder.ins().sextend(types::I32, trunc);
let val_i32 = builder.ins().select(is_signed, sext, raw_half);
@@ -3656,15 +3679,14 @@ pub(crate) fn emit_psq_dequant_element(
builder.ins().fpromote(types::F64, scaled_f32)
}
-pub(crate) fn emit_psq_quant_element(
+pub(crate) fn emit_psq_quant_element<const SYSTEM: SystemId>(
builder: &mut FunctionBuilder,
ctx_ptr: Value,
ea: Value,
value_f64: Value,
type_v: Value,
scale_f32: Value,
- write_u8_fn: FuncRef,
- write_u16_fn: FuncRef,
+ local: &BlockEmitContext,
) {
let val_f32 = builder.ins().fdemote(types::F32, value_f64);
let scaled_f32 = builder.ins().fmul(val_f32, scale_f32);
@@ -3674,24 +3696,21 @@ pub(crate) fn emit_psq_quant_element(
let signed_bit = builder.ins().band_imm(type_v, 2);
let is_signed = builder.ins().icmp_imm(IntCC::NotEqual, signed_bit, 0);
- let zero_f = builder.ins().f32const(0.0_f32);
- let u8_max = builder.ins().f32const(255.0_f32);
- let u16_max = builder.ins().f32const(65535.0_f32);
- let i8_min = builder.ins().f32const(-128.0_f32);
- let i8_max = builder.ins().f32const(127.0_f32);
- let i16_min = builder.ins().f32const(-32768.0_f32);
- let i16_max = builder.ins().f32const(32767.0_f32);
-
+ let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, scaled_f32);
+ let zero = builder.ins().iconst(types::I32, 0);
+ let u8_max = builder.ins().iconst(types::I32, 255);
+ let u16_max = builder.ins().iconst(types::I32, 65535);
+ let i8_min = builder.ins().iconst(types::I32, -128);
+ let i8_max = builder.ins().iconst(types::I32, 127);
+ let i16_min = builder.ins().iconst(types::I32, -32768);
+ let i16_max = builder.ins().iconst(types::I32, 32767);
let signed_min = builder.ins().select(is_half, i16_min, i8_min);
- let min_f32 = builder.ins().select(is_signed, signed_min, zero_f);
+ let min = builder.ins().select(is_signed, signed_min, zero);
let signed_max = builder.ins().select(is_half, i16_max, i8_max);
let unsigned_max = builder.ins().select(is_half, u16_max, u8_max);
- let max_f32 = builder.ins().select(is_signed, signed_max, unsigned_max);
-
- let lo_clamped = builder.ins().fmax(scaled_f32, min_f32);
- let clamped = builder.ins().fmin(lo_clamped, max_f32);
-
- let val_i32 = builder.ins().fcvt_to_sint_sat(types::I32, clamped);
+ let max = builder.ins().select(is_signed, signed_max, unsigned_max);
+ let clamped = builder.ins().smax(val_i32, min);
+ let val_i32 = builder.ins().smin(clamped, max);
let mask_byte = builder.ins().iconst(types::I32, 0xFFi64);
let mask_half = builder.ins().iconst(types::I32, 0xFFFFi64);
@@ -3705,12 +3724,12 @@ pub(crate) fn emit_psq_quant_element(
builder.switch_to_block(byte_block);
builder.seal_block(byte_block);
- builder.ins().call(write_u8_fn, &[ctx_ptr, ea, val_to_write]);
+ quantized::store::<SYSTEM>(builder, ctx_ptr, ea, val_to_write, MemSize::U8, local);
builder.ins().jump(merge, &[]);
builder.switch_to_block(half_block);
builder.seal_block(half_block);
- builder.ins().call(write_u16_fn, &[ctx_ptr, ea, val_to_write]);
+ quantized::store::<SYSTEM>(builder, ctx_ptr, ea, val_to_write, MemSize::U16, local);
builder.ins().jump(merge, &[]);
builder.switch_to_block(merge);
@@ -3757,13 +3776,13 @@ pub(crate) fn emit_psq_load_quantized<const SYSTEM: SystemId>(
let scale_addr = builder.ins().iadd(table_addr, scale_offset);
let scale_f32 = builder.ins().load(types::F32, vmctx_flags(), scale_addr, 0);
- let ps0 = emit_psq_dequant_element(builder, ctx_ptr, ea, type_v, scale_f32, local.read_u8, local.read_u16);
+ let ps0 = emit_psq_dequant_element::<SYSTEM>(builder, ctx_ptr, ea, type_v, scale_f32, local);
let ps1 = if w {
builder.ins().f64const(1.0)
} else {
let elem_size = emit_psq_elem_size(builder, type_v);
let ea1 = builder.ins().iadd(ea, elem_size);
- emit_psq_dequant_element(builder, ctx_ptr, ea1, type_v, scale_f32, local.read_u8, local.read_u16)
+ emit_psq_dequant_element::<SYSTEM>(builder, ctx_ptr, ea1, type_v, scale_f32, local)
};
fpr_store::<SYSTEM>(builder, ctx_ptr, fd, ps0);
ps1_store::<SYSTEM>(builder, ctx_ptr, fd, ps1);
@@ -3842,30 +3861,12 @@ pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(
let scale_f32 = builder.ins().load(types::F32, vmctx_flags(), scale_addr, 0);
let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, fs);
- emit_psq_quant_element(
- builder,
- ctx_ptr,
- ea,
- ps0,
- type_v,
- scale_f32,
- local.write_u8,
- local.write_u16,
- );
+ emit_psq_quant_element::<SYSTEM>(builder, ctx_ptr, ea, ps0, type_v, scale_f32, local);
if !w {
let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, fs);
let elem_size = emit_psq_elem_size(builder, type_v);
let ea1 = builder.ins().iadd(ea, elem_size);
- emit_psq_quant_element(
- builder,
- ctx_ptr,
- ea1,
- ps1,
- type_v,
- scale_f32,
- local.write_u8,
- local.write_u16,
- );
+ emit_psq_quant_element::<SYSTEM>(builder, ctx_ptr, ea1, ps1, type_v, scale_f32, local);
}
builder.ins().jump(merge, &[]);
@@ -4722,28 +4723,36 @@ pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(
slow: FuncRef,
update: bool,
) {
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ let val = emit_fastmem_load(builder, ctx_ptr, ea, host_ptr, host_addr, size, slow);
+ gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);
+
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
+ }
+}
+
+pub(crate) fn emit_fastmem_load(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ host_ptr: Value,
+ host_addr: Value,
+ size: MemSize,
+ slow: FuncRef,
+) -> Value {
let fast_block = builder.create_block();
let slow_block = builder.create_block();
builder.set_cold_block(slow_block);
let merge = builder.create_block();
builder.append_block_param(merge, types::I32);
-
- let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
- let host_addr = builder.ins().iadd(host_ptr, page_off);
- builder
- .ins()
- .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
- builder.append_block_param(fast_block, types::I64);
+ builder.ins().brif(host_ptr, fast_block, &[], slow_block, &[]);
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
- let addr = builder.block_params(fast_block)[0];
- let flags = heap_flags();
- let raw_val = builder.ins().load(size.ir_type(), flags, addr, 0);
- let val_swapped = match size {
- MemSize::U8 => raw_val,
- MemSize::U16 | MemSize::U32 => builder.ins().bswap(raw_val),
- };
+ let raw_val = builder.ins().load(size.ir_type(), heap_flags(), host_addr, 0);
+ let val_swapped = size.swap(builder, raw_val);
let val32 = match size {
MemSize::U32 => val_swapped,
MemSize::U16 | MemSize::U8 => builder.ins().uextend(types::I32, val_swapped),
@@ -4758,12 +4767,7 @@ pub(crate) fn emit_load_at_ea<const SYSTEM: SystemId>(
builder.switch_to_block(merge);
builder.seal_block(merge);
- let val = builder.block_params(merge)[0];
- gpr_store::<SYSTEM>(builder, ctx_ptr, rd, val);
-
- if update {
- gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
- }
+ builder.block_params(merge)[0]
}
pub(crate) fn emit_store<const SYSTEM: SystemId>(
@@ -4824,39 +4828,51 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
update: bool,
) {
let val32 = gpr_load::<SYSTEM>(builder, ctx_ptr, rs);
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ emit_fastmem_store::<SYSTEM>(
+ builder,
+ ctx_ptr,
+ ea,
+ host_ptr,
+ host_addr,
+ val32,
+ size,
+ slow,
+ cause_smc_write,
+ );
+ if update {
+ gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
+ }
+}
+
+pub(crate) fn emit_fastmem_store<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ host_ptr: Value,
+ host_addr: Value,
+ val32: Value,
+ size: MemSize,
+ slow: FuncRef,
+ cause_smc_write: FuncRef,
+) {
let fast_block = builder.create_block();
let slow_block = builder.create_block();
builder.set_cold_block(slow_block);
let merge = builder.create_block();
-
- let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
- let host_addr = builder.ins().iadd(host_ptr, page_off);
- builder
- .ins()
- .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
- builder.append_block_param(fast_block, types::I64);
+ builder.ins().brif(host_ptr, fast_block, &[], slow_block, &[]);
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
- let addr = builder.block_params(fast_block)[0];
- let flags = heap_flags();
let truncated = match size {
MemSize::U32 => val32,
- MemSize::U16 => builder.ins().ireduce(types::I16, val32),
- MemSize::U8 => builder.ins().ireduce(types::I8, val32),
+ MemSize::U16 | MemSize::U8 => builder.ins().ireduce(size.ir_type(), val32),
};
- let val_to_store = match size {
- MemSize::U8 => truncated,
- MemSize::U16 | MemSize::U32 => builder.ins().bswap(truncated),
- };
- builder.ins().store(flags, val_to_store, addr, 0);
- let size_bytes = match size {
- MemSize::U8 => 1,
- MemSize::U16 => 2,
- MemSize::U32 => 4,
- };
- emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size_bytes, cause_smc_write);
+ let val_to_store = size.swap(builder, truncated);
+ builder.ins().store(heap_flags(), val_to_store, host_addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, size.bytes(), cause_smc_write);
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -4866,10 +4882,6 @@ pub(crate) fn emit_store_at_ea<const SYSTEM: SystemId>(
builder.switch_to_block(merge);
builder.seal_block(merge);
-
- if update {
- gpr_store::<SYSTEM>(builder, ctx_ptr, ra, ea);
- }
}
pub(crate) fn emit_addi<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, instr: Instruction) {
diff --git a/crates/gecko/src/gekko/jit/translator/gqr.rs b/crates/gecko/src/gekko/jit/translator/gqr.rs
new file mode 100644
index 0000000..8f068cd
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/translator/gqr.rs
@@ -0,0 +1,88 @@
+use cranelift_codegen::ir::condcodes::IntCC;
+use cranelift_codegen::ir::{InstBuilder, SigRef, Value, types};
+use cranelift_frontend::FunctionBuilder;
+
+use crate::gekko::instruction::Instruction;
+use crate::gekko::interpreter::psq::{gqr_ld_type, gqr_st_type};
+use crate::gekko::jit::BlockEntry;
+use crate::gekko::jit::block::BlockSpec;
+use crate::system::SystemId;
+
+use super::{BlockEmitContext, gqr_cache_store, gqr_load};
+
+const MIN_QUANTIZED_ACCESSES: usize = 4;
+
+pub(crate) struct GqrAssumptions {
+ expected: [Option<u32>; 8],
+}
+
+impl GqrAssumptions {
+ pub(crate) fn for_block(spec: &BlockSpec, gqrs: &[u32; 8]) -> Option<Self> {
+ if cfg!(feature = "hooks") {
+ return None;
+ }
+
+ let mut values = [None; 8];
+ let mut count = 0;
+ for &raw in &spec.instrs {
+ let instr = Instruction(raw);
+ let primary = instr.primary_opcode();
+ if primary == 31 && instr.xo10() == 467 && matches!(instr.spr_swapped(), 912..=919) {
+ return None;
+ }
+
+ let (index, store) = match primary {
+ 56 | 57 => (instr.psq_i(), false),
+ 60 | 61 => (instr.psq_i(), true),
+ 4 => match instr.xo6() {
+ 6 | 38 => (instr.psq_ix(), false),
+ 7 | 39 => (instr.psq_ix(), true),
+ _ => continue,
+ },
+ _ => continue,
+ };
+ let gqr = gqrs[index as usize];
+ let qtype = if store { gqr_st_type(gqr) } else { gqr_ld_type(gqr) };
+ if qtype >= 4 {
+ values[index as usize] = Some(gqr);
+ count += 1;
+ }
+ }
+
+ (count >= MIN_QUANTIZED_ACCESSES).then_some(Self { expected: values })
+ }
+
+ pub(super) fn emit_guard<const SYSTEM: SystemId>(
+ &self,
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ host_return_pc: Value,
+ block_sig_ref: SigRef,
+ local: &mut BlockEmitContext,
+ fallback_entry: BlockEntry,
+ ) {
+ let mut matches = builder.ins().iconst(types::I8, 1);
+ for (index, expected) in self.expected.iter().enumerate() {
+ let Some(expected) = *expected else { continue };
+ let actual = gqr_load::<SYSTEM>(builder, ctx_ptr, local, index as u8);
+ let same = builder.ins().icmp_imm(IntCC::Equal, actual, expected as i64);
+ matches = builder.ins().band(matches, same);
+ let constant = builder.ins().iconst(types::I32, expected as i64);
+ gqr_cache_store(local, index as u8, constant);
+ }
+
+ let specialized = builder.create_block();
+ let fallback = builder.create_block();
+ builder.ins().brif(matches, specialized, &[], fallback, &[]);
+
+ builder.switch_to_block(fallback);
+ builder.seal_block(fallback);
+ let target = builder.ins().iconst(types::I64, fallback_entry as i64);
+ builder
+ .ins()
+ .return_call_indirect(block_sig_ref, target, &[ctx_ptr, host_return_pc]);
+
+ builder.switch_to_block(specialized);
+ builder.seal_block(specialized);
+ }
+}
diff --git a/crates/gecko/src/gekko/jit/translator/quantized.rs b/crates/gecko/src/gekko/jit/translator/quantized.rs
new file mode 100644
index 0000000..6d93543
--- /dev/null
+++ b/crates/gecko/src/gekko/jit/translator/quantized.rs
@@ -0,0 +1,92 @@
+use cranelift_codegen::ir::condcodes::IntCC;
+use cranelift_codegen::ir::{InstBuilder, Value, types};
+use cranelift_frontend::FunctionBuilder;
+
+use crate::gekko::jit::abi;
+use crate::mmio::FASTMEM_PAGE_BYTES;
+use crate::mmio::constants::{LCACHE_BASE, LCACHE_SIZE};
+use crate::system::SystemId;
+
+use super::{BlockEmitContext, MemSize, emit_fastmem_load, emit_fastmem_lookup, emit_fastmem_store, vmctx_flags};
+
+fn pointer<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, ea: Value, size: MemSize) -> Value {
+ let bytes = size.bytes() as usize;
+
+ let (base, offset) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let top = builder.ins().ushr_imm(ea, 28);
+ let cached = builder.ins().icmp_imm(IntCC::Equal, top, 8);
+ let uncached = builder.ins().icmp_imm(IntCC::Equal, top, 12);
+ let alias = builder.ins().bor(cached, uncached);
+ let mapped = builder.ins().icmp_imm(IntCC::NotEqual, base, 0);
+ let in_page = builder.ins().icmp_imm(
+ IntCC::UnsignedLessThanOrEqual,
+ offset,
+ (FASTMEM_PAGE_BYTES - bytes) as i64,
+ );
+ let valid = builder.ins().band(mapped, alias);
+ let valid = builder.ins().band(valid, in_page);
+ let addr = builder.ins().iadd(base, offset);
+ let zero = builder.ins().iconst(types::I64, 0);
+ let ram = builder.ins().select(valid, addr, zero);
+
+ let lc_offset = builder.ins().iadd_imm(ea, -(LCACHE_BASE as i64));
+ let in_cache = builder
+ .ins()
+ .icmp_imm(IntCC::UnsignedLessThanOrEqual, lc_offset, (LCACHE_SIZE - bytes) as i64);
+ let lc_base = builder.ins().load(
+ types::I64,
+ vmctx_flags(),
+ ctx_ptr,
+ abi::lcache_fastmem_ptr_offset::<SYSTEM>() as i32,
+ );
+ let lc_mapped = builder.ins().icmp_imm(IntCC::NotEqual, lc_base, 0);
+ let lc_valid = builder.ins().band(in_cache, lc_mapped);
+ let lc_offset = builder.ins().uextend(types::I64, lc_offset);
+ let lc_addr = builder.ins().iadd(lc_base, lc_offset);
+ builder.ins().select(lc_valid, lc_addr, ram)
+}
+
+pub(super) fn load<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ size: MemSize,
+ local: &BlockEmitContext,
+) -> Value {
+ let slow = match size {
+ MemSize::U8 => local.read_u8,
+ MemSize::U16 => local.read_u16,
+ MemSize::U32 => local.read_u32,
+ };
+
+ if cfg!(feature = "hooks") {
+ let call = builder.ins().call(slow, &[ctx_ptr, ea]);
+ return builder.inst_results(call)[0];
+ }
+
+ let ptr = self::pointer::<SYSTEM>(builder, ctx_ptr, ea, size);
+ emit_fastmem_load(builder, ctx_ptr, ea, ptr, ptr, size, slow)
+}
+
+pub(super) fn store<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ value: Value,
+ size: MemSize,
+ local: &BlockEmitContext,
+) {
+ let slow = match size {
+ MemSize::U8 => local.write_u8,
+ MemSize::U16 => local.write_u16,
+ MemSize::U32 => local.write_u32,
+ };
+
+ if cfg!(feature = "hooks") {
+ builder.ins().call(slow, &[ctx_ptr, ea, value]);
+ return;
+ }
+
+ let ptr = self::pointer::<SYSTEM>(builder, ctx_ptr, ea, size);
+ emit_fastmem_store::<SYSTEM>(builder, ctx_ptr, ea, ptr, ptr, value, size, slow, local.cause_smc_write);
+}
diff --git a/crates/gecko/src/gekko/spr.rs b/crates/gecko/src/gekko/spr.rs
index 6ce678d..7eb4b5e 100644
--- a/crates/gecko/src/gekko/spr.rs
+++ b/crates/gecko/src/gekko/spr.rs
@@ -137,6 +137,10 @@ pub struct Spr {
}
impl Spr {
+ pub const fn is_dbat(spr_num: u32) -> bool {
+ matches!(spr_num, 536..=543 | 568..=575)
+ }
+
pub fn read_gqr(&self, index: u8) -> u32 {
match index {
0 => self.gqr0,
diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rs
index 7bd7607..c0e422d 100644
--- a/crates/gecko/src/mmio.rs
+++ b/crates/gecko/src/mmio.rs
@@ -37,6 +37,7 @@ pub struct Mmio<const SYSTEM: SystemId> {
pub ram_ptr: usize,
pub fastmem_lut: Vec<usize>,
pub fastmem_lut_ptr: usize,
+ pub(crate) lcache_fastmem_ptr: usize,
pub code_refcount: Box<[u8]>,
pub code_refcount_ptr: usize,
memory_write_generation: u64,
@@ -193,6 +194,7 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
ram_ptr,
fastmem_lut,
fastmem_lut_ptr,
+ lcache_fastmem_ptr: 0,
code_refcount,
code_refcount_ptr,
memory_write_generation: 0,
diff --git a/crates/gecko/src/mmio/bus.rs b/crates/gecko/src/mmio/bus.rs
index 727e12a..adf67be 100644
--- a/crates/gecko/src/mmio/bus.rs
+++ b/crates/gecko/src/mmio/bus.rs
@@ -80,6 +80,15 @@ macro_rules! bus_write_hooks {
}
impl<const SYSTEM: SystemId> System<SYSTEM> {
+ pub(crate) fn refresh_lcache_fastmem(&mut self) {
+ let base = crate::mmio::constants::LCACHE_BASE;
+ self.mmio.lcache_fastmem_ptr = if self.bat_translate(base) == base {
+ self.mmio.lcache.as_ptr() as usize
+ } else {
+ 0
+ };
+ }
+
#[cold]
pub(crate) fn sync_deferred_efb_writeback(&mut self, phys: u32, len: usize) {
if !self.mmio.efb_writeback_needed(phys, len) {
diff --git a/crates/gecko/src/savestate.rs b/crates/gecko/src/savestate.rs
index 4d67053..8548c80 100644
--- a/crates/gecko/src/savestate.rs
+++ b/crates/gecko/src/savestate.rs
@@ -293,6 +293,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {
self.gekko = r.pod()?;
self.mmio.load_state(&mut r)?;
+ self.refresh_lcache_fastmem();
self.vi = r.pod()?;
self.pe = r.pod()?;