A GameCube/Wii emulator and debugger written in Rust.
fix dense arena on macos arm and refactor jit a bit. also use arena for dsp and vtx
Laylecb43ac475739
diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rsindex 9cd3938..c3e45cb 100644--- a/crates/gecko/src/flipper/dsp.rs+++ b/crates/gecko/src/flipper/dsp.rs@@ -842,12 +842,12 @@ pub trait DspJitHandle {fn dump_hot_blocks(&self, top_k: usize);fn dump_hot_blocks_csv(&self, top_k: usize, path: &std::path::Path) -> std::io::Result<()>;fn dump_top_clif(&mut self, top_k: usize, iram: &[u8], irom: &[u8]);- fn cached_blocks(&self) -> Vec<crate::jit_cache::CachedBlockDsp>;+ fn cached_blocks(&self) -> Vec<crate::jit::cache::CachedBlockDsp>;fn precompile_blocks(&mut self,iram: &[u8],irom: &[u8],- blocks: &[crate::jit_cache::CachedBlockDsp],+ blocks: &[crate::jit::cache::CachedBlockDsp],) -> (usize, usize);}@@ -883,7 +883,7 @@ impl<const SYSTEM: SystemId> DspJitHandle for jit::JitEngine<SYSTEM> {Ok(())}- fn cached_blocks(&self) -> Vec<crate::jit_cache::CachedBlockDsp> {+ fn cached_blocks(&self) -> Vec<crate::jit::cache::CachedBlockDsp> {Self::cached_blocks(self)}@@ -891,7 +891,7 @@ impl<const SYSTEM: SystemId> DspJitHandle for jit::JitEngine<SYSTEM> {&mut self,iram: &[u8],irom: &[u8],- blocks: &[crate::jit_cache::CachedBlockDsp],+ blocks: &[crate::jit::cache::CachedBlockDsp],) -> (usize, usize) {Self::precompile_blocks(self, iram, irom, blocks)}diff --git a/crates/gecko/src/flipper/dsp/jit.rs b/crates/gecko/src/flipper/dsp/jit.rsindex f66acc0..7b6ab04 100644--- a/crates/gecko/src/flipper/dsp/jit.rs+++ b/crates/gecko/src/flipper/dsp/jit.rs@@ -36,6 +36,7 @@ const CHAIN_DEPTH_HISTOGRAM_LEN: usize = super::DSP_JIT_CHAIN_BUDGET as usize +pub struct JitEngine<const SYSTEM: SystemId> {module: JITModule,+ arena: crate::jit::arena::DenseArena,ctx: Context,builder_ctx: FunctionBuilderContext,cache: FxHashMap<u16, BlockEntry>,@@ -108,6 +109,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {jit_builder.symbol(syms.read_reg_full.0, syms.read_reg_full.1);jit_builder.symbol(syms.write_reg_full.0, syms.write_reg_full.1);+ let arena = crate::jit::arena::DenseArena::new(256 << 20).expect("reserve DSP JIT code arena");+ jit_builder.memory_provider(Box::new(arena.clone()));+let mut module = JITModule::new(jit_builder);let pointer_type = module.target_config().pointer_type();@@ -141,6 +145,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let trampoline_fn = build_trampoline(&mut module, host_cc, pointer_type, &block_sig);+ arena.set_floor();+let table = vec![DspBlockLookupSlot {pc: 0,@@ -154,6 +160,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {Self {module,+ arena,ctx: Context::new(),builder_ctx: FunctionBuilderContext::new(),cache: FxHashMap::default(),@@ -288,15 +295,15 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {entry}- pub fn cached_blocks(&self) -> Vec<crate::jit_cache::CachedBlockDsp> {+ pub fn cached_blocks(&self) -> Vec<crate::jit::cache::CachedBlockDsp> {self.cache.keys().filter_map(|&pc| {let spec = self.block_specs.get(&pc)?;- Some(crate::jit_cache::CachedBlockDsp {+ Some(crate::jit::cache::CachedBlockDsp {pc,instr_count: spec.instrs.len() as u16,- hash: crate::jit_cache::hash_words(spec.instrs.iter().map(|e| e.raw)),+ hash: crate::jit::cache::hash_words(spec.instrs.iter().map(|e| e.raw)),})}).collect()@@ -306,7 +313,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {&mut self,iram: &[u8],irom: &[u8],- blocks: &[crate::jit_cache::CachedBlockDsp],+ blocks: &[crate::jit::cache::CachedBlockDsp],) -> (usize, usize) {let mut compiled = 0usize;let mut skipped = 0usize;@@ -327,7 +334,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {continue;}- let actual = crate::jit_cache::hash_words(spec.instrs.iter().map(|e| e.raw));+ let actual = crate::jit::cache::hash_words(spec.instrs.iter().map(|e| e.raw));if actual != b.hash {skipped += 1;continue;@@ -429,6 +436,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.loop_end_table_built = false;self.clear_lookup_table();+ self.arena.reset();}#[cfg_attr(feature = "hotpath", hotpath::measure(label = "dsp_jit_run_block"))]diff --git a/crates/gecko/src/flipper/gx/jit.rs b/crates/gecko/src/flipper/gx/jit.rsindex 1775a4c..38c5a47 100644--- a/crates/gecko/src/flipper/gx/jit.rs+++ b/crates/gecko/src/flipper/gx/jit.rs@@ -190,6 +190,9 @@ impl JitVertexEngine {runtime::gecko_gx_jit_apply_texgens as *const u8,);+ let arena = crate::jit::arena::DenseArena::new(64 << 20).expect("reserve VTX JIT code arena");+ jit_builder.memory_provider(Box::new(arena));+let module = JITModule::new(jit_builder);let ptr = module.target_config().pointer_type();let host_cc = module.target_config().default_call_conv;@@ -209,10 +212,10 @@ impl JitVertexEngine {}}- pub fn cached_keys(&self) -> Vec<crate::jit_cache::CachedVtxKey> {+ pub fn cached_keys(&self) -> Vec<crate::jit::cache::CachedVtxKey> {self.cache.keys()- .map(|k| crate::jit_cache::CachedVtxKey {+ .map(|k| crate::jit::cache::CachedVtxKey {vcd_lo: k.vcd_lo,vcd_hi: k.vcd_hi,vat_a: k.vat_a,@@ -222,7 +225,7 @@ impl JitVertexEngine {.collect()}- pub fn precompile_keys(&mut self, keys: &[crate::jit_cache::CachedVtxKey]) -> (usize, usize) {+ pub fn precompile_keys(&mut self, keys: &[crate::jit::cache::CachedVtxKey]) -> (usize, usize) {let mut compiled = 0;let mut skipped = 0;for k in keys {diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rsindex ec2b190..2243a23 100644--- a/crates/gecko/src/gekko/jit.rs+++ b/crates/gecko/src/gekko/jit.rs@@ -1,6 +1,5 @@pub mod abi;pub mod block;-pub mod dense_arena;pub mod handlers;pub mod idle;pub mod insn;@@ -314,7 +313,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {jit_builder.symbol(name, addr);}- let arena = dense_arena::DenseArena::new(512 << 20).expect("reserve JIT code arena");+ let arena = crate::jit::arena::DenseArena::new(512 << 20).expect("reserve JIT code arena");jit_builder.memory_provider(Box::new(arena));let mut module = JITModule::new(jit_builder);@@ -550,15 +549,15 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}}- pub fn cached_blocks(&self) -> Vec<crate::jit_cache::CachedBlockPpc> {+ pub fn cached_blocks(&self) -> Vec<crate::jit::cache::CachedBlockPpc> {self.cache.keys().filter_map(|&pc| {let spec = self.block_specs.get(&pc)?;- Some(crate::jit_cache::CachedBlockPpc {+ Some(crate::jit::cache::CachedBlockPpc {pc,instr_count: spec.instrs.len() as u16,- hash: crate::jit_cache::hash_words(spec.instrs.iter().copied()),+ hash: crate::jit::cache::hash_words(spec.instrs.iter().copied()),})}).collect()@@ -567,7 +566,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {pub fn precompile_blocks(&mut self,sys: &mut System<SYSTEM>,- blocks: &[crate::jit_cache::CachedBlockPpc],+ blocks: &[crate::jit::cache::CachedBlockPpc],) -> (usize, usize) {let mut compiled = 0usize;let mut skipped = 0usize;@@ -589,7 +588,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {buf.push(sys.mmio.fetch_instruction(instr_pc));}- let actual = crate::jit_cache::hash_words(buf.into_iter());+ let actual = crate::jit::cache::hash_words(buf.into_iter());if actual != b.hash {skipped += 1;continue;diff --git a/crates/gecko/src/gekko/jit/dense_arena.rs b/crates/gecko/src/gekko/jit/dense_arena.rsdeleted file mode 100644index 8106c28..0000000--- a/crates/gecko/src/gekko/jit/dense_arena.rs+++ /dev/null@@ -1,50 +0,0 @@-use std::io;--use cranelift_jit::{BranchProtection, JITMemoryKind, JITMemoryProvider};-use cranelift_module::ModuleResult;--pub struct DenseArena {- alloc: region::Allocation,- offset: usize,-}--unsafe impl Send for DenseArena {}--impl DenseArena {- pub fn new(reserved: usize) -> io::Result<Self> {- let alloc = region::alloc(reserved, region::Protection::READ_WRITE_EXECUTE).map_err(io::Error::other)?;-- #[cfg(target_os = "linux")]- unsafe {- libc::madvise(- alloc.as_ptr::<u8>() as *mut libc::c_void,- alloc.len(),- libc::MADV_HUGEPAGE,- );- }-- Ok(DenseArena { alloc, offset: 0 })- }-}--impl JITMemoryProvider for DenseArena {- fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> {- debug_assert!(matches!(kind, JITMemoryKind::Executable));-- let align = (align as usize).max(1);- let start = (self.offset + align - 1) & !(align - 1);- let end = start + size;- if end > self.alloc.len() {- return Err(io::Error::new(io::ErrorKind::OutOfMemory, "JIT code arena exhausted"));- }-- self.offset = end;- Ok(unsafe { (self.alloc.as_mut_ptr::<u8>()).add(start) })- }-- unsafe fn free_memory(&mut self) {}-- fn finalize(&mut self, _branch_protection: BranchProtection) -> ModuleResult<()> {- Ok(())- }-}diff --git a/crates/gecko/src/jit.rs b/crates/gecko/src/jit.rsnew file mode 100644index 0000000..7182249--- /dev/null+++ b/crates/gecko/src/jit.rs@@ -0,0 +1,3 @@+#[cfg(feature = "jit")]+pub mod arena;+pub mod cache;diff --git a/crates/gecko/src/jit/arena.rs b/crates/gecko/src/jit/arena.rsnew file mode 100644index 0000000..811a1c6--- /dev/null+++ b/crates/gecko/src/jit/arena.rs@@ -0,0 +1,128 @@+use std::io;+use std::sync::{Arc, Mutex};++use cranelift_jit::{BranchProtection, JITMemoryKind, JITMemoryProvider};+use cranelift_module::ModuleResult;++#[derive(Clone)]+pub struct DenseArena {+ inner: Arc<Mutex<Inner>>,+}++struct Inner {+ alloc: region::Allocation,+ offset: usize,+ floor: usize,++ #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+ clean: usize,+}++unsafe impl Send for Inner {}++impl DenseArena {+ pub fn new(reserved: usize) -> io::Result<Self> {+ let alloc = region::alloc(reserved, region::Protection::READ_WRITE_EXECUTE).map_err(io::Error::other)?;++ #[cfg(target_os = "linux")]+ unsafe {+ libc::madvise(+ alloc.as_ptr::<u8>() as *mut libc::c_void,+ alloc.len(),+ libc::MADV_HUGEPAGE,+ );+ }++ let inner = Inner {+ alloc,+ offset: 0,+ floor: 0,++ #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+ clean: 0,+ };++ Ok(DenseArena {+ inner: Arc::new(Mutex::new(inner)),+ })+ }++ pub fn set_floor(&self) {+ let mut inner = self.inner.lock().unwrap();+ inner.floor = inner.offset;+ }++ pub fn reset(&self) {+ let mut inner = self.inner.lock().unwrap();+ inner.offset = inner.floor;++ #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+ {+ inner.clean = inner.floor;+ }+ }+}++impl JITMemoryProvider for DenseArena {+ fn allocate(&mut self, size: usize, align: u64, _kind: JITMemoryKind) -> io::Result<*mut u8> {+ let mut inner = self.inner.lock().unwrap();++ let align = (align as usize).max(1);+ let start = (inner.offset + align - 1) & !(align - 1);+ let end = start + size;+ if end > inner.alloc.len() {+ return Err(io::Error::new(io::ErrorKind::OutOfMemory, "JIT code arena exhausted"));+ }++ #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+ apple::set_writable();++ inner.offset = end;+ Ok(unsafe { inner.alloc.as_mut_ptr::<u8>().add(start) })+ }++ unsafe fn free_memory(&mut self) {}++ fn finalize(&mut self, _branch_protection: BranchProtection) -> ModuleResult<()> {+ #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+ {+ let mut inner = self.inner.lock().unwrap();++ let dirty = unsafe { inner.alloc.as_ptr::<u8>().add(inner.clean) };+ let len = inner.offset - inner.clean;++ apple::set_executable();+ apple::flush_icache(dirty, len);++ inner.clean = inner.offset;+ }++ Ok(())+ }+}++#[cfg(all(target_vendor = "apple", target_arch = "aarch64"))]+mod apple {+ use core::ffi::c_void;++ unsafe extern "C" {+ fn sys_icache_invalidate(start: *const c_void, len: usize);+ }++ #[inline]+ pub fn set_writable() {+ unsafe { libc::pthread_jit_write_protect_np(0) };+ }++ #[inline]+ pub fn set_executable() {+ unsafe { libc::pthread_jit_write_protect_np(1) };+ }++ #[inline]+ pub fn flush_icache(start: *const u8, len: usize) {+ if len != 0 {+ unsafe { sys_icache_invalidate(start as *const c_void, len) };+ }+ }+}diff --git a/crates/gecko/src/jit_cache.rs b/crates/gecko/src/jit/cache.rssimilarity index 100%rename from crates/gecko/src/jit_cache.rsrename to crates/gecko/src/jit/cache.rsdiff --git a/crates/gecko/src/lib.rs b/crates/gecko/src/lib.rsindex 246f7d1..2ea5d77 100644--- a/crates/gecko/src/lib.rs+++ b/crates/gecko/src/lib.rs@@ -10,7 +10,7 @@ pub mod hollywood;pub mod host;pub mod input;pub mod ipl;-pub mod jit_cache;+pub mod jit;pub mod mmio;#[cfg(any(feature = "jit-stats", feature = "profile", feature = "gx-stats"))]pub mod profile;diff --git a/crates/gecko/src/system.rs b/crates/gecko/src/system.rsindex db2242b..969d070 100644--- a/crates/gecko/src/system.rs+++ b/crates/gecko/src/system.rs@@ -246,6 +246,10 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[cfg(feature = "jit")]pub fn load_jit_cache(&mut self, game_id: &str) -> (usize, usize, usize, usize, usize, usize) {+ if self.execution_mode != ExecutionMode::Jit {+ return (0, 0, 0, 0, 0, 0);+ }+let mut ppc_compiled = 0;let mut ppc_skipped = 0;let mut dsp_compiled = 0;@@ -253,8 +257,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let mut vtx_compiled = 0;let mut vtx_skipped = 0;- let ppc_path = crate::jit_cache::ppc_cache_path(game_id);- if let Ok(blocks) = crate::jit_cache::load_ppc_blocks(&ppc_path) {+ let ppc_path = crate::jit::cache::ppc_cache_path(game_id);+ if let Ok(blocks) = crate::jit::cache::load_ppc_blocks(&ppc_path) {tracing::info!(count = blocks.len(), "loaded PPC JIT block cache");if self.jit.is_none() {@@ -269,8 +273,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {self.jit = Some(jit);}- let dsp_path = crate::jit_cache::dsp_cache_path(game_id);- if let Ok(blocks) = crate::jit_cache::load_dsp_blocks(&dsp_path) {+ let dsp_path = crate::jit::cache::dsp_cache_path(game_id);+ if let Ok(blocks) = crate::jit::cache::load_dsp_blocks(&dsp_path) {tracing::info!(count = blocks.len(), "loaded DSP JIT block cache");if self.dsp.jit.is_none() {@@ -289,8 +293,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {dsp_skipped = s;}- let vtx_path = crate::jit_cache::vtx_cache_path(game_id);- if let Ok(keys) = crate::jit_cache::load_vtx_keys(&vtx_path) {+ let vtx_path = crate::jit::cache::vtx_cache_path(game_id);+ if let Ok(keys) = crate::jit::cache::load_vtx_keys(&vtx_path) {tracing::info!(count = keys.len(), "loaded vertex JIT key cache");let (c, s) = self.gx.jit_vtx.precompile_keys(&keys);vtx_compiled = c;@@ -310,9 +314,9 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[cfg(feature = "jit")]pub fn save_jit_cache(&self, game_id: &str) -> std::io::Result<(usize, usize, usize)> {let cached_system = if SYSTEM == WII {- crate::jit_cache::CachedSystem::Wii+ crate::jit::cache::CachedSystem::Wii} else {- crate::jit_cache::CachedSystem::Gc+ crate::jit::cache::CachedSystem::Gc};let mut ppc_count = 0;@@ -321,18 +325,18 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {if let Some(jit) = self.jit.as_ref() {let blocks = jit.cached_blocks();ppc_count = blocks.len();- crate::jit_cache::save_ppc_blocks(&crate::jit_cache::ppc_cache_path(game_id), cached_system, &blocks)?;+ crate::jit::cache::save_ppc_blocks(&crate::jit::cache::ppc_cache_path(game_id), cached_system, &blocks)?;}if let Some(jit) = self.dsp.jit.as_ref() {let blocks = jit.cached_blocks();dsp_count = blocks.len();- crate::jit_cache::save_dsp_blocks(&crate::jit_cache::dsp_cache_path(game_id), cached_system, &blocks)?;+ crate::jit::cache::save_dsp_blocks(&crate::jit::cache::dsp_cache_path(game_id), cached_system, &blocks)?;}let keys = self.gx.jit_vtx.cached_keys();let vtx_count = keys.len();- crate::jit_cache::save_vtx_keys(&crate::jit_cache::vtx_cache_path(game_id), cached_system, &keys)?;+ crate::jit::cache::save_vtx_keys(&crate::jit::cache::vtx_cache_path(game_id), cached_system, &keys)?;Ok((ppc_count, dsp_count, vtx_count))}