A GameCube/Wii emulator and debugger written in Rust.
optimize ppc/dsp jit exec, batch gx draws, reduce fb readbacks and fix presentation for uncapped runner
Layled57ac9ee4d52
diff --git a/README.md b/README.mdindex dbebde3..5604abe 100644--- a/README.md+++ b/README.md@@ -44,6 +44,7 @@ Gecko is developed with homebrew development and reverse engineering in mind, bu- FIFO recorder, player and [debugger](https://www.youtube.com/watch?v=8mTPH3iAjy4)- Compatible with Dolphin- GameCube memory cards+- Savestates- Frame pacing- Modular audio backend, defaults to `cpal`- Supports mixing audio sinks@@ -51,6 +52,7 @@ Gecko is developed with homebrew development and reverse engineering in mind, bu- MCP server- Lua scripting system for runtime introspection- A beautiful yet advanced egui-based debugging UI+- Experimental freecam in the debugger- Symbol parsing from ELFs and IDA Pro databases- IDA Pro loaders for DOL and Apploader- RenderDoc captures with all sorts of debug markers@@ -114,38 +116,38 @@ Keyboard and player-window hotkeys are configurable the same way under `Settings#### GameCube-| Gamepad | Action |-| ----------- | --------------- |-| Left stick | Main stick |-| Right stick | C-stick |-| D-pad | D-pad |-| `South` | A |-| `West` | B |-| `East` | X |-| `North` | Y |-| `Start` | Start |-| `L2` `R2` | L / R (analog) |-| `R1` | Z |+| Gamepad | Action |+| ----------- | -------------- |+| Left stick | Main stick |+| Right stick | C-stick |+| D-pad | D-pad |+| `South` | A |+| `West` | B |+| `East` | X |+| `North` | Y |+| `Start` | Start |+| `L2` `R2` | L / R (analog) |+| `R1` | Z |#### Wii-| Gamepad | Action |-| ------------ | ---------------------------- |-| Gyro | IR pointer (if available) |-| Right stick | IR pointer or D-pad |-| Left stick | Nunchuk stick |-| D-pad | D-pad |-| `South` | A |-| `R2` | B |-| `West` | 1 |-| `North` | 2 |-| `Start` | Plus |-| `Select` | Minus |-| `Guide` | Home |-| `L1` | Nunchuk C |-| `L2` | Nunchuk Z |-| `R1` | Shake (pads without sensors) |-| `R3` | Recenter pointer |+| Gamepad | Action |+| ----------- | ---------------------------- |+| Gyro | IR pointer (if available) |+| Right stick | IR pointer or D-pad |+| Left stick | Nunchuk stick |+| D-pad | D-pad |+| `South` | A |+| `R2` | B |+| `West` | 1 |+| `North` | 2 |+| `Start` | Plus |+| `Select` | Minus |+| `Guide` | Home |+| `L1` | Nunchuk C |+| `L2` | Nunchuk Z |+| `R1` | Shake (pads without sensors) |+| `R3` | Recenter pointer |</details>@@ -157,6 +159,8 @@ Keyboard and player-window hotkeys are configurable the same way under `Settings| Key | Action || ------- | ------------------------------------------------------------------ || `Space` | Start emulation from the splash screen (`--wait` flag) |+| `F5` | Save state (overwrites the game's existing state) |+| `F7` | Load state || `F10` | Trigger a RenderDoc capture (requires `renderdoc-capture` feature) || `F11` | Screenshot the full window || `F12` | Screenshot the emulated framebuffer only |@@ -167,6 +171,8 @@ Keyboard and player-window hotkeys are configurable the same way under `Settings| ------- | --------------------------------------------------------------------- || `Space` | Pause / resume emulation || `Tab` | Toggle the speed limit (run uncapped) |+| `F5` | Save state (overwrites the game's existing state) |+| `F7` | Load state || `F10` | Toggle borderless fullscreen || `F11` | Toggle the FPS and emulation speed overlay || `F12` | Screenshot the emulated framebuffer to `<gecko exe dir>/screenshots/` |@@ -218,6 +224,28 @@ Nunchuk:</details>+### Savestates++`tinyapp`, `debugger` and the `gecko` player support saving and loading GameCube and Wii emulation state with `F5` and `F7`. The debugger also provides Save State and Load State menu actions. In `gecko`, these hotkeys are configurable under `Settings > Keyboard Bindings`; requests made while paused are processed after emulation resumes.++Each game has one state slot at `<data dir>/cache/<game ID>/state.gkst`; saving again overwrites it. The data directory defaults to the executable's directory and can be overridden with `GECKO_DATA_DIR`. Sessions without a game ID use `cache/default/state.gkst`.++`tinyapp` and `debugger` can also load a state immediately after boot with `--load-state <file>`.++### Freecam (experimental)++In `debugger`, open `Windows > GX`, expand `Freecam` and enable `Unlock camera`. Freecam controls are not exposed in `tinyapp` or the `gecko` player.++| Input | Action |+| -------------------------------- | -------------------------------------- |+| Hold right mouse button and move | Look around |+| `W` `A` `S` `D` | Move forward, left, backward and right |+| `Q` / `E` | Move down / up |+| `Shift` | Move 5x faster |+| Mouse wheel | Adjust movement speed |++The panel also provides a speed slider and `Reset view` button. Game input is suspended while freecam is enabled; disable `Unlock camera` to return to normal controls.+## Required filesGecko does not ship any system files.@@ -383,4 +411,4 @@ Gecko is being developed alongside other amazing emulators that shaped how Gecko- [solstice](https://codeberg.org/hazelwiss/solstice) by hazelwiss- [beanwii](https://github.com/zaydlang/beanwii) by zayd-Besides these "sister projects", [Dolphin](https://github.com/dolphin-emu/dolphin) has also been a major contributor in many ways.\ No newline at end of file+Besides these "sister projects", [Dolphin](https://github.com/dolphin-emu/dolphin) has also been a major contributor in many ways.diff --git a/crates/backend-wgpu/src/action.rs b/crates/backend-wgpu/src/action.rsindex 732db9e..31860b3 100644--- a/crates/backend-wgpu/src/action.rs+++ b/crates/backend-wgpu/src/action.rs@@ -6,7 +6,7 @@ use crate::{};use gecko::flipper::gx::draw::Primitive;use gecko::flipper::gx::regs::{CullMode, MagFilter, MinFilter, WrapMode};-use gecko::host::{DrawData, DrawState, GxAction};+use gecko::host::{DrawSegment, DrawState, GxAction};use glam::{Mat4, UVec4, Vec4};#[cfg(feature = "gx-stats")]use std::sync::atomic::Ordering;@@ -133,7 +133,7 @@ impl GxRenderer {/// Append `draw`'s vertices and indices onto the draw already recorded at/// `previous`, rebasing the indices onto the merged vertex range.- fn merge_into_previous(&mut self, previous: usize, draw: &DrawData) {+ fn merge_into_previous(&mut self, previous: usize, draw: &DrawSegment) {let vertex_bias = self.scratch_draws[previous].vertex_count;let (_, additional_indices) = self::emit_draw_indices(draw, &mut self.scratch_indices, vertex_bias);@@ -246,6 +246,13 @@ impl GxRenderer {.efb_copy_cache.get(&tid.ram_addr).is_some_and(|e| e.matches(*fmt, *width, *height));+ let reference_only = rgba.is_empty();++ assert!(+ !reference_only || keep_cached,+ "pending EFB texture missing from renderer cache"+ );+if !keep_cached && let Some(entry) = self.efb_copy_cache.remove(&tid.ram_addr) {self.return_to_pool(entry.texture, entry.view);}@@ -253,11 +260,13 @@ impl GxRenderer {if let Some((_, cached_tex, _)) = self.texture_cache.get(&tid) {let size = cached_tex.size();if size.width == *width && size.height == *height {- let cached_tex = cached_tex.clone();- let staged = self.stage_texture_upload(device, &cached_tex, rgba, *width, *height);+ if !reference_only {+ let cached_tex = cached_tex.clone();+ let staged = self.stage_texture_upload(device, &cached_tex, rgba, *width, *height);- if !staged {- queue.write_texture(cached_tex.as_image_copy(), rgba, direct_copy_layout, copy_size);+ if !staged {+ queue.write_texture(cached_tex.as_image_copy(), rgba, direct_copy_layout, copy_size);+ }}if let Some((cached_fmt, _, _)) = self.texture_cache.get_mut(&tid) {@@ -289,9 +298,12 @@ impl GxRenderer {})});- let staged = self.stage_texture_upload(device, &tex, rgba, *width, *height);- if !staged {- queue.write_texture(tex.as_image_copy(), rgba, direct_copy_layout, copy_size);+ if !reference_only {+ let staged = self.stage_texture_upload(device, &tex, rgba, *width, *height);++ if !staged {+ queue.write_texture(tex.as_image_copy(), rgba, direct_copy_layout, copy_size);+ }}let view = tex.create_view(&Default::default());@@ -307,7 +319,7 @@ impl GxRenderer {}GxAction::InvalidateCaches => {self.flush_pending_draws(device, queue);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);self.texture_cache.clear();let drained: Vec<_> = self.efb_copy_cache.drain().map(|(_, e)| (e.texture, e.view)).collect();for (tex, view) in drained {@@ -339,7 +351,8 @@ impl GxRenderer {}GxAction::Draw(draw) => {- let state_changed = draw.state.is_some();+ let batch_state_changed = draw.state.is_some();+if let Some(state) = draw.state.as_ref() {self.current_draw_state.clone_from(state);let shader_key = ShaderKey::from_draw(state, self.current_alpha_compare);@@ -349,218 +362,229 @@ impl GxRenderer {self.current_active_texture_mask = draw_active_slot_mask(state);}- if cull_all_suppresses_draw(self.current_cull_mode, draw.primitive) {- self.last_frame_uniform_index = None;- self.draw_fast_path_compatible = false;- return;- }+ for (segment_index, draw) in draw.segments.iter().enumerate() {+ let state_changed = batch_state_changed && segment_index == 0;- if !draw_emits_triangles(draw) {- tracing::warn!("draw call produced zero vertices, skipping");- self.draw_fast_path_compatible = false;- return;- }+ if cull_all_suppresses_draw(self.current_cull_mode, draw.primitive) {+ self.last_frame_uniform_index = None;+ self.draw_fast_path_compatible = false;- if !state_changed- && self.draw_fast_path_compatible- && let Some(previous) = self.scratch_draws.len().checked_sub(1)- && let Some(shader_key) = self.current_shader_key- {- let stride = crate::packed_vertex_stride(shader_key.active_texcoords);- let previous_draw = self.scratch_draws[previous];- if previous_draw.src_vertex_index + previous_draw.vertex_count == draw.base_vertex- && previous_draw.packed_vertex_stride == stride- {- self.merge_into_previous(previous, draw);- return;+ continue;}- }- let state = &self.current_draw_state;+ if !draw_emits_triangles(draw) {+ tracing::debug!("draw segment produced no triangles, skipping");+ self.draw_fast_path_compatible = false;- let shader_key = self- .current_shader_key- .unwrap_or_else(|| ShaderKey::from_draw(state, self.current_alpha_compare));- let specialization = if self.current_shader_key.is_some() {- self.current_specialization_key- } else {- ShaderSpecializationKey::from_draw(state, self.current_alpha_compare, shader_key)- };- let ztex_enabled = state.ztex_op != 0;- let active = if self.current_shader_key.is_some() {- self.current_active_texture_mask- } else {- draw_active_slot_mask(state)- };- let num_indirect_stages = state.num_indirect_stages;- let next_frame_uniform = (state_changed || self.last_frame_uniform_index.is_none())- .then(|| build_frame_uniform(state, self.current_alpha_compare));- if !self.shader_cache.contains_key(&shader_key) {- #[cfg(feature = "gx-stats")]- let shader_started = std::time::Instant::now();- let wgsl = shader_specialization::compile_variant(shader_key);- let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {- label: Some(&format!("gx_shader_{shader_key:?}")),- source: wgpu::ShaderSource::Wgsl(wgsl.into()),- });- self.shader_cache.insert(shader_key, module);- #[cfg(feature = "gx-stats")]+ continue;+ }++ if !state_changed+ && self.draw_fast_path_compatible+ && let Some(previous) = self.scratch_draws.len().checked_sub(1)+ && let Some(shader_key) = self.current_shader_key{- self.stats.shader_modules_created.fetch_add(1, Ordering::Relaxed);- self.stats- .shader_create_cpu_ns- .fetch_add(shader_started.elapsed().as_nanos() as u64, Ordering::Relaxed);+ let stride = crate::packed_vertex_stride(shader_key.active_texcoords);+ let previous_draw = self.scratch_draws[previous];++ if previous_draw.src_vertex_index + previous_draw.vertex_count == draw.base_vertex+ && previous_draw.packed_vertex_stride == stride+ {+ self.merge_into_previous(previous, draw);++ continue;+ }}- tracing::info!(?shader_key, "compiled specialized shader variant");- }- let pipeline_key = self.current_pipeline_key(ztex_enabled);- let full_key = FullPipelineKey {- shader: shader_key,- specialization,- fixed: pipeline_key,- };- if !self.pipeline_cache.contains_key(&full_key) {- let uber_key = UberPipelineKey::from(full_key);- if !self.uber_pipeline_cache.contains_key(&uber_key) {++ let state = &self.current_draw_state;++ let shader_key = self+ .current_shader_key+ .unwrap_or_else(|| ShaderKey::from_draw(state, self.current_alpha_compare));+ let specialization = if self.current_shader_key.is_some() {+ self.current_specialization_key+ } else {+ ShaderSpecializationKey::from_draw(state, self.current_alpha_compare, shader_key)+ };+ let ztex_enabled = state.ztex_op != 0;+ let active = if self.current_shader_key.is_some() {+ self.current_active_texture_mask+ } else {+ draw_active_slot_mask(state)+ };+ let num_indirect_stages = state.num_indirect_stages;+ let next_frame_uniform = (state_changed || self.last_frame_uniform_index.is_none())+ .then(|| build_frame_uniform(state, self.current_alpha_compare));+ if !self.shader_cache.contains_key(&shader_key) {#[cfg(feature = "gx-stats")]- let pipeline_started = std::time::Instant::now();- let module = &self.shader_cache[&shader_key];- let pipeline = crate::pipeline::create_uber_pipeline(- device,- &self.pipeline_layout,- self.surface_format,- module,- &uber_key,- );- self.uber_pipeline_cache.insert(uber_key, pipeline);+ let shader_started = std::time::Instant::now();+ let wgsl = shader_specialization::compile_variant(shader_key);+ let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {+ label: Some(&format!("gx_shader_{shader_key:?}")),+ source: wgpu::ShaderSource::Wgsl(wgsl.into()),+ });+ self.shader_cache.insert(shader_key, module);#[cfg(feature = "gx-stats")]{- self.stats.pipelines_created.fetch_add(1, Ordering::Relaxed);+ self.stats.shader_modules_created.fetch_add(1, Ordering::Relaxed);self.stats- .pipeline_create_cpu_ns- .fetch_add(pipeline_started.elapsed().as_nanos() as u64, Ordering::Relaxed);+ .shader_create_cpu_ns+ .fetch_add(shader_started.elapsed().as_nanos() as u64, Ordering::Relaxed);}+ tracing::info!(?shader_key, "compiled specialized shader variant");}- self.request_specialized_pipeline(full_key);- }-- let first_vertex = draw.base_vertex;- let vertex_count = draw.vertex_count;- // Build per-draw uniform using tracked projection. The- // freecam view transform only applies to perspective draws- // so 2D/HUD passes stay glued to the screen.- let mvp = if self.freelook_enabled && self.current_projection_perspective {- self.current_projection * self.freelook- } else {- self.current_projection- };- let mut tex_dims = [UVec4::splat(1); 4];- if num_indirect_stages > 0 {- for slot in 0..8 {- if (active >> slot) & 1 == 0 {- continue;+ let pipeline_key = self.current_pipeline_key(ztex_enabled);+ let full_key = FullPipelineKey {+ shader: shader_key,+ specialization,+ fixed: pipeline_key,+ };++ if !self.pipeline_cache.contains_key(&full_key) {+ let uber_key = UberPipelineKey::from(full_key);++ if !self.uber_pipeline_cache.contains_key(&uber_key) {+ #[cfg(feature = "gx-stats")]+ let pipeline_started = std::time::Instant::now();+ let module = &self.shader_cache[&shader_key];+ let pipeline = crate::pipeline::create_uber_pipeline(+ device,+ &self.pipeline_layout,+ self.surface_format,+ module,+ &uber_key,+ );+ self.uber_pipeline_cache.insert(uber_key, pipeline);+ #[cfg(feature = "gx-stats")]+ {+ self.stats.pipelines_created.fetch_add(1, Ordering::Relaxed);+ self.stats+ .pipeline_create_cpu_ns+ .fetch_add(pipeline_started.elapsed().as_nanos() as u64, Ordering::Relaxed);+ }}- let Some(tid) = &self.current_texture_ids[slot] else {- continue;- };- let Some((_, tex, _)) = self.texture_cache.get(tid) else {- continue;- };- let size = tex.size();- let c = (slot % 2) * 2;- tex_dims[slot / 2][c] = size.width;- tex_dims[slot / 2][c + 1] = size.height;+ self.request_specialized_pipeline(full_key);}- }- let draw_uniform = DrawUniforms { mvp, tex_dims };- let draw_uniform_bytes = bytemuck::bytes_of(&draw_uniform);- let bg_key = trim_bg_key(self.current_bind_group_key(), active);- let stride = crate::packed_vertex_stride(shader_key.active_texcoords);-- // Build a fresh FrameUniforms slot only when the producer- // signaled state changed since the last draw or this is the- // first draw of the flush.- let frame_bytes_len_before = self.frame_uniform_bytes.len();- let mut frame_was_pushed = false;- let frame_idx = if let Some(frame_uniform) = next_frame_uniform {- let fstart = self.frame_uniform_bytes.len();- self.frame_uniform_bytes.resize(fstart + frame_stride, 0);- self.frame_uniform_bytes[fstart..fstart + frame_struct_size]- .copy_from_slice(bytemuck::bytes_of(&frame_uniform));- let idx = (fstart / frame_stride) as u32;- self.last_frame_uniform_index = Some(idx);- frame_was_pushed = true;- idx- } else {- self.last_frame_uniform_index.unwrap()- };+ let first_vertex = draw.base_vertex;+ let vertex_count = draw.vertex_count;- let previous_index = self.scratch_draws.len().checked_sub(1);- let can_merge = previous_index.is_some_and(|previous| {- let previous_draw = self.scratch_draws[previous];- let previous_draw_start = previous * draw_stride;- previous_draw.src_vertex_index + previous_draw.vertex_count == first_vertex- && previous_draw.packed_vertex_stride == stride- && self.draw_pipeline_keys[previous] == full_key- && self.draw_bg_keys[previous] == bg_key- && self.draw_viewports[previous] == self.current_viewport- && self.draw_scissors[previous] == self.current_scissor- && uniform_slots_equal(- &self.frame_uniform_bytes,- frame_stride,- frame_struct_size,- self.draw_frame_indices[previous],- frame_idx,- )- && self.scratch_uniform_bytes[previous_draw_start..previous_draw_start + draw_struct_size]- == *draw_uniform_bytes- });+ let mvp = if self.freelook_enabled && self.current_projection_perspective {+ self.current_projection * self.freelook+ } else {+ self.current_projection+ };++ let mut tex_dims = [UVec4::splat(1); 4];++ if num_indirect_stages > 0 {+ for slot in 0..8 {+ if (active >> slot) & 1 == 0 {+ continue;+ }- if let Some(previous) = previous_index.filter(|_| can_merge) {- let previous_frame_idx = self.draw_frame_indices[previous];- if frame_was_pushed {- self.frame_uniform_bytes.truncate(frame_bytes_len_before);- self.last_frame_uniform_index = Some(previous_frame_idx);- frame_was_pushed = false;+ let Some(tid) = &self.current_texture_ids[slot] else {+ continue;+ };++ let Some((_, tex, _)) = self.texture_cache.get(tid) else {+ continue;+ };++ let size = tex.size();+ let c = (slot % 2) * 2;+ tex_dims[slot / 2][c] = size.width;+ tex_dims[slot / 2][c + 1] = size.height;+ }}- self.merge_into_previous(previous, draw);- } else {- let first_index = self.scratch_indices.len() as u32;- let (_, index_count) = emit_draw_indices(draw, &mut self.scratch_indices, 0);-- let start = self.scratch_draws.len() * draw_stride;- self.scratch_uniform_bytes.resize(start + draw_stride, 0);- self.scratch_uniform_bytes[start..start + draw_struct_size].copy_from_slice(draw_uniform_bytes);-- self.draw_pipeline_keys.push(full_key);- self.draw_bg_keys.push(bg_key);- self.draw_viewports.push(self.current_viewport);- self.draw_scissors.push(self.current_scissor);- self.draw_frame_indices.push(frame_idx);- #[cfg(feature = "renderdoc-capture")]- self.draw_primitives.push(draw.primitive);-- let packed_byte_offset = self- .scratch_draws- .last()- .map(|d| d.packed_vertex_byte_offset + d.vertex_count * d.packed_vertex_stride)- .unwrap_or(0);- self.scratch_draws.push(crate::DrawRecord {- src_vertex_index: first_vertex,- vertex_count,- first_index,- index_count,- packed_vertex_byte_offset: packed_byte_offset,- packed_vertex_stride: stride,+ let draw_uniform = DrawUniforms { mvp, tex_dims };+ let draw_uniform_bytes = bytemuck::bytes_of(&draw_uniform);+ let bg_key = trim_bg_key(self.current_bind_group_key(), active);+ let stride = crate::packed_vertex_stride(shader_key.active_texcoords);++ let frame_bytes_len_before = self.frame_uniform_bytes.len();+ let mut frame_was_pushed = false;+ let frame_idx = if let Some(frame_uniform) = next_frame_uniform {+ let fstart = self.frame_uniform_bytes.len();+ self.frame_uniform_bytes.resize(fstart + frame_stride, 0);+ self.frame_uniform_bytes[fstart..fstart + frame_struct_size]+ .copy_from_slice(bytemuck::bytes_of(&frame_uniform));+ let idx = (fstart / frame_stride) as u32;+ self.last_frame_uniform_index = Some(idx);+ frame_was_pushed = true;+ idx+ } else {+ self.last_frame_uniform_index.unwrap()+ };++ let previous_index = self.scratch_draws.len().checked_sub(1);+ let can_merge = previous_index.is_some_and(|previous| {+ let previous_draw = self.scratch_draws[previous];+ let previous_draw_start = previous * draw_stride;+ previous_draw.src_vertex_index + previous_draw.vertex_count == first_vertex+ && previous_draw.packed_vertex_stride == stride+ && self.draw_pipeline_keys[previous] == full_key+ && self.draw_bg_keys[previous] == bg_key+ && self.draw_viewports[previous] == self.current_viewport+ && self.draw_scissors[previous] == self.current_scissor+ && uniform_slots_equal(+ &self.frame_uniform_bytes,+ frame_stride,+ frame_struct_size,+ self.draw_frame_indices[previous],+ frame_idx,+ )+ && self.scratch_uniform_bytes[previous_draw_start..previous_draw_start + draw_struct_size]+ == *draw_uniform_bytes});- }- debug_assert!(!frame_was_pushed || self.last_frame_uniform_index == Some(frame_idx));- self.draw_fast_path_compatible = true;+ if let Some(previous) = previous_index.filter(|_| can_merge) {+ let previous_frame_idx = self.draw_frame_indices[previous];++ if frame_was_pushed {+ self.frame_uniform_bytes.truncate(frame_bytes_len_before);+ self.last_frame_uniform_index = Some(previous_frame_idx);+ frame_was_pushed = false;+ }++ self.merge_into_previous(previous, draw);+ } else {+ let first_index = self.scratch_indices.len() as u32;+ let (_, index_count) = emit_draw_indices(draw, &mut self.scratch_indices, 0);++ let start = self.scratch_draws.len() * draw_stride;+ self.scratch_uniform_bytes.resize(start + draw_stride, 0);+ self.scratch_uniform_bytes[start..start + draw_struct_size].copy_from_slice(draw_uniform_bytes);++ self.draw_pipeline_keys.push(full_key);+ self.draw_bg_keys.push(bg_key);+ self.draw_viewports.push(self.current_viewport);+ self.draw_scissors.push(self.current_scissor);+ self.draw_frame_indices.push(frame_idx);+ #[cfg(feature = "renderdoc-capture")]+ self.draw_primitives.push(draw.primitive);++ let packed_byte_offset = self+ .scratch_draws+ .last()+ .map(|d| d.packed_vertex_byte_offset + d.vertex_count * d.packed_vertex_stride)+ .unwrap_or(0);+ self.scratch_draws.push(crate::DrawRecord {+ src_vertex_index: first_vertex,+ vertex_count,+ first_index,+ index_count,+ packed_vertex_byte_offset: packed_byte_offset,+ packed_vertex_stride: stride,+ });+ }++ debug_assert!(!frame_was_pushed || self.last_frame_uniform_index == Some(frame_idx));+ self.draw_fast_path_compatible = true;+ }}GxAction::CopyXfb {@@ -682,7 +706,7 @@ impl GxRenderer {);if self.draw_bufs_write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}let fub = std::mem::take(&mut self.frame_uniform_bytes);@@ -1169,12 +1193,13 @@ fn uniform_slots_equal(bytes: &[u8], stride: usize, size: usize, a: u32, b: u32)bytes[a..a + size] == bytes[b..b + size]}-fn draw_emits_triangles(draw: &DrawData) -> bool {+fn draw_emits_triangles(draw: &DrawSegment) -> bool {use gecko::flipper::gx::draw::Primitive;match draw.primitive {- Primitive::Triangles | Primitive::TriangleStrip | Primitive::TriangleFan => draw.vertex_count >= 3,- Primitive::Quads => draw.vertex_count >= 4,+ Primitive::Triangles | Primitive::TriangleStrip | Primitive::TriangleFan | Primitive::Quads => {+ draw.vertex_count >= 3+ }_ => {tracing::error!(?draw.primitive, "draw_emits_triangles: skipping unsupported primitive");false@@ -1182,15 +1207,17 @@ fn draw_emits_triangles(draw: &DrawData) -> bool {}}-fn emit_draw_indices(draw: &DrawData, indices_out: &mut Vec<u32>, vertex_bias: u32) -> (u32, u32) {+fn emit_draw_indices(draw: &DrawSegment, indices_out: &mut Vec<u32>, vertex_bias: u32) -> (u32, u32) {use gecko::flipper::gx::draw::Primitive;let n = draw.vertex_count;match draw.primitive {Primitive::Triangles => {- indices_out.extend((0..n).map(|i| vertex_bias + i));- (n, n)+ let emitted = n - n % 3;+ indices_out.extend((0..emitted).map(|i| vertex_bias + i));++ (n, emitted)}Primitive::Quads => {let chunks = n / 4;@@ -1198,7 +1225,16 @@ fn emit_draw_indices(draw: &DrawData, indices_out: &mut Vec<u32>, vertex_bias: ulet base = vertex_bias + q * 4;indices_out.extend_from_slice(&[base, base + 1, base + 2, base, base + 2, base + 3]);}- (n, chunks * 6)++ let mut index_count = chunks * 6;++ if n % 4 == 3 {+ let base = vertex_bias + chunks * 4;+ indices_out.extend_from_slice(&[base, base + 1, base + 2]);+ index_count += 3;+ }++ (n, index_count)}Primitive::TriangleStrip => {if n < 3 {diff --git a/crates/backend-wgpu/src/lib.rs b/crates/backend-wgpu/src/lib.rsindex d9cb85c..07af1ac 100644--- a/crates/backend-wgpu/src/lib.rs+++ b/crates/backend-wgpu/src/lib.rs@@ -421,7 +421,6 @@ pub(crate) struct PendingWriteback {pub copy_format: gecko::flipper::gx::texture::CopyFormat,pub stride: u32,pub swap_bgra: bool,- pub box_filter_downsample: bool,}impl GxRenderer {@@ -1051,12 +1050,12 @@ impl GxRenderer {pub fn debug_flush(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {self.flush_pending_draws(device, queue);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}pub fn reset_efb(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {self.flush_pending_draws(device, queue);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);self.clear_efb_region(device,@@ -1072,7 +1071,7 @@ impl GxRenderer {true,);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}pub fn efb_view(&self) -> &wgpu::TextureView {@@ -1092,7 +1091,7 @@ impl GxRenderer {]}- pub(crate) fn submit_pending(&mut self, queue: &wgpu::Queue) {+ pub(crate) fn submit_pending(&mut self, queue: &wgpu::Queue) -> Option<wgpu::SubmissionIndex> {// Ship any staged texture-upload bytes through ONE `write_buffer_with`// before finishing the encoder. wgpu orders queue writes ahead of// submitted commands, so the encoder's `copy_buffer_to_texture`@@ -1113,7 +1112,7 @@ impl GxRenderer {self.pending_command_buffers.push(encoder.finish());}if self.pending_command_buffers.is_empty() {- return;+ return None;}#[cfg(feature = "gx-stats")]@@ -1121,7 +1120,7 @@ impl GxRenderer {#[cfg(feature = "gx-stats")]let submit_started = std::time::Instant::now();- queue.submit(self.pending_command_buffers.drain(..));+ let submission_index = queue.submit(self.pending_command_buffers.drain(..));#[cfg(feature = "gx-stats")]{@@ -1140,6 +1139,8 @@ impl GxRenderer {self.efb_clear_uniform_write_pending = false;self.efb_depth_resolve_uniform_write_pending = false;self.efb_depth_cache_uniform_write_pending = false;++ Some(submission_index)}/// Append `rgba` (W*H*4 tight, no row padding) into@@ -1241,7 +1242,7 @@ impl GxRenderer {z_update: bool,) {if self.efb_clear_uniform_write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}let mut encoder = self.take_or_create_encoder(device);diff --git a/crates/backend-wgpu/src/render.rs b/crates/backend-wgpu/src/render.rsindex 45832a7..503c689 100644--- a/crates/backend-wgpu/src/render.rs+++ b/crates/backend-wgpu/src/render.rs@@ -209,7 +209,7 @@ impl GxRenderer {let needs_shader_copy = dst_h != height || (gamma - 1.0).abs() > f32::EPSILON;if needs_shader_copy && self.xfb_copy_uniform_write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}let mut encoder = self.take_or_create_encoder(device);@@ -394,7 +394,7 @@ impl GxRenderer {EfbCopySource::Depth => self.efb_depth_cache_uniform_write_pending,};if write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}let uniforms = XfbCopyUniforms {@@ -643,7 +643,7 @@ impl GxRenderer {encoder.pop_debug_group();self.current_encoder = Some(encoder);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);// The staging buffer's in-flight references have just been submitted;// now's the safe moment to grow it if this frame exceeded capacity.self.maybe_grow_texture_staging(device);@@ -689,7 +689,7 @@ impl GxRenderer {},);- self.submit_pending(queue);+ let _ = self.submit_pending(queue);self.xfb_has_content = true;}@@ -777,34 +777,37 @@ impl GxRenderer {return;}- // wgpu requires 256-byte row alignment for texture<->buffer copies.- let bytes_per_row = align_up(width as u64 * 4, 256);- let staging_size = bytes_per_row * height as u64;+ let readback_width = if mipmap { (width / 2).max(1) } else { width };+ let readback_height = if mipmap { (height / 2).max(1) } else { height };+ self.discard_superseded_writebacks(dest_addr, readback_width, readback_height, copy_format_enum, stride);++ let bytes_per_row = align_up(readback_width as u64 * 4, 256);+ let staging_size = bytes_per_row * readback_height as u64;let (staging, staging_capacity) = self.acquire_readback_staging(device, staging_size);let group_label = format!("CopyEfbToTexture addr={dest_addr:#010x} src=({src_x},{src_y} {width}x{height}) fmt={copy_format_enum:?} mip={mipmap} stride={stride} depth={depth_copy}");- if self.efb_scale > 1 {+ if self.efb_scale > 1 || mipmap {if self.efb_color_readback_uniform_write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}Self::ensure_writeback_target(device,&mut self.efb_color_readback_target,- width,- height,+ readback_width,+ readback_height,self.surface_format,"efb_color_readback_target",);let uniforms = XfbCopyUniforms {src_rect: self.scaled_src_rect(src_x, src_y, width, height),- dst_size: [width as f32, height as f32],+ dst_size: [readback_width as f32, readback_height as f32],gamma: 1.0,- filter_mode: 0,+ filter_mode: if mipmap { 2 } else { 0 },};queue.write_buffer(&self.efb_color_readback_uniform_buffer,@@ -816,7 +819,7 @@ impl GxRenderer {let mut encoder = self.take_or_create_encoder(device);encoder.push_debug_group(&group_label);- let (src_texture, src_origin) = if self.efb_scale == 1 {+ let (src_texture, src_origin) = if self.efb_scale == 1 && !mipmap {(&self.efb_texture,wgpu::Origin3d {@@ -845,8 +848,8 @@ impl GxRenderer {timestamp_writes: None,multiview_mask: None,});- rpass.set_viewport(0.0, 0.0, width as f32, height as f32, 0.0, 1.0);- rpass.set_scissor_rect(0, 0, width, height);+ rpass.set_viewport(0.0, 0.0, readback_width as f32, readback_height as f32, 0.0, 1.0);+ rpass.set_scissor_rect(0, 0, readback_width, readback_height);rpass.set_pipeline(&self.xfb_copy_pipeline);rpass.set_bind_group(0, &self.efb_color_readback_bind_group, &[]);rpass.draw(0..3, 0..1);@@ -867,19 +870,19 @@ impl GxRenderer {layout: wgpu::TexelCopyBufferLayout {offset: 0,bytes_per_row: Some(bytes_per_row as u32),- rows_per_image: Some(height),+ rows_per_image: Some(readback_height),},},wgpu::Extent3d {- width,- height,+ width: readback_width,+ height: readback_height,depth_or_array_layers: 1,},);encoder.pop_debug_group();self.current_encoder = Some(encoder);- if self.efb_scale > 1 {+ if self.efb_scale > 1 || mipmap {self.efb_color_readback_uniform_write_pending = true;}@@ -893,12 +896,11 @@ impl GxRenderer {staging_capacity,bytes_per_row,staging_size,- width,- height,+ width: readback_width,+ height: readback_height,copy_format: copy_format_enum,stride,swap_bgra,- box_filter_downsample: mipmap,});self.cache_efb_copy(@@ -933,6 +935,7 @@ impl GxRenderer {let divisor = if mipmap { 2 } else { 1 };let encode_w = (width / divisor).max(1);let encode_h = (height / divisor).max(1);+ self.discard_superseded_writebacks(dest_addr, encode_w, encode_h, copy_format_enum, stride);Self::ensure_writeback_target(device,@@ -944,7 +947,7 @@ impl GxRenderer {);if self.efb_depth_resolve_uniform_write_pending {- self.submit_pending(queue);+ let _ = self.submit_pending(queue);}let uniforms = XfbCopyUniforms {@@ -1023,7 +1026,6 @@ impl GxRenderer {copy_format: copy_format_enum,stride,swap_bgra: false,- box_filter_downsample: false,});}@@ -1079,6 +1081,33 @@ impl GxRenderer {(buf, capacity)}+ fn discard_superseded_writebacks(+ &mut self,+ dest_addr: Address,+ width: u32,+ height: u32,+ copy_format: texture::CopyFormat,+ stride: u32,+ ) {+ let mut i = 0;++ while i < self.pending_writebacks.len() {+ let pending = &self.pending_writebacks[i];++ if pending.dest_addr == dest_addr+ && pending.width == width+ && pending.height == height+ && pending.copy_format == copy_format+ && pending.stride == stride+ {+ let old = self.pending_writebacks.swap_remove(i);+ self.return_readback_staging(old.staging, old.staging_capacity);+ } else {+ i += 1;+ }+ }+ }+pub(crate) fn return_readback_staging(&mut self, buf: wgpu::Buffer, capacity: u64) {const MAX_PER_BUCKET: usize = 8;let bucket = self.efb_readback_staging_pool.entry(capacity).or_default();@@ -1097,7 +1126,7 @@ impl GxRenderer {return;}- self.submit_pending(queue);+ let submission_index = self.submit_pending(queue);for pending in &self.pending_writebacks {pending@@ -1107,7 +1136,7 @@ impl GxRenderer {}if let Err(err) = device.poll(wgpu::PollType::Wait {- submission_index: None,+ submission_index,timeout: Some(std::time::Duration::from_secs(5)),}) {tracing::warn!(?err, "efb writeback drain: device poll failed");@@ -1146,19 +1175,9 @@ impl GxRenderer {}w.staging.unmap();- let (encode_w, encode_h, encode_src) = if w.box_filter_downsample {- (- w.width / 2,- w.height / 2,- texture::downsample_box_2x(&rgba, w.width, w.height),- )- } else {- (w.width, w.height, rgba)- };-- let encoded = texture::encode_from_rgba(&encode_src, encode_w as usize, encode_h as usize, w.copy_format);- let row_bytes = texture::encoded_row_bytes(encode_w, w.copy_format);- let row_count = texture::encoded_row_count(encode_h, w.copy_format);+ let encoded = texture::encode_from_rgba(&rgba, w.width as usize, w.height as usize, w.copy_format);+ let row_bytes = texture::encoded_row_bytes(w.width, w.copy_format);+ let row_count = texture::encoded_row_count(w.height, w.copy_format);let dest_stride_bytes = w.stride as usize;texture::write_strided_copy_to_ram(ram, w.dest_addr, &encoded, row_bytes, row_count, dest_stride_bytes);diff --git a/crates/backend-wgpu/src/shaders/xfb_copy.wgsl b/crates/backend-wgpu/src/shaders/xfb_copy.wgslindex 8933b46..85784aa 100644--- a/crates/backend-wgpu/src/shaders/xfb_copy.wgsl+++ b/crates/backend-wgpu/src/shaders/xfb_copy.wgsl@@ -30,6 +30,21 @@ fn vs_main(@builtin(vertex_index) vi: u32) -> VsOut {fn fs_main(@builtin(position) position: vec4<f32>) -> @location(0) vec4<f32> {let tex_size = vec2<f32>(textureDimensions(efb_tex));let dst_size = max(u.dst_size, vec2<f32>(1.0, 1.0));++ if (u.filter_mode == 2u) {+ let src_step = u.src_rect.zw / dst_size;+ let first = u.src_rect.xy + (position.xy - vec2<f32>(0.5)) * src_step;+ let coord = vec2<i32>(floor(first));+ let half_step = vec2<i32>(src_step * 0.5);++ return (+ textureLoad(efb_tex, coord, 0) ++ textureLoad(efb_tex, coord + vec2<i32>(half_step.x, 0), 0) ++ textureLoad(efb_tex, coord + vec2<i32>(0, half_step.y), 0) ++ textureLoad(efb_tex, coord + half_step, 0)+ ) * 0.25;+ }+let src_pixel = u.src_rect.xy + (position.xy / dst_size) * u.src_rect.zw;let uv = src_pixel / tex_size;diff --git a/crates/backend-wgpu/src/sink.rs b/crates/backend-wgpu/src/sink.rsindex 3820bcb..ce3844d 100644--- a/crates/backend-wgpu/src/sink.rs+++ b/crates/backend-wgpu/src/sink.rs@@ -3,7 +3,7 @@ use gecko::host::GxAction;#[cfg(not(target_arch = "wasm32"))]use crate::GxRenderer;#[cfg(not(target_arch = "wasm32"))]-use gecko::host::{DrawData, DrawVertex, RenderSink};+use gecko::host::{DrawData, DrawSegment, DrawState, DrawVertex, RenderSink};#[cfg(all(not(target_arch = "wasm32"), feature = "gx-stats"))]use std::sync::atomic::Ordering;#[cfg(not(target_arch = "wasm32"))]@@ -66,7 +66,18 @@ pub struct ThreadedSink {worker_thread: Option<JoinHandle<()>>,pending_actions: Vec<GxAction>,scratch: Vec<DrawVertex>,+ immediate_draw_data: Option<Box<DrawData>>,efb_writeback_pending: bool,+ pending_efb_textures: Vec<PendingEfbTexture>,+}++#[cfg(not(target_arch = "wasm32"))]+struct PendingEfbTexture {+ addr: u32,+ end: u32,+ width: u32,+ height: u32,+ format: gecko::flipper::gx::draw::TextureFormat,}#[cfg(not(target_arch = "wasm32"))]@@ -117,7 +128,7 @@ impl RenderWorker {}}- self.gx.submit_pending(&self.queue);+ let _ = self.gx.submit_pending(&self.queue);self.gx.poll_compiled_pipelines();#[cfg(feature = "renderdoc-capture")]if let Ok(mut rd) = self.renderdoc.lock() {@@ -136,6 +147,10 @@ impl RenderWorker {fn exec_action(&mut self, action: GxAction) {self.gx.process_action(&self.device, &self.queue, &action);+ if matches!(&action, GxAction::CopyEfbToTexture { .. }) {+ let _ = self.gx.submit_pending(&self.queue);+ }+match action {GxAction::PresentXfb { .. } | GxAction::PresentRawXfb { .. } => {#[cfg(feature = "renderdoc-capture")]@@ -204,7 +219,11 @@ impl RenderWorker {// ensures all prior EFB copy commands have reached the worker.let mem1 = std::slice::from_raw_parts_mut(request.mem1_addr as *mut u8, request.mem1_len);let mem2 = std::slice::from_raw_parts_mut(request.mem2_addr as *mut u8, request.mem2_len);- gecko::mmio::RamViewMut { mem1, mem2 }+ gecko::mmio::RamViewMut {+ mem1,+ mem2,+ memory_page_generations: &[],+ }};#[cfg(feature = "gx-stats")]let writeback_count = self.gx.pending_writebacks.len() as u64;@@ -286,8 +305,63 @@ impl RenderSink for ThreadedSink {#[cfg(feature = "gx-stats")]self.stats.actions_sent.fetch_add(1, Ordering::Relaxed);- if matches!(&action, GxAction::CopyEfbToTexture { .. }) {+ let action = match action {+ GxAction::Draw(mut draw) if draw.state.is_none() => {+ if let Some(GxAction::Draw(previous)) = self.pending_actions.last_mut() {+ previous.segments.append(&mut draw.segments);+ self.immediate_draw_data = Some(draw);++ return;+ }++ GxAction::Draw(draw)+ }+ action => action,+ };++ if let GxAction::CopyEfbToTexture {+ dest_addr,+ src_x,+ src_y,+ src_w,+ src_h,+ copy_format,+ mipmap,+ stride,+ depth_copy,+ ..+ } = &action+ {+ use gecko::flipper::gx::texture::{CopyFormat, encoded_row_bytes, encoded_row_count};self.efb_writeback_pending = true;+ let format = if *depth_copy {+ CopyFormat::from_u8_depth(*copy_format)+ } else {+ CopyFormat::from_u8_color(*copy_format)+ };++ if let Some(format) = format {+ let divisor = if *mipmap { 2 } else { 1 };+ let width = src_w.min(&crate::EFB_WIDTH.saturating_sub(*src_x)) / divisor;+ let height = src_h.min(&crate::EFB_HEIGHT.saturating_sub(*src_y)) / divisor;+ let row_bytes = encoded_row_bytes(width, format);+ let rows = encoded_row_count(height, format);+ let len = rows.saturating_sub(1) * *stride as usize + row_bytes;+ let end = dest_addr.saturating_add(len as u32);+ self.pending_efb_textures+ .retain(|copy| copy.end <= *dest_addr || end <= copy.addr);+ if width > 0 && height > 0 && *stride as usize == row_bytes {+ self.pending_efb_textures.push(PendingEfbTexture {+ addr: *dest_addr,+ end,+ width,+ height,+ format: format.base_texture_format(),+ });+ }+ }+ } else if matches!(&action, GxAction::InvalidateCaches) {+ self.pending_efb_textures.clear();}let resets_scratch = action_resets_vertex_scratch(&action);@@ -298,6 +372,49 @@ impl RenderSink for ThreadedSink {}}+ fn exec_draw(&mut self, segment: DrawSegment, state: Option<DrawState>) {+ self.exec_draw_batch(std::slice::from_ref(&segment), state);+ }++ fn exec_draw_batch(&mut self, segments: &[DrawSegment], state: Option<DrawState>) {+ if segments.is_empty() {+ return;+ }++ #[cfg(feature = "gx-stats")]+ self.stats+ .actions_sent+ .fetch_add(segments.len() as u64, Ordering::Relaxed);++ self.reclaim_batch();++ if state.is_none()+ && let Some(GxAction::Draw(previous)) = self.pending_actions.last_mut()+ {+ previous.segments.extend_from_slice(segments);++ return;+ }++ let mut draw = self.take_draw_data();+ draw.segments.clear();+ draw.segments.extend_from_slice(segments);+ draw.state = state;+ self.pending_actions.push(GxAction::Draw(draw));+ }++ fn has_pending_efb_texture(+ &self,+ addr: u32,+ width: u32,+ height: u32,+ fmt: gecko::flipper::gx::draw::TextureFormat,+ ) -> bool {+ self.pending_efb_textures+ .iter()+ .any(|copy| copy.addr == addr && copy.width == width && copy.height == height && copy.format == fmt)+ }+fn flush_efb_copies(&mut self, ram: &mut gecko::mmio::RamViewMut<'_>) {if !self.efb_writeback_pending {return;@@ -333,6 +450,7 @@ impl RenderSink for ThreadedSink {self.pending_actions = batch.actions;self.scratch = batch.vertices;self.efb_writeback_pending = false;+ self.pending_efb_textures.clear();}fn vertex_scratch(&mut self) -> &mut Vec<DrawVertex> {@@ -342,7 +460,10 @@ impl RenderSink for ThreadedSink {fn take_draw_data(&mut self) -> Box<DrawData> {self.reclaim_batch();- self.recycled_draw_data_rx.try_recv().unwrap_or_default()+ self.immediate_draw_data+ .take()+ .or_else(|| self.recycled_draw_data_rx.try_recv().ok())+ .unwrap_or_default()}fn render_stats(&self) -> gecko::host::RenderStats {@@ -585,7 +706,9 @@ impl Renderer {worker_thread: Some(worker_thread),pending_actions: Vec::new(),scratch: Vec::new(),+ immediate_draw_data: None,efb_writeback_pending: false,+ pending_efb_textures: Vec::new(),};let renderer = Renderer {diff --git a/crates/gecko/src/flipper/cp.rs b/crates/gecko/src/flipper/cp.rsindex 4fa4161..e36b7fb 100644--- a/crates/gecko/src/flipper/cp.rs+++ b/crates/gecko/src/flipper/cp.rs@@ -297,6 +297,7 @@ pub fn pump_fifo<const SYSTEM: SystemId>(sys: &mut System<SYSTEM>) {if consumed > 0 {sys.gx.drain_fifo(&mut sys.mmio, sys.render_sink.as_mut());+ sys.gx.defer_pending_efb_writebacks(&mut sys.mmio);sys.check_gx_pe_interrupts();sys.cp.refresh_status();refresh_interrupts(sys);diff --git a/crates/gecko/src/flipper/dsp.rs b/crates/gecko/src/flipper/dsp.rsindex ee873d9..2f7609b 100644--- a/crates/gecko/src/flipper/dsp.rs+++ b/crates/gecko/src/flipper/dsp.rs@@ -21,7 +21,9 @@ pub mod lut_wii {use crate::flipper::dsp::instruction::Instruction;use crate::mmio::Mmio;-use crate::system::{ExecutionMode, System, SystemId};+#[cfg(feature = "jit")]+use crate::system::ExecutionMode;+use crate::system::{System, SystemId};#[cfg(feature = "jit")]pub const DSP_JIT_CHAIN_BUDGET: u32 = 64;diff --git a/crates/gecko/src/flipper/dsp/jit.rs b/crates/gecko/src/flipper/dsp/jit.rsindex 7b6ab04..5acf750 100644--- a/crates/gecko/src/flipper/dsp/jit.rs+++ b/crates/gecko/src/flipper/dsp/jit.rs@@ -425,6 +425,13 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let entry = self.module.get_finalized_function(func_id) as usize;+ crate::jit::register_jit_code(+ "dsp",+ spec.start_pc as u32,+ entry,+ self.ctx.compiled_code().unwrap().code_buffer().len(),+ );+self.register_in_lookup_table(spec.start_pc, entry);entrydiff --git a/crates/gecko/src/flipper/dsp/jit/block.rs b/crates/gecko/src/flipper/dsp/jit/block.rsindex 0637865..38120bb 100644--- a/crates/gecko/src/flipper/dsp/jit/block.rs+++ b/crates/gecko/src/flipper/dsp/jit/block.rs@@ -24,6 +24,7 @@ pub struct BlockSpec {pub instrs: Vec<InstrEntry>,pub terminator: TermKind,pub fallthrough_pc: u16,+ pub unrolled_loop: bool,}fn read_imem_word(iram: &[u8], irom: &[u8], addr: u16) -> Option<u16> {@@ -108,10 +109,95 @@ pub fn discover(iram: &[u8], irom: &[u8], start_pc: u16) -> BlockSpec {pc = pc.wrapping_add(size as u16);}- BlockSpec {+ let mut spec = BlockSpec {start_pc,instrs,terminator: term,fallthrough_pc: pc,+ unrolled_loop: false,+ };+ try_unroll_immediate_loop(iram, irom, &mut spec);++ spec+}++fn try_unroll_immediate_loop(iram: &[u8], irom: &[u8], spec: &mut BlockSpec) {+ const MAX_EXPANDED_INSTRS: usize = 256;++ if spec.terminator != TermKind::LoopSetup {+ return;+ }++ let Some(setup) = spec.instrs.last() else {+ return;+ };++ let primary = setup.raw as u16;+ if primary & 0xFF00 != 0x1100 {+ return;+ }++ let count = (primary & 0x00FF) as usize;++ if count == 0 {+ return;+ }++ let end_pc = (setup.raw >> 16) as u16;+ let mut body = Vec::new();+ let mut pc = spec.fallthrough_pc;++ loop {+ let Some(w0) = read_imem_word(iram, irom, pc) else {+ return;+ };++ if classify(w0).is_some() {+ return;+ }++ let size = approx_size(w0);+ let raw = if size == 2 {+ let Some(w1) = read_imem_word(iram, irom, pc.wrapping_add(1)) else {+ return;+ };+ (w0 as u32) | ((w1 as u32) << 16)+ } else {+ w0 as u32+ };+ body.push(InstrEntry { pc, raw, size });++ let last_word = pc.wrapping_add(size as u16 - 1);++ if last_word == end_pc {+ break;+ }++ let next = pc.wrapping_add(size as u16);++ if next <= pc || next > end_pc || body.len() > MAX_EXPANDED_INSTRS {+ return;+ }+ pc = next;+ }++ let prefix_len = spec.instrs.len() - 1;+ let Some(expanded_len) = body.len().checked_mul(count).and_then(|n| n.checked_add(prefix_len)) else {+ return;+ };++ if expanded_len > MAX_EXPANDED_INSTRS {+ return;}++ spec.instrs.pop();+ spec.instrs.reserve(body.len() * count);++ for _ in 0..count {+ spec.instrs.extend(body.iter().cloned());+ }++ spec.terminator = TermKind::LengthLimit;+ spec.fallthrough_pc = end_pc.wrapping_add(1);+ spec.unrolled_loop = true;}diff --git a/crates/gecko/src/flipper/dsp/jit/translate.rs b/crates/gecko/src/flipper/dsp/jit/translate.rsindex 271d229..2483344 100644--- a/crates/gecko/src/flipper/dsp/jit/translate.rs+++ b/crates/gecko/src/flipper/dsp/jit/translate.rs@@ -1,5 +1,5 @@use cranelift_codegen::ir::{InstBuilder, MemFlagsData, Value, types};-use cranelift_frontend::FunctionBuilder;+use cranelift_frontend::{FunctionBuilder, Variable};use cranelift_module::Module;use crate::flipper::dsp::instruction::{GcDspExt, Instruction};@@ -9,13 +9,287 @@ use crate::system::SystemId;use super::abi;use super::translator::ExternFuncs;+const REG_CACHE_LEN: usize = 15;++#[inline]+fn reg_cache_offsets() -> [i32; REG_CACHE_LEN] {+ [+ abi::dsp_ac0_high_offset() as i32,+ abi::dsp_ac1_high_offset() as i32,+ abi::dsp_ac0_mid_offset() as i32,+ abi::dsp_ac1_mid_offset() as i32,+ abi::dsp_ac0_low_offset() as i32,+ abi::dsp_ac1_low_offset() as i32,+ abi::dsp_product_low_offset() as i32,+ abi::dsp_product_mid1_offset() as i32,+ abi::dsp_product_high_offset() as i32,+ abi::dsp_product_mid2_offset() as i32,+ abi::dsp_ax_base_offset() as i32,+ (abi::dsp_ax_base_offset() + 2) as i32,+ abi::dsp_axh_base_offset() as i32,+ (abi::dsp_axh_base_offset() + 2) as i32,+ abi::dsp_status_offset() as i32,+ ]+}++fn split_ac(builder: &mut FunctionBuilder, value: Value) -> (Value, Value, Value) {+ let mid_shift = builder.ins().sshr_imm(value, 16);+ let mid = builder.ins().ireduce(types::I16, mid_shift);+ let low = builder.ins().ireduce(types::I16, value);+ let high_shift = builder.ins().sshr_imm(value, 32);+ let high_byte = builder.ins().ireduce(types::I8, high_shift);+ let high = builder.ins().sextend(types::I16, high_byte);++ (high, mid, low)+}++fn compose_ac_parts(builder: &mut FunctionBuilder, high: Value, mid: Value, low: Value) -> Value {+ let high64 = builder.ins().sextend(types::I64, high);+ let high_byte = builder.ins().band_imm(high64, 0xFF);+ let mid64 = builder.ins().uextend(types::I64, mid);+ let low64 = builder.ins().uextend(types::I64, low);+ let h_shl = builder.ins().ishl_imm(high_byte, 32);+ let m_shl = builder.ins().ishl_imm(mid64, 16);+ let hm = builder.ins().bor(h_shl, m_shl);+ let merged = builder.ins().bor(hm, low64);+ let shl = builder.ins().ishl_imm(merged, 24);++ builder.ins().sshr_imm(shl, 24)+}++pub(crate) struct DspRegCache {+ vars: [Variable; REG_CACHE_LEN],+ ac_full: [Variable; 2],+ product_full: Variable,+ ac_full_stale: [bool; 2],+ product_full_stale: bool,+ dmem_bases: [Value; 2],+ dirty: [bool; REG_CACHE_LEN],+}++impl DspRegCache {+ pub(crate) fn new(builder: &mut FunctionBuilder, sys_ptr: Value) -> Self {+ let offsets = reg_cache_offsets();+ let vars = std::array::from_fn(|idx| {+ let var = builder.declare_var(types::I16);+ let value = builder+ .ins()+ .load(types::I16, MemFlagsData::trusted(), sys_ptr, offsets[idx]);+ builder.def_var(var, value);++ var+ });++ let dmem_base_offsets = [abi::dsp_dram_ptr_offset(), abi::dsp_coef_ptr_offset()];+ let dmem_bases = std::array::from_fn(|idx| {+ builder.ins().load(+ types::I64,+ MemFlagsData::trusted(),+ sys_ptr,+ dmem_base_offsets[idx] as i32,+ )+ });++ let mut cache = Self {+ vars,+ ac_full: std::array::from_fn(|_| builder.declare_var(types::I64)),+ product_full: builder.declare_var(types::I64),+ ac_full_stale: [false; 2],+ product_full_stale: false,+ dmem_bases,+ dirty: [false; REG_CACHE_LEN],+ };+ cache.rebuild_composites(builder);++ cache+ }++ fn compose_ac(&self, builder: &mut FunctionBuilder, d: u8) -> Value {+ let (high_off, mid_off, low_off) = TranslatorCtx::ac_offsets(d);+ let high = builder.use_var(self.vars[Self::index(high_off).unwrap()]);+ let mid = builder.use_var(self.vars[Self::index(mid_off).unwrap()]);+ let low = builder.use_var(self.vars[Self::index(low_off).unwrap()]);++ compose_ac_parts(builder, high, mid, low)+ }++ fn compose_product(&self, builder: &mut FunctionBuilder) -> Value {+ let ph = builder.use_var(self.vars[Self::index(abi::dsp_product_high_offset() as i32).unwrap()]);+ let pm1 = builder.use_var(self.vars[Self::index(abi::dsp_product_mid1_offset() as i32).unwrap()]);+ let pm2 = builder.use_var(self.vars[Self::index(abi::dsp_product_mid2_offset() as i32).unwrap()]);+ let pl = builder.use_var(self.vars[Self::index(abi::dsp_product_low_offset() as i32).unwrap()]);++ let ph_u8 = builder.ins().band_imm(ph, 0xFF);+ let ph_shl = builder.ins().ishl_imm(ph_u8, 8);+ let ph_i16 = builder.ins().sshr_imm(ph_shl, 8);+ let ph_i64 = builder.ins().sextend(types::I64, ph_i16);+ let ph_shifted = builder.ins().ishl_imm(ph_i64, 32);+ let pm1_64 = builder.ins().uextend(types::I64, pm1);+ let pm2_64 = builder.ins().uextend(types::I64, pm2);+ let mid_sum = builder.ins().iadd(pm1_64, pm2_64);+ let mid_shifted = builder.ins().ishl_imm(mid_sum, 16);+ let pl_64 = builder.ins().uextend(types::I64, pl);+ let upper = builder.ins().iadd(ph_shifted, mid_shifted);++ builder.ins().iadd(upper, pl_64)+ }++ fn rebuild_composites(&mut self, builder: &mut FunctionBuilder) {+ for d in 0..2 {+ let value = self.compose_ac(builder, d);+ builder.def_var(self.ac_full[d as usize], value);+ self.ac_full_stale[d as usize] = false;+ }++ let product = self.compose_product(builder);+ builder.def_var(self.product_full, product);+ self.product_full_stale = false;+ }++ #[inline]+ fn index(offset: i32) -> Option<usize> {+ reg_cache_offsets().iter().position(|&candidate| candidate == offset)+ }++ #[inline]+ pub(crate) fn load(&self, builder: &mut FunctionBuilder, sys_ptr: Value, offset: i32) -> Value {+ if let Some(idx) = Self::index(offset) {+ builder.use_var(self.vars[idx])+ } else {+ builder.ins().load(types::I16, MemFlagsData::trusted(), sys_ptr, offset)+ }+ }++ #[inline]+ fn store(&mut self, builder: &mut FunctionBuilder, sys_ptr: Value, value: Value, offset: i32) {+ if let Some(idx) = Self::index(offset) {+ builder.def_var(self.vars[idx], value);+ self.dirty[idx] = true;++ if [+ abi::dsp_ac0_high_offset() as i32,+ abi::dsp_ac0_mid_offset() as i32,+ abi::dsp_ac0_low_offset() as i32,+ ]+ .contains(&offset)+ {+ self.ac_full_stale[0] = true;+ } else if [+ abi::dsp_ac1_high_offset() as i32,+ abi::dsp_ac1_mid_offset() as i32,+ abi::dsp_ac1_low_offset() as i32,+ ]+ .contains(&offset)+ {+ self.ac_full_stale[1] = true;+ } else if [+ abi::dsp_product_low_offset() as i32,+ abi::dsp_product_mid1_offset() as i32,+ abi::dsp_product_high_offset() as i32,+ abi::dsp_product_mid2_offset() as i32,+ ]+ .contains(&offset)+ {+ self.product_full_stale = true;+ }+ } else {+ builder.ins().store(MemFlagsData::trusted(), value, sys_ptr, offset);+ }+ }++ pub(crate) fn flush(&self, builder: &mut FunctionBuilder, sys_ptr: Value) {+ let offsets = reg_cache_offsets();++ for (idx, &dirty) in self.dirty.iter().enumerate() {+ if dirty {+ let value = builder.use_var(self.vars[idx]);+ builder+ .ins()+ .store(MemFlagsData::trusted(), value, sys_ptr, offsets[idx]);+ }+ }+ }++ fn reload(&mut self, builder: &mut FunctionBuilder, sys_ptr: Value) {+ for (idx, offset) in reg_cache_offsets().into_iter().enumerate() {+ let value = builder.ins().load(types::I16, MemFlagsData::trusted(), sys_ptr, offset);+ builder.def_var(self.vars[idx], value);+ }++ self.rebuild_composites(builder);+ }++ #[inline]+ fn set_piece(&mut self, builder: &mut FunctionBuilder, offset: i32, value: Value) {+ let idx = Self::index(offset).expect("composite register piece must be cached");+ builder.def_var(self.vars[idx], value);+ self.dirty[idx] = true;+ }++ fn read_ac_i64(&mut self, builder: &mut FunctionBuilder, d: u8) -> Value {+ if self.ac_full_stale[d as usize] {+ let value = self.compose_ac(builder, d);+ builder.def_var(self.ac_full[d as usize], value);+ self.ac_full_stale[d as usize] = false;+ }++ builder.use_var(self.ac_full[d as usize])+ }++ fn write_ac_i64(&mut self, builder: &mut FunctionBuilder, d: u8, value: Value) {+ let shifted = builder.ins().ishl_imm(value, 24);+ let value = builder.ins().sshr_imm(shifted, 24);+ builder.def_var(self.ac_full[d as usize], value);+ let (high_off, mid_off, low_off) = TranslatorCtx::ac_offsets(d);+ let (high, mid, low) = split_ac(builder, value);+ self.set_piece(builder, high_off, high);+ self.set_piece(builder, mid_off, mid);+ self.set_piece(builder, low_off, low);+ self.ac_full_stale[d as usize] = false;+ }++ fn read_product_i64(&mut self, builder: &mut FunctionBuilder) -> Value {+ if self.product_full_stale {+ let value = self.compose_product(builder);+ builder.def_var(self.product_full, value);+ self.product_full_stale = false;+ }++ builder.use_var(self.product_full)+ }++ fn write_product_i64(&mut self, builder: &mut FunctionBuilder, value: Value) {+ let shifted = builder.ins().ishl_imm(value, 24);+ let value = builder.ins().sshr_imm(shifted, 24);+ builder.def_var(self.product_full, value);+ let (high, pm1, low) = split_ac(builder, value);+ let zero = builder.ins().iconst(types::I16, 0);+ self.set_piece(builder, abi::dsp_product_high_offset() as i32, high);+ self.set_piece(builder, abi::dsp_product_mid1_offset() as i32, pm1);+ self.set_piece(builder, abi::dsp_product_low_offset() as i32, low);+ self.set_piece(builder, abi::dsp_product_mid2_offset() as i32, zero);+ self.product_full_stale = false;+ }++ #[inline]+ fn dmem_base(&self, ptr_field_offset: usize) -> Option<Value> {+ match ptr_field_offset {+ offset if offset == abi::dsp_dram_ptr_offset() => Some(self.dmem_bases[0]),+ offset if offset == abi::dsp_coef_ptr_offset() => Some(self.dmem_bases[1]),+ _ => None,+ }+ }+}+pub struct TranslatorCtx<'a, 'b> {pub builder: &'a mut FunctionBuilder<'b>,pub module: &'a mut cranelift_jit::JITModule,pub extern_funcs: ExternFuncs,pub sys_ptr: Value,+ pub(crate) reg_cache: &'a mut DspRegCache,pub pc: u16,pub size: u8,+ pub szat_needed: bool,}impl<'a, 'b> TranslatorCtx<'a, 'b> {@@ -26,16 +300,12 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {#[inline]fn load_u16(&mut self, offset: i32) -> Value {- self.builder- .ins()- .load(types::I16, MemFlagsData::trusted(), self.sys_ptr, offset)+ self.reg_cache.load(self.builder, self.sys_ptr, offset)}#[inline]fn store_u16(&mut self, val: Value, offset: i32) {- self.builder- .ins()- .store(MemFlagsData::trusted(), val, self.sys_ptr, offset);+ self.reg_cache.store(self.builder, self.sys_ptr, val, offset);}fn simple_reg_offset(slot: u8) -> Option<i32> {@@ -64,6 +334,16 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}pub fn try_emit_reg_read(&mut self, slot: u8) -> Option<Value> {+ if (30..=31).contains(&slot) {+ let (high_off, mid_off, _) = Self::ac_offsets(slot - 30);+ let high = self.load_u16(high_off);+ let mid = self.load_u16(mid_off);+ let sxm = self.read_status_bit(14);+ let saturated = super::translator::emit_saturate_ac_mid(self.builder, high, mid);++ return Some(self.builder.ins().select(sxm, saturated, mid));+ }+let off = Self::simple_reg_offset(slot)?;let raw = self.load_u16(off);let v = match slot {@@ -76,6 +356,22 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}pub fn try_emit_reg_write(&mut self, slot: u8, value: Value) -> bool {+ if (30..=31).contains(&slot) {+ let (high_off, mid_off, low_off) = Self::ac_offsets(slot - 30);+ let high = self.load_u16(high_off);+ let low = self.load_u16(low_off);+ let sxm = self.read_status_bit(14);+ let sign = self.builder.ins().sshr_imm(value, 15);+ let zero = self.iconst16(0);+ let high = self.builder.ins().select(sxm, sign, high);+ let low = self.builder.ins().select(sxm, zero, low);+ self.store_u16(high, high_off);+ self.store_u16(value, mid_off);+ self.store_u16(low, low_off);++ return true;+ }+let Some(off) = Self::simple_reg_offset(slot) else {return false;};@@ -137,76 +433,19 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}pub fn read_ac_i64(&mut self, d: u8) -> Value {- let (h_off, m_off, l_off) = Self::ac_offsets(d);- let high = self.load_u16(h_off);- let mid = self.load_u16(m_off);- let low = self.load_u16(l_off);-- let high64 = self.builder.ins().sextend(types::I64, high);- let high_byte = self.builder.ins().band_imm(high64, 0xFFi64);- let mid64 = self.builder.ins().uextend(types::I64, mid);- let low64 = self.builder.ins().uextend(types::I64, low);- let h_shl = self.builder.ins().ishl_imm(high_byte, 32);- let m_shl = self.builder.ins().ishl_imm(mid64, 16);- let hm = self.builder.ins().bor(h_shl, m_shl);- let merged = self.builder.ins().bor(hm, low64);-- let shl = self.builder.ins().ishl_imm(merged, 24);- self.builder.ins().sshr_imm(shl, 24)+ self.reg_cache.read_ac_i64(self.builder, d)}pub fn write_ac_i64(&mut self, d: u8, val: Value) {- let (h_off, m_off, l_off) = Self::ac_offsets(d);- let mid_shift = self.builder.ins().sshr_imm(val, 16);- let mid = self.builder.ins().ireduce(types::I16, mid_shift);- let low = self.builder.ins().ireduce(types::I16, val);- let high_shift = self.builder.ins().sshr_imm(val, 32);-- let high_byte = self.builder.ins().ireduce(types::I8, high_shift);- let high = self.builder.ins().sextend(types::I16, high_byte);- self.store_u16(high, h_off);- self.store_u16(mid, m_off);- self.store_u16(low, l_off);+ self.reg_cache.write_ac_i64(self.builder, d, val);}pub fn read_product_i64(&mut self) -> Value {- let ph = self.load_u16(abi::dsp_product_high_offset() as i32);- let pm1 = self.load_u16(abi::dsp_product_mid1_offset() as i32);- let pm2 = self.load_u16(abi::dsp_product_mid2_offset() as i32);- let pl = self.load_u16(abi::dsp_product_low_offset() as i32);-- let ph_u8 = self.builder.ins().band_imm(ph, 0xFF);- let ph_shl = self.builder.ins().ishl_imm(ph_u8, 8);- let ph_i16 = self.builder.ins().sshr_imm(ph_shl, 8);- let ph_i64 = self.builder.ins().sextend(types::I64, ph_i16);- let ph_shifted = self.builder.ins().ishl_imm(ph_i64, 32);-- let pm1_64 = self.builder.ins().uextend(types::I64, pm1);- let pm2_64 = self.builder.ins().uextend(types::I64, pm2);- let mid_sum = self.builder.ins().iadd(pm1_64, pm2_64);- let mid_shifted = self.builder.ins().ishl_imm(mid_sum, 16);-- let pl_64 = self.builder.ins().uextend(types::I64, pl);-- let part1 = self.builder.ins().iadd(ph_shifted, mid_shifted);- self.builder.ins().iadd(part1, pl_64)+ self.reg_cache.read_product_i64(self.builder)}pub fn write_product_i64(&mut self, val: Value) {- let high_shift = self.builder.ins().sshr_imm(val, 32);- let high_i8 = self.builder.ins().ireduce(types::I8, high_shift);- let high = self.builder.ins().sextend(types::I16, high_i8);- self.store_u16(high, abi::dsp_product_high_offset() as i32);-- let pm1_shift = self.builder.ins().sshr_imm(val, 16);- let pm1 = self.builder.ins().ireduce(types::I16, pm1_shift);- self.store_u16(pm1, abi::dsp_product_mid1_offset() as i32);-- let pl = self.builder.ins().ireduce(types::I16, val);- self.store_u16(pl, abi::dsp_product_low_offset() as i32);-- let zero = self.iconst16(0);- self.store_u16(zero, abi::dsp_product_mid2_offset() as i32);+ self.reg_cache.write_product_i64(self.builder, val);}pub fn multiply_i64(&mut self, a_i16: Value, b_i16: Value) -> Value {@@ -467,6 +706,9 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}fn flags40_szat(&mut self, r40: Value) -> Value {+ if !self.szat_needed {+ return self.iconst16(0);+ }use cranelift_codegen::ir::condcodes::IntCC;let sign64 = self.builder.ins().ushr_imm(r40, 39);@@ -575,12 +817,14 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}fn emit_dmem_word_addr(&mut self, ptr_field_offset: usize, addr: Value, word_bias: i64) -> Value {- let base = self.builder.ins().load(- types::I64,- MemFlagsData::trusted(),- self.sys_ptr,- ptr_field_offset as i32,- );+ let base = self.reg_cache.dmem_base(ptr_field_offset).unwrap_or_else(|| {+ self.builder.ins().load(+ types::I64,+ MemFlagsData::trusted(),+ self.sys_ptr,+ ptr_field_offset as i32,+ )+ });let biased = if word_bias != 0 {self.builder.ins().iadd_imm(addr, word_bias)@@ -999,6 +1243,7 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}pub fn emit_read_reg_full(&mut self, slot: u32) -> Value {+ self.reg_cache.flush(self.builder, self.sys_ptr);let f = self.module.declare_func_in_func(self.extern_funcs.read_reg_full, self.builder.func);@@ -1008,12 +1253,14 @@ impl<'a, 'b> TranslatorCtx<'a, 'b> {}pub fn emit_write_reg_full(&mut self, slot: u32, value: Value) {+ self.reg_cache.flush(self.builder, self.sys_ptr);let f = self.module.declare_func_in_func(self.extern_funcs.write_reg_full, self.builder.func);let slot_v = self.builder.ins().iconst(types::I32, slot as i64);let value_u32 = self.builder.ins().uextend(types::I32, value);self.builder.ins().call(f, &[self.sys_ptr, slot_v, value_u32]);+ self.reg_cache.reload(self.builder, self.sys_ptr);}pub fn emit_read_reg(&mut self, slot: u8) -> Value {diff --git a/crates/gecko/src/flipper/dsp/jit/translator.rs b/crates/gecko/src/flipper/dsp/jit/translator.rsindex f102885..f5c12a8 100644--- a/crates/gecko/src/flipper/dsp/jit/translator.rs+++ b/crates/gecko/src/flipper/dsp/jit/translator.rs@@ -14,6 +14,88 @@ pub struct ExternFuncs {pub write_reg_full: FuncId,}+fn szat_liveness(spec: &BlockSpec, loop_end_table: &[u8; 0x10000]) -> Vec<bool> {+ use disasm::dsp::GcDspInstruction as I;+ let mut live = true;+ let mut needed = vec![true; spec.instrs.len()];+ let is_flag_independent_register = |reg: u8| reg != 19 && !(12..=15).contains(®);++ for (idx, entry) in spec.instrs.iter().enumerate().rev() {+ if !spec.unrolled_loop && loop_end_table[entry.pc as usize] != 0 {+ live = true;+ }++ let bytes = [+ (entry.raw >> 8) as u8,+ entry.raw as u8,+ (entry.raw >> 24) as u8,+ (entry.raw >> 16) as u8,+ ];+ let Some((insn, _)) = I::decode(&bytes) else {+ live = true;++ continue;+ };++ match insn {+ I::Add { .. }+ | I::Addax { .. }+ | I::Addr { .. }+ | I::Addp { .. }+ | I::Sub { .. }+ | I::Subax { .. }+ | I::Subr { .. }+ | I::Subp { .. }+ | I::Neg { .. }+ | I::Movp { .. }+ | I::Movpz { .. }+ | I::Tst { .. }+ | I::Clr { .. }+ | I::Lsl16 { .. }+ | I::Asr16 { .. }+ | I::Lsr16 { .. }+ | I::Mulac { .. }+ | I::Mulmv { .. }+ | I::Mulmvz { .. }+ | I::Mulxac { .. }+ | I::Mulxmv { .. }+ | I::Mulxmvz { .. }+ | I::Mulcac { .. }+ | I::Mulcmv { .. }+ | I::Mulcmvz { .. } => {+ needed[idx] = live;+ live = false;+ }+ I::Nop+ | I::Dar { .. }+ | I::Iar { .. }+ | I::Addarn { .. }+ | I::Subarn { .. }+ | I::Mul { .. }+ | I::Mulx { .. }+ | I::Mulc { .. }+ | I::Nx0 { .. }+ | I::Nx1 { .. }+ | I::M0 { .. }+ | I::M2 { .. }+ | I::Set15 { .. }+ | I::Clr15 { .. }+ | I::Set16 { .. }+ | I::Set40 { .. } => {}+ I::Lri { rd, .. } | I::Lr { rd, .. } if is_flag_independent_register(rd) => {}+ I::Sr { rs, .. } if is_flag_independent_register(rs) => {}+ I::Lrr { d, .. } | I::Lrrd { d, .. } | I::Lrri { d, .. } | I::Lrrn { d, .. }+ if is_flag_independent_register(d) => {}+ I::Srr { s, .. } | I::Srrd { s, .. } | I::Srri { s, .. } | I::Srrn { s, .. }+ if is_flag_independent_register(s) => {}+ I::Mrr { dst, src } if is_flag_independent_register(dst) && is_flag_independent_register(src) => {}+ _ => live = true,+ }+ }++ needed+}+pub fn translate(ctx: &mut Context,builder_ctx: &mut FunctionBuilderContext,@@ -55,16 +137,34 @@ pub fn translate(}let block_sig_ref = builder.import_signature(block_signature(pointer_type));+ let mut reg_cache = t::DspRegCache::new(&mut builder, ctx_ptr);+ let szat_needed = szat_liveness(spec, loop_end_table);for (idx, entry) in spec.instrs.iter().enumerate() {let natural_nia = entry.pc.wrapping_add(entry.size as u16);- let nia_v = builder.ins().iconst(types::I16, natural_nia as i64);- builder.ins().store(MemFlagsData::trusted(), nia_v, ctx_ptr, nia_offset);+ let is_last = idx + 1 == spec.instrs.len();+ let dynamic_nia = is_last && spec.terminator != TermKind::LengthLimit;+ let marked_loop_end = !spec.unrolled_loop && loop_end_table[entry.pc as usize] != 0;++ if is_last || marked_loop_end {+ let nia_v = builder.ins().iconst(types::I16, natural_nia as i64);+ builder.ins().store(MemFlagsData::trusted(), nia_v, ctx_ptr, nia_offset);+ }let primary = (entry.raw & 0xFFFF) as u16;let has_ext = ((primary >> 12) & 0xF) >= 3;- if has_ext {- emit_cache_ext_ac_inline(&mut builder, ctx_ptr);+ let ext_byte = has_ext.then_some({+ if ((primary >> 12) & 0xF) == 3 {+ primary & 0x7F+ } else {+ primary & 0xFF+ }+ });+ let ext_ac_source =+ ext_byte.and_then(|ext| emit_ext_ac_source_value(&mut builder, ctx_ptr, ®_cache, ext as u8));+ if let Some((source, value)) = ext_ac_source {+ let cache_off = super::abi::dsp_ext_ac_cache_base_offset() as i32 + source as i32 * 2;+ builder.ins().store(MemFlagsData::trusted(), value, ctx_ptr, cache_off);}let mut tctx = t::TranslatorCtx {@@ -72,36 +172,35 @@ pub fn translate(module,extern_funcs: *extern_funcs,sys_ptr: ctx_ptr,+ reg_cache: &mut reg_cache,pc: entry.pc,size: entry.size,+ szat_needed: szat_needed[idx],};jit_lut::dispatch(&mut tctx, Instruction(entry.raw));if has_ext {- let ext_byte = if ((primary >> 12) & 0xF) == 3 {- primary & 0x7F- } else {- primary & 0xFF- };-let mut tctx2 = t::TranslatorCtx {builder: &mut builder,module,extern_funcs: *extern_funcs,sys_ptr: ctx_ptr,+ reg_cache: &mut reg_cache,pc: entry.pc,size: entry.size,+ szat_needed: szat_needed[idx],};- jit_lut::dispatch_gc_dsp_ext(&mut tctx2, crate::flipper::dsp::instruction::GcDspExt(ext_byte as u8));+ jit_lut::dispatch_gc_dsp_ext(+ &mut tctx2,+ crate::flipper::dsp::instruction::GcDspExt(ext_byte.unwrap() as u8),+ );}- let is_last = idx + 1 == spec.instrs.len();- let dynamic_nia = is_last && spec.terminator != TermKind::LengthLimit;- let marked_loop_end = loop_end_table[entry.pc as usize] != 0;-if !dynamic_nia && !marked_loop_end {- let pc_v = builder.ins().iconst(types::I16, natural_nia as i64);- builder.ins().store(MemFlagsData::trusted(), pc_v, ctx_ptr, pc_offset);+ if is_last {+ let pc_v = builder.ins().iconst(types::I16, natural_nia as i64);+ builder.ins().store(MemFlagsData::trusted(), pc_v, ctx_ptr, pc_offset);+ }continue;}@@ -232,6 +331,7 @@ pub fn translate(block_sig_ref,block_lookup_table_addr,idx as i64 + 1,+ ®_cache,);builder.switch_to_block(continue_block);@@ -244,6 +344,7 @@ pub fn translate(block_sig_ref,block_lookup_table_addr,spec.instrs.len() as i64,+ ®_cache,);builder.finalize();@@ -255,6 +356,7 @@ fn emit_block_tail_chain(block_sig_ref: cranelift_codegen::ir::SigRef,block_lookup_table_addr: i64,block_instr_count: i64,+ reg_cache: &super::translate::DspRegCache,) {use cranelift_codegen::ir::MemFlagsData;use cranelift_codegen::ir::condcodes::IntCC;@@ -263,6 +365,7 @@ fn emit_block_tail_chain(let chain_budget_offset = super::abi::dsp_chain_budget_offset() as i32;let instr_count_offset = super::abi::dsp_instr_count_offset() as i32;+ reg_cache.flush(builder, ctx_ptr);let instr_count = builder.ins().load(types::I32, MemFlagsData::trusted(), ctx_ptr, instr_count_offset);@@ -319,72 +422,49 @@ fn emit_block_tail_chain(.return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr]);}-fn emit_cache_ext_ac_inline(builder: &mut FunctionBuilder, ctx_ptr: Value) {- use cranelift_codegen::ir::MemFlagsData;+fn emit_ext_ac_source_value(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ reg_cache: &super::translate::DspRegCache,+ ext: u8,+) -> Option<(u8, Value)> {use cranelift_codegen::ir::condcodes::IntCC;- let ac0_low_off = super::abi::dsp_ac0_low_offset() as i32;- let ac1_low_off = super::abi::dsp_ac1_low_offset() as i32;- let ac0_mid_off = super::abi::dsp_ac0_mid_offset() as i32;- let ac1_mid_off = super::abi::dsp_ac1_mid_offset() as i32;- let ac0_hi_off = super::abi::dsp_ac0_high_offset() as i32;- let ac1_hi_off = super::abi::dsp_ac1_high_offset() as i32;- let status_off = super::abi::dsp_status_offset() as i32;- let cache_base = super::abi::dsp_ext_ac_cache_base_offset() as i32;-- let ac0_low = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac0_low_off);- let ac1_low = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac1_low_off);- let ac0_mid = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac0_mid_off);- let ac1_mid = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac1_mid_off);- let ac0_hi = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac0_hi_off);- let ac1_hi = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, ac1_hi_off);-- builder- .ins()- .store(MemFlagsData::trusted(), ac0_low, ctx_ptr, cache_base);- builder- .ins()- .store(MemFlagsData::trusted(), ac1_low, ctx_ptr, cache_base + 2);- builder- .ins()- .store(MemFlagsData::trusted(), ac0_mid, ctx_ptr, cache_base + 8);- builder- .ins()- .store(MemFlagsData::trusted(), ac1_mid, ctx_ptr, cache_base + 10);+ let source = match ext {+ 0x10..=0x1F => ext & 0x3,+ 0x20..=0x3F => (ext >> 3) & 0x3,+ 0x80..=0xBF => 4 + (ext & 0x1),+ _ => return None,+ };- let status = builder- .ins()- .load(types::I16, MemFlagsData::trusted(), ctx_ptr, status_off);- let sxm_shifted = builder.ins().ushr_imm(status, 14);- let sxm = builder.ins().band_imm(sxm_shifted, 1);- let sxm_set = builder.ins().icmp_imm(IntCC::NotEqual, sxm, 0);+ let ac = (source & 1) as usize;+ let mid_off = [super::abi::dsp_ac0_mid_offset(), super::abi::dsp_ac1_mid_offset()][ac] as i32;+ let value = match source {+ 0 | 1 => {+ let low_off = [super::abi::dsp_ac0_low_offset(), super::abi::dsp_ac1_low_offset()][ac] as i32;- let sat0 = emit_saturate_ac_mid(builder, ac0_hi, ac0_mid);- let sat1 = emit_saturate_ac_mid(builder, ac1_hi, ac1_mid);+ reg_cache.load(builder, ctx_ptr, low_off)+ }+ 2 | 3 => {+ let high_off = [super::abi::dsp_ac0_high_offset(), super::abi::dsp_ac1_high_offset()][ac] as i32;+ let mid = reg_cache.load(builder, ctx_ptr, mid_off);+ let high = reg_cache.load(builder, ctx_ptr, high_off);+ let status = reg_cache.load(builder, ctx_ptr, super::abi::dsp_status_offset() as i32);+ let sxm_shifted = builder.ins().ushr_imm(status, 14);+ let sxm = builder.ins().band_imm(sxm_shifted, 1);+ let sxm_set = builder.ins().icmp_imm(IntCC::NotEqual, sxm, 0);+ let saturated = emit_saturate_ac_mid(builder, high, mid);++ builder.ins().select(sxm_set, saturated, mid)+ }+ 4 | 5 => reg_cache.load(builder, ctx_ptr, mid_off),+ _ => unreachable!(),+ };- let sel0 = builder.ins().select(sxm_set, sat0, ac0_mid);- let sel1 = builder.ins().select(sxm_set, sat1, ac1_mid);- builder- .ins()- .store(MemFlagsData::trusted(), sel0, ctx_ptr, cache_base + 4);- builder- .ins()- .store(MemFlagsData::trusted(), sel1, ctx_ptr, cache_base + 6);+ Some((source, value))}-fn emit_saturate_ac_mid(builder: &mut FunctionBuilder, high: Value, mid: Value) -> Value {+pub(super) fn emit_saturate_ac_mid(builder: &mut FunctionBuilder, high: Value, mid: Value) -> Value {use cranelift_codegen::ir::condcodes::IntCC;let sign_ext = builder.ins().sshr_imm(mid, 15);diff --git a/crates/gecko/src/flipper/dsp/regs.rs b/crates/gecko/src/flipper/dsp/regs.rsindex 96e3b7c..e1dba9c 100644--- a/crates/gecko/src/flipper/dsp/regs.rs+++ b/crates/gecko/src/flipper/dsp/regs.rs@@ -4,6 +4,8 @@ use crate::scheduler;use crate::system::{System, SystemId};use chapa::BitEnum;+const MAILBOX_DRAIN_MAX_STEPS: u32 = 256 * 1024;+// 0xCC00500A 2 [R/W] CSR - DSP Control/Status Register#[derive(BitEnum, Debug, PartialEq, Eq)]@@ -182,7 +184,7 @@ impl<const SYSTEM: SystemId> MmioAccess<System<SYSTEM>> for MailboxToDspLo {// Based off of FFCC, thanks SpinningCube, zayd, JustinCase// TODO: WHAT THE FUCK KINDA NUMBER SHOULD I USE// used to be 64 * 1024 and worked well on many games...- sys.drain_dsp_synchronous(64 * 1024);+ sys.drain_dsp_synchronous(MAILBOX_DRAIN_MAX_STEPS);crate::flipper::dsp::wake_dsp_scheduler::<SYSTEM>(sys);}}diff --git a/crates/gecko/src/flipper/gx.rs b/crates/gecko/src/flipper/gx.rsindex b042a4c..9766986 100644--- a/crates/gecko/src/flipper/gx.rs+++ b/crates/gecko/src/flipper/gx.rs@@ -34,6 +34,10 @@ pub struct GraphicsProcessor {pub xf_mem: Vec<u32>,pub fifo: Vec<u8>,pub dl_scratch: Vec<u8>,+ draw_batch_data: Vec<u8>,+ draw_batch_entries: Vec<vertex::DrawBatchEntry>,+ draw_segments_scratch: Vec<crate::host::DrawSegment>,+ pub(crate) pending_efb_writeback_ranges: Vec<(u32, usize)>,// FIFO recording stuffpub recorder: Option<Box<recorder::FifoRecorder>>,@@ -102,6 +106,10 @@ pub struct GraphicsProcessor {pub jit_vtx: jit::JitVertexEngine,#[cfg(feature = "jit")]pub jit_vtx_arrays: jit::ResolvedArrays,+ #[cfg(feature = "jit")]+ pub jit_vtx_arrays_key: Option<jit::VtxKey>,+ #[cfg(feature = "jit")]+ pub jit_vtx_arrays_ok: bool,#[cfg(feature = "vtx-jit-validate")]pub jit_vtx_validator: jit::validate::VertexJitValidator,pub lighting_dirty: bool,@@ -114,11 +122,7 @@ pub struct GraphicsProcessor {pub cached_lights: [LightData; 8],#[cfg(feature = "gx-stats")]pub(crate) stats: GxStats,- // Hash of the raw texture data at each cache key; used to detect when- // texture content changes and avoid redundant decodes + LoadTexture- // sends. Keyed by the same `TextureKey` sent to the renderer in- // [`GxAction::LoadTexture`].- pub texture_hashes: FxHashMap<TextureKey, u64>,+ pub texture_hashes: FxHashMap<TextureKey, (u64, u64)>,pub execution_mode: ExecutionMode,}@@ -153,6 +157,15 @@ pub(crate) struct GxStats {}impl GraphicsProcessor {+ pub(crate) fn defer_pending_efb_writebacks<const SYSTEM: SystemId>(+ &mut self,+ mmio: &mut crate::mmio::Mmio<SYSTEM>,+ ) {+ for (addr, len) in self.pending_efb_writeback_ranges.drain(..) {+ mmio.defer_efb_writeback(addr, len);+ }+ }+pub fn new() -> Self {GraphicsProcessor {raise_interrupt: false,@@ -165,6 +178,10 @@ impl GraphicsProcessor {xf_mem: vec![0; XF_MEM_SIZE],fifo: Vec::with_capacity(256),dl_scratch: Vec::with_capacity(4096),+ draw_batch_data: Vec::with_capacity(4096),+ draw_batch_entries: Vec::with_capacity(32),+ draw_segments_scratch: Vec::with_capacity(32),+ pending_efb_writeback_ranges: Vec::new(),recorder: None,projection: Matrix4::default(),cur_textures: Default::default(),@@ -209,6 +226,10 @@ impl GraphicsProcessor {jit_vtx: jit::JitVertexEngine::new(),#[cfg(feature = "jit")]jit_vtx_arrays: jit::ResolvedArrays::default(),+ #[cfg(feature = "jit")]+ jit_vtx_arrays_key: None,+ #[cfg(feature = "jit")]+ jit_vtx_arrays_ok: false,#[cfg(feature = "vtx-jit-validate")]jit_vtx_validator: jit::validate::VertexJitValidator::new(),lighting_dirty: true,@@ -651,6 +672,18 @@ impl GraphicsProcessor {self.xfb_fields_since_emit = r.u32()?;self.xfb_last_emit_gen = r.u64()?;+ self.xfb_regions.clear();+ self.xfb_dirty = false;+ self.xfb_copy_seq = 0;+ self.xfb_present_seq = 0;+ self.xfb_last_present_base = 0;+ self.xfb_last_seen_base = 0;+ self.xfb_prev_base = 0;+ self.xfb_fields_since_flip = 0;+ self.xfb_flip_interval = 1;+ self.xfb_fields_since_emit = 0;+ self.xfb_last_emit_gen = 0;+self.cached_color_ctrl = r.pod()?;self.cached_alpha_ctrl = r.pod()?;self.cached_ambient_color = r.pod()?;@@ -658,7 +691,16 @@ impl GraphicsProcessor {self.cached_lights = r.pod()?;self.dl_scratch.clear();+ self.draw_batch_data.clear();+ self.draw_batch_entries.clear();+ self.draw_segments_scratch.clear();self.texture_hashes.clear();+ #[cfg(feature = "jit")]+ {+ self.jit_vtx_arrays_key = None;+ self.jit_vtx_arrays_ok = false;+ }+self.tex_dirty = 0xFF;self.lighting_dirty = true;self.konst_dirty = true;diff --git a/crates/gecko/src/flipper/gx/bp.rs b/crates/gecko/src/flipper/gx/bp.rsindex 97622d1..ac4f574 100644--- a/crates/gecko/src/flipper/gx/bp.rs+++ b/crates/gecko/src/flipper/gx/bp.rs@@ -3,7 +3,7 @@ use super::recorder::MemoryUpdateType;use super::regs::*;use super::{GraphicsProcessor, draw, texture};use crate::host::{GxAction, RenderSink, TextureKey};-use crate::mmio::{RamView, RamViewMut};+use crate::mmio::{Mmio, RamView, RamViewMut};impl GraphicsProcessor {#[cfg_attr(feature = "hotpath", hotpath::measure)]@@ -219,16 +219,22 @@ impl GraphicsProcessor {// PE finishif idx == BP_PE_DONE && (val & BP_PE_DONE_FINISH_BIT) != 0 {self.raise_interrupt = true;- renderer.flush_efb_copies(ram);- self.sync_recorder_efb_shadow(ram);++ if self.recorder.is_some() {+ renderer.flush_efb_copies(ram);+ self.sync_recorder_efb_shadow(ram);+ }}// PE token (0x47): store token value onlyif idx == BP_PE_TOKEN {self.pending_token = (val & 0xFFFF) as u16;self.token_dirty = true;- renderer.flush_efb_copies(ram);- self.sync_recorder_efb_shadow(ram);++ if self.recorder.is_some() {+ renderer.flush_efb_copies(ram);+ self.sync_recorder_efb_shadow(ram);+ }}// PE token interrupt (0x48): store token value + raise interrupt@@ -237,8 +243,11 @@ impl GraphicsProcessor {self.pending_token = (val & 0xFFFF) as u16;self.token_dirty = true;self.raise_token_interrupt = true;- renderer.flush_efb_copies(ram);- self.sync_recorder_efb_shadow(ram);++ if self.recorder.is_some() {+ renderer.flush_efb_copies(ram);+ self.sync_recorder_efb_shadow(ram);+ }}// Scissor registers (BP 0x20 = TL, 0x21 = BR, 0x59 = OFFSET).@@ -308,17 +317,39 @@ impl GraphicsProcessor {rec.use_memory(ram, ram_addr as u32, raw_size, MemoryUpdateType::TextureMap);}- let changed = match tex_slice {- Some(tex) => self::texture_data_changed(&mut self.texture_hashes, tex, cache_id, palette, tlut, format),- None => {- tracing::warn!(- addr = format!("{ram_addr:#010X}"),- raw_size,- "texture: address not mapped to MEM1/MEM2, skipping decode"- );- false- }- };+ let gpu_copy =+ self.recorder.is_none() && renderer.has_pending_efb_texture(ram_addr as u32, width, height, format);+ let changed = !gpu_copy+ && match tex_slice {+ Some(tex) => self::texture_data_changed(+ &mut self.texture_hashes,+ tex,+ cache_id,+ palette,+ tlut,+ format,+ ram.range_generation(ram_addr, raw_size).unwrap_or(u64::MAX),+ ),+ None => {+ tracing::warn!(+ addr = format!("{ram_addr:#010X}"),+ raw_size,+ "texture: address not mapped to MEM1/MEM2, skipping decode"+ );++ false+ }+ };++ if gpu_copy {+ renderer.exec(GxAction::LoadTexture {+ id: cache_id,+ width,+ height,+ fmt: format,+ rgba: Vec::new(),+ });+ }if changed {let desc = draw::TextureDescriptor {@@ -392,6 +423,39 @@ impl GraphicsProcessor {}}+ pub(crate) fn dirty_texture_overlaps_efb_writeback<const SYSTEM: crate::system::SystemId>(+ &self,+ mmio: &Mmio<SYSTEM>,+ renderer: &dyn RenderSink,+ ) -> bool {+ let mut dirty = self.tex_dirty;++ while dirty != 0 {+ let slot = dirty.trailing_zeros() as usize;+ dirty &= !(1 << slot);++ let image0 =+ TxSetImage0::from_raw(self.bp_regs[self::tx_slot_reg(BP_TX_SETIMAGE0_I0, BP_TX_SETIMAGE0_I4, slot)]);+ let image3 =+ TxSetImage3::from_raw(self.bp_regs[self::tx_slot_reg(BP_TX_SETIMAGE3_I0, BP_TX_SETIMAGE3_I4, slot)]);+ let width = (image0.width() + 1) as u32;+ let height = (image0.height() + 1) as u32;+ let len = texture::raw_data_size(width, height, image0.format());++ if self.recorder.is_none()+ && renderer.has_pending_efb_texture(image3.ram_addr() as u32, width, height, image0.format())+ {+ continue;+ }++ if mmio.efb_writeback_needed(image3.ram_addr() as u32, len) {+ return true;+ }+ }++ false+ }+fn load_tev_env(&mut self, idx: usize, val: u32) {let stage = (idx - BP_TEV_COLOR_ENV_0) / 2;if idx % 2 == 0 {@@ -490,6 +554,7 @@ impl GraphicsProcessor {src_w,src_h,dest_addr = format!("{dest_addr:#010X}"),+ dest_stride,copy_to_xfb,clear,half,@@ -571,22 +636,25 @@ impl GraphicsProcessor {let depth_copy = self.cur_pe_control.pixel_format().is_depth_only();- if let Some(rec) = self.recorder.as_deref_mut()- && rec.is_recording()- {- let resolved = if depth_copy {- texture::CopyFormat::from_u8_depth(copy_format)- } else {- texture::CopyFormat::from_u8_color(copy_format)- };- if let Some(fmt) = resolved {- let (w, h) = if half { (src_w / 2, src_h / 2) } else { (src_w, src_h) };- let row_bytes = texture::encoded_row_bytes(w, fmt);- let row_count = texture::encoded_row_count(h, fmt);-- if row_count > 0 {- let len = (row_count - 1) * dest_stride as usize + row_bytes;- rec.note_efb_copy(dest_addr as u32, len);+ let resolved = if depth_copy {+ texture::CopyFormat::from_u8_depth(copy_format)+ } else {+ texture::CopyFormat::from_u8_color(copy_format)+ };++ if let Some(fmt) = resolved {+ let (w, h) = if half { (src_w / 2, src_h / 2) } else { (src_w, src_h) };+ let row_bytes = texture::encoded_row_bytes(w, fmt);+ let row_count = texture::encoded_row_count(h, fmt);++ if row_count > 0 {+ let len = (row_count - 1) * dest_stride as usize + row_bytes;+ self.pending_efb_writeback_ranges.push((dest_addr, len));++ if let Some(rec) = self.recorder.as_deref_mut()+ && rec.is_recording()+ {+ rec.note_efb_copy(dest_addr, len);}}}@@ -715,13 +783,21 @@ fn texture_cache_id(ram_addr: usize, format: draw::TextureFormat, tlut: draw::Tl/// `cache_id` so paletted textures bound with multiple TLUTs are tracked/// independently.fn texture_data_changed(- hashes: &mut rustc_hash::FxHashMap<TextureKey, u64>,+ hashes: &mut rustc_hash::FxHashMap<TextureKey, (u64, u64)>,tex: &[u8],cache_id: TextureKey,palette: &[u16],tlut: draw::TlutRef,format: draw::TextureFormat,+ memory_generation: u64,) -> bool {+ if hashes+ .get(&cache_id)+ .is_some_and(|&(_, previous_generation)| previous_generation == memory_generation)+ {+ return false;+ }+let mut hash = twox_hash::xxhash3_64::Hasher::oneshot(tex);if format.is_paletted() {let max_entries = match format {@@ -737,8 +813,9 @@ fn texture_data_changed(// into the hash or a format switch alone wouldn't force a redecode.hash ^= tlut.format as u64;}- let prev = hashes.insert(cache_id, hash);- prev != Some(hash)++ let prev = hashes.insert(cache_id, (hash, memory_generation));+ prev.is_none_or(|(previous_hash, _)| previous_hash != hash)}impl GraphicsProcessor {diff --git a/crates/gecko/src/flipper/gx/constants.rs b/crates/gecko/src/flipper/gx/constants.rsindex a5773cb..3987b40 100644--- a/crates/gecko/src/flipper/gx/constants.rs+++ b/crates/gecko/src/flipper/gx/constants.rs@@ -13,6 +13,7 @@ pub const DRAW_COMMANDS_START: u8 = 0x80;pub const DRAW_COMMANDS_END: u8 = 0xBF; // TODO: Double check thispub const DRAW_QUADS_CMD: u8 = 0x80;+pub const DRAW_QUADS2_CMD: u8 = 0x88;pub const DRAW_TRIANGLES_CMD: u8 = 0x90;pub const DRAW_TRIANGLE_STRIP_CMD: u8 = 0x98;pub const DRAW_TRIANGLE_FAN_CMD: u8 = 0xA0;diff --git a/crates/gecko/src/flipper/gx/draw.rs b/crates/gecko/src/flipper/gx/draw.rsindex 4147b65..7a2e6ce 100644--- a/crates/gecko/src/flipper/gx/draw.rs+++ b/crates/gecko/src/flipper/gx/draw.rs@@ -18,7 +18,7 @@ impl Primitive {use super::constants::*;match cmd & !0b111 {- DRAW_QUADS_CMD => Some(Primitive::Quads),+ DRAW_QUADS_CMD | DRAW_QUADS2_CMD => Some(Primitive::Quads),DRAW_TRIANGLES_CMD => Some(Primitive::Triangles),DRAW_TRIANGLE_STRIP_CMD => Some(Primitive::TriangleStrip),DRAW_TRIANGLE_FAN_CMD => Some(Primitive::TriangleFan),diff --git a/crates/gecko/src/flipper/gx/fifo.rs b/crates/gecko/src/flipper/gx/fifo.rsindex 82ed0d3..45788a9 100644--- a/crates/gecko/src/flipper/gx/fifo.rs+++ b/crates/gecko/src/flipper/gx/fifo.rs@@ -158,9 +158,67 @@ fn drain_buf<const SYSTEM: SystemId>(break;}- let vertex_data = &fifo[pos + 3..pos + total];- gp.create_draw_call(mmio, renderer, cmd, vertex_data);- pos += total;+ let batch_enabled = !gp.recorder.as_ref().is_some_and(|rec| rec.is_recording());++ if batch_enabled && vertex_data_len != 0 {+ let stride = vertex_data_len / count.max(1);+ let mut batch_data = std::mem::take(&mut gp.draw_batch_data);+ let mut batch_entries = std::mem::take(&mut gp.draw_batch_entries);+ batch_data.clear();+ batch_entries.clear();++ let mut scan = pos;++ loop {+ let scan_remaining = fifo.len() - scan;++ if scan_remaining < 3 {+ break;+ }++ let scan_cmd = fifo[scan];++ if !matches!(scan_cmd, DRAW_COMMANDS_START..=DRAW_COMMANDS_END)+ || (scan_cmd & 7) as usize != vertex_format_index+ {+ break;+ }++ let scan_count = u16::from_be_bytes([fifo[scan + 1], fifo[scan + 2]]) as usize;+ let Some(scan_data_len) = scan_count.checked_mul(stride) else {+ break;+ };++ let scan_total = 3 + scan_data_len;++ if scan_remaining < scan_total {+ break;+ }++ batch_entries.push(super::vertex::DrawBatchEntry {+ cmd: scan_cmd,+ vertex_count: scan_count,+ });+ batch_data.extend_from_slice(&fifo[scan + 3..scan + scan_total]);+ scan += scan_total;+ }++ #[cfg(feature = "jit")]+ batch_data.reserve(crate::flipper::gx::jit::VEC_OVERREAD_BYTES);++ gp.create_draw_batch(mmio, renderer, &batch_data, &batch_entries);+ gp.draw_batch_data = batch_data;+ gp.draw_batch_entries = batch_entries;+ pos = scan;+ } else {+ let vertex_data = &fifo[pos + 3..pos + total];+ let entry = [super::vertex::DrawBatchEntry {+ cmd,+ vertex_count: count,+ }];+ gp.create_draw_batch(mmio, renderer, vertex_data, &entry);+ pos += total;+ }}_ => {tracing::error!(cmd = cmd, "unknown FIFO command");diff --git a/crates/gecko/src/flipper/gx/vertex.rs b/crates/gecko/src/flipper/gx/vertex.rsindex 049a802..d950a5f 100644--- a/crates/gecko/src/flipper/gx/vertex.rs+++ b/crates/gecko/src/flipper/gx/vertex.rs@@ -2,14 +2,22 @@ use super::constants::*;use super::math::{Vec3, unpack_rgba};use super::regs::{self, *};use super::{GraphicsProcessor, draw};-use crate::host::{DrawData, DrawState, DrawVertex, GxAction, RenderSink};+use crate::host::{DrawSegment, DrawState, DrawVertex, RenderSink};use crate::mmio::{Mmio, RamView};-use crate::system::{ExecutionMode, SystemId};+#[cfg(feature = "jit")]+use crate::system::ExecutionMode;+use crate::system::SystemId;use std::io::{Cursor, Read};#[cfg(feature = "jit")]use super::jit;+#[derive(Clone, Copy)]+pub(super) struct DrawBatchEntry {+ pub cmd: u8,+ pub vertex_count: usize,+}+/// Parsed vertex format descriptor from CP/VAT registers.struct VertexFormat {vat_a: VatA,@@ -45,29 +53,33 @@ struct VertexFormat {impl GraphicsProcessor {#[cfg_attr(feature = "hotpath", hotpath::measure)]- pub fn create_draw_call<const SYSTEM: SystemId>(+ pub(super) fn create_draw_batch<const SYSTEM: SystemId>(&mut self,mmio: &mut Mmio<SYSTEM>,renderer: &mut dyn RenderSink,- cmd: u8,data: &[u8],+ entries: &[DrawBatchEntry],) {#[cfg(feature = "gx-stats")]let _gx_stats_t0 = std::time::Instant::now();- let Some(primitive) = draw::Primitive::from_cmd(cmd) else {- tracing::error!(cmd, "goofy draw command");+ let Some(first) = entries.first() else {return;};- let vf = self.build_vertex_format(cmd);+ let cmd = first.cmd;++ let vertex_format_index = (cmd & 0b111) as usize;+ let vertex_stride = super::fifo::vertex_stride_from_cp(&self.cp_regs, vertex_format_index);- if vf.vertex_stride == 0 {+ if vertex_stride == 0 {tracing::warn!("draw call with zero vertex stride, skipping");return;}- let vertex_count = data.len() / vf.vertex_stride;+ let vertex_count: usize = entries.iter().map(|entry| entry.vertex_count).sum();+ debug_assert_eq!(data.len(), vertex_count * vertex_stride);+ debug_assert!(entries.iter().all(|entry| (entry.cmd & 7) == (cmd & 7)));// Resolve any texture slots whose descriptor regs changed since the// last draw. Done lazily here (not at BP write time) because games@@ -75,14 +87,28 @@ impl GraphicsProcessor {// the descriptor guaranteed consistent. Runs before the recorder so// recorded draws reference the resolved textures.if self.tex_dirty != 0 {+ self.defer_pending_efb_writebacks(mmio);++ if self.dirty_texture_overlaps_efb_writeback(mmio, renderer) {+ renderer.flush_efb_copies(&mut mmio.ram_view_mut());+ mmio.clear_deferred_efb_writebacks();+ }+self.snapshot_dirty_textures(renderer, &mmio.ram_view());}- if let Some(rec) = self.recorder.as_deref_mut()- && rec.is_recording()- {+ if self.recorder.as_ref().is_some_and(|rec| rec.is_recording()) {+ let vf = self.build_vertex_format(cmd);let view = mmio.ram_view();- self::record_draw_arrays(rec, &view, &vf, data, vertex_count);+ let rec = self.recorder.as_deref_mut().unwrap();+ let mut offset = 0;++ for entry in entries {+ let len = entry.vertex_count * vertex_stride;+ self::record_draw_arrays(rec, &view, &vf, &data[offset..offset + len], entry.vertex_count);+ offset += len;+ }+for desc in self.cur_textures.iter().flatten() {let len = super::texture::raw_data_size(desc.width, desc.height, desc.format);rec.use_draw_texture(&view, desc.ram_addr as u32, len);@@ -91,13 +117,16 @@ impl GraphicsProcessor {#[cfg(feature = "gx-stats")]{- self.stats.draw_calls += 1;+ self.stats.draw_calls += entries.len() as u64;self.stats.vertices += vertex_count as u64;self.stats.fifo_bytes += data.len() as u64;- self.stats.draws_by_primitive[(primitive as usize) & 0x7] += 1;- }- let mut boxed: Box<DrawData> = renderer.take_draw_data();+ for entry in entries {+ if let Some(primitive) = draw::Primitive::from_cmd(entry.cmd) {+ self.stats.draws_by_primitive[(primitive as usize) & 0x7] += 1;+ }+ }+ }// Decode directly into the renderer's vertex scratch. No// intermediate `draw_vertices_scratch`, no append memcpy. The@@ -106,12 +135,9 @@ impl GraphicsProcessor {let verts = renderer.vertex_scratch();let base_vertex = verts.len() as u32;verts.reserve(vertex_count);- self::dispatch_decode(self, mmio, cmd, data, vertex_count, &vf, verts);+ self::dispatch_decode(self, mmio, cmd, data, vertex_count, verts);- boxed.primitive = primitive;- boxed.base_vertex = base_vertex;- boxed.vertex_count = vertex_count as u32;- boxed.state = self.frame_state_dirty.then(|| {+ let state = self.frame_state_dirty.then(|| {let tev_color_regs = self.resolve_tev_color_regs();let tev_orders = self.resolve_tev_orders();@@ -179,7 +205,26 @@ impl GraphicsProcessor {}});self.frame_state_dirty = false;- renderer.exec(GxAction::Draw(boxed));+ self.draw_segments_scratch.clear();+ let mut segment_base = base_vertex;++ for entry in entries {+ let Some(primitive) = draw::Primitive::from_cmd(entry.cmd) else {+ tracing::error!(cmd = entry.cmd, "goofy draw command");+ segment_base = segment_base.wrapping_add(entry.vertex_count as u32);++ continue;+ };++ self.draw_segments_scratch.push(DrawSegment {+ primitive,+ base_vertex: segment_base,+ vertex_count: entry.vertex_count as u32,+ });+ segment_base = segment_base.wrapping_add(entry.vertex_count as u32);+ }++ renderer.exec_draw_batch(&self.draw_segments_scratch, state);#[cfg(feature = "gx-stats")]{@@ -814,15 +859,21 @@ fn dispatch_decode<const SYSTEM: SystemId>(#[cfg_attr(not(feature = "jit"), allow(unused_variables))] cmd: u8,data: &[u8],vertex_count: usize,- vf: &VertexFormat,verts: &mut Vec<DrawVertex>,) {#[cfg(feature = "jit")]if gp.execution_mode == ExecutionMode::Jit {let vat_index = (cmd & 0b111) as usize;let key = jit::VtxKey::from_cp_regs(&gp.cp_regs, vat_index);- let view = mmio.ram_view();- let arrays_ok = jit::resolve_arrays_for_draw(&gp.cp_regs, &key, view.mem1, view.mem2, &mut gp.jit_vtx_arrays);+ let arrays_ok = if gp.jit_vtx_arrays_key == Some(key) {+ gp.jit_vtx_arrays_ok+ } else {+ let view = mmio.ram_view();+ let ok = jit::resolve_arrays_for_draw(&gp.cp_regs, &key, view.mem1, view.mem2, &mut gp.jit_vtx_arrays);+ gp.jit_vtx_arrays_key = Some(key);+ gp.jit_vtx_arrays_ok = ok;+ ok+ };let parser = if arrays_ok {gp.jit_vtx.lookup_or_compile(key)} else {@@ -841,12 +892,16 @@ fn dispatch_decode<const SYSTEM: SystemId>(}#[cfg(feature = "vtx-jit-validate")]- self::run_validator(gp, mmio, cmd, key, data, vertex_count, vf, verts, base);+ {+ let vf = gp.build_vertex_format(cmd);+ self::run_validator(gp, mmio, cmd, key, data, vertex_count, &vf, verts, base);+ }return;}}- self::run_interpreter(gp, mmio, data, vertex_count, vf, verts);+ let vf = gp.build_vertex_format(cmd);+ self::run_interpreter(gp, mmio, data, vertex_count, &vf, verts);}#[cfg_attr(feature = "hotpath", hotpath::measure(label = "vtx_run_interpreter"))]diff --git a/crates/gecko/src/flipper/gx/xf.rs b/crates/gecko/src/flipper/gx/xf.rsindex 4f679ca..0dda05a 100644--- a/crates/gecko/src/flipper/gx/xf.rs+++ b/crates/gecko/src/flipper/gx/xf.rs@@ -149,8 +149,18 @@ impl GraphicsProcessor {pub fn load_cp(&mut self, data: &[u8]) {let idx = data[0] as usize;let val = u32::from_be_bytes([data[1], data[2], data[3], data[4]]);+ #[cfg(feature = "jit")]+ let changed = self.cp_regs[idx] != val;self.cp_regs[idx] = val;+ #[cfg(feature = "jit")]+ if changed+ && ((ARRAY_BASE_REG..ARRAY_BASE_REG + super::jit::RESOLVED_ARRAY_COUNT).contains(&idx)+ || (ARRAY_STRIDE_REG..ARRAY_STRIDE_REG + super::jit::RESOLVED_ARRAY_COUNT).contains(&idx))+ {+ self.jit_vtx_arrays_key = None;+ }+tracing::debug!(reg_idx = format!("{idx:02X}"),value = format!("{val:08X}"),diff --git a/crates/gecko/src/gekko/dec.rs b/crates/gecko/src/gekko/dec.rsindex efec2b5..654e95b 100644--- a/crates/gecko/src/gekko/dec.rs+++ b/crates/gecko/src/gekko/dec.rs@@ -10,6 +10,11 @@ pub struct Decrementer {interrupt_pending: bool,}+#[inline(always)]+pub const fn interrupt_pending_offset() -> usize {+ core::mem::offset_of!(Decrementer, interrupt_pending)+}+impl Default for Decrementer {fn default() -> Self {Self {diff --git a/crates/gecko/src/gekko/interpreter/system.rs b/crates/gecko/src/gekko/interpreter/system.rsindex 7a2ae9a..7016a6f 100644--- a/crates/gecko/src/gekko/interpreter/system.rs+++ b/crates/gecko/src/gekko/interpreter/system.rs@@ -155,8 +155,17 @@ pub fn tw<const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instruction)}#[inline(always)]-pub fn nop<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, _instr: Instruction) {+pub fn nop<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, instr: Instruction) {ctx.scheduler.cycles += crate::gekko::cycles::cycles_for_op(OP) as u64;++ if matches!(OP, OP_DCBF | OP_DCBST) {+ let base = ctx.gekko.read_gpr_or_zero(instr.ra());+ let ea = base.wrapping_add(ctx.gekko.read_gpr(instr.rb()));+ ctx.mmio.mark_memory_dirty_range(+ crate::mmio::virt_to_phys(ea) & crate::mmio::CODE_LINE_MASK,+ crate::mmio::CODE_LINE_BYTES,+ );+ }}/// dcbz / dcbz_l zero the targeted 32-byte cache line via `System::dcbz_line`.diff --git a/crates/gecko/src/gekko/jit.rs b/crates/gecko/src/gekko/jit.rsindex 2296252..71ada83 100644--- a/crates/gecko/src/gekko/jit.rs+++ b/crates/gecko/src/gekko/jit.rs@@ -17,7 +17,7 @@ pub mod lut_wii {}use cranelift_codegen::Context;-use cranelift_codegen::ir::{AbiParam, InstBuilder, Signature, types};+use cranelift_codegen::ir::{AbiParam, InstBuilder, Signature, condcodes::IntCC, types};use cranelift_codegen::isa::CallConv;use cranelift_codegen::settings::{self, Configurable};use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};@@ -42,6 +42,53 @@ pub struct BlockLookupSlot {pub const BLOCK_LOOKUP_TABLE_SIZE: usize = 131072;pub const BLOCK_LOOKUP_TABLE_MASK: u32 = (BLOCK_LOOKUP_TABLE_SIZE as u32) - 1;+const DIV2I_SIZE: u32 = 312;+const DIV2I_CHECKSUM: u32 = 0xFB31_C123;++fn signature_checksum<const SYSTEM: SystemId>(sys: &System<SYSTEM>, start: u32, size: u32) -> u32 {+ let mut sum = 0u32;++ for offset in (0..size).step_by(4) {+ let opcode = sys.mmio.fetch_instruction(start.wrapping_add(offset));+ let primary = opcode >> 26;+ let op = opcode & 0xFC00_0000;+ let mut op2 = 0;+ let mut op3 = 0;++ match primary {+ 4 => {+ op2 = opcode & 0x3F;++ if matches!(op2, 0 | 8 | 16 | 21 | 22) {+ op3 = opcode & 0x7C0;+ }+ }+ 7 | 8 | 10..=15 => op2 = opcode & 0x03FF_0000,+ 19 | 31 | 63 => op2 = opcode & 0x7FF,+ 59 => {+ op2 = opcode & 0x3F;++ if op2 < 16 {+ op3 = opcode & 0x7C0;+ }+ }+ 32..=55 => op2 = opcode & 0x03FF_0000,+ _ => {}+ }++ sum = sum.rotate_left(17) ^ (op | op2 | op3);+ }++ sum+}++fn is_div2i<const SYSTEM: SystemId>(sys: &System<SYSTEM>, pc: u32) -> bool {+ sys.mmio.fetch_instruction(pc) == 0x9421_FFF0+ && sys.mmio.fetch_instruction(pc.wrapping_add(4)) == 0x5469_0001+ && sys.mmio.fetch_instruction(pc.wrapping_add(8)) == 0x4182_000C+ && signature_checksum(sys, pc, DIV2I_SIZE) == DIV2I_CHECKSUM+}+#[derive(Clone, Copy)]pub struct ExternFuncs {pub cause_invalid_opcode: FuncId,@@ -75,6 +122,7 @@ pub struct ExternFuncs {pub do_psq_store: FuncId,pub read_timebase: FuncId,pub cause_icbi: FuncId,+ pub mark_memory_writeback: FuncId,pub cause_smc_write: FuncId,pub dcbz: FuncId,}@@ -85,6 +133,7 @@ pub struct JitEngine<const SYSTEM: SystemId> {builder_ctx: FunctionBuilderContext,cache: FxHashMap<u32, BlockEntry>,block_func_ids: FxHashMap<u32, FuncId>,+ hle_code_sizes: FxHashMap<u32, u32>,target_slots: std::cell::RefCell<FxHashMap<u32, usize>>,blocks_by_line: FxHashMap<u32, smallvec::SmallVec<[u32; 2]>>,pub(crate) drain_scratch: Vec<u32>,@@ -155,6 +204,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_store: (&'static str, *const u8),read_timebase: (&'static str, *const u8),cause_icbi: (&'static str, *const u8),+ mark_memory_writeback: (&'static str, *const u8),cause_smc_write: (&'static str, *const u8),dcbz: (&'static str, *const u8),}@@ -209,6 +259,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_store: ("gecko_jit_do_psq_store_gc", runtime::do_psq_store_gc as *const u8),read_timebase: ("gecko_jit_read_timebase_gc", runtime::read_timebase_gc as *const u8),cause_icbi: ("gecko_jit_cause_icbi_gc", runtime::cause_icbi_gc as *const u8),+ mark_memory_writeback: (+ "gecko_jit_mark_memory_writeback_gc",+ runtime::mark_memory_writeback_gc as *const u8,+ ),cause_smc_write: ("gecko_jit_cause_smc_write_gc", runtime::cause_smc_write_gc as *const u8),dcbz: ("gecko_jit_dcbz_gc", runtime::dcbz_gc as *const u8),},@@ -265,6 +319,10 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {do_psq_store: ("gecko_jit_do_psq_store_wii", runtime::do_psq_store_wii as *const u8),read_timebase: ("gecko_jit_read_timebase_wii", runtime::read_timebase_wii as *const u8),cause_icbi: ("gecko_jit_cause_icbi_wii", runtime::cause_icbi_wii as *const u8),+ mark_memory_writeback: (+ "gecko_jit_mark_memory_writeback_wii",+ runtime::mark_memory_writeback_wii as *const u8,+ ),cause_smc_write: ("gecko_jit_cause_smc_write_wii",runtime::cause_smc_write_wii as *const u8,@@ -306,6 +364,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {syms.do_psq_store,syms.read_timebase,syms.cause_icbi,+ syms.mark_memory_writeback,syms.cause_smc_write,syms.dcbz,] {@@ -322,6 +381,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let mut block_sig = Signature::new(CallConv::Tail);block_sig.params.push(AbiParam::new(pointer_type));+ block_sig.params.push(AbiParam::new(types::I32));block_sig.returns.push(AbiParam::new(types::I32));let mut fallback_sig = Signature::new(call_conv);@@ -481,6 +541,9 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {cause_icbi: module.declare_function(syms.cause_icbi.0, Linkage::Import, &ctx_u32_sig).expect("declare cause_icbi"),+ mark_memory_writeback: module+ .declare_function(syms.mark_memory_writeback.0, Linkage::Import, &ctx_u32_sig)+ .expect("declare mark_memory_writeback"),dcbz: module.declare_function(syms.dcbz.0, Linkage::Import, &ctx_u32_sig).expect("declare dcbz"),@@ -508,6 +571,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {builder_ctx: FunctionBuilderContext::new(),cache: FxHashMap::default(),block_func_ids: FxHashMap::default(),+ hle_code_sizes: FxHashMap::default(),target_slots: std::cell::RefCell::new(FxHashMap::default()),blocks_by_line: FxHashMap::default(),drain_scratch: Vec::new(),@@ -624,10 +688,17 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {return entry;}+ let hle_div2i = is_div2i(sys, pc);let spec = block::discover::<SYSTEM>(sys, pc);let gprs_snapshot = sys.gekko.gprs;- let entry = self.compile(&spec, &gprs_snapshot);+ let entry = if hle_div2i {+ self.hle_code_sizes.insert(pc, DIV2I_SIZE);+ self.compile_div2i(pc)+ } else {+ self.compile(&spec, &gprs_snapshot)+ };+self.cache.insert(pc, entry);self.block_specs.insert(pc, spec);self.register_block(&mut sys.mmio, pc);@@ -637,8 +708,21 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {fn register_block(&mut self, mmio: &mut crate::mmio::Mmio<SYSTEM>, pc: u32) {let Some(spec) = self.block_specs.get(&pc) else { return };++ let tracked_pcs = if let Some(&size) = self.hle_code_sizes.get(&pc) {+ std::borrow::Cow::Owned(+ (0..size)+ .step_by(4)+ .map(|offset| pc.wrapping_add(offset))+ .collect::<Vec<_>>(),+ )+ } else {+ std::borrow::Cow::Borrowed(spec.pcs.as_slice())+ };+let mut last = u32::MAX;- for &vpc in &spec.pcs {++ for &vpc in tracked_pcs.iter() {let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;if line == last {continue;@@ -654,6 +738,18 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {let Some(spec) = self.block_specs.remove(&pc) else {return;};++ let tracked_pcs = if let Some(size) = self.hle_code_sizes.remove(&pc) {+ std::borrow::Cow::Owned(+ (0..size)+ .step_by(4)+ .map(|offset| pc.wrapping_add(offset))+ .collect::<Vec<_>>(),+ )+ } else {+ std::borrow::Cow::Borrowed(spec.pcs.as_slice())+ };+self.cache.remove(&pc);self.block_func_ids.remove(&pc);@@ -672,7 +768,8 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}let mut last = u32::MAX;- for &vpc in &spec.pcs {++ for &vpc in tracked_pcs.iter() {let line = crate::mmio::virt_to_phys(vpc) & crate::mmio::CODE_LINE_MASK;if line == last {continue;@@ -901,6 +998,7 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {pub fn flush(&mut self) {self.cache.clear();self.block_func_ids.clear();+ self.hle_code_sizes.clear();self.block_specs.clear();self.blocks_by_line.clear();@@ -927,12 +1025,141 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {}#[cfg_attr(feature = "hotpath", hotpath::measure(label = "ppc_jit_compile"))]- fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {- let func_id = self.func_id_for(spec.start_pc);+ fn compile_div2i(&mut self, pc: u32) -> BlockEntry {+ let func_id = self.func_id_for(pc);++ #[cfg(feature = "jit-stats")]+ let entry_counter_addr = self.block_entry_counter_slot(pc);self.ctx.clear();self.ctx.func.signature = self.block_sig.clone();+ let mut builder = FunctionBuilder::new(&mut self.ctx.func, &mut self.builder_ctx);+ translator::register_alias_regions(&mut builder);+ let entry = builder.create_block();+ builder.append_block_params_for_function_params(entry);+ builder.switch_to_block(entry);+ builder.seal_block(entry);+ let ctx_ptr = builder.block_params(entry)[0];++ #[cfg(feature = "jit-stats")]+ {+ let slot = builder.ins().iconst(types::I64, entry_counter_addr as i64);+ let count = builder.ins().load(types::I64, translator::vmctx_flags(), slot, 0);+ let next = builder.ins().iadd_imm(count, 1);+ builder.ins().store(translator::vmctx_flags(), next, slot, 0);+ }++ let load_gpr = |builder: &mut FunctionBuilder, index: usize| {+ builder.ins().load(+ types::I32,+ translator::vmctx_flags(),+ ctx_ptr,+ (abi::gpr_base_offset::<SYSTEM>() + index * 4) as i32,+ )+ };++ let hi_lhs = load_gpr(&mut builder, 3);+ let lo_lhs = load_gpr(&mut builder, 4);+ let hi_rhs = load_gpr(&mut builder, 5);+ let lo_rhs = load_gpr(&mut builder, 6);++ let hi_lhs = builder.ins().uextend(types::I64, hi_lhs);+ let lo_lhs = builder.ins().uextend(types::I64, lo_lhs);+ let hi_rhs = builder.ins().uextend(types::I64, hi_rhs);+ let lo_rhs = builder.ins().uextend(types::I64, lo_rhs);+ let lhs_hi = builder.ins().ishl_imm(hi_lhs, 32);+ let rhs_hi = builder.ins().ishl_imm(hi_rhs, 32);+ let lhs = builder.ins().bor(lhs_hi, lo_lhs);+ let rhs = builder.ins().bor(rhs_hi, lo_rhs);++ let zero_block = builder.create_block();+ let nonzero_block = builder.create_block();+ let overflow_block = builder.create_block();+ let divide_block = builder.create_block();+ let merge_block = builder.create_block();+ builder.append_block_param(merge_block, types::I64);++ let rhs_zero = builder.ins().icmp_imm(IntCC::Equal, rhs, 0);+ builder.ins().brif(rhs_zero, zero_block, &[], nonzero_block, &[]);++ builder.switch_to_block(zero_block);+ builder.seal_block(zero_block);+ let zero = builder.ins().iconst(types::I64, 0);+ builder.ins().jump(merge_block, &[zero.into()]);++ builder.switch_to_block(nonzero_block);+ builder.seal_block(nonzero_block);+ let lhs_min = builder.ins().icmp_imm(IntCC::Equal, lhs, i64::MIN);+ let rhs_neg_one = builder.ins().icmp_imm(IntCC::Equal, rhs, -1);+ let overflows = builder.ins().band(lhs_min, rhs_neg_one);+ builder.ins().brif(overflows, overflow_block, &[], divide_block, &[]);++ builder.switch_to_block(overflow_block);+ builder.seal_block(overflow_block);+ builder.ins().jump(merge_block, &[lhs.into()]);++ builder.switch_to_block(divide_block);+ builder.seal_block(divide_block);+ let quotient = builder.ins().sdiv(lhs, rhs);+ builder.ins().jump(merge_block, &[quotient.into()]);++ builder.switch_to_block(merge_block);+ builder.seal_block(merge_block);+ let quotient = builder.block_params(merge_block)[0];+ let quotient_hi = builder.ins().ushr_imm(quotient, 32);+ let quotient_hi = builder.ins().ireduce(types::I32, quotient_hi);+ let quotient_lo = builder.ins().ireduce(types::I32, quotient);+ builder.ins().store(+ translator::vmctx_flags(),+ quotient_hi,+ ctx_ptr,+ (abi::gpr_base_offset::<SYSTEM>() + 3 * 4) as i32,+ );+ builder.ins().store(+ translator::vmctx_flags(),+ quotient_lo,+ ctx_ptr,+ (abi::gpr_base_offset::<SYSTEM>() + 4 * 4) as i32,+ );++ let cycles_off = abi::cycles_offset::<SYSTEM>() as i32;+ let cycles = builder+ .ins()+ .load(types::I64, translator::vmctx_flags(), ctx_ptr, cycles_off);+ let cycles = builder.ins().iadd_imm(cycles, 300);+ builder+ .ins()+ .store(translator::vmctx_flags(), cycles, ctx_ptr, cycles_off);++ let lr = builder.ins().load(+ types::I32,+ translator::vmctx_flags(),+ ctx_ptr,+ abi::lr_offset::<SYSTEM>() as i32,+ );+ builder.ins().return_(&[lr]);+ builder.finalize();++ self.module+ .define_function(func_id, &mut self.ctx)+ .expect("define __div2i intrinsic");+ self.module.finalize_definitions().expect("finalize __div2i intrinsic");+ let entry = self.module.get_finalized_function(func_id) as usize;++ let slot_addr = self.target_slot_addr(pc);+ unsafe {+ *(slot_addr as *mut usize) = entry;+ }++ self.register_in_lookup_table(pc, entry);++ entry+ }++ fn compile(&mut self, spec: &block::BlockSpec, gprs: &[u32; 32]) -> BlockEntry {+ let func_id = self.func_id_for(spec.start_pc);+let _ = self.target_slot_addr(spec.start_pc);#[cfg(feature = "jit-stats")]@@ -940,6 +1167,28 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {#[cfg(not(feature = "jit-stats"))]let entry_counter_addr: Option<usize> = None;+ let entry = self.compile_function(spec, gprs, func_id, entry_counter_addr);++ let slot_addr = self.target_slot_addr(spec.start_pc);+ unsafe {+ *(slot_addr as *mut usize) = entry;+ }++ self.register_in_lookup_table(spec.start_pc, entry);++ entry+ }++ fn compile_function(+ &mut self,+ spec: &block::BlockSpec,+ gprs: &[u32; 32],+ func_id: FuncId,+ entry_counter_addr: Option<usize>,+ ) -> BlockEntry {+ self.ctx.clear();+ self.ctx.func.signature = self.block_sig.clone();+let block_lookup_table_addr = self.block_lookup_table_addr as i64;let chain = translator::ChainContext {self_pc: spec.start_pc,@@ -966,13 +1215,12 @@ impl<const SYSTEM: SystemId> JitEngine<SYSTEM> {self.module.finalize_definitions().expect("finalize");let entry = self.module.get_finalized_function(func_id) as usize;-- let slot_addr = self.target_slot_addr(spec.start_pc);- unsafe {- *(slot_addr as *mut usize) = entry;- }-- self.register_in_lookup_table(spec.start_pc, entry);+ crate::jit::register_jit_code(+ "ppc",+ spec.start_pc,+ entry,+ self.ctx.compiled_code().unwrap().code_buffer().len(),+ );entry}@@ -1061,7 +1309,10 @@ fn build_trampoline(let ctx_ptr = builder.block_params(entry)[0];let block_ptr = builder.block_params(entry)[1];- let call = builder.ins().call_indirect(block_sig_ref, block_ptr, &[ctx_ptr]);+ let no_host_return = builder.ins().iconst(types::I32, u32::MAX as i64);+ let call = builder+ .ins()+ .call_indirect(block_sig_ref, block_ptr, &[ctx_ptr, no_host_return]);let result = builder.inst_results(call)[0];builder.ins().return_(&[result]);builder.finalize();diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rsindex a1fca83..343417b 100644--- a/crates/gecko/src/gekko/jit/abi.rs+++ b/crates/gecko/src/gekko/jit/abi.rs@@ -1,5 +1,6 @@use core::mem::{offset_of, size_of};+use crate::flipper::pi::ProcessorInterface;use crate::gekko::condition::ConditionRegister;use crate::gekko::msr::Msr;use crate::gekko::spr::{Spr, Xer};@@ -69,6 +70,21 @@ pub const fn ps1_base_offset<const SYSTEM: SystemId>() -> usize {gekko_offset::<SYSTEM>() + offset_of!(Gekko, fprs) + size_of::<f64>()}+#[inline(always)]+pub const fn pi_intsr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, pi) + offset_of!(ProcessorInterface, intsr)+}++#[inline(always)]+pub const fn pi_intmr_offset<const SYSTEM: SystemId>() -> usize {+ offset_of!(System<SYSTEM>, pi) + offset_of!(ProcessorInterface, intmr)+}++#[inline(always)]+pub const fn dec_interrupt_pending_offset<const SYSTEM: SystemId>() -> usize {+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, dec) + crate::gekko::dec::interrupt_pending_offset()+}+#[inline(always)]pub const fn cycles_offset<const SYSTEM: SystemId>() -> usize {scheduler_offset::<SYSTEM>() + offset_of!(Scheduler<SYSTEM>, cycles)diff --git a/crates/gecko/src/gekko/jit/handlers.rs b/crates/gecko/src/gekko/jit/handlers.rsindex 9e605e5..9b8f097 100644--- a/crates/gecko/src/gekko/jit/handlers.rs+++ b/crates/gecko/src/gekko/jit/handlers.rs@@ -1,26 +1,62 @@use cranelift_codegen::ir::condcodes::IntCC;-use cranelift_codegen::ir::{InstBuilder, types};+use cranelift_codegen::ir::{InstBuilder, Value, types};use cranelift_frontend::FunctionBuilder;use crate::gekko::instruction::Instruction;use crate::gekko::jit::translator::{- self, AddmeSubfmeKind, AddzeSubfzeKind, ImmLogical, JitTranslator, LocalFuncs, LogicalFullOp, LogicalOp, MemSize,- ShiftKind, TermEmit, vmctx_flags,+ self, AddmeSubfmeKind, AddzeSubfzeKind, BlockEmitContext, ImmLogical, JitTranslator, LogicalFullOp, LogicalOp,+ MemSize, ShiftKind, TermEmit, vmctx_flags,};use crate::gekko::jit::{abi, lut};use crate::system::SystemId;#[inline(always)]-unsafe fn parts<'a>(builder_ptr: usize, local_ptr: usize) -> (&'a mut FunctionBuilder<'a>, &'a LocalFuncs) {+unsafe fn parts<'a>(+ builder_ptr: usize,+ emit_ctx_ptr: usize,+) -> (&'a mut FunctionBuilder<'a>, &'a mut BlockEmitContext) {(unsafe { &mut *(builder_ptr as *mut FunctionBuilder<'a>) }, unsafe {- &*(local_ptr as *const LocalFuncs)+ &mut *(emit_ctx_ptr as *mut BlockEmitContext)})}+fn emit_must_deliver_external_interrupt<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ msr: Value,+) -> Value {+ let ee = builder.ins().band_imm(msr, 0x8000);+ let enables_ee = builder.ins().icmp_imm(IntCC::NotEqual, ee, 0);+ let pi_cause = builder.ins().load(+ types::I32,+ vmctx_flags(),+ ctx_ptr,+ abi::pi_intsr_offset::<SYSTEM>() as i32,+ );+ let pi_mask = builder.ins().load(+ types::I32,+ vmctx_flags(),+ ctx_ptr,+ abi::pi_intmr_offset::<SYSTEM>() as i32,+ );+ let pi_enabled = builder.ins().band(pi_cause, pi_mask);+ let pi_pending = builder.ins().icmp_imm(IntCC::NotEqual, pi_enabled, 0);+ let dec_pending = builder.ins().load(+ types::I8,+ vmctx_flags(),+ ctx_ptr,+ abi::dec_interrupt_pending_offset::<SYSTEM>() as i32,+ );+ let dec_pending = builder.ins().icmp_imm(IntCC::NotEqual, dec_pending, 0);+ let pending = builder.ins().bor(pi_pending, dec_pending);++ builder.ins().band(enables_ee, pending)+}+#[inline(always)]pub fn alu<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_ADDI => translator::emit_addi::<SYSTEM>(builder, t.ctx_ptr, instr),@@ -270,7 +306,7 @@ pub fn alu<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:#[inline(always)]pub fn rotate<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let r = match OP {lut::OP_RLWIMIX => translator::emit_rlwimi::<SYSTEM>(builder, t.ctx_ptr, instr),@@ -289,7 +325,7 @@ pub fn rotate<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, inst#[inline(always)]pub fn logical<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let r = match OP {lut::OP_ANDX => translator::emit_logical_xform::<SYSTEM>(builder, t.ctx_ptr, instr, LogicalOp::And),@@ -324,7 +360,7 @@ pub fn logical<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, ins#[inline(always)]pub fn compare<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_CMPI => translator::emit_cmp_imm::<SYSTEM>(builder, t.ctx_ptr, instr, true),@@ -340,7 +376,7 @@ pub fn compare<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, ins#[inline(always)]pub fn cr_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_MFCR => {@@ -384,7 +420,7 @@ pub fn cr_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, inst#[inline(always)]pub fn segment<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_MFSR => {@@ -406,7 +442,7 @@ pub fn segment<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, ins#[inline(always)]pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_MFMSR => {@@ -424,11 +460,34 @@ pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:.store(vmctx_flags(), val, t.ctx_ptr, abi::msr_offset::<SYSTEM>() as i32);unsafe {- (&*(t.local_ptr as *const LocalFuncs)).fp_guard_emitted.set(false);+ (&mut *(t.emit_ctx_ptr as *mut BlockEmitContext)).fp_guard_emitted = false;}if t.is_terminator {- t.last_terminator_nia = Some(builder.ins().iconst(types::I32, t.pc.wrapping_add(4) as i64));+ let next_pc = t.pc.wrapping_add(4);+ let next_nia = builder.ins().iconst(types::I32, next_pc as i64);++ if let Some(slot_addr) = t.chain_fall_addr {+ let must_deliver = emit_must_deliver_external_interrupt::<SYSTEM>(builder, t.ctx_ptr, val);+ let disabled_block = builder.create_block();+ builder+ .ins()+ .brif(must_deliver, t.exit_block, &[next_nia.into()], disabled_block, &[]);+ builder.switch_to_block(disabled_block);+ builder.seal_block(disabled_block);+ translator::emit_chain_or_exit::<SYSTEM>(+ builder,+ t.ctx_ptr,+ t.host_return_pc,+ slot_addr,+ t.block_sig_ref,+ next_nia,+ t.exit_block,+ );+ t.chained = true;+ } else {+ t.last_terminator_nia = Some(next_nia);+ }}t.handled_natively = true;}@@ -452,7 +511,24 @@ pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:let nia = builder.ins().band_imm(srr0_raw, 0xFFFF_FFFCi64);builder.ins().store(vmctx_flags(), nia, t.ctx_ptr, nia_off);- t.last_terminator_nia = Some(nia);+ let must_deliver = emit_must_deliver_external_interrupt::<SYSTEM>(builder, t.ctx_ptr, new_msr_clr);+ let dispatch_block = builder.create_block();+ builder+ .ins()+ .brif(must_deliver, t.exit_block, &[nia.into()], dispatch_block, &[]);+ builder.switch_to_block(dispatch_block);+ builder.seal_block(dispatch_block);++ translator::emit_inline_cache_dispatch::<SYSTEM>(+ builder,+ t.ctx_ptr,+ t.host_return_pc,+ nia,+ t.exit_block,+ t.block_sig_ref,+ t.block_lookup_table_addr,+ );+ t.chained = true;t.handled_natively = true;}_ => {}@@ -462,7 +538,7 @@ pub fn msr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:#[inline(always)]pub fn spr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_MFSPR => {@@ -485,7 +561,7 @@ pub fn spr<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:#[inline(always)]pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_LWZ => translator::emit_load::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false),@@ -663,7 +739,7 @@ pub fn store_load<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator,#[inline(always)]pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, t.op_cycles);@@ -714,7 +790,7 @@ pub fn store_load_fp<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslato#[inline(always)]pub fn store_load_psq<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, t.op_cycles);@@ -743,15 +819,18 @@ pub fn store_load_psq<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslatpub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);let _ = (instr, SYSTEM);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {+ lut::OP_DCBF | lut::OP_DCBST => {+ let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);+ builder.ins().call(local.mark_memory_writeback, &[t.ctx_ptr, ea]);+ t.handled_natively = true;+ }lut::OP_ISYNC| lut::OP_SYNC| lut::OP_EIEIO- | lut::OP_DCBF| lut::OP_DCBI- | lut::OP_DCBST| lut::OP_DCBT| lut::OP_DCBTST| lut::OP_DCBA@@ -764,7 +843,8 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:}_ => {}}- let _ = builder;++ let _ = (builder, local);}/// dcbz / dcbz_l zero the targeted 32-byte cache line. We have no dcache, so@@ -773,7 +853,7 @@ pub fn nop<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:#[inline(always)]pub fn dcbz<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);builder.ins().call(local.dcbz, &[t.ctx_ptr, ea]);@@ -784,7 +864,7 @@ pub fn dcbz<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr:#[inline(always)]pub fn icbi<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let ra = instr.ra();let rb = instr.rb();@@ -819,7 +899,7 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instreturn;}- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };match OP {lut::OP_BX => {@@ -840,15 +920,19 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, insttranslator::emit_call_or_exit::<SYSTEM>(builder,t.ctx_ptr,+ t.host_return_pc,slot_addr,t.block_sig_ref,target_v,+ t.pc.wrapping_add(4),+ t.chain_fall_addr,t.exit_block,);} else {translator::emit_chain_or_exit::<SYSTEM>(builder,t.ctx_ptr,+ t.host_return_pc,slot_addr,t.block_sig_ref,target_v,@@ -868,12 +952,14 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instmatch translator::emit_bc_with_chain::<SYSTEM>(builder,t.ctx_ptr,+ t.host_return_pc,instr,t.pc,t.exit_block,t.chain_slot_addr,t.chain_fall_addr,t.block_sig_ref,+ t.self_loop_header.map(|header| (t.self_pc, header)),) {TermEmit::HandledNia(nia) => {t.last_terminator_nia = Some(nia);@@ -891,6 +977,7 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instmatch translator::emit_bclr_dispatch::<SYSTEM>(builder,t.ctx_ptr,+ t.host_return_pc,instr,t.pc,t.exit_block,@@ -914,6 +1001,7 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instmatch translator::emit_bcctr_dispatch::<SYSTEM>(builder,t.ctx_ptr,+ t.host_return_pc,instr,t.pc,t.exit_block,@@ -940,7 +1028,7 @@ pub fn branch<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, inst#[inline(always)]pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, t.op_cycles);@@ -955,7 +1043,7 @@ pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, inst#[inline(always)]pub fn ps_ops<const OP: u32, const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(OP);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_msr_fp_guard::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local, t.op_cycles);@@ -973,7 +1061,7 @@ pub fn sc<const SYSTEM: SystemId>(t: &mut JitTranslator, _instr: Instruction) {return;}- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let cia_off = abi::cia_offset::<SYSTEM>() as i32;let pc_const = builder.ins().iconst(types::I32, t.pc as i64);@@ -994,7 +1082,7 @@ pub fn sc<const SYSTEM: SystemId>(t: &mut JitTranslator, _instr: Instruction) {#[inline(always)]pub fn mcrxr<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_MCRXR);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_mcrxr::<SYSTEM>(builder, t.ctx_ptr, instr);@@ -1004,7 +1092,7 @@ pub fn mcrxr<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn lwarx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_LWARX);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_lwarx::<SYSTEM>(builder, t.ctx_ptr, instr, local);@@ -1014,7 +1102,7 @@ pub fn lwarx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn stwcx_dot<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_STWCX_DOT);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_stwcx_dot::<SYSTEM>(builder, t.ctx_ptr, instr, local);@@ -1024,7 +1112,7 @@ pub fn stwcx_dot<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instructi#[inline(always)]pub fn lswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_LSWI);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };let ea = if instr.ra() == 0 {@@ -1043,7 +1131,7 @@ pub fn lswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {#[inline(always)]pub fn stswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_STSWI);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let n = if instr.nb() == 0 { 32 } else { instr.nb() as u32 };let ea = if instr.ra() == 0 {@@ -1062,7 +1150,7 @@ pub fn stswi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn lswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_LSWX);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);let rd_v = builder.ins().iconst(types::I32, instr.rd() as i64);@@ -1074,7 +1162,7 @@ pub fn lswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {#[inline(always)]pub fn stswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_STSWX);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let ea = translator::emit_x_form_ea::<SYSTEM>(builder, t.ctx_ptr, instr);let rs_v = builder.ins().iconst(types::I32, instr.rs() as i64);@@ -1086,7 +1174,7 @@ pub fn stswx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_ECIWX);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_load_xform::<SYSTEM>(builder, t.ctx_ptr, instr, MemSize::U32, local.read_u32, false);@@ -1096,7 +1184,7 @@ pub fn eciwx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_ECOWX);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_store_xform::<SYSTEM>(builder,@@ -1114,7 +1202,7 @@ pub fn ecowx<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn mfsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_MFSRIN);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());let idx = builder.ins().ushr_imm(b, 28);@@ -1136,7 +1224,7 @@ pub fn mfsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn mtsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_MTSRIN);- let (builder, _) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, _) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let b = translator::gpr_load::<SYSTEM>(builder, t.ctx_ptr, instr.rb());let idx = builder.ins().ushr_imm(b, 28);@@ -1158,7 +1246,7 @@ pub fn mtsrin<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction)#[inline(always)]pub fn mftb<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_MFTB);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };let tbr = instr.spr_swapped() as u16;@@ -1199,7 +1287,7 @@ pub fn mftb<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {#[inline(always)]pub fn tw<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_TW);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_tw::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);@@ -1209,7 +1297,7 @@ pub fn tw<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {#[inline(always)]pub fn twi<const SYSTEM: SystemId>(t: &mut JitTranslator, instr: Instruction) {t.op_cycles = crate::gekko::cycles::cycles_for_op(lut::OP_TWI);- let (builder, local) = unsafe { parts(t.builder_ptr, t.local_ptr) };+ let (builder, local) = unsafe { parts(t.builder_ptr, t.emit_ctx_ptr) };translator::emit_twi::<SYSTEM>(builder, t.ctx_ptr, instr, t.pc, t.exit_block, local);diff --git a/crates/gecko/src/gekko/jit/runtime.rs b/crates/gecko/src/gekko/jit/runtime.rsindex 94d92d8..4b6041b 100644--- a/crates/gecko/src/gekko/jit/runtime.rs+++ b/crates/gecko/src/gekko/jit/runtime.rs@@ -575,6 +575,23 @@ pub extern "C" fn cause_icbi_wii(sys: *mut core::ffi::c_void, ea: u32) {cause_icbi::<WII>(sys.cast(), ea);}+#[inline(always)]+fn mark_memory_writeback<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) {+ let sys = unsafe { &mut *sys };+ sys.mmio.mark_memory_dirty_range(+ crate::mmio::virt_to_phys(ea) & crate::mmio::CODE_LINE_MASK,+ crate::mmio::CODE_LINE_BYTES,+ );+}++pub extern "C" fn mark_memory_writeback_gc(sys: *mut core::ffi::c_void, ea: u32) {+ mark_memory_writeback::<GC>(sys.cast(), ea);+}++pub extern "C" fn mark_memory_writeback_wii(sys: *mut core::ffi::c_void, ea: u32) {+ mark_memory_writeback::<WII>(sys.cast(), ea);+}+#[inline(always)]fn do_dcbz<const SYSTEM: SystemId>(sys: *mut System<SYSTEM>, ea: u32) {unsafe { (*sys).dcbz_line(ea) }diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rsindex 06fa3fc..b634890 100644--- a/crates/gecko/src/gekko/jit/translator.rs+++ b/crates/gecko/src/gekko/jit/translator.rs@@ -13,6 +13,16 @@ use crate::gekko::jit::idle::{self, IdleClass};use crate::gekko::jit::{ExternFuncs, abi};use crate::system::SystemId;+struct GqrCache {+ values: [Option<Value>; 8],+}++fn should_use_native_self_loop(spec: &BlockSpec, idle_class: IdleClass) -> bool {+ matches!(idle_class, IdleClass::None)+ && spec.terminator == TermKind::BranchCond+ && terminator_static_taken_pc(spec) == Some(spec.start_pc)+}+pub(crate) fn register_alias_regions(builder: &mut FunctionBuilder) {builder.func.dfg.alias_regions.insert(AliasRegionData {user_id: 0,@@ -59,7 +69,7 @@ impl<'a> ChainContext<'a> {pub(crate) const DEFAULT_CYCLES: i64 = crate::gekko::cycles::DEFAULT_CYCLES;#[allow(dead_code)]-pub(crate) struct LocalFuncs {+pub(crate) struct BlockEmitContext {pub(crate) cause_invalid_opcode: FuncRef,pub(crate) advance_to_deadline: FuncRef,pub(crate) read_u8: FuncRef,@@ -91,21 +101,51 @@ pub(crate) struct LocalFuncs {pub(crate) do_psq_store: FuncRef,pub(crate) read_timebase: FuncRef,pub(crate) cause_icbi: FuncRef,+ pub(crate) mark_memory_writeback: FuncRef,pub(crate) cause_smc_write: FuncRef,pub(crate) dcbz: FuncRef,- pub(crate) fp_guard_emitted: std::cell::Cell<bool>,+ pub(crate) fp_guard_emitted: bool,+ gqr_cache: GqrCache,+}++fn gqr_load<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ emit: &mut BlockEmitContext,+ index: u8,+) -> Value {+ if let Some(value) = emit.gqr_cache.values[index as usize] {+ return value;+ }++ let value = builder.ins().load(+ types::I32,+ vmctx_flags(),+ ctx_ptr,+ abi::gqr_offset::<SYSTEM>(index) as i32,+ );+ emit.gqr_cache.values[index as usize] = Some(value);++ value+}++fn gqr_cache_store(emit: &mut BlockEmitContext, index: u8, value: Value) {+ emit.gqr_cache.values[index as usize] = Some(value);}pub struct JitTranslator {pub(crate) builder_ptr: usize,- pub(crate) local_ptr: usize,+ pub(crate) emit_ctx_ptr: usize,pub ctx_ptr: Value,+ pub host_return_pc: Value,pub pc: u32,pub exit_block: cranelift_codegen::ir::Block,pub chain_slot_addr: Option<i64>,pub chain_fall_addr: Option<i64>,pub block_sig_ref: cranelift_codegen::ir::SigRef,pub block_lookup_table_addr: i64,+ pub self_pc: u32,+ pub self_loop_header: Option<Block>,pub is_terminator: bool,pub chained: bool,pub last_terminator_nia: Option<Value>,@@ -123,7 +163,7 @@ pub fn translate<const SYSTEM: SystemId>(chain: &ChainContext<'_>,entry_counter_addr: Option<usize>,) {- let local = LocalFuncs {+ let mut local = BlockEmitContext {cause_invalid_opcode: module.declare_func_in_func(extern_funcs.cause_invalid_opcode, &mut ctx.func),advance_to_deadline: module.declare_func_in_func(extern_funcs.advance_to_deadline, &mut ctx.func),read_u8: module.declare_func_in_func(extern_funcs.read_u8, &mut ctx.func),@@ -155,9 +195,11 @@ pub fn translate<const SYSTEM: SystemId>(do_psq_store: module.declare_func_in_func(extern_funcs.do_psq_store, &mut ctx.func),read_timebase: module.declare_func_in_func(extern_funcs.read_timebase, &mut ctx.func),cause_icbi: module.declare_func_in_func(extern_funcs.cause_icbi, &mut ctx.func),+ mark_memory_writeback: module.declare_func_in_func(extern_funcs.mark_memory_writeback, &mut ctx.func),cause_smc_write: module.declare_func_in_func(extern_funcs.cause_smc_write, &mut ctx.func),dcbz: module.declare_func_in_func(extern_funcs.dcbz, &mut ctx.func),- fp_guard_emitted: std::cell::Cell::new(false),+ fp_guard_emitted: false,+ gqr_cache: GqrCache { values: [None; 8] },};let idle_class = idle::classify::<SYSTEM>(spec, gprs);@@ -175,13 +217,15 @@ pub fn translate<const SYSTEM: SystemId>(let mut builder = FunctionBuilder::new(&mut ctx.func, builder_ctx);register_alias_regions(&mut builder);-let entry = builder.create_block();builder.append_block_params_for_function_params(entry);builder.switch_to_block(entry);builder.seal_block(entry);let ctx_ptr: Value = builder.block_params(entry)[0];+ let host_return_pc: Value = builder.block_params(entry)[1];++ let is_self_loop = should_use_native_self_loop(spec, idle_class);if let Some(addr) = entry_counter_addr {let slot_v = builder.ins().iconst(types::I64, addr as i64);@@ -192,13 +236,17 @@ pub fn translate<const SYSTEM: SystemId>(let exit_block = builder.create_block();builder.append_block_param(exit_block, types::I32);+ let dirty_exit_block = builder.create_block();+ builder.append_block_param(dirty_exit_block, types::I32);let dirty_off = abi::jit_dirty_offset::<SYSTEM>() as i32;let dirty = builder.ins().load(types::I8, vmctx_flags(), ctx_ptr, dirty_off);let nz = builder.ins().icmp_imm(IntCC::NotEqual, dirty, 0);let alive = builder.create_block();let start_nia = builder.ins().iconst(types::I32, spec.start_pc as i64);- builder.ins().brif(nz, exit_block, &[start_nia.into()], alive, &[]);+ builder+ .ins()+ .brif(nz, dirty_exit_block, &[start_nia.into()], alive, &[]);builder.switch_to_block(alive);builder.seal_block(alive);@@ -229,7 +277,15 @@ pub fn translate<const SYSTEM: SystemId>(let final_nia = builder.ins().iconst(types::I32, spec.start_pc.wrapping_add(12) as i64);if let Some(slot_addr) = chain_fall_addr {- emit_chain_or_exit::<SYSTEM>(&mut builder, ctx_ptr, slot_addr, block_sig_ref, final_nia, exit_block);+ emit_chain_or_exit::<SYSTEM>(+ &mut builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ final_nia,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[final_nia.into()]);}@@ -238,6 +294,11 @@ pub fn translate<const SYSTEM: SystemId>(builder.seal_block(exit_block);let exit_nia = builder.block_params(exit_block)[0];builder.ins().return_(&[exit_nia]);++ builder.switch_to_block(dirty_exit_block);+ builder.seal_block(dirty_exit_block);+ let dirty_nia = builder.block_params(dirty_exit_block)[0];+ builder.ins().return_(&[dirty_nia]);builder.finalize();return;}@@ -246,16 +307,27 @@ pub fn translate<const SYSTEM: SystemId>(let mut chained = false;let mut pending_cycles: i64 = 0;+ let self_loop_header = is_self_loop.then(|| {+ let header = builder.create_block();+ builder.ins().jump(header, &[]);+ builder.switch_to_block(header);++ header+ });+let mut t = JitTranslator {builder_ptr: &mut builder as *mut FunctionBuilder<'_> as *mut () as usize,- local_ptr: &local as *const LocalFuncs as usize,+ emit_ctx_ptr: &mut local as *mut BlockEmitContext as usize,ctx_ptr,+ host_return_pc,pc: 0,exit_block,chain_slot_addr,chain_fall_addr,block_sig_ref,block_lookup_table_addr: chain.block_lookup_table_addr,+ self_pc: spec.start_pc,+ self_loop_header,is_terminator: false,chained: false,last_terminator_nia: None,@@ -326,15 +398,15 @@ pub fn translate<const SYSTEM: SystemId>(}drop(t);+ if let Some(header) = self_loop_header {+ builder.seal_block(header);+ }+if pending_cycles > 0 {self::emit_cycles_add::<SYSTEM>(&mut builder, ctx_ptr, pending_cycles);}if !chained {- if idle_class != IdleClass::None {- builder.ins().call(local.advance_to_deadline, &[ctx_ptr]);- }-let final_nia = match spec.terminator {TermKind::LengthCap => builder.ins().iconst(types::I32, spec.end_pc() as i64),_ => last_terminator_nia.unwrap_or_else(|| {@@ -343,8 +415,38 @@ pub fn translate<const SYSTEM: SystemId>(builder.ins().load(types::I32, vmctx_flags(), nia_addr, 0)}),};++ match idle_class {+ IdleClass::None | IdleClass::PointerIterLoop { .. } => {}+ IdleClass::BranchToSelf => {+ builder.ins().call(local.advance_to_deadline, &[ctx_ptr]);+ }+ IdleClass::PollingLoop => {+ let taken_pc =+ terminator_static_taken_pc(spec).expect("polling-loop classifier requires a static branch target");+ let taken = builder.ins().icmp_imm(IntCC::Equal, final_nia, taken_pc as i64);+ let advance = builder.create_block();+ let resume = builder.create_block();+ builder.ins().brif(taken, advance, &[], resume, &[]);+ builder.switch_to_block(advance);+ builder.seal_block(advance);+ builder.ins().call(local.advance_to_deadline, &[ctx_ptr]);+ builder.ins().jump(resume, &[]);+ builder.switch_to_block(resume);+ builder.seal_block(resume);+ }+ }+if let Some(slot_addr) = chain_slot_addr {- emit_chain_or_exit::<SYSTEM>(&mut builder, ctx_ptr, slot_addr, block_sig_ref, final_nia, exit_block);+ emit_chain_or_exit::<SYSTEM>(+ &mut builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ final_nia,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[final_nia.into()]);}@@ -356,6 +458,11 @@ pub fn translate<const SYSTEM: SystemId>(let exit_nia = builder.block_params(exit_block)[0];builder.ins().return_(&[exit_nia]);+ builder.switch_to_block(dirty_exit_block);+ builder.seal_block(dirty_exit_block);+ let dirty_nia = builder.block_params(dirty_exit_block)[0];+ builder.ins().return_(&[dirty_nia]);+builder.finalize();}@@ -616,7 +723,7 @@ pub(crate) fn emit_twi<const SYSTEM: SystemId>(instr: Instruction,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let to = (instr.bo() as u32) as u32;let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());@@ -630,7 +737,7 @@ pub(crate) fn emit_tw<const SYSTEM: SystemId>(instr: Instruction,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let to = (instr.bo() as u32) as u32;let a = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());@@ -646,7 +753,7 @@ pub(crate) fn emit_trap_check<const SYSTEM: SystemId>(to: u32,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let _ = local;let mut cond: Option<Value> = None;@@ -919,7 +1026,7 @@ pub(crate) fn emit_mfspr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &BlockEmitContext,) -> bool {let spr_num = instr.spr_swapped() as u16;if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {@@ -938,12 +1045,16 @@ pub(crate) fn emit_mtspr<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &mut BlockEmitContext,) -> bool {let spr_num = instr.spr_swapped() as u16;let val = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());if let Some(off) = abi::spr_field_offset::<SYSTEM>(spr_num).map(|off| off as i32) {builder.ins().store(vmctx_flags(), val, ctx_ptr, off);++ if matches!(spr_num, 912..=919) {+ gqr_cache_store(local, (spr_num - 912) as u8, val);+ }} else {let num_v = builder.ins().iconst(types::I32, spr_num as i64);builder.ins().call(local.write_spr, &[ctx_ptr, num_v, val]);@@ -1072,14 +1183,16 @@ pub(crate) fn emit_b<const SYSTEM: SystemId>(pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,instr: Instruction,pc: u32,exit_block: Block,taken_slot: Option<i64>,fall_slot: Option<i64>,block_sig_ref: cranelift_codegen::ir::SigRef,+ self_loop: Option<(u32, Block)>,) -> TermEmit {- if taken_slot.is_none() && fall_slot.is_none() {+ if taken_slot.is_none() && fall_slot.is_none() && self_loop.is_none() {return TermEmit::HandledNia(emit_bc::<SYSTEM>(builder, ctx_ptr, instr, pc));}@@ -1137,8 +1250,19 @@ pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(lr_store::<SYSTEM>(builder, ctx_ptr, lr_val);}let taken_v = builder.ins().iconst(types::I32, taken_target as i64);- if let Some(slot_addr) = taken_slot {- emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, taken_v, exit_block);++ if let Some((_, header)) = self_loop.filter(|(self_pc, _)| *self_pc == taken_target && !instr.lk()) {+ emit_loop_or_exit::<SYSTEM>(builder, ctx_ptr, header, taken_v, exit_block);+ } else if let Some(slot_addr) = taken_slot {+ emit_chain_or_exit::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ taken_v,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[taken_v.into()]);}@@ -1148,7 +1272,15 @@ pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(let fall_v = builder.ins().iconst(types::I32, fall_target as i64);if let Some(slot_addr) = fall_slot {- emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, fall_v, exit_block);+ emit_chain_or_exit::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ fall_v,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[fall_v.into()]);}@@ -1156,6 +1288,31 @@ pub(crate) fn emit_bc_with_chain<const SYSTEM: SystemId>(TermEmit::HandledChained}+fn emit_loop_or_exit<const SYSTEM: SystemId>(+ builder: &mut FunctionBuilder,+ ctx_ptr: Value,+ loop_header: Block,+ nia_if_exit: Value,+ exit_block: Block,+) {+ let cycles = builder.ins().load(+ types::I64,+ vmctx_flags(),+ ctx_ptr,+ abi::cycles_offset::<SYSTEM>() as i32,+ );+ let deadline = builder.ins().load(+ types::I64,+ vmctx_flags(),+ ctx_ptr,+ abi::next_deadline_offset::<SYSTEM>() as i32,+ );+ let exhausted = builder.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, cycles, deadline);+ builder+ .ins()+ .brif(exhausted, exit_block, &[nia_if_exit.into()], loop_header, &[]);+}+pub(crate) fn emit_bc<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,@@ -1232,6 +1389,7 @@ pub(crate) fn lr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctxpub(crate) fn emit_bclr_dispatch<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,instr: Instruction,pc: u32,exit_block: Block,@@ -1243,11 +1401,20 @@ pub(crate) fn emit_bclr_dispatch<const SYSTEM: SystemId>(let unconditional = (bo & 0x14) == 0x14;if unconditional && !instr.lk() {- emit_blr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)+ emit_blr_with_ic::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ pc,+ exit_block,+ block_sig_ref,+ lookup_table_addr,+ )} else if !unconditional {self::emit_bclr_chain_fall::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,instr,pc,exit_block,@@ -1263,6 +1430,7 @@ pub(crate) fn emit_bclr_dispatch<const SYSTEM: SystemId>(pub(crate) fn emit_bcctr_dispatch<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,instr: Instruction,pc: u32,exit_block: Block,@@ -1273,11 +1441,20 @@ pub(crate) fn emit_bcctr_dispatch<const SYSTEM: SystemId>(let bo = (instr.bo() as u32) as u8;let unconditional = (bo & 0x14) == 0x14;if unconditional && !instr.lk() {- emit_bctr_with_ic::<SYSTEM>(builder, ctx_ptr, pc, exit_block, block_sig_ref, lookup_table_addr)+ emit_bctr_with_ic::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ pc,+ exit_block,+ block_sig_ref,+ lookup_table_addr,+ )} else if !unconditional {self::emit_bcctr_chain_fall::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,instr,pc,exit_block,@@ -1293,6 +1470,7 @@ pub(crate) fn emit_bcctr_dispatch<const SYSTEM: SystemId>(fn emit_bclr_chain_fall<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,instr: Instruction,pc: u32,exit_block: Block,@@ -1353,6 +1531,7 @@ fn emit_bclr_chain_fall<const SYSTEM: SystemId>(emit_inline_cache_dispatch::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,target_pc,exit_block,block_sig_ref,@@ -1365,7 +1544,15 @@ fn emit_bclr_chain_fall<const SYSTEM: SystemId>(let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);if let Some(slot_addr) = fall_slot {- emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, fall_v, exit_block);+ emit_chain_or_exit::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ fall_v,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[fall_v.into()]);}@@ -1376,6 +1563,7 @@ fn emit_bclr_chain_fall<const SYSTEM: SystemId>(fn emit_bcctr_chain_fall<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,instr: Instruction,pc: u32,exit_block: Block,@@ -1409,6 +1597,7 @@ fn emit_bcctr_chain_fall<const SYSTEM: SystemId>(emit_inline_cache_dispatch::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,target_pc,exit_block,block_sig_ref,@@ -1419,7 +1608,15 @@ fn emit_bcctr_chain_fall<const SYSTEM: SystemId>(builder.seal_block(fall_block);let fall_v = builder.ins().iconst(types::I32, pc.wrapping_add(4) as i64);if let Some(slot_addr) = fall_slot {- emit_chain_or_exit::<SYSTEM>(builder, ctx_ptr, slot_addr, block_sig_ref, fall_v, exit_block);+ emit_chain_or_exit::<SYSTEM>(+ builder,+ ctx_ptr,+ host_return_pc,+ slot_addr,+ block_sig_ref,+ fall_v,+ exit_block,+ );} else {builder.ins().jump(exit_block, &[fall_v.into()]);}@@ -1430,6 +1627,7 @@ fn emit_bcctr_chain_fall<const SYSTEM: SystemId>(pub(crate) fn emit_blr_with_ic<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,_pc: u32,exit_block: Block,block_sig_ref: cranelift_codegen::ir::SigRef,@@ -1437,9 +1635,21 @@ pub(crate) fn emit_blr_with_ic<const SYSTEM: SystemId>() -> TermEmit {let lr = lr_load::<SYSTEM>(builder, ctx_ptr);let target_pc = builder.ins().band_imm(lr, !3i64);+ let direct = builder.ins().icmp(IntCC::Equal, target_pc, host_return_pc);+ let return_block = builder.create_block();+ let dispatch_block = builder.create_block();+ builder.ins().brif(direct, return_block, &[], dispatch_block, &[]);++ builder.switch_to_block(return_block);+ builder.seal_block(return_block);+ builder.ins().return_(&[target_pc]);++ builder.switch_to_block(dispatch_block);+ builder.seal_block(dispatch_block);emit_inline_cache_dispatch::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,target_pc,exit_block,block_sig_ref,@@ -1451,6 +1661,7 @@ pub(crate) fn emit_blr_with_ic<const SYSTEM: SystemId>(pub(crate) fn emit_bctr_with_ic<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,_pc: u32,exit_block: Block,block_sig_ref: cranelift_codegen::ir::SigRef,@@ -1461,6 +1672,7 @@ pub(crate) fn emit_bctr_with_ic<const SYSTEM: SystemId>(emit_inline_cache_dispatch::<SYSTEM>(builder,ctx_ptr,+ host_return_pc,target_pc,exit_block,block_sig_ref,@@ -1472,6 +1684,7 @@ pub(crate) fn emit_bctr_with_ic<const SYSTEM: SystemId>(pub(crate) fn emit_inline_cache_dispatch<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,target_pc: Value,exit_block: Block,block_sig_ref: cranelift_codegen::ir::SigRef,@@ -1515,7 +1728,7 @@ pub(crate) fn emit_inline_cache_dispatch<const SYSTEM: SystemId>(builder.seal_block(hit_block);builder.ins()- .return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr]);+ .return_call_indirect(block_sig_ref, slot_entry, &[ctx_ptr, host_return_pc]);}pub(crate) fn emit_bclr<const SYSTEM: SystemId>(@@ -1748,7 +1961,12 @@ fn terminator_fall_pc(spec: &BlockSpec) -> Option<u32> {let fall_pc = pc.wrapping_add(4);match spec.terminator {- TermKind::BranchCond => Some(fall_pc),+ TermKind::Branch => {+ let instr = Instruction(spec.instrs[last_idx]);++ instr.lk().then_some(fall_pc)+ }+ TermKind::BranchCond | TermKind::Mtmsr => Some(fall_pc),TermKind::BranchToReg => {let raw = spec.instrs[last_idx];let bo = (raw >> 21) & 0x1F;@@ -1794,6 +2012,7 @@ pub(crate) enum TermEmit {pub(crate) fn emit_chain_or_exit<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ host_return_pc: Value,slot_addr: i64,block_sig_ref: cranelift_codegen::ir::SigRef,nia_if_exit: Value,@@ -1829,15 +2048,18 @@ pub(crate) fn emit_chain_or_exit<const SYSTEM: SystemId>(builder.seal_block(tailcall_block);builder.ins()- .return_call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);+ .return_call_indirect(block_sig_ref, target_ptr, &[ctx_ptr, host_return_pc]);}pub(crate) fn emit_call_or_exit<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ caller_host_return_pc: Value,slot_addr: i64,block_sig_ref: cranelift_codegen::ir::SigRef,nia_if_exit: Value,+ return_pc: u32,+ return_slot_addr: Option<i64>,exit_block: Block,) {let cyc_off = abi::cycles_offset::<SYSTEM>() as i32;@@ -1869,10 +2091,40 @@ pub(crate) fn emit_call_or_exit<const SYSTEM: SystemId>(builder.switch_to_block(call_block);builder.seal_block(call_block);- let call = builder.ins().call_indirect(block_sig_ref, target_ptr, &[ctx_ptr]);+ let return_pc_v = builder.ins().iconst(types::I32, return_pc as i64);+ let call = builder+ .ins()+ .call_indirect(block_sig_ref, target_ptr, &[ctx_ptr, return_pc_v]);let returned_nia = builder.inst_results(call)[0];+ let is_expected_return = builder.ins().icmp(IntCC::Equal, returned_nia, return_pc_v);+ let continue_block = builder.create_block();+ let bubble_block = builder.create_block();+ builder.set_cold_block(bubble_block);+ builder+ .ins()+ .brif(is_expected_return, continue_block, &[], bubble_block, &[]);++ builder.switch_to_block(bubble_block);+ builder.seal_block(bubble_block);builder.ins().jump(exit_block, &[returned_nia.into()]);++ builder.switch_to_block(continue_block);+ builder.seal_block(continue_block);++ if let Some(return_slot_addr) = return_slot_addr {+ emit_chain_or_exit::<SYSTEM>(+ builder,+ ctx_ptr,+ caller_host_return_pc,+ return_slot_addr,+ block_sig_ref,+ returned_nia,+ exit_block,+ );+ } else {+ builder.ins().jump(exit_block, &[returned_nia.into()]);+ }}fn gpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {@@ -2290,7 +2542,7 @@ pub(crate) fn emit_lfx_update<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,single: bool,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {@@ -2306,7 +2558,7 @@ pub(crate) fn emit_stfx_update<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,single: bool,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {@@ -2374,7 +2626,7 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &mut BlockEmitContext,store: bool,update: bool,) {@@ -2382,7 +2634,7 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(let b = gpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());let ea = builder.ins().iadd(base, b);- let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_22_24(), store);+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, local, instr.i_22_24(), store);let fast_block = builder.create_block();let slow_block = builder.create_block();builder.set_cold_block(slow_block);@@ -2448,7 +2700,7 @@ pub(crate) fn emit_lwarx<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);emit_load_at_ea::<SYSTEM>(@@ -2469,7 +2721,7 @@ pub(crate) fn emit_stwcx_dot<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_x_form_ea::<SYSTEM>(builder, ctx_ptr, instr);@@ -2525,7 +2777,7 @@ pub(crate) fn emit_lmw_stmw<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,- local: &LocalFuncs,+ local: &BlockEmitContext,store: bool,) {let ea_base = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);@@ -2766,13 +3018,13 @@ pub(crate) fn emit_msr_fp_guard<const SYSTEM: SystemId>(_instr: Instruction,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &mut BlockEmitContext,op_cycles: i64,) {- if local.fp_guard_emitted.get() {+ if local.fp_guard_emitted {return;}- local.fp_guard_emitted.set(true);+ local.fp_guard_emitted = true;let msr_off = abi::msr_offset::<SYSTEM>() as i32;let msr = builder.ins().load(types::I32, vmctx_flags(), ctx_ptr, msr_off);@@ -2806,7 +3058,7 @@ pub(crate) fn emit_lf_d_form_update<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,single: bool,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {@@ -2822,7 +3074,7 @@ pub(crate) fn emit_stf_d_form_update<const SYSTEM: SystemId>(ctx_ptr: Value,instr: Instruction,single: bool,- local: &LocalFuncs,+ local: &BlockEmitContext,) {let ea = emit_d_form_ea::<SYSTEM>(builder, ctx_ptr, instr);if single {@@ -2858,10 +3110,7 @@ pub(crate) fn emit_lfs<const SYSTEM: SystemId>(builder.switch_to_block(fast_block);builder.seal_block(fast_block);let addr = builder.block_params(fast_block)[0];- let raw_u32 = builder.ins().load(types::I32, heap_flags(), addr, 0);- let be_u32 = builder.ins().bswap(raw_u32);- let f32v = builder.ins().bitcast(types::F32, MemFlagsData::new(), be_u32);- let f64v = builder.ins().fpromote(types::F64, f32v);+ let f64v = emit_direct_read_f32(builder, addr);builder.ins().jump(merge, &[f64v.into()]);builder.switch_to_block(slow_block);@@ -2944,10 +3193,7 @@ pub(crate) fn emit_stfs<const SYSTEM: SystemId>(builder.switch_to_block(fast_block);builder.seal_block(fast_block);let addr = builder.block_params(fast_block)[0];- let f32v = builder.ins().fdemote(types::F32, f64v);- let bits = builder.ins().bitcast(types::I32, MemFlagsData::new(), f32v);- let swapped = builder.ins().bswap(bits);- builder.ins().store(heap_flags(), swapped, addr, 0);+ emit_direct_write_f32(builder, addr, f64v);emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 4, cause_smc_write);builder.ins().jump(merge, &[]);@@ -3204,19 +3450,53 @@ pub(crate) fn emit_stfd_at_ea<const SYSTEM: SystemId>(builder.seal_block(merge);}+fn emit_direct_read_f32(builder: &mut FunctionBuilder, addr: Value) -> Value {+ let raw = builder.ins().load(types::I32, heap_flags(), addr, 0);+ let be = builder.ins().bswap(raw);+ let value = builder.ins().bitcast(types::F32, MemFlagsData::new(), be);++ builder.ins().fpromote(types::F64, value)+}++fn emit_direct_read_ps_pair(builder: &mut FunctionBuilder, addr: Value) -> Value {+ let raw = builder.ins().load(types::I64, heap_flags(), addr, 0);+ let swapped = builder.ins().bswap(raw);+ let rotated = builder.ins().rotl_imm(swapped, 32);+ let vector = builder.ins().scalar_to_vector(types::I64X2, rotated);+ let f32s = builder.ins().bitcast(types::F32X4, le_vec_flags(), vector);++ builder.ins().fvpromote_low(f32s)+}++fn emit_direct_write_f32(builder: &mut FunctionBuilder, addr: Value, value: Value) {+ let value = builder.ins().fdemote(types::F32, value);+ let bits = builder.ins().bitcast(types::I32, MemFlagsData::new(), value);+ let swapped = builder.ins().bswap(bits);+ builder.ins().store(heap_flags(), swapped, addr, 0);+}++fn emit_direct_write_ps_pair(builder: &mut FunctionBuilder, addr: Value, pair: Value) {+ let f32s = builder.ins().fvdemote(pair);+ let vector = builder.ins().bitcast(types::I64X2, le_vec_flags(), f32s);+ let low = builder.ins().extractlane(vector, 0);+ let rotated = builder.ins().rotl_imm(low, 32);+ let swapped = builder.ins().bswap(rotated);+ builder.ins().store(heap_flags(), swapped, addr, 0);+}+pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,instr: Instruction,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &mut BlockEmitContext,update: bool,) {let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);- let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), false);+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, local, instr.i_17_19(), false);let fast_block = builder.create_block();let slow_block = builder.create_block();@@ -3266,13 +3546,13 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(instr: Instruction,pc: u32,exit_block: Block,- local: &LocalFuncs,+ local: &mut BlockEmitContext,update: bool,) {let base = gpr_load_or_zero::<SYSTEM>(builder, ctx_ptr, instr.ra());let ea = builder.ins().iadd_imm(base, instr.d_20_31() as i64);- let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, instr.i_17_19(), true);+ let easy = emit_gqr_type_zero_check::<SYSTEM>(builder, ctx_ptr, local, instr.i_17_19(), true);let fast_block = builder.create_block();let slow_block = builder.create_block();@@ -3317,11 +3597,11 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(pub(crate) fn emit_gqr_type_zero_check<const SYSTEM: SystemId>(builder: &mut FunctionBuilder,ctx_ptr: Value,+ local: &mut BlockEmitContext,gqr_idx: u8,store: bool,) -> Value {- let off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;- let gqr = builder.ins().load(types::I32, vmctx_flags(), ctx_ptr, off);+ let gqr = gqr_load::<SYSTEM>(builder, ctx_ptr, local, gqr_idx);let mask = if store { 0x7 } else { 0x70000 };let bits = builder.ins().band_imm(gqr, mask);builder.ins().icmp_imm(IntCC::Equal, bits, 0)@@ -3453,10 +3733,9 @@ pub(crate) fn emit_psq_load_quantized<const SYSTEM: SystemId>(w: bool,gqr_idx: u8,merge: Block,- local: &LocalFuncs,+ local: &mut BlockEmitContext,) {- let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;- let gqr_v = builder.ins().load(types::I32, vmctx_flags(), ctx_ptr, gqr_off);+ let gqr_v = gqr_load::<SYSTEM>(builder, ctx_ptr, local, gqr_idx);let type_shifted = builder.ins().ushr_imm(gqr_v, 16);let type_v = builder.ins().band_imm(type_shifted, 0x7);@@ -3539,10 +3818,9 @@ pub(crate) fn emit_psq_store_quantized<const SYSTEM: SystemId>(w: bool,gqr_idx: u8,merge: Block,- local: &LocalFuncs,+ local: &mut BlockEmitContext,) {- let gqr_off = abi::gqr_offset::<SYSTEM>(gqr_idx) as i32;- let gqr_v = builder.ins().load(types::I32, vmctx_flags(), ctx_ptr, gqr_off);+ let gqr_v = gqr_load::<SYSTEM>(builder, ctx_ptr, local, gqr_idx);let type_v = builder.ins().band_imm(gqr_v, 0x7);let q_bit = builder.ins().band_imm(type_v, 0x4);@@ -3741,12 +4019,7 @@ pub(crate) fn emit_read_ps_pair<const SYSTEM: SystemId>(builder.switch_to_block(fast_block);builder.seal_block(fast_block);let addr = builder.block_params(fast_block)[0];- let raw = builder.ins().load(types::I64, heap_flags(), addr, 0);- let sw = builder.ins().bswap(raw);- let ro = builder.ins().rotl_imm(sw, 32);- let v64 = builder.ins().scalar_to_vector(types::I64X2, ro);- let f32s = builder.ins().bitcast(types::F32X4, le_vec_flags(), v64);- let pair = builder.ins().fvpromote_low(f32s);+ let pair = emit_direct_read_ps_pair(builder, addr);builder.ins().jump(merge, &[pair.into()]);builder.switch_to_block(slow_block);@@ -3787,12 +4060,7 @@ pub(crate) fn emit_write_ps_pair<const SYSTEM: SystemId>(builder.switch_to_block(fast_block);builder.seal_block(fast_block);let addr = builder.block_params(fast_block)[0];- let f32s = builder.ins().fvdemote(pair);- let v64 = builder.ins().bitcast(types::I64X2, le_vec_flags(), f32s);- let lo = builder.ins().extractlane(v64, 0);- let ro = builder.ins().rotl_imm(lo, 32);- let sw = builder.ins().bswap(ro);- builder.ins().store(heap_flags(), sw, addr, 0);+ emit_direct_write_ps_pair(builder, addr, pair);emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);builder.ins().jump(merge, &[]);diff --git a/crates/gecko/src/host.rs b/crates/gecko/src/host.rsindex 188c830..9513efc 100644--- a/crates/gecko/src/host.rs+++ b/crates/gecko/src/host.rs@@ -164,16 +164,16 @@ pub struct XfbPart {pub offset_y: u32,}-/// Per-draw data: primitive type, vertex range, and an optional snapshot of-/// TEV/lighting state. Vertices live in the renderer's scratch buffer (see-/// [`RenderSink::vertex_scratch`]); `base_vertex` is the index into that-/// buffer where this draw's vertices start and `vertex_count` is how many of-/// them belong to this draw.-#[derive(Debug, Default)]-pub struct DrawData {+#[derive(Debug, Clone, Copy)]+pub struct DrawSegment {pub primitive: Primitive,pub base_vertex: u32,pub vertex_count: u32,+}++#[derive(Debug, Default)]+pub struct DrawData {+ pub segments: Vec<DrawSegment>,/// Present only when GX frame state changed since the preceding draw./// Renderers retain the last snapshot across draw-buffer flushes.pub state: Option<DrawState>,@@ -282,13 +282,26 @@ pub trait RenderSink: Send {/// Submit a single action.fn exec(&mut self, action: GxAction);- /// Mutable handle to the sink's vertex scratch buffer. Callers append- /// per-draw vertices here before issuing [`GxAction::Draw`] and store the- /// pre-append length as [`DrawData::base_vertex`]. Real renderers- /// (e.g. wgpu) keep this buffer alive across draws and upload it in one- /// shot at flush time; headless sinks can use a throwaway local.+ fn exec_draw(&mut self, segment: DrawSegment, state: Option<DrawState>) {+ let mut draw = self.take_draw_data();+ draw.segments.clear();+ draw.segments.push(segment);+ draw.state = state;+ self.exec(GxAction::Draw(draw));+ }++ fn exec_draw_batch(&mut self, segments: &[DrawSegment], mut state: Option<DrawState>) {+ for &segment in segments {+ self.exec_draw(segment, state.take());+ }+ }+fn vertex_scratch(&mut self) -> &mut Vec<DrawVertex>;+ fn has_pending_efb_texture(&self, _addr: u32, _width: u32, _height: u32, _fmt: TextureFormat) -> bool {+ false+ }+fn flush_efb_copies(&mut self, ram: &mut crate::mmio::RamViewMut<'_>) {let _ = ram;}diff --git a/crates/gecko/src/jit.rs b/crates/gecko/src/jit.rsindex 7182249..9c73450 100644--- a/crates/gecko/src/jit.rs+++ b/crates/gecko/src/jit.rs@@ -1,3 +1,28 @@#[cfg(feature = "jit")]pub mod arena;pub mod cache;++#[cfg(feature = "jit")]+pub(crate) fn register_jit_code(kind: &str, pc: u32, address: usize, size: usize) {+ use std::io::Write;+ use std::sync::{Mutex, OnceLock};+ static MAP: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();+ let map = MAP.get_or_init(|| {+ let path = std::env::var_os("GECKO_JIT_MAP")?;++ match std::fs::File::create(path) {+ Ok(file) => Some(Mutex::new(file)),+ Err(err) => {+ tracing::warn!(%err, "failed to create JIT symbol map");++ None+ }+ }+ });++ if let Some(map) = map {+ if let Ok(mut file) = map.lock() {+ let _ = writeln!(file, "{address:x} {size:x} {kind}_{pc:08x}");+ }+ }+}diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rsindex 219a74c..85f34b6 100644--- a/crates/gecko/src/mmio.rs+++ b/crates/gecko/src/mmio.rs@@ -17,6 +17,11 @@ pub const CODE_LINE_BYTES: u32 = 32;pub const CODE_LINE_SHIFT: u32 = 5;pub const CODE_LINE_MASK: u32 = !(CODE_LINE_BYTES - 1);+pub const MEMORY_DIRTY_PAGE_SHIFT: u32 = 12;+pub const MEMORY_DIRTY_PAGE_BYTES: usize = 1 << MEMORY_DIRTY_PAGE_SHIFT;+const MEM1_DIRTY_PAGES: usize = RAM_SIZE.div_ceil(MEMORY_DIRTY_PAGE_BYTES);+const MEM2_DIRTY_PAGES: usize = MEM2_SIZE.div_ceil(MEMORY_DIRTY_PAGE_BYTES);+pub const PHYS_MASK: u32 = 0x3FFF_FFFF;pub const MEM1_LINES: usize = RAM_SIZE >> CODE_LINE_SHIFT as usize;@@ -34,6 +39,9 @@ pub struct Mmio<const SYSTEM: SystemId> {pub fastmem_lut_ptr: usize,pub code_refcount: Box<[u8]>,pub code_refcount_ptr: usize,+ memory_write_generation: u64,+ memory_page_generations: Box<[u64]>,+ efb_writeback_pages: FxHashSet<u32>,#[cfg(feature = "jit")]pub pending_icbi: FxHashSet<u32>,#[cfg(feature = "jit")]@@ -46,6 +54,7 @@ pub struct Mmio<const SYSTEM: SystemId> {pub struct RamView<'a> {pub mem1: &'a [u8],pub mem2: &'a [u8],+ pub memory_page_generations: &'a [u64],}impl<'a> RamView<'a> {@@ -65,12 +74,33 @@ impl<'a> RamView<'a> {None}}++ #[inline]+ pub fn range_generation(&self, addr: usize, len: usize) -> Option<u64> {+ if len == 0 {+ return Some(0);+ }++ let end = addr.checked_add(len - 1)?;+ let (first, last) = if end < self.mem1.len() {+ (addr >> MEMORY_DIRTY_PAGE_SHIFT, end >> MEMORY_DIRTY_PAGE_SHIFT)+ } else if addr >= MEM2_BASE as usize && end < (MEM2_BASE as usize).checked_add(self.mem2.len())? {+ let first = MEM1_DIRTY_PAGES + ((addr - MEM2_BASE as usize) >> MEMORY_DIRTY_PAGE_SHIFT);+ let last = MEM1_DIRTY_PAGES + ((end - MEM2_BASE as usize) >> MEMORY_DIRTY_PAGE_SHIFT);+ (first, last)+ } else {+ return None;+ };++ self.memory_page_generations.get(first..=last)?.iter().copied().max()+ }}/// Mutable counterpart of `RamView`. Used by EFB writeback into RAM.pub struct RamViewMut<'a> {pub mem1: &'a mut [u8],pub mem2: &'a mut [u8],+ pub memory_page_generations: &'a [u64],}impl<'a> RamViewMut<'a> {@@ -79,6 +109,7 @@ impl<'a> RamViewMut<'a> {RamView {mem1: self.mem1,mem2: self.mem2,+ memory_page_generations: self.memory_page_generations,}}@@ -145,6 +176,11 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {};let code_refcount: Box<[u8]> = vec![0u8; refcount_len].into_boxed_slice();let code_refcount_ptr = code_refcount.as_ptr() as usize;+ let dirty_page_count = if SYSTEM == WII {+ MEM1_DIRTY_PAGES + MEM2_DIRTY_PAGES+ } else {+ MEM1_DIRTY_PAGES+ };Mmio {ram,@@ -158,6 +194,9 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {fastmem_lut_ptr,code_refcount,code_refcount_ptr,+ memory_write_generation: 0,+ memory_page_generations: vec![0; dirty_page_count].into_boxed_slice(),+ efb_writeback_pages: FxHashSet::default(),#[cfg(feature = "jit")]pending_icbi: FxHashSet::default(),#[cfg(feature = "jit")]@@ -165,6 +204,92 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {}}+ pub fn defer_efb_writeback(&mut self, addr: u32, len: usize) {+ if len == 0 {+ return;+ }++ let Ok(len) = u32::try_from(len) else {+ return;+ };++ let Some(end) = addr.checked_add(len) else {+ return;+ };++ let first_page = addr >> FASTMEM_PAGE_SHIFT;+ let last_page = (end - 1) >> FASTMEM_PAGE_SHIFT;++ for page in first_page..=last_page {+ if !self.efb_writeback_pages.insert(page) {+ continue;+ }++ let phys_base = page << FASTMEM_PAGE_SHIFT;+ let is_fastmem = phys_base <= RAM_END || (SYSTEM == WII && (MEM2_BASE..=MEM2_END).contains(&phys_base));+ let aliases: &[u32] = if is_fastmem {+ &[0x0000_0000, 0x8000_0000, 0xC000_0000]+ } else {+ &[]+ };++ for &alias in aliases {+ self.fastmem_lut[((phys_base | alias) >> FASTMEM_PAGE_SHIFT) as usize] = 0;+ }+ }+ }++ #[inline(always)]+ pub fn efb_writeback_needed(&self, phys: u32, len: usize) -> bool {+ if self.efb_writeback_pages.is_empty() {+ return false;+ }++ let end = phys.saturating_add(len.saturating_sub(1) as u32);+ let first_page = phys >> FASTMEM_PAGE_SHIFT;+ let last_page = end >> FASTMEM_PAGE_SHIFT;++ (first_page..=last_page).any(|page| self.efb_writeback_pages.contains(&page))+ }++ pub fn clear_deferred_efb_writebacks(&mut self) {+ if !self.efb_writeback_pages.is_empty() {+ self.memory_write_generation = self.memory_write_generation.wrapping_add(1).max(1);+ }++ let write_generation = self.memory_write_generation;++ for page in self.efb_writeback_pages.drain() {+ let phys_base = page << FASTMEM_PAGE_SHIFT;+ let phys_end = phys_base + FASTMEM_PAGE_BYTES as u32 - 1;++ if let (Some(first), Some(last)) = (+ Self::memory_dirty_page_index(phys_base),+ Self::memory_dirty_page_index(phys_end),+ ) {+ self.memory_page_generations[first..=last].fill(write_generation);+ }++ let (host_addr, aliases): (usize, &[u32]) = if phys_base <= RAM_END {+ (+ self.ram_ptr + phys_base as usize,+ &[0x0000_0000, 0x8000_0000, 0xC000_0000],+ )+ } else if SYSTEM == WII && (MEM2_BASE..=MEM2_END).contains(&phys_base) {+ (+ self.mem2.as_ptr() as usize + (phys_base - MEM2_BASE) as usize,+ &[0x0000_0000, 0x8000_0000, 0xC000_0000],+ )+ } else {+ continue;+ };++ for &alias in aliases {+ self.fastmem_lut[((phys_base | alias) >> FASTMEM_PAGE_SHIFT) as usize] = host_addr;+ }+ }+ }+/// Resolve a physical address to a `(backing_slice, offset)` pair/// This is the one place that maps physical addresses to memory regions#[inline(always)]@@ -376,9 +501,41 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {self.code_refcount.fill(0);}+ #[inline]+ fn memory_dirty_page_index(phys: u32) -> Option<usize> {+ match phys {+ RAM_BASE..=RAM_END => Some((phys as usize) >> MEMORY_DIRTY_PAGE_SHIFT),+ MEM2_BASE..=MEM2_END if SYSTEM == WII => {+ Some(MEM1_DIRTY_PAGES + (((phys - MEM2_BASE) as usize) >> MEMORY_DIRTY_PAGE_SHIFT))+ }+ _ => None,+ }+ }++ pub fn mark_memory_dirty_range(&mut self, phys: u32, len: u32) {+ if len == 0 {+ return;+ }++ let Some(last_phys) = phys.checked_add(len - 1) else {+ return;+ };++ let (Some(first), Some(last)) = (+ Self::memory_dirty_page_index(phys),+ Self::memory_dirty_page_index(last_phys),+ ) else {+ return;+ };++ self.memory_write_generation = self.memory_write_generation.wrapping_add(1).max(1);+ self.memory_page_generations[first..=last].fill(self.memory_write_generation);+ }+#[cfg(feature = "jit")]#[inline(always)]pub fn queue_icbi_for_range(&mut self, phys: u32, len: u32) {+ self.mark_memory_dirty_range(phys, len);Self::for_each_code_line(phys, len, |line| {if self.is_code_chunk(line) {self.pending_icbi.insert(line);@@ -394,6 +551,7 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {RamView {mem1: &self.ram,mem2: &self.mem2,+ memory_page_generations: &self.memory_page_generations,}}@@ -403,6 +561,7 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {RamViewMut {mem1: &mut self.ram,mem2: &mut self.mem2,+ memory_page_generations: &self.memory_page_generations,}}@@ -633,6 +792,9 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {r.bytes_into(&mut self.lcache)?;r.bytes_into(&mut self.ipl)?;+ self.memory_write_generation = 0;+ self.memory_page_generations.fill(0);+#[cfg(feature = "jit")]{self.pending_icbi.clear();diff --git a/crates/gecko/src/mmio/bus.rs b/crates/gecko/src/mmio/bus.rsindex 86c3980..f327bb8 100644--- a/crates/gecko/src/mmio/bus.rs+++ b/crates/gecko/src/mmio/bus.rs@@ -80,6 +80,17 @@ macro_rules! bus_write_hooks {}impl<const SYSTEM: SystemId> System<SYSTEM> {+ #[cold]+ pub(crate) fn sync_deferred_efb_writeback(&mut self, phys: u32, len: usize) {+ if !self.mmio.efb_writeback_needed(phys, len) {+ return;+ }++ let mut ram = self.mmio.ram_view_mut();+ self.render_sink.flush_efb_copies(&mut ram);+ self.mmio.clear_deferred_efb_writebacks();+ }+/// Translate a virtual address to physical using DBAT registers./// Falls back to simple masking if no BAT matches.#[inline(always)]@@ -153,6 +164,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn read_u8(&mut self, addr: u32) -> u8 {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 1);bus_read_hooks!(self, addr, phys, 1, {if phys <= RAM_END {self.mmio.ram_read_u8(phys)@@ -165,6 +177,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn read_u16(&mut self, addr: u32) -> u16 {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 2);bus_read_hooks!(self, addr, phys, 2, {if phys <= RAM_END - 1 {self.mmio.ram_read_u16(phys)@@ -177,6 +190,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn read_u32(&mut self, addr: u32) -> u32 {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 4);bus_read_hooks!(self, addr, phys, 4, {if phys <= RAM_END - 3 {self.mmio.ram_read_u32(phys)@@ -191,6 +205,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn write_u8(&mut self, addr: u32, val: u8) {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 1);bus_write_hooks!(self, addr, phys, 1, val, {if phys <= RAM_END {self.mmio.ram_write_u8(phys, val);@@ -205,6 +220,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn write_u16(&mut self, addr: u32, val: u16) {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 2);bus_write_hooks!(self, addr, phys, 2, val, {if phys <= RAM_END - 1 {self.mmio.ram_write_u16(phys, val);@@ -219,6 +235,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {#[inline(always)]pub fn write_u32(&mut self, addr: u32, val: u32) {let phys = self.translate_addr(addr);+ self.sync_deferred_efb_writeback(phys, 4);bus_write_hooks!(self, addr, phys, 4, val, {if phys <= RAM_END - 3 {self.mmio.ram_write_u32(phys, val);diff --git a/crates/gecko/src/mmio/fp.rs b/crates/gecko/src/mmio/fp.rsindex 5059e75..2a79aff 100644--- a/crates/gecko/src/mmio/fp.rs+++ b/crates/gecko/src/mmio/fp.rs@@ -7,6 +7,8 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {pub fn read_f64(&mut self, addr: u32) -> f64 {let phys = crate::mmio::virt_to_phys(addr);if phys <= RAM_END - 7 {+ self.sync_deferred_efb_writeback(phys, 8);+return f64::from_bits(self.mmio.ram_read_u64(phys));}@@ -21,6 +23,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {let phys = crate::mmio::virt_to_phys(addr);let bits = val.to_bits();if phys <= RAM_END - 7 {+ self.sync_deferred_efb_writeback(phys, 8);self.mmio.ram_write_u64(phys, bits);#[cfg(feature = "jit")]self.mmio.queue_icbi_for_range(phys, 8);diff --git a/crates/gecko/src/savestate.rs b/crates/gecko/src/savestate.rsindex a68b00d..cd1e82b 100644--- a/crates/gecko/src/savestate.rs+++ b/crates/gecko/src/savestate.rs@@ -240,6 +240,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {pub fn save_state(&mut self) -> Result<Vec<u8>, StateError> {let mut ram = self.mmio.ram_view_mut();self.render_sink.flush_efb_copies(&mut ram);+ self.mmio.clear_deferred_efb_writebacks();let mut w = StateWriter::new();@@ -273,6 +274,7 @@ impl<const SYSTEM: SystemId> System<SYSTEM> {}pub fn load_state(&mut self, data: &[u8]) -> Result<(), StateError> {+ self.mmio.clear_deferred_efb_writebacks();let payload = self::unpack(SYSTEM, data)?;let mut r = StateReader::new(&payload);diff --git a/crates/tinyapp/src/main.rs b/crates/tinyapp/src/main.rsindex 7319ff0..692afb0 100644--- a/crates/tinyapp/src/main.rs+++ b/crates/tinyapp/src/main.rs@@ -216,8 +216,8 @@ fn main() {let args = Args::parse();- let present_mode = if args.immediate {- wgpu::PresentMode::Immediate+ let present_mode = if args.immediate || args.no_cap {+ wgpu::PresentMode::AutoNoVsync} else {wgpu::PresentMode::Fifo};diff --git a/crates/tinybench/src/bin/wii_bench.rs b/crates/tinybench/src/bin/wii_bench.rsindex f088c1d..ee57a47 100644--- a/crates/tinybench/src/bin/wii_bench.rs+++ b/crates/tinybench/src/bin/wii_bench.rs@@ -15,6 +15,9 @@ struct Args {#[arg(long)]coef: Option<String>,+ #[arg(long)]+ load_state: Option<std::path::PathBuf>,+#[arg(long, default_value_t = 60.0)]seconds: f64,@@ -48,6 +51,10 @@ fn main() {emulator.dsp.load_coef(&coef_data);}+ if let Some(path) = &args.load_state {+ emulator.load_state_from_file(path).expect("failed to load savestate");+ }+if !args.no_jit_cache {let (ppc_c, ppc_s, dsp_c, dsp_s, vtx_c, vtx_s) = emulator.load_jit_cache(&game_id);if ppc_c + dsp_c + vtx_c + ppc_s + dsp_s + vtx_s > 0 {