Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

convert ci8 textures on gpu directly + optimize efb readbacks and vertex uploads

Laylef1bbc3c680f7
diff --git a/crates/backend-wgpu/src/action.rs b/crates/backend-wgpu/src/action.rs
index df4b5fd..3505c68 100644
--- a/crates/backend-wgpu/src/action.rs
+++ b/crates/backend-wgpu/src/action.rs
@@ -359,6 +359,70 @@ impl GxRenderer {
self.return_load_texture_to_pool(old_tex);
}
}
+ GxAction::LoadEfbPalette {
+ id,
+ width,
+ height,
+ palette,
+ } => {
+ if self
+ .draw_bg_keys
+ .iter()
+ .any(|k| k.tex_keys.iter().flatten().any(|t| t.ram_addr == id.ram_addr))
+ {
+ self.flush_draws_keep_vertices(device, queue);
+ }
+
+ let entry = self.efb_copy_cache.get(&id.ram_addr).expect("missing palette EFB copy");
+ let source = entry
+ .palette_source
+ .as_ref()
+ .expect("missing native EFB snapshot")
+ .clone();
+ debug_assert_eq!((source.width(), source.height()), (*width, *height));
+
+ let existing = self.texture_cache.get(id).filter(|(_, texture, _)| {
+ texture.width() == *width
+ && texture.height() == *height
+ && texture.mip_level_count() == 1
+ && texture.usage().contains(wgpu::TextureUsages::RENDER_ATTACHMENT)
+ });
+
+ let (texture, view) = if let Some((_, texture, view)) = existing {
+ (texture.clone(), view.clone())
+ } else {
+ if self.texture_cache.contains_key(id) {
+ self.bind_group_cache
+ .retain(|key, _| !key.tex_keys.contains(&Some(*id)));
+ }
+ let texture = device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("efb_ci8_decoded"),
+ size: wgpu::Extent3d {
+ width: *width,
+ height: *height,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format: wgpu::TextureFormat::Rgba8Unorm,
+ usage: wgpu::TextureUsages::TEXTURE_BINDING
+ | wgpu::TextureUsages::RENDER_ATTACHMENT
+ | wgpu::TextureUsages::COPY_SRC
+ | wgpu::TextureUsages::COPY_DST,
+ view_formats: &[],
+ });
+ let view = texture.create_view(&Default::default());
+ (texture, view)
+ };
+
+ let mut encoder = self.take_or_create_encoder(device);
+ self.palette_converter
+ .encode(device, queue, &mut encoder, &source, entry.format, palette, &view);
+ self.current_encoder = Some(encoder);
+ self.texture_cache
+ .insert(*id, (gecko::flipper::gx::draw::TextureFormat::CI8, texture, view));
+ }
GxAction::InvalidateCaches => {
self.flush_pending_draws(device, queue);
let _ = self.submit_pending(queue);
diff --git a/crates/backend-wgpu/src/lib.rs b/crates/backend-wgpu/src/lib.rs
index 2bc2760..bba76a1 100644
--- a/crates/backend-wgpu/src/lib.rs
+++ b/crates/backend-wgpu/src/lib.rs
@@ -4,6 +4,7 @@ mod clear;
#[cfg(not(target_arch = "wasm32"))]
mod dump;
mod helpers;
+mod palette;
mod pipeline;
mod render;
#[cfg(feature = "renderdoc-capture")]
@@ -244,6 +245,7 @@ pub(crate) struct EfbCopyEntry {
pub(crate) native_h: u32,
pub(crate) texture: wgpu::Texture,
pub(crate) view: wgpu::TextureView,
+ pub(crate) palette_source: Option<wgpu::Texture>,
}
impl EfbCopyEntry {
@@ -304,6 +306,7 @@ pub struct GxRenderer {
// Retired LoadTexture allocations grouped by (w, h, mip levels).
pub(crate) texture_pool: FxHashMap<(u32, u32, u32), Vec<wgpu::Texture>>,
pub(crate) efb_copy_cache: FxHashMap<Address, EfbCopyEntry>,
+ pub(crate) palette_converter: palette::PaletteConverter,
pub(crate) efb_copy_pool: FxHashMap<(u32, u32), Vec<(wgpu::Texture, wgpu::TextureView)>>,
pub(crate) efb_pack_pipelines: render::EfbPackPipelines,
pub(crate) efb_depth_pack_pipelines: render::EfbDepthPackPipelines,
@@ -321,6 +324,7 @@ pub struct GxRenderer {
pub(crate) scratch_vertices: Vec<GpuVertex>,
pub(crate) scratch_draws: Vec<DrawRecord>,
pub(crate) scratch_uniform_bytes: Vec<u8>,
+ pub(crate) scratch_vertex_bytes: Vec<u8>,
pub(crate) bind_group_cache: FxHashMap<BindGroupCacheKey, wgpu::BindGroup>,
// Per-frame draw accumulation (persists across process_action calls,
// flushed by flush_pending_draws).
@@ -913,6 +917,7 @@ impl GxRenderer {
r8: depth_pack("efb_depth_pack_r8", "fs_r8"),
rg8: depth_pack("efb_depth_pack_rg8", "fs_rg8"),
};
+ let palette_converter = palette::PaletteConverter::new(device, pack_formats_wgsl, make_pack_pipeline);
let efb_clear = clear::EfbClear::new(
device,
surface_format,
@@ -973,6 +978,7 @@ impl GxRenderer {
texture_cache: FxHashMap::default(),
texture_pool: FxHashMap::default(),
efb_copy_cache: FxHashMap::default(),
+ palette_converter,
efb_copy_pool: FxHashMap::default(),
efb_pack_pipelines,
efb_depth_pack_pipelines,
@@ -984,6 +990,7 @@ impl GxRenderer {
scratch_vertices: Vec::new(),
scratch_draws: Vec::new(),
scratch_uniform_bytes: Vec::new(),
+ scratch_vertex_bytes: Vec::new(),
bind_group_cache: FxHashMap::default(),
frame_uniform_bytes: Vec::new(),
draw_pipeline_keys: Vec::new(),
diff --git a/crates/backend-wgpu/src/palette.rs b/crates/backend-wgpu/src/palette.rs
new file mode 100644
index 0000000..ea53880
--- /dev/null
+++ b/crates/backend-wgpu/src/palette.rs
@@ -0,0 +1,121 @@
+use gecko::flipper::gx::texture::CopyFormat;
+
+pub(crate) struct PaletteConverter {
+ layout: wgpu::BindGroupLayout,
+ i8: wgpu::RenderPipeline,
+ r8: wgpu::RenderPipeline,
+}
+
+impl PaletteConverter {
+ pub(crate) fn new(
+ device: &wgpu::Device,
+ pack_formats_wgsl: &str,
+ make_pipeline: impl Fn(&str, &str, &wgpu::PipelineLayout, &wgpu::ShaderModule) -> wgpu::RenderPipeline,
+ ) -> Self {
+ let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
+ label: Some("efb_palette"),
+ source: wgpu::ShaderSource::Wgsl(
+ format!("{}\n{}", include_str!("shaders/efb_palette.wgsl"), pack_formats_wgsl).into(),
+ ),
+ });
+ let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: Some("efb_palette"),
+ entries: &std::array::from_fn::<_, 2, _>(|binding| wgpu::BindGroupLayoutEntry {
+ binding: binding as u32,
+ visibility: wgpu::ShaderStages::FRAGMENT,
+ ty: wgpu::BindingType::Texture {
+ sample_type: wgpu::TextureSampleType::Float { filterable: false },
+ view_dimension: wgpu::TextureViewDimension::D2,
+ multisampled: false,
+ },
+ count: None,
+ }),
+ });
+ let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
+ label: Some("efb_palette"),
+ bind_group_layouts: &[Some(&layout)],
+ immediate_size: 0,
+ });
+
+ Self {
+ i8: make_pipeline("efb_palette_i8", "fs_palette_i8", &pipeline_layout, &shader),
+ r8: make_pipeline("efb_palette_r8", "fs_palette_r8", &pipeline_layout, &shader),
+ layout,
+ }
+ }
+
+ pub(crate) fn encode(
+ &self,
+ device: &wgpu::Device,
+ queue: &wgpu::Queue,
+ encoder: &mut wgpu::CommandEncoder,
+ source: &wgpu::Texture,
+ format: CopyFormat,
+ palette: &[[u8; 4]; 256],
+ target: &wgpu::TextureView,
+ ) {
+ let palette_texture = device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("efb_ci8_palette"),
+ size: wgpu::Extent3d {
+ width: 256,
+ height: 1,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format: wgpu::TextureFormat::Rgba8Unorm,
+ usage: wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING,
+ view_formats: &[],
+ });
+ queue.write_texture(
+ palette_texture.as_image_copy(),
+ palette.as_flattened(),
+ wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(1024),
+ rows_per_image: None,
+ },
+ palette_texture.size(),
+ );
+ let source_view = source.create_view(&Default::default());
+ let palette_view = palette_texture.create_view(&Default::default());
+ let bindings = device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: Some("efb_ci8_palette"),
+ layout: &self.layout,
+ entries: &[
+ wgpu::BindGroupEntry {
+ binding: 0,
+ resource: wgpu::BindingResource::TextureView(&source_view),
+ },
+ wgpu::BindGroupEntry {
+ binding: 1,
+ resource: wgpu::BindingResource::TextureView(&palette_view),
+ },
+ ],
+ });
+ let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
+ label: Some("efb_ci8_palette"),
+ color_attachments: &[Some(wgpu::RenderPassColorAttachment {
+ view: target,
+ resolve_target: None,
+ depth_slice: None,
+ ops: wgpu::Operations {
+ load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
+ store: wgpu::StoreOp::Store,
+ },
+ })],
+ depth_stencil_attachment: None,
+ occlusion_query_set: None,
+ timestamp_writes: None,
+ multiview_mask: None,
+ });
+ pass.set_pipeline(match format {
+ CopyFormat::I8 => &self.i8,
+ CopyFormat::R8 => &self.r8,
+ _ => panic!("unsupported CI8 source format"),
+ });
+ pass.set_bind_group(0, &bindings, &[]);
+ pass.draw(0..3, 0..1);
+ }
+}
diff --git a/crates/backend-wgpu/src/render.rs b/crates/backend-wgpu/src/render.rs
index c646f80..d409321 100644
--- a/crates/backend-wgpu/src/render.rs
+++ b/crates/backend-wgpu/src/render.rs
@@ -129,45 +129,28 @@ impl GxRenderer {
}
let layout = self.draw_buffer_layout;
- let total_used = layout.index_offset + index_used;
- let Some(size) = std::num::NonZeroU64::new(total_used) else {
- return;
- };
- let mut view = queue
- .write_buffer_with(&self.draw_buffer, 0, size)
- .expect("gx_draw_buffer too small for write_buffer_with");
- let frame_off = layout.frame_offset as usize;
- let draw_off = layout.draw_offset as usize;
- let vertex_off = layout.vertex_offset as usize;
- let index_off = layout.index_offset as usize;
- view.slice(frame_off..frame_off + frame_uniform_bytes.len())
- .copy_from_slice(frame_uniform_bytes);
- view.slice(draw_off..draw_off + self.scratch_uniform_bytes.len())
- .copy_from_slice(&self.scratch_uniform_bytes);
+ queue.write_buffer(&self.draw_buffer, layout.frame_offset, frame_uniform_bytes);
+ queue.write_buffer(&self.draw_buffer, layout.draw_offset, &self.scratch_uniform_bytes);
+ self.scratch_vertex_bytes.resize(vertex_used as usize, 0);
for draw in &self.scratch_draws {
let stride = draw.packed_vertex_stride as usize;
- let texcoord_bytes = stride - 68;
- let mut cursor = vertex_off + draw.packed_vertex_byte_offset as usize;
+ let mut cursor = draw.packed_vertex_byte_offset as usize;
let src_base = draw.src_vertex_index as usize;
let src_end = src_base + draw.vertex_count as usize;
for src_v in &self.scratch_vertices[src_base..src_end] {
- let src_bytes = bytemuck::bytes_of(src_v);
-
- view.slice(cursor..cursor + 68).copy_from_slice(&src_bytes[..68]);
- if texcoord_bytes > 0 {
- view.slice(cursor + 68..cursor + 68 + texcoord_bytes)
- .copy_from_slice(&src_bytes[68..68 + texcoord_bytes]);
- }
-
+ self.scratch_vertex_bytes[cursor..cursor + stride]
+ .copy_from_slice(&bytemuck::bytes_of(src_v)[..stride]);
cursor += stride;
}
}
+ queue.write_buffer(&self.draw_buffer, layout.vertex_offset, &self.scratch_vertex_bytes);
- if !self.scratch_indices.is_empty() {
- view.slice(index_off..index_off + index_used as usize)
- .copy_from_slice(bytemuck::cast_slice(&self.scratch_indices));
- }
+ queue.write_buffer(
+ &self.draw_buffer,
+ layout.index_offset,
+ bytemuck::cast_slice(&self.scratch_indices),
+ );
}
pub(crate) fn execute_copy_xfb(
@@ -378,6 +361,7 @@ impl GxRenderer {
half: bool,
copy_format: CopyFormat,
source: EfbCopySource,
+ palette_source: Option<wgpu::Texture>,
) {
if let EfbCopySource::Color { intensity, .. } = source {
debug_assert!(
@@ -480,6 +464,7 @@ impl GxRenderer {
native_h: dst_h,
texture: tex,
view,
+ palette_source,
},
);
}
@@ -764,6 +749,9 @@ impl GxRenderer {
return;
};
+ let readback_width = if mipmap { (width / 2).max(1) } else { width };
+ let readback_height = if mipmap { (height / 2).max(1) } else { height };
+
if depth_copy {
self.execute_depth_writeback(
device,
@@ -777,6 +765,20 @@ impl GxRenderer {
stride,
copy_format_enum,
);
+
+ let mut encoder = self.take_or_create_encoder(device);
+ let palette_source = self.snapshot_palette_source(
+ device,
+ &mut encoder,
+ dest_addr,
+ copy_format_enum,
+ self.efb_depth_writeback_target.as_ref().unwrap().0.as_image_copy(),
+ readback_width,
+ readback_height,
+ wgpu::TextureFormat::Rgba8Unorm,
+ );
+ self.current_encoder = Some(encoder);
+
self.cache_efb_copy(
device,
queue,
@@ -788,12 +790,11 @@ impl GxRenderer {
mipmap,
copy_format_enum,
EfbCopySource::Depth,
+ palette_source,
);
return;
}
- let readback_width = if mipmap { (width / 2).max(1) } else { width };
- let readback_height = if mipmap { (height / 2).max(1) } else { height };
self.discard_superseded_writebacks(dest_addr, readback_width, readback_height, copy_format_enum, stride);
let bytes_per_row = align_up(readback_width as u64 * 4, 256);
@@ -875,13 +876,25 @@ impl GxRenderer {
(readback_tex, wgpu::Origin3d::ZERO)
};
+ let src = wgpu::TexelCopyTextureInfo {
+ texture: src_texture,
+ mip_level: 0,
+ origin: src_origin,
+ aspect: wgpu::TextureAspect::All,
+ };
+ let palette_source = self.snapshot_palette_source(
+ device,
+ &mut encoder,
+ dest_addr,
+ copy_format_enum,
+ src,
+ readback_width,
+ readback_height,
+ self.surface_format.remove_srgb_suffix(),
+ );
+
encoder.copy_texture_to_buffer(
- wgpu::TexelCopyTextureInfo {
- texture: src_texture,
- mip_level: 0,
- origin: src_origin,
- aspect: wgpu::TextureAspect::default(),
- },
+ src,
wgpu::TexelCopyBufferInfo {
buffer: &staging,
layout: wgpu::TexelCopyBufferLayout {
@@ -935,6 +948,7 @@ impl GxRenderer {
intensity: is_intensity,
alpha_supported,
},
+ palette_source,
);
}
@@ -975,6 +989,47 @@ impl GxRenderer {
});
}
+ fn snapshot_palette_source(
+ &self,
+ device: &wgpu::Device,
+ encoder: &mut wgpu::CommandEncoder,
+ dest_addr: Address,
+ copy_format: CopyFormat,
+ src: wgpu::TexelCopyTextureInfo<'_>,
+ width: u32,
+ height: u32,
+ format: wgpu::TextureFormat,
+ ) -> Option<wgpu::Texture> {
+ if !copy_format.is_palette_index() {
+ return None;
+ }
+
+ let texture = self
+ .efb_copy_cache
+ .get(&dest_addr)
+ .and_then(|entry| entry.palette_source.as_ref())
+ .filter(|texture| texture.width() == width && texture.height() == height && texture.format() == format)
+ .cloned()
+ .unwrap_or_else(|| {
+ device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("efb_native_palette_source"),
+ size: wgpu::Extent3d {
+ width,
+ height,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format,
+ usage: wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING,
+ view_formats: &[],
+ })
+ });
+ encoder.copy_texture_to_texture(src, texture.as_image_copy(), texture.size());
+ Some(texture)
+ }
+
fn encode_depth_readback(
&mut self,
device: &wgpu::Device,
@@ -1193,7 +1248,7 @@ impl GxRenderer {
&& pending.copy_format == copy_format
&& pending.stride == stride
{
- let old = self.pending_writebacks.swap_remove(i);
+ let old = self.pending_writebacks.remove(i);
self.return_readback_staging(old.staging, old.staging_capacity);
} else {
i += 1;
diff --git a/crates/backend-wgpu/src/shaders/efb_palette.wgsl b/crates/backend-wgpu/src/shaders/efb_palette.wgsl
new file mode 100644
index 0000000..53c2270
--- /dev/null
+++ b/crates/backend-wgpu/src/shaders/efb_palette.wgsl
@@ -0,0 +1,20 @@
+@group(0) @binding(0) var source: texture_2d<f32>;
+@group(0) @binding(1) var palette: texture_2d<f32>;
+
+fn fetch(pos: vec2<f32>) -> vec4<f32> {
+ return textureLoad(source, vec2<i32>(pos), 0);
+}
+
+fn lookup(index: u32) -> vec4<f32> {
+ return textureLoad(palette, vec2<i32>(i32(index), 0), 0);
+}
+
+@fragment
+fn fs_palette_r8(@builtin(position) p: vec4<f32>) -> @location(0) vec4<f32> {
+ return lookup(u32(round(fetch(p.xy).r * 255.0)));
+}
+
+@fragment
+fn fs_palette_i8(@builtin(position) p: vec4<f32>) -> @location(0) vec4<f32> {
+ return lookup(luma_u8(fetch(p.xy)));
+}
diff --git a/crates/backend-wgpu/src/sink.rs b/crates/backend-wgpu/src/sink.rs
index ac8648d..672c5cf 100644
--- a/crates/backend-wgpu/src/sink.rs
+++ b/crates/backend-wgpu/src/sink.rs
@@ -86,7 +86,7 @@ struct PendingEfbTexture {
end: u32,
width: u32,
height: u32,
- format: gecko::flipper::gx::draw::TextureFormat,
+ format: gecko::flipper::gx::texture::CopyFormat,
}
#[cfg(not(target_arch = "wasm32"))]
@@ -403,7 +403,7 @@ impl RenderSink for ThreadedSink {
end,
width,
height,
- format: format.base_texture_format(),
+ format,
});
}
}
@@ -457,9 +457,13 @@ impl RenderSink for ThreadedSink {
height: u32,
fmt: gecko::flipper::gx::draw::TextureFormat,
) -> bool {
- self.pending_efb_textures
- .iter()
- .any(|copy| copy.addr == addr && copy.width == width && copy.height == height && copy.format == fmt)
+ self.pending_efb_textures.iter().any(|copy| {
+ copy.addr == addr
+ && copy.width == width
+ && copy.height == height
+ && (copy.format.base_texture_format() == fmt
+ || (copy.format.is_palette_index() && fmt == gecko::flipper::gx::draw::TextureFormat::CI8))
+ })
}
fn flush_efb_copies(&mut self, ram: &mut gecko::mmio::RamViewMut<'_>) {
diff --git a/crates/gecko/src/flipper/gx/bp.rs b/crates/gecko/src/flipper/gx/bp.rs
index 4d284b4..366dcac 100644
--- a/crates/gecko/src/flipper/gx/bp.rs
+++ b/crates/gecko/src/flipper/gx/bp.rs
@@ -347,7 +347,16 @@ impl GraphicsProcessor {
}
};
- if gpu_copy {
+ if gpu_copy && format == draw::TextureFormat::CI8 {
+ renderer.exec(GxAction::LoadEfbPalette {
+ id: cache_id,
+ width,
+ height,
+ palette: Box::new(std::array::from_fn(|i| {
+ texture::palette_lookup(palette, i, tlut.format)
+ })),
+ });
+ } else if gpu_copy {
renderer.exec(GxAction::LoadTexture {
id: cache_id,
width,
@@ -928,4 +937,3 @@ impl GraphicsProcessor {
tracing::debug!(ram_base = format!("{ram_base:#010X}"), tmem_offset, entries, "LOADTLUT");
}
}
-
diff --git a/crates/gecko/src/flipper/gx/texture.rs b/crates/gecko/src/flipper/gx/texture.rs
index 9598963..d12ba0c 100644
--- a/crates/gecko/src/flipper/gx/texture.rs
+++ b/crates/gecko/src/flipper/gx/texture.rs
@@ -519,7 +519,7 @@ fn tlut_entry_to_rgba(packed: u16, format: TlutFormat) -> [u8; 4] {
}
#[inline(always)]
-fn palette_lookup(palette: &[u16], index: usize, format: TlutFormat) -> [u8; 4] {
+pub fn palette_lookup(palette: &[u16], index: usize, format: TlutFormat) -> [u8; 4] {
let entry = palette.get(index).copied().unwrap_or(0);
tlut_entry_to_rgba(entry, format)
}
@@ -774,6 +774,10 @@ impl CopyFormat {
CopyFormat::RGBA8 | CopyFormat::Z24X8 => TextureFormat::RGBA8,
}
}
+
+ pub const fn is_palette_index(self) -> bool {
+ matches!(self, CopyFormat::I8 | CopyFormat::R8)
+ }
}
pub fn encoded_size(w: u32, h: u32, format: CopyFormat) -> usize {
diff --git a/crates/gecko/src/host.rs b/crates/gecko/src/host.rs
index 25c4bef..e056f92 100644
--- a/crates/gecko/src/host.rs
+++ b/crates/gecko/src/host.rs
@@ -68,6 +68,13 @@ pub enum GxAction {
rgba: Vec<u8>,
},
+ LoadEfbPalette {
+ id: TextureKey,
+ width: u32,
+ height: u32,
+ palette: Box<[[u8; 4]; 256]>,
+ },
+
/// Debug action: Drop every cached pipeline, bind group, and texture on
/// the renderer side. Used by the GX debug window to force fresh decodes.
InvalidateCaches,
diff --git a/crates/gecko/src/mmio.rs b/crates/gecko/src/mmio.rs
index 85f34b6..ce5ee89 100644
--- a/crates/gecko/src/mmio.rs
+++ b/crates/gecko/src/mmio.rs
@@ -42,6 +42,7 @@ pub struct Mmio<const SYSTEM: SystemId> {
memory_write_generation: u64,
memory_page_generations: Box<[u64]>,
efb_writeback_pages: FxHashSet<u32>,
+ efb_writeback_ranges: Vec<std::ops::Range<u32>>,
#[cfg(feature = "jit")]
pub pending_icbi: FxHashSet<u32>,
#[cfg(feature = "jit")]
@@ -197,6 +198,7 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
memory_write_generation: 0,
memory_page_generations: vec![0; dirty_page_count].into_boxed_slice(),
efb_writeback_pages: FxHashSet::default(),
+ efb_writeback_ranges: Vec::new(),
#[cfg(feature = "jit")]
pending_icbi: FxHashSet::default(),
#[cfg(feature = "jit")]
@@ -217,6 +219,16 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
return;
};
+ let mut range = addr..end;
+ self.efb_writeback_ranges.retain(|other| {
+ let touches = range.start <= other.end && other.start <= range.end;
+ if touches {
+ range = range.start.min(other.start)..range.end.max(other.end);
+ }
+ !touches
+ });
+ self.efb_writeback_ranges.push(range);
+
let first_page = addr >> FASTMEM_PAGE_SHIFT;
let last_page = (end - 1) >> FASTMEM_PAGE_SHIFT;
@@ -241,18 +253,18 @@ impl<const SYSTEM: SystemId> Mmio<SYSTEM> {
#[inline(always)]
pub fn efb_writeback_needed(&self, phys: u32, len: usize) -> bool {
- if self.efb_writeback_pages.is_empty() {
+ if len == 0 || self.efb_writeback_ranges.is_empty() {
return false;
}
- let end = phys.saturating_add(len.saturating_sub(1) as u32);
- let first_page = phys >> FASTMEM_PAGE_SHIFT;
- let last_page = end >> FASTMEM_PAGE_SHIFT;
-
- (first_page..=last_page).any(|page| self.efb_writeback_pages.contains(&page))
+ let end = phys as u64 + len as u64;
+ self.efb_writeback_ranges
+ .iter()
+ .any(|range| phys < range.end && (range.start as u64) < end)
}
pub fn clear_deferred_efb_writebacks(&mut self) {
+ self.efb_writeback_ranges.clear();
if !self.efb_writeback_pages.is_empty() {
self.memory_write_generation = self.memory_write_generation.wrapping_add(1).max(1);
}