Skip to content

armgit / gecko

Public Imported from GitHub
0

A GameCube/Wii emulator and debugger written in Rust.

SPEED SPEED SPEED. simd paired singles, fused madd

Layleface83131d60
diff --git a/crates/dbglib/src/trace.rs b/crates/dbglib/src/trace.rs
index d5c0808..093d86e 100644
--- a/crates/dbglib/src/trace.rs
+++ b/crates/dbglib/src/trace.rs
@@ -16,7 +16,7 @@ pub fn format_trace_line<const SYSTEM: SystemId>(emulator: &System<SYSTEM>) -> S
if let Some((instr, _)) = GekkoInstruction::decode(emulator.mmio.virt_slice(pc, 4)) {
let text = format!("{}", instr);
- let comment = reg_comment(&text, &emulator.gekko.gprs, &emulator.gekko.fprs);
+ let comment = reg_comment(&text, &emulator.gekko.gprs, &emulator.gekko.fpr_array());
let pad = COMMENT_COL.saturating_sub(DISASM_COL + text.len());
if comment.is_empty() {
format!("{:08X} {:08X} {}", pc, raw, text)
diff --git a/crates/dbglib/src/windows/cpu.rs b/crates/dbglib/src/windows/cpu.rs
index a0085af..594fc89 100644
--- a/crates/dbglib/src/windows/cpu.rs
+++ b/crates/dbglib/src/windows/cpu.rs
@@ -208,7 +208,7 @@ pub fn show_cpu<const SYSTEM: SystemId>(
.scroll_bar_visibility(egui::scroll_area::ScrollBarVisibility::VisibleWhenNeeded)
.show(ui, |ui| {
Grid::new("fprs").num_columns(2).striped(true).show(ui, |ui| {
- for (i, &val) in cpu.fprs.iter().enumerate() {
+ for (i, val) in cpu.fpr_array().into_iter().enumerate() {
ui.label(format!("f{i:<2}"));
ui.monospace(format!("{:+.6e}", val));
ui.end_row();
diff --git a/crates/gecko/src/gekko.rs b/crates/gecko/src/gekko.rs
index 35a1e20..797a015 100644
--- a/crates/gecko/src/gekko.rs
+++ b/crates/gecko/src/gekko.rs
@@ -26,10 +26,12 @@ pub mod lut_wii {
pub const IPL_RESET_VECTOR: u32 = 0xFFF0_0100;
+#[repr(C, align(16))]
+pub struct FprFile(pub [[f64; 2]; 32]);
+
pub struct Gekko {
pub gprs: [u32; 32],
- pub fprs: [f64; 32], // PS0
- pub ps1s: [f64; 32], // PS1 (paired single slot 1)
+ pub fprs: FprFile,
pub pc: u32,
pub cr: ConditionRegister,
pub fpscr: fpscr::Fpscr,
@@ -55,8 +57,7 @@ impl Gekko {
Gekko {
gprs: [0; 32],
- fprs: [0.0; 32],
- ps1s: [1.0; 32],
+ fprs: FprFile([[0.0, 1.0]; 32]),
pc: initial_pc,
cia: initial_pc,
nia: initial_pc.wrapping_add(4),
@@ -85,25 +86,33 @@ impl Gekko {
#[inline(always)]
pub fn read_fpr(&self, index: u8) -> f64 {
debug_assert!(index < 32);
- unsafe { *self.fprs.get_unchecked(index as usize) }
+ unsafe { self.fprs.0.get_unchecked(index as usize)[0] }
}
#[inline(always)]
pub fn write_fpr(&mut self, index: u8, value: f64) {
debug_assert!(index < 32);
- unsafe { *self.fprs.get_unchecked_mut(index as usize) = value }
+ unsafe { self.fprs.0.get_unchecked_mut(index as usize)[0] = value }
}
#[inline(always)]
pub fn read_ps1(&self, index: u8) -> f64 {
debug_assert!(index < 32);
- unsafe { *self.ps1s.get_unchecked(index as usize) }
+ unsafe { self.fprs.0.get_unchecked(index as usize)[1] }
}
#[inline(always)]
pub fn write_ps1(&mut self, index: u8, value: f64) {
debug_assert!(index < 32);
- unsafe { *self.ps1s.get_unchecked_mut(index as usize) = value }
+ unsafe { self.fprs.0.get_unchecked_mut(index as usize)[1] = value }
+ }
+
+ pub fn fpr_array(&self) -> [f64; 32] {
+ core::array::from_fn(|i| self.fprs.0[i][0])
+ }
+
+ pub fn ps1_array(&self) -> [f64; 32] {
+ core::array::from_fn(|i| self.fprs.0[i][1])
}
#[inline(always)]
diff --git a/crates/gecko/src/gekko/interpreter.rs b/crates/gecko/src/gekko/interpreter.rs
index 0039213..8c9816a 100644
--- a/crates/gecko/src/gekko/interpreter.rs
+++ b/crates/gecko/src/gekko/interpreter.rs
@@ -20,6 +20,20 @@ pub use rotate::rotate;
pub use store_load::{eciwx, ecowx, lswi, lswx, lwarx, store_load, store_load_fp, stswi, stswx, stwcx_dot};
pub use system::{dcbz, mfsrin, mftb, msr, mtsrin, nop, sc, segment, spr, tw, twi};
+pub const FRC_KEEP_MASK: u64 = 0xFFFF_FFFF_F800_0000;
+pub const FRC_ROUND_BIT: u64 = 0x0800_0000;
+
+#[inline(always)]
+pub fn round_frc(d: f64) -> f64 {
+ let bits = d.to_bits();
+ f64::from_bits((bits & FRC_KEEP_MASK) + (bits & FRC_ROUND_BIT))
+}
+
+#[inline(always)]
+pub fn neg_unless_nan(d: f64) -> f64 {
+ if d.is_nan() { d } else { -d }
+}
+
#[cold]
#[inline(never)]
pub fn invalid<const SYSTEM: crate::system::SystemId>(
diff --git a/crates/gecko/src/gekko/interpreter/fp_ops.rs b/crates/gecko/src/gekko/interpreter/fp_ops.rs
index 03d394e..a497fb3 100644
--- a/crates/gecko/src/gekko/interpreter/fp_ops.rs
+++ b/crates/gecko/src/gekko/interpreter/fp_ops.rs
@@ -184,23 +184,31 @@ pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, i
OP_FMADDX => fp_write(
ctx,
&instr,
- ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) + ctx.gekko.read_fpr(instr.rb()),
+ ctx.gekko
+ .read_fpr(instr.ra())
+ .mul_add(ctx.gekko.read_fpr(instr.fc()), ctx.gekko.read_fpr(instr.rb())),
),
OP_FMSUBX => fp_write(
ctx,
&instr,
- ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) - ctx.gekko.read_fpr(instr.rb()),
- ),
- OP_FNMADDX => fp_write(
- ctx,
- &instr,
- -(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) + ctx.gekko.read_fpr(instr.rb())),
- ),
- OP_FNMSUBX => fp_write(
- ctx,
- &instr,
- -(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) - ctx.gekko.read_fpr(instr.rb())),
+ ctx.gekko
+ .read_fpr(instr.ra())
+ .mul_add(ctx.gekko.read_fpr(instr.fc()), -ctx.gekko.read_fpr(instr.rb())),
),
+ OP_FNMADDX => {
+ let t = ctx
+ .gekko
+ .read_fpr(instr.ra())
+ .mul_add(ctx.gekko.read_fpr(instr.fc()), ctx.gekko.read_fpr(instr.rb()));
+ fp_write(ctx, &instr, super::neg_unless_nan(t));
+ }
+ OP_FNMSUBX => {
+ let t = ctx
+ .gekko
+ .read_fpr(instr.ra())
+ .mul_add(ctx.gekko.read_fpr(instr.fc()), -ctx.gekko.read_fpr(instr.rb()));
+ fp_write(ctx, &instr, super::neg_unless_nan(t));
+ }
OP_FADDSX => fp_write_single(
ctx,
&instr,
@@ -214,7 +222,7 @@ pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, i
OP_FMULSX => fp_write_single(
ctx,
&instr,
- (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc())) as f32 as f64,
+ (ctx.gekko.read_fpr(instr.ra()) * super::round_frc(ctx.gekko.read_fpr(instr.fc()))) as f32 as f64,
),
OP_FDIVSX => fp_write_single(
ctx,
@@ -224,27 +232,33 @@ pub fn fp_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, i
OP_FMADDSX => fp_write_single(
ctx,
&instr,
- (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) + ctx.gekko.read_fpr(instr.rb())) as f32
- as f64,
+ ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64,
),
OP_FMSUBSX => fp_write_single(
ctx,
&instr,
- (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) - ctx.gekko.read_fpr(instr.rb())) as f32
- as f64,
- ),
- OP_FNMADDSX => fp_write_single(
- ctx,
- &instr,
- (-(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) + ctx.gekko.read_fpr(instr.rb()))) as f32
- as f64,
- ),
- OP_FNMSUBSX => fp_write_single(
- ctx,
- &instr,
- (-(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) - ctx.gekko.read_fpr(instr.rb()))) as f32
- as f64,
+ ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ -ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64,
),
+ OP_FNMADDSX => {
+ let t = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ fp_write_single(ctx, &instr, super::neg_unless_nan(t));
+ }
+ OP_FNMSUBSX => {
+ let t = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ -ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ fp_write_single(ctx, &instr, super::neg_unless_nan(t));
+ }
OP_FSQRTX => fp_write(ctx, &instr, ctx.gekko.read_fpr(instr.rb()).sqrt()),
OP_FSQRTSX => fp_write_single(ctx, &instr, (ctx.gekko.read_fpr(instr.rb()).sqrt()) as f32 as f64),
OP_FRESX => fp_write_single(ctx, &instr, (1.0f32 / ctx.gekko.read_fpr(instr.rb()) as f32) as f64),
diff --git a/crates/gecko/src/gekko/interpreter/ps_ops.rs b/crates/gecko/src/gekko/interpreter/ps_ops.rs
index af8d12c..3894bfb 100644
--- a/crates/gecko/src/gekko/interpreter/ps_ops.rs
+++ b/crates/gecko/src/gekko/interpreter/ps_ops.rs
@@ -22,8 +22,8 @@ pub fn ps_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, i
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MUL => {
- let ps0 = (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc())) as f32 as f64;
- let ps1 = (ctx.gekko.read_ps1(instr.ra()) * ctx.gekko.read_ps1(instr.fc())) as f32 as f64;
+ let ps0 = (ctx.gekko.read_fpr(instr.ra()) * super::round_frc(ctx.gekko.read_fpr(instr.fc()))) as f32 as f64;
+ let ps1 = (ctx.gekko.read_ps1(instr.ra()) * super::round_frc(ctx.gekko.read_ps1(instr.fc()))) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_DIV => {
@@ -32,55 +32,87 @@ pub fn ps_ops<const OP: u32, const SYSTEM: SystemId>(ctx: &mut System<SYSTEM>, i
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MADD => {
- let ps0 = (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) + ctx.gekko.read_fpr(instr.rb()))
- as f32 as f64;
- let ps1 = (ctx.gekko.read_ps1(instr.ra()) * ctx.gekko.read_ps1(instr.fc()) + ctx.gekko.read_ps1(instr.rb()))
- as f32 as f64;
+ let ps0 = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ let ps1 = ctx.gekko.read_ps1(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_ps1(instr.fc())),
+ ctx.gekko.read_ps1(instr.rb()),
+ ) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MSUB => {
- let ps0 = (ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc()) - ctx.gekko.read_fpr(instr.rb()))
- as f32 as f64;
- let ps1 = (ctx.gekko.read_ps1(instr.ra()) * ctx.gekko.read_ps1(instr.fc()) - ctx.gekko.read_ps1(instr.rb()))
- as f32 as f64;
+ let ps0 = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ -ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ let ps1 = ctx.gekko.read_ps1(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_ps1(instr.fc())),
+ -ctx.gekko.read_ps1(instr.rb()),
+ ) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_NMADD => {
- let ps0 = -(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc())
- + ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
- let ps1 = -(ctx.gekko.read_ps1(instr.ra()) * ctx.gekko.read_ps1(instr.fc())
- + ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
+ let t0 = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ let t1 = ctx.gekko.read_ps1(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_ps1(instr.fc())),
+ ctx.gekko.read_ps1(instr.rb()),
+ ) as f32 as f64;
+ let ps0 = super::neg_unless_nan(t0);
+ let ps1 = super::neg_unless_nan(t1);
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_NMSUB => {
- let ps0 = -(ctx.gekko.read_fpr(instr.ra()) * ctx.gekko.read_fpr(instr.fc())
- - ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
- let ps1 = -(ctx.gekko.read_ps1(instr.ra()) * ctx.gekko.read_ps1(instr.fc())
- - ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
+ let t0 = ctx.gekko.read_fpr(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_fpr(instr.fc())),
+ -ctx.gekko.read_fpr(instr.rb()),
+ ) as f32 as f64;
+ let t1 = ctx.gekko.read_ps1(instr.ra()).mul_add(
+ super::round_frc(ctx.gekko.read_ps1(instr.fc())),
+ -ctx.gekko.read_ps1(instr.rb()),
+ ) as f32 as f64;
+ let ps0 = super::neg_unless_nan(t0);
+ let ps1 = super::neg_unless_nan(t1);
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MULS0 => {
- let c0 = ctx.gekko.read_fpr(instr.fc());
+ let c0 = super::round_frc(ctx.gekko.read_fpr(instr.fc()));
let ps0 = (ctx.gekko.read_fpr(instr.ra()) * c0) as f32 as f64;
let ps1 = (ctx.gekko.read_ps1(instr.ra()) * c0) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MULS1 => {
- let c1 = ctx.gekko.read_ps1(instr.fc());
+ let c1 = super::round_frc(ctx.gekko.read_ps1(instr.fc()));
let ps0 = (ctx.gekko.read_fpr(instr.ra()) * c1) as f32 as f64;
let ps1 = (ctx.gekko.read_ps1(instr.ra()) * c1) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MADDS0 => {
- let c0 = ctx.gekko.read_fpr(instr.fc());
- let ps0 = (ctx.gekko.read_fpr(instr.ra()) * c0 + ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
- let ps1 = (ctx.gekko.read_ps1(instr.ra()) * c0 + ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
+ let c0 = super::round_frc(ctx.gekko.read_fpr(instr.fc()));
+ let ps0 = ctx
+ .gekko
+ .read_fpr(instr.ra())
+ .mul_add(c0, ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
+ let ps1 = ctx
+ .gekko
+ .read_ps1(instr.ra())
+ .mul_add(c0, ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_MADDS1 => {
- let c1 = ctx.gekko.read_ps1(instr.fc());
- let ps0 = (ctx.gekko.read_fpr(instr.ra()) * c1 + ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
- let ps1 = (ctx.gekko.read_ps1(instr.ra()) * c1 + ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
+ let c1 = super::round_frc(ctx.gekko.read_ps1(instr.fc()));
+ let ps0 = ctx
+ .gekko
+ .read_fpr(instr.ra())
+ .mul_add(c1, ctx.gekko.read_fpr(instr.rb())) as f32 as f64;
+ let ps1 = ctx
+ .gekko
+ .read_ps1(instr.ra())
+ .mul_add(c1, ctx.gekko.read_ps1(instr.rb())) as f32 as f64;
ps_write(ctx, &instr, ps0, ps1);
}
OP_PS_SUM0 => {
diff --git a/crates/gecko/src/gekko/jit/abi.rs b/crates/gecko/src/gekko/jit/abi.rs
index f268f93..a1fca83 100644
--- a/crates/gecko/src/gekko/jit/abi.rs
+++ b/crates/gecko/src/gekko/jit/abi.rs
@@ -1,9 +1,9 @@
-use core::mem::offset_of;
+use core::mem::{offset_of, size_of};
-use crate::gekko::Gekko;
use crate::gekko::condition::ConditionRegister;
use crate::gekko::msr::Msr;
use crate::gekko::spr::{Spr, Xer};
+use crate::gekko::{FprFile, Gekko};
use crate::mmio::Mmio;
use crate::scheduler::Scheduler;
use crate::system::{System, SystemId};
@@ -21,6 +21,12 @@ const _: () = {
if core::mem::size_of::<Msr>() != 4 {
panic!("Msr is no longer 4 bytes; JIT MSR access needs to be revisited");
}
+ if core::mem::size_of::<FprFile>() != 32 * 16 {
+ panic!("FprFile is no longer 32 16-byte pairs; JIT FPR stride needs to be revisited");
+ }
+ if core::mem::align_of::<FprFile>() != 16 {
+ panic!("FprFile alignment changed; JIT FPR access needs to be revisited");
+ }
};
#[inline(always)]
@@ -60,7 +66,7 @@ pub const fn fpr_base_offset<const SYSTEM: SystemId>() -> usize {
#[inline(always)]
pub const fn ps1_base_offset<const SYSTEM: SystemId>() -> usize {
- gekko_offset::<SYSTEM>() + offset_of!(Gekko, ps1s)
+ gekko_offset::<SYSTEM>() + offset_of!(Gekko, fprs) + size_of::<f64>()
}
#[inline(always)]
diff --git a/crates/gecko/src/gekko/jit/translator.rs b/crates/gecko/src/gekko/jit/translator.rs
index e248274..802b098 100644
--- a/crates/gecko/src/gekko/jit/translator.rs
+++ b/crates/gecko/src/gekko/jit/translator.rs
@@ -1889,7 +1889,7 @@ pub(crate) fn gpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, c
fn fpr_offset<const SYSTEM: SystemId>(i: u8) -> i32 {
debug_assert!(i < 32);
- (abi::fpr_base_offset::<SYSTEM>() + 8 * i as usize) as i32
+ (abi::fpr_base_offset::<SYSTEM>() + 16 * i as usize) as i32
}
pub(crate) fn fpr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
@@ -1904,7 +1904,7 @@ pub(crate) fn fpr_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, c
fn ps1_offset<const SYSTEM: SystemId>(i: u8) -> i32 {
debug_assert!(i < 32);
- (abi::ps1_base_offset::<SYSTEM>() + 8 * i as usize) as i32
+ (abi::ps1_base_offset::<SYSTEM>() + 16 * i as usize) as i32
}
pub(crate) fn ps1_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
@@ -1922,18 +1922,77 @@ pub(crate) fn round_to_single(builder: &mut FunctionBuilder, val: Value) -> Valu
builder.ins().fpromote(types::F64, f32v)
}
-pub(crate) fn f32_reciprocal(builder: &mut FunctionBuilder, b_f64: Value) -> Value {
- let b32 = builder.ins().fdemote(types::F32, b_f64);
- let one32 = builder.ins().f32const(1.0f32);
- let q32 = builder.ins().fdiv(one32, b32);
- builder.ins().fpromote(types::F64, q32)
+pub(crate) fn round_frc_value(builder: &mut FunctionBuilder, c: Value) -> Value {
+ use crate::gekko::interpreter::{FRC_KEEP_MASK, FRC_ROUND_BIT};
+
+ let bits = builder.ins().bitcast(types::I64, MemFlags::new(), c);
+ let keep = builder.ins().band_imm(bits, FRC_KEEP_MASK as i64);
+ let round = builder.ins().band_imm(bits, FRC_ROUND_BIT as i64);
+ let sum = builder.ins().iadd(keep, round);
+ builder.ins().bitcast(types::F64, MemFlags::new(), sum)
}
-pub(crate) fn f32_rsqrte(builder: &mut FunctionBuilder, b_f64: Value) -> Value {
+pub(crate) fn neg_unless_nan(builder: &mut FunctionBuilder, v: Value) -> Value {
+ use cranelift_codegen::ir::condcodes::FloatCC;
+
+ let negged = builder.ins().fneg(v);
+ let nan = builder.ins().fcmp(FloatCC::Unordered, v, v);
+ builder.ins().select(nan, v, negged)
+}
+
+pub(crate) fn fpr_pair_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8) -> Value {
+ let off = fpr_offset::<SYSTEM>(i);
+ builder.ins().load(types::F64X2, vmctx_flags(), ctx_ptr, off)
+}
+
+pub(crate) fn fpr_pair_store<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value, i: u8, v: Value) {
+ let off = fpr_offset::<SYSTEM>(i);
+ builder.ins().store(vmctx_flags(), v, ctx_ptr, off);
+}
+
+pub(crate) fn pair_round_to_single(builder: &mut FunctionBuilder, v: Value) -> Value {
+ let f32v = builder.ins().fvdemote(v);
+ builder.ins().fvpromote_low(f32v)
+}
+
+pub(crate) fn pair_round_frc(builder: &mut FunctionBuilder, c: Value) -> Value {
+ use crate::gekko::interpreter::{FRC_KEEP_MASK, FRC_ROUND_BIT};
+
+ let bits = builder.ins().bitcast(types::I64X2, MemFlags::new(), c);
+ let keep_mask = builder.ins().iconst(types::I64, FRC_KEEP_MASK as i64);
+ let keep_splat = builder.ins().splat(types::I64X2, keep_mask);
+ let round_mask = builder.ins().iconst(types::I64, FRC_ROUND_BIT as i64);
+ let round_splat = builder.ins().splat(types::I64X2, round_mask);
+ let keep = builder.ins().band(bits, keep_splat);
+ let round = builder.ins().band(bits, round_splat);
+ let sum = builder.ins().iadd(keep, round);
+ builder.ins().bitcast(types::F64X2, MemFlags::new(), sum)
+}
+
+pub(crate) fn pair_neg_unless_nan(builder: &mut FunctionBuilder, v: Value) -> Value {
+ use cranelift_codegen::ir::condcodes::FloatCC;
+
+ let negged = builder.ins().fneg(v);
+ let nan = builder.ins().fcmp(FloatCC::Unordered, v, v);
+ pair_bitselect(builder, nan, v, negged)
+}
+
+pub(crate) fn pair_bitselect(builder: &mut FunctionBuilder, mask: Value, x: Value, y: Value) -> Value {
+ let xi = builder.ins().bitcast(types::I64X2, MemFlags::new(), x);
+ let yi = builder.ins().bitcast(types::I64X2, MemFlags::new(), y);
+ let ri = builder.ins().bitselect(mask, xi, yi);
+ builder.ins().bitcast(types::F64X2, MemFlags::new(), ri)
+}
+
+pub(crate) fn build_pair(builder: &mut FunctionBuilder, ps0: Value, ps1: Value) -> Value {
+ let v = builder.ins().scalar_to_vector(types::F64X2, ps0);
+ builder.ins().insertlane(v, ps1, 1)
+}
+
+pub(crate) fn f32_reciprocal(builder: &mut FunctionBuilder, b_f64: Value) -> Value {
let b32 = builder.ins().fdemote(types::F32, b_f64);
- let s32 = builder.ins().sqrt(b32);
let one32 = builder.ins().f32const(1.0f32);
- let q32 = builder.ins().fdiv(one32, s32);
+ let q32 = builder.ins().fdiv(one32, b32);
builder.ins().fpromote(types::F64, q32)
}
@@ -1968,8 +2027,8 @@ pub(crate) fn fpr_store_paired<const SYSTEM: SystemId>(
i: u8,
val: Value,
) {
- fpr_store::<SYSTEM>(builder, ctx_ptr, i, val);
- ps1_store::<SYSTEM>(builder, ctx_ptr, i, val);
+ let v = builder.ins().splat(types::F64X2, val);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, i, v);
}
pub(crate) fn cr_load<const SYSTEM: SystemId>(builder: &mut FunctionBuilder, ctx_ptr: Value) -> Value {
@@ -2322,23 +2381,21 @@ pub(crate) fn emit_psq_x<const SYSTEM: SystemId>(
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
if store {
- let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
- if !instr.w_21_21() {
- let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
- let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);
+ if instr.w_21_21() {
+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
+ } else {
+ let pair = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rd());
+ emit_write_ps_pair::<SYSTEM>(builder, ctx_ptr, ea, pair, local.write_f32, local.cause_smc_write);
}
- } else {
+ } else if instr.w_21_21() {
let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
- let ps1 = if instr.w_21_21() {
- builder.ins().f64const(1.0)
- } else {
- let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)
- };
- fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);
- ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);
+ let one = builder.ins().f64const(1.0);
+ let pair = build_pair(builder, ps0, one);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), pair);
+ } else {
+ let pair = emit_read_ps_pair::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), pair);
}
builder.ins().jump(merge, &[]);
@@ -3158,15 +3215,15 @@ pub(crate) fn emit_psq_l<const SYSTEM: SystemId>(
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
- let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
- let ps1 = if instr.w_16_16() {
- builder.ins().f64const(1.0)
+ if instr.w_16_16() {
+ let ps0 = emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
+ let one = builder.ins().f64const(1.0);
+ let pair = build_pair(builder, ps0, one);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), pair);
} else {
- let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_read_f32::<SYSTEM>(builder, ctx_ptr, ea1, local.read_f32)
- };
- fpr_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps0);
- ps1_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), ps1);
+ let pair = emit_read_ps_pair::<SYSTEM>(builder, ctx_ptr, ea, local.read_f32);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), pair);
+ }
builder.ins().jump(merge, &[]);
builder.switch_to_block(slow_block);
@@ -3214,12 +3271,12 @@ pub(crate) fn emit_psq_st<const SYSTEM: SystemId>(
builder.switch_to_block(fast_block);
builder.seal_block(fast_block);
- let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
- if !instr.w_16_16() {
- let ps1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
- let ea1 = builder.ins().iadd_imm(ea, 4);
- emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea1, ps1, local.write_f32, local.cause_smc_write);
+ if instr.w_16_16() {
+ let ps0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ emit_write_f32::<SYSTEM>(builder, ctx_ptr, ea, ps0, local.write_f32, local.cause_smc_write);
+ } else {
+ let pair = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rs());
+ emit_write_ps_pair::<SYSTEM>(builder, ctx_ptr, ea, pair, local.write_f32, local.cause_smc_write);
}
builder.ins().jump(merge, &[]);
@@ -3647,6 +3704,100 @@ pub(crate) fn emit_write_f32<const SYSTEM: SystemId>(
builder.seal_block(merge);
}
+fn le_vec_flags() -> MemFlags {
+ MemFlags::new().with_endianness(cranelift_codegen::ir::Endianness::Little)
+}
+
+pub(crate) fn emit_read_ps_pair<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ slow_f32: FuncRef,
+) -> Value {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ builder.set_cold_block(slow_block);
+ let merge = builder.create_block();
+ builder.append_block_param(merge, types::F64X2);
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let raw = builder.ins().load(types::I64, heap_flags(), addr, 0);
+ let sw = builder.ins().bswap(raw);
+ let ro = builder.ins().rotl_imm(sw, 32);
+ let v64 = builder.ins().scalar_to_vector(types::I64X2, ro);
+ let f32s = builder.ins().bitcast(types::F32X4, le_vec_flags(), v64);
+ let pair = builder.ins().fvpromote_low(f32s);
+ builder.ins().jump(merge, &[pair.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let c0 = builder.ins().call(slow_f32, &[ctx_ptr, ea]);
+ let p0 = builder.inst_results(c0)[0];
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ let c1 = builder.ins().call(slow_f32, &[ctx_ptr, ea1]);
+ let p1 = builder.inst_results(c1)[0];
+ let slow_pair = build_pair(builder, p0, p1);
+ builder.ins().jump(merge, &[slow_pair.into()]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+ builder.block_params(merge)[0]
+}
+
+pub(crate) fn emit_write_ps_pair<const SYSTEM: SystemId>(
+ builder: &mut FunctionBuilder,
+ ctx_ptr: Value,
+ ea: Value,
+ pair: Value,
+ slow_f32: FuncRef,
+ cause_smc_write: FuncRef,
+) {
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ builder.set_cold_block(slow_block);
+ let merge = builder.create_block();
+
+ let (host_ptr, page_off) = emit_fastmem_lookup::<SYSTEM>(builder, ctx_ptr, ea);
+ let host_addr = builder.ins().iadd(host_ptr, page_off);
+ builder
+ .ins()
+ .brif(host_ptr, fast_block, &[host_addr.into()], slow_block, &[]);
+ builder.append_block_param(fast_block, types::I64);
+
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+ let addr = builder.block_params(fast_block)[0];
+ let f32s = builder.ins().fvdemote(pair);
+ let v64 = builder.ins().bitcast(types::I64X2, le_vec_flags(), f32s);
+ let lo = builder.ins().extractlane(v64, 0);
+ let ro = builder.ins().rotl_imm(lo, 32);
+ let sw = builder.ins().bswap(ro);
+ builder.ins().store(heap_flags(), sw, addr, 0);
+ emit_smc_check::<SYSTEM>(builder, ctx_ptr, ea, 8, cause_smc_write);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ let p0 = builder.ins().extractlane(pair, 0);
+ builder.ins().call(slow_f32, &[ctx_ptr, ea, p0]);
+ let p1 = builder.ins().extractlane(pair, 1);
+ let ea1 = builder.ins().iadd_imm(ea, 4);
+ builder.ins().call(slow_f32, &[ctx_ptr, ea1, p1]);
+ builder.ins().jump(merge, &[]);
+
+ builder.switch_to_block(merge);
+ builder.seal_block(merge);
+}
+
pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(
builder: &mut FunctionBuilder,
ctx_ptr: Value,
@@ -3656,257 +3807,212 @@ pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(
match OP {
lut::OP_PS_ADD => {
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s0 = builder.ins().fadd(a0, b0);
- let r0 = round_to_single(builder, s0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s1 = builder.ins().fadd(a1, b1);
- let r1 = round_to_single(builder, s1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fadd(a, b);
+ let r = pair_round_to_single(builder, s);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_SUB => {
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s0 = builder.ins().fsub(a0, b0);
- let r0 = round_to_single(builder, s0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s1 = builder.ins().fsub(a1, b1);
- let r1 = round_to_single(builder, s1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fsub(a, b);
+ let r = pair_round_to_single(builder, s);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MUL => {
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let m0 = builder.ins().fmul(a0, c0);
- let r0 = round_to_single(builder, m0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let m1 = builder.ins().fmul(a1, c1);
- let r1 = round_to_single(builder, m1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = pair_round_frc(builder, c);
+ let m = builder.ins().fmul(a, c25);
+ let r = pair_round_to_single(builder, m);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_DIV => {
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let q0 = builder.ins().fdiv(a0, b0);
- let r0 = round_to_single(builder, q0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let q1 = builder.ins().fdiv(a1, b1);
- let r1 = round_to_single(builder, q1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let q = builder.ins().fdiv(a, b);
+ let r = pair_round_to_single(builder, q);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MADD => {
- let r0 = ps_fma::<SYSTEM>(
- builder,
- ctx_ptr,
- instr.ra(),
- instr.fc(),
- instr.rb(),
- FmaSign::Pos,
- false,
- );
- let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let c25 = pair_round_frc(builder, c);
+ let t = builder.ins().fma(a, c25, b);
+ let r = pair_round_to_single(builder, t);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MSUB => {
- let r0 = ps_fma::<SYSTEM>(
- builder,
- ctx_ptr,
- instr.ra(),
- instr.fc(),
- instr.rb(),
- FmaSign::Neg,
- false,
- );
- let r1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let c25 = pair_round_frc(builder, c);
+ let nb = builder.ins().fneg(b);
+ let t = builder.ins().fma(a, c25, nb);
+ let r = pair_round_to_single(builder, t);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_NMADD => {
- let v0 = ps_fma::<SYSTEM>(
- builder,
- ctx_ptr,
- instr.ra(),
- instr.fc(),
- instr.rb(),
- FmaSign::Pos,
- false,
- );
- let r0 = builder.ins().fneg(v0);
- let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Pos, true);
- let r1 = builder.ins().fneg(v1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let c25 = pair_round_frc(builder, c);
+ let t = builder.ins().fma(a, c25, b);
+ let r = pair_round_to_single(builder, t);
+ let n = pair_neg_unless_nan(builder, r);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), n);
+ true
}
lut::OP_PS_NMSUB => {
- let v0 = ps_fma::<SYSTEM>(
- builder,
- ctx_ptr,
- instr.ra(),
- instr.fc(),
- instr.rb(),
- FmaSign::Neg,
- false,
- );
- let r0 = builder.ins().fneg(v0);
- let v1 = ps_fma::<SYSTEM>(builder, ctx_ptr, instr.ra(), instr.fc(), instr.rb(), FmaSign::Neg, true);
- let r1 = builder.ins().fneg(v1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let c25 = pair_round_frc(builder, c);
+ let nb = builder.ins().fneg(b);
+ let t = builder.ins().fma(a, c25, nb);
+ let r = pair_round_to_single(builder, t);
+ let n = pair_neg_unless_nan(builder, r);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), n);
+ true
}
lut::OP_PS_MULS0 => {
- let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let m0 = builder.ins().fmul(a0, c0);
- let r0 = round_to_single(builder, m0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let m1 = builder.ins().fmul(a1, c0);
- let r1 = round_to_single(builder, m1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = round_frc_value(builder, c);
+ let cs = builder.ins().splat(types::F64X2, c25);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m = builder.ins().fmul(a, cs);
+ let r = pair_round_to_single(builder, m);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MULS1 => {
- let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let m0 = builder.ins().fmul(a0, c1);
- let r0 = round_to_single(builder, m0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let m1 = builder.ins().fmul(a1, c1);
- let r1 = round_to_single(builder, m1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let c = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = round_frc_value(builder, c);
+ let cs = builder.ins().splat(types::F64X2, c25);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let m = builder.ins().fmul(a, cs);
+ let r = pair_round_to_single(builder, m);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MADDS0 => {
- let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let m0 = builder.ins().fmul(a0, c0);
- let s0 = builder.ins().fadd(m0, b0);
- let r0 = round_to_single(builder, s0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let m1 = builder.ins().fmul(a1, c0);
- let s1 = builder.ins().fadd(m1, b1);
- let r1 = round_to_single(builder, s1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = round_frc_value(builder, c);
+ let cs = builder.ins().splat(types::F64X2, c25);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let t = builder.ins().fma(a, cs, b);
+ let r = pair_round_to_single(builder, t);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MADDS1 => {
- let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let m0 = builder.ins().fmul(a0, c1);
- let s0 = builder.ins().fadd(m0, b0);
- let r0 = round_to_single(builder, s0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let m1 = builder.ins().fmul(a1, c1);
- let s1 = builder.ins().fadd(m1, b1);
- let r1 = round_to_single(builder, s1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let c = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = round_frc_value(builder, c);
+ let cs = builder.ins().splat(types::F64X2, c25);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let t = builder.ins().fma(a, cs, b);
+ let r = pair_round_to_single(builder, t);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_SUM0 => {
- let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s = builder.ins().fadd(a, b);
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fadd(a0, b1);
let r0 = round_to_single(builder, s);
- let r1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, c1);
+ true
}
lut::OP_PS_SUM1 => {
- let r0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let b = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let s = builder.ins().fadd(a, b);
+ let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let s = builder.ins().fadd(a0, b1);
let r1 = round_to_single(builder, s);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), c0, r1);
+ true
}
lut::OP_PS_SEL => {
use cranelift_codegen::ir::condcodes::FloatCC;
+
let zero = builder.ins().f64const(0.0);
- let a0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let c0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let cond0 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a0, zero);
- let r0 = builder.ins().select(cond0, c0, b0);
- let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let c1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let cond1 = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a1, zero);
- let r1 = builder.ins().select(cond1, c1, b1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let zs = builder.ins().splat(types::F64X2, zero);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let mask = builder.ins().fcmp(FloatCC::GreaterThanOrEqual, a, zs);
+ let r = pair_bitselect(builder, mask, c, b);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_RES => {
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let r0 = f32_reciprocal(builder, b0);
- let r1 = f32_reciprocal(builder, b1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let b32 = builder.ins().fvdemote(b);
+ let one = builder.ins().f32const(1.0);
+ let ones = builder.ins().splat(types::F32X4, one);
+ let q = builder.ins().fdiv(ones, b32);
+ let r = builder.ins().fvpromote_low(q);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_RSQRTE => {
- let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let r0 = f32_rsqrte(builder, b0);
- let r1 = f32_rsqrte(builder, b1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), r0, r1);
- return true;
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let b32 = builder.ins().fvdemote(b);
+ let s = builder.ins().sqrt(b32);
+ let one = builder.ins().f32const(1.0);
+ let ones = builder.ins().splat(types::F32X4, one);
+ let q = builder.ins().fdiv(ones, s);
+ let r = builder.ins().fvpromote_low(q);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ true
}
lut::OP_PS_MR => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ let v = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);
true
}
lut::OP_PS_NEG => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let n0 = builder.ins().fneg(p0);
- let n1 = builder.ins().fneg(p1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);
+ let v = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let n = builder.ins().fneg(v);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), n);
true
}
lut::OP_PS_ABS => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let a0 = builder.ins().fabs(p0);
- let a1 = builder.ins().fabs(p1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), a0, a1);
+ let v = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a = builder.ins().fabs(v);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), a);
true
}
lut::OP_PS_NABS => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let a0 = builder.ins().fabs(p0);
- let a1 = builder.ins().fabs(p1);
- let n0 = builder.ins().fneg(a0);
- let n1 = builder.ins().fneg(a1);
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), n0, n1);
+ let v = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a = builder.ins().fabs(v);
+ let n = builder.ins().fneg(a);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), n);
true
}
lut::OP_PS_MERGE00 => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let p1 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let v = builder.ins().insertlane(a, b0, 1);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);
true
}
lut::OP_PS_MERGE01 => {
- let p0 = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ let a = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let b1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let v = builder.ins().insertlane(a, b1, 1);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);
true
}
lut::OP_PS_MERGE10 => {
@@ -3916,9 +4022,10 @@ pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(
true
}
lut::OP_PS_MERGE11 => {
- let p0 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
- let p1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- ps_write::<SYSTEM>(builder, ctx_ptr, instr.rd(), p0, p1);
+ let b = fpr_pair_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
+ let a1 = ps1_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let v = builder.ins().insertlane(b, a1, 0);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, instr.rd(), v);
true
}
lut::OP_PS_CMPU0 | lut::OP_PS_CMPO0 => {
@@ -3937,44 +4044,6 @@ pub(crate) fn emit_ps_arith<const OP: u32, const SYSTEM: SystemId>(
}
}
-#[derive(Clone, Copy)]
-pub(crate) enum FmaSign {
- Pos,
- Neg,
-}
-
-pub(crate) fn ps_fma<const SYSTEM: SystemId>(
- builder: &mut FunctionBuilder,
- ctx_ptr: Value,
- fra: u8,
- frc: u8,
- frb: u8,
- sign: FmaSign,
- pick_ps1: bool,
-) -> Value {
- let a = if pick_ps1 {
- ps1_load::<SYSTEM>(builder, ctx_ptr, fra)
- } else {
- fpr_load::<SYSTEM>(builder, ctx_ptr, fra)
- };
- let c = if pick_ps1 {
- ps1_load::<SYSTEM>(builder, ctx_ptr, frc)
- } else {
- fpr_load::<SYSTEM>(builder, ctx_ptr, frc)
- };
- let b = if pick_ps1 {
- ps1_load::<SYSTEM>(builder, ctx_ptr, frb)
- } else {
- fpr_load::<SYSTEM>(builder, ctx_ptr, frb)
- };
- let m = builder.ins().fmul(a, c);
- let r = match sign {
- FmaSign::Pos => builder.ins().fadd(m, b),
- FmaSign::Neg => builder.ins().fsub(m, b),
- };
- round_to_single(builder, r)
-}
-
pub(crate) fn ps_write<const SYSTEM: SystemId>(
builder: &mut FunctionBuilder,
ctx_ptr: Value,
@@ -3982,8 +4051,8 @@ pub(crate) fn ps_write<const SYSTEM: SystemId>(
ps0: Value,
ps1: Value,
) {
- fpr_store::<SYSTEM>(builder, ctx_ptr, rd, ps0);
- ps1_store::<SYSTEM>(builder, ctx_ptr, rd, ps1);
+ let v = build_pair(builder, ps0, ps1);
+ fpr_pair_store::<SYSTEM>(builder, ctx_ptr, rd, v);
}
pub(crate) fn emit_fp_compare<const SYSTEM: SystemId>(
@@ -4056,7 +4125,16 @@ pub(crate) fn emit_fp_arith<const OP: u32, const SYSTEM: SystemId>(
);
return true;
}
- lut::OP_FMULSX | lut::OP_FMULX => {
+ lut::OP_FMULSX => {
+ let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
+ let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
+ let c25 = round_frc_value(builder, c);
+ let m = builder.ins().fmul(a, c25);
+ let r = round_to_single(builder, m);
+ fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), r);
+ return true;
+ }
+ lut::OP_FMULX => {
emit_fp_binop::<SYSTEM>(
builder,
ctx_ptr,
@@ -4099,18 +4177,14 @@ pub(crate) fn emit_fp_arith<const OP: u32, const SYSTEM: SystemId>(
let a = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.ra());
let c = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.fc());
let b = fpr_load::<SYSTEM>(builder, ctx_ptr, instr.rb());
- let m = builder.ins().fmul(a, c);
- let core = if is_add {
- builder.ins().fadd(m, b)
- } else {
- builder.ins().fsub(m, b)
- };
- let signed = if neg { builder.ins().fneg(core) } else { core };
- let stored = if single {
- round_to_single(builder, signed)
- } else {
- signed
- };
+
+ let c_in = if single { round_frc_value(builder, c) } else { c };
+ let b_in = if is_add { b } else { builder.ins().fneg(b) };
+ let core = builder.ins().fma(a, c_in, b_in);
+
+ let rounded = if single { round_to_single(builder, core) } else { core };
+ let stored = if neg { neg_unless_nan(builder, rounded) } else { rounded };
+
if single {
fpr_store_paired::<SYSTEM>(builder, ctx_ptr, instr.rd(), stored);
} else {
diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs
index fd26066..032ae94 100644
--- a/crates/mcp/src/server.rs
+++ b/crates/mcp/src/server.rs
@@ -719,8 +719,8 @@ fn registers_of<const SYSTEM: gecko::SystemId>(sys: &gecko::System<{ SYSTEM }>)
srr0: g.spr.srr0.raw(),
srr1: g.spr.srr1,
gprs: g.gprs,
- fprs: g.fprs,
- ps1s: g.ps1s,
+ fprs: g.fpr_array(),
+ ps1s: g.ps1_array(),
}
}
diff --git a/crates/tinytracer/src/snaptshot.rs b/crates/tinytracer/src/snaptshot.rs
index 03865c3..0a51106 100644
--- a/crates/tinytracer/src/snaptshot.rs
+++ b/crates/tinytracer/src/snaptshot.rs
@@ -13,7 +13,7 @@ impl CpuSnapshot {
pub fn from_cpu(cpu: &gecko::gekko::Gekko) -> Self {
Self {
gprs: cpu.gprs,
- fprs: cpu.fprs,
+ fprs: cpu.fpr_array(),
lr: cpu.spr.lr,
ctr: cpu.spr.ctr,
cr: cpu.cr,