VERY experimental Nintendo Switch 2 emulator
C++
[dynarmic] avoid stable_vector<> reallocations for shared labels (#3717)
lizziecf7086de7cd0
diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cppindex 049a58432e..bbbecada63 100644--- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp+++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp@@ -59,8 +59,10 @@ static Xbyak::Address MJitStateExtReg(A32::ExtReg reg) {UNREACHABLE();}-A32EmitContext::A32EmitContext(const A32::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block)- : EmitContext(reg_alloc, block), conf(conf) {}+A32EmitContext::A32EmitContext(const A32::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels)+ : EmitContext(reg_alloc, block, shared_labels)+ , conf(conf)+{}A32::LocationDescriptor A32EmitContext::Location() const {return A32::LocationDescriptor{block.Location()};@@ -109,7 +111,8 @@ A32EmitX64::BlockDescriptor A32EmitX64::Emit(IR::Block& block) {gprs.reset(size_t(HostLoc::R14));return gprs;}(), any_xmm);- A32EmitContext ctx{conf, reg_alloc, block};++ A32EmitContext ctx{conf, reg_alloc, block, shared_labels};// Start emitting.code.align();@@ -168,6 +171,7 @@ A32EmitX64::BlockDescriptor A32EmitX64::Emit(IR::Block& block) {auto const bdesc = RegisterBlock(descriptor, entrypoint, size);code.DisableWriting();+ shared_labels.clear();return bdesc;}diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.hindex 5ec78ff50e..8e97dc7737 100644--- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h+++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h@@ -1,4 +1,4 @@-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project// SPDX-License-Identifier: GPL-3.0-or-later/* This file is part of the dynarmic project.@@ -29,7 +29,7 @@ namespace Dynarmic::Backend::X64 {class RegAlloc;struct A32EmitContext final : public EmitContext {- A32EmitContext(const A32::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block);+ A32EmitContext(const A32::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels);A32::LocationDescriptor Location() const;A32::LocationDescriptor EndLocation() const;@@ -130,6 +130,7 @@ public:ankerl::unordered_dense::map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks;ankerl::unordered_dense::map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks;ankerl::unordered_dense::set<DoNotFastmemMarker> do_not_fastmem;+ boost::container::stable_vector<Xbyak::Label> shared_labels;void (*memory_read_128)() = nullptr; // Dummyvoid (*memory_write_128)() = nullptr; // Dummyconst void* terminal_handler_pop_rsb_hint;diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cppindex 7c43ea1ab2..8edeb29aed 100644--- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp+++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp@@ -37,8 +37,10 @@ namespace Dynarmic::Backend::X64 {using namespace Xbyak::util;-A64EmitContext::A64EmitContext(const A64::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block)- : EmitContext(reg_alloc, block), conf(conf) {}+A64EmitContext::A64EmitContext(const A64::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels)+ : EmitContext(reg_alloc, block, shared_labels)+ , conf(conf)+{}A64::LocationDescriptor A64EmitContext::Location() const {return A64::LocationDescriptor{block.Location()};@@ -83,7 +85,8 @@ A64EmitX64::BlockDescriptor A64EmitX64::Emit(IR::Block& block) noexcept {gprs.reset(size_t(HostLoc::R14));return gprs;}(), any_xmm};- A64EmitContext ctx{conf, reg_alloc, block};++ A64EmitContext ctx{conf, reg_alloc, block, shared_labels};// Start emitting.code.align();@@ -160,6 +163,7 @@ finish_this_inst:auto bdesc = RegisterBlock(descriptor, entrypoint, size);code.DisableWriting();+ shared_labels.clear();return bdesc;}diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.hindex dd556e36ce..d57b1d81b9 100644--- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h+++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h@@ -1,4 +1,4 @@-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project// SPDX-License-Identifier: GPL-3.0-or-later/* This file is part of the dynarmic project.@@ -27,7 +27,7 @@namespace Dynarmic::Backend::X64 {struct A64EmitContext final : public EmitContext {- A64EmitContext(const A64::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block);+ A64EmitContext(const A64::UserConfig& conf, RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels);A64::LocationDescriptor Location() const;bool IsSingleStep() const;@@ -126,6 +126,7 @@ public:ankerl::unordered_dense::map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks;ankerl::unordered_dense::map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks;ankerl::unordered_dense::set<DoNotFastmemMarker> do_not_fastmem;+ boost::container::stable_vector<Xbyak::Label> shared_labels;const void* terminal_handler_pop_rsb_hint = nullptr;const void* terminal_handler_fast_dispatch_hint = nullptr;FastDispatchEntry& (*fast_dispatch_table_lookup)(u64) = nullptr;diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cppindex 4e515fef2f..4ed198e09f 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp@@ -32,8 +32,11 @@ namespace Dynarmic::Backend::X64 {using namespace Xbyak::util;-EmitContext::EmitContext(RegAlloc& reg_alloc, IR::Block& block)- : reg_alloc(reg_alloc), block(block) {}+EmitContext::EmitContext(RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels)+ : reg_alloc(reg_alloc)+ , block(block)+ , shared_labels(shared_labels)+{}EmitContext::~EmitContext() = default;diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.hindex 301f4ffc89..619945e19a 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h@@ -16,11 +16,12 @@#include <type_traits>#include <vector>-#include "dynarmic/mcl/bit.hpp"#include <ankerl/unordered_dense.h>-#include "dynarmic/backend/x64/xbyak.h"+#include <boost/container/stable_vector.hpp>#include <boost/container/small_vector.hpp>+#include "dynarmic/backend/x64/xbyak.h"+#include "dynarmic/mcl/bit.hpp"#include "dynarmic/backend/exception_handler.h"#include "dynarmic/backend/x64/reg_alloc.h"#include "dynarmic/common/fp/fpcr.h"@@ -52,24 +53,23 @@ using VectorArray = std::array<T, A64FullVectorWidth::value / mcl::bitsizeof<T>>template<typename T>using HalfVectorArray = std::array<T, A64FullVectorWidth::value / mcl::bitsizeof<T> / 2>;+using SharedLabel = Xbyak::Label*;struct EmitContext {- EmitContext(RegAlloc& reg_alloc, IR::Block& block);+ EmitContext(RegAlloc& reg_alloc, IR::Block& block, boost::container::stable_vector<Xbyak::Label>& shared_labels);virtual ~EmitContext();virtual FP::FPCR FPCR(bool fpcr_controlled = true) const = 0;virtual bool HasOptimization(OptimizationFlag flag) const = 0;- RegAlloc& reg_alloc;- IR::Block& block;+ [[nodiscard]] inline Xbyak::Label* GenSharedLabel() noexcept {+ return &shared_labels.emplace_back();+ }std::vector<std::function<void()>> deferred_emits;+ RegAlloc& reg_alloc;+ IR::Block& block;+ boost::container::stable_vector<Xbyak::Label>& shared_labels;};-using SharedLabel = std::shared_ptr<Xbyak::Label>;--inline SharedLabel GenSharedLabel() {- return std::make_shared<Xbyak::Label>();-}-class EmitX64 {public:struct BlockDescriptor {diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_floating_point.cpp b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_floating_point.cppindex d073991fbe..6a3ab005f3 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_floating_point.cpp+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_floating_point.cpp@@ -136,7 +136,7 @@ void ForceToDefaultNaN(BlockOfCode& code, Xbyak::Xmm result) {template<size_t fsize>SharedLabel ProcessNaN(BlockOfCode& code, EmitContext& ctx, Xbyak::Xmm a) {- SharedLabel nan = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel nan = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();FCODE(ucomis)(a, a);code.jp(*nan, code.T_NEAR);@@ -251,7 +251,7 @@ template<size_t fsize, typename Function>void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) {auto args = ctx.reg_alloc.GetArgumentInfo(inst);- SharedLabel end = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel();Xbyak::Xmm result = ctx.reg_alloc.UseScratchXmm(code, args[0]);@@ -304,7 +304,7 @@ void FPThreeOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn)const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(code);const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr(code);- SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), nan = ctx.GenSharedLabel();code.movaps(result, op1);if constexpr (std::is_member_function_pointer_v<Function>) {@@ -413,7 +413,7 @@ static void EmitFPMinMax(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, boDenormalsAreZero<fsize>(code, ctx, {result, operand});- SharedLabel equal = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel equal = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();FCODE(ucomis)(result, operand);code.jz(*equal, code.T_NEAR);@@ -484,7 +484,7 @@ static inline void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::}};- SharedLabel end = GenSharedLabel(), z = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), z = ctx.GenSharedLabel();FCODE(ucomis)(op1, op2);code.jz(*z, code.T_NEAR);@@ -632,7 +632,7 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, bo}if (code.HasHostFeature(HostFeature::FMA | HostFeature::AVX)) {- SharedLabel fallback = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel fallback = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(code, args[0]);const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(code, args[1]);@@ -843,7 +843,7 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(code);const Xbyak::Reg64 tmp = do_default_nan ? INVALID_REG : ctx.reg_alloc.ScratchGpr(code);- SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), nan = ctx.GenSharedLabel();if (code.HasHostFeature(HostFeature::AVX)) {FCODE(vmuls)(result, op1, op2);@@ -981,7 +981,7 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*}if (code.HasHostFeature(HostFeature::FMA)) {- SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), fallback = ctx.GenSharedLabel();const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(code, args[0]);const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(code, args[1]);@@ -1129,7 +1129,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* iconst Xbyak::Xmm value = ctx.reg_alloc.ScratchXmm(code);[[maybe_unused]] const Xbyak::Reg32 tmp = ctx.reg_alloc.ScratchGpr(code).cvt32();- SharedLabel bad_values = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel bad_values = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();code.movaps(value, operand);@@ -1296,7 +1296,7 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*}if (code.HasHostFeature(HostFeature::FMA | HostFeature::AVX)) {- SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), fallback = ctx.GenSharedLabel();const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(code, args[0]);const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(code, args[1]);@@ -1641,7 +1641,7 @@ static void EmitFPToFixed(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {const Xbyak::Xmm scratch = ctx.reg_alloc.ScratchXmm(code);if (!unsigned_) {- SharedLabel saturate_max = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel saturate_max = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();ZeroIfNaN<64>(code, src, scratch);diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.incindex 54fc595214..4fa14d504b 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc@@ -86,7 +86,7 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) {const auto wrapped_fn = read_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];- SharedLabel abort = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel abort = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();if (fastmem_marker) {// Use fastmem@@ -108,7 +108,7 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) {conf.recompile_on_fastmem_failure,});- EmitCheckMemoryAbort(ctx, inst, end.get());+ EmitCheckMemoryAbort(ctx, inst, end);code.jmp(*end, code.T_NEAR);});} else {@@ -120,7 +120,7 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) {ctx.deferred_emits.emplace_back([=, this, &ctx] {code.L(*abort);code.call(wrapped_fn);- EmitCheckMemoryAbort(ctx, inst, end.get());+ EmitCheckMemoryAbort(ctx, inst, end);code.jmp(*end, code.T_NEAR);});}@@ -173,7 +173,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) {const auto wrapped_fn = write_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];- SharedLabel abort = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel abort = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();if (fastmem_marker) {// Use fastmem@@ -195,7 +195,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) {conf.recompile_on_fastmem_failure,});- EmitCheckMemoryAbort(ctx, inst, end.get());+ EmitCheckMemoryAbort(ctx, inst, end);code.jmp(*end, code.T_NEAR);});} else {@@ -207,7 +207,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) {ctx.deferred_emits.emplace_back([=, this, &ctx] {code.L(*abort);code.call(wrapped_fn);- EmitCheckMemoryAbort(ctx, inst, end.get());+ EmitCheckMemoryAbort(ctx, inst, end);code.jmp(*end, code.T_NEAR);});}@@ -352,7 +352,7 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* inconst auto fastmem_marker = ShouldFastmem(ctx, inst);if (fastmem_marker) {- SharedLabel abort = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel abort = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();bool require_abort_handling = false;const auto src_ptr = EmitFastmemVAddr(code, ctx, *abort, vaddr, require_abort_handling);@@ -427,7 +427,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* iEmitExclusiveLock(code, conf, tmp, tmp2.cvt32());- SharedLabel end = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel();code.mov(status, u32(1));code.movzx(tmp.cvt32(), code.byte[code.ABI_JIT_PTR + offsetof(AxxJitState, exclusive_state)]);@@ -460,7 +460,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* iconst auto fastmem_marker = ShouldFastmem(ctx, inst);if (fastmem_marker) {- SharedLabel abort = GenSharedLabel();+ SharedLabel abort = ctx.GenSharedLabel();bool require_abort_handling = false;const auto dest_ptr = EmitFastmemVAddr(code, ctx, *abort, vaddr, require_abort_handling, tmp);diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.hindex b354efcb51..3ac078f1d7 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h@@ -54,7 +54,7 @@ void EmitDetectMisalignedVAddr(BlockOfCode& code, EmitContext& ctx, size_t bitsiif (ctx.conf.only_detect_misalignment_via_page_table_on_page_boundary) {const u32 page_align_mask = static_cast<u32>(page_table_const_size - 1) & ~align_mask;- SharedLabel detect_boundary = GenSharedLabel(), resume = GenSharedLabel();+ SharedLabel detect_boundary = ctx.GenSharedLabel(), resume = ctx.GenSharedLabel();code.jnz(*detect_boundary, code.T_NEAR);code.L(*resume);diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_vector_floating_point.cpp b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_vector_floating_point.cppindex 70edfbd0bc..046ecc78d6 100644--- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_vector_floating_point.cpp+++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_vector_floating_point.cpp@@ -24,6 +24,7 @@#include "dynarmic/common/fp/fpcr.h"#include "dynarmic/common/fp/info.h"#include "dynarmic/common/fp/op.h"+#include "dynarmic/common/fp/rounding_mode.h"#include "dynarmic/common/fp/util.h"#include "dynarmic/interface/optimization_flags.h"#include "dynarmic/ir/basic_block.h"@@ -93,7 +94,7 @@ void HandleNaNs(BlockOfCode& code, EmitContext& ctx, bool fpcr_controlled, std::code.cmp(bitmask, 0);}- SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), nan = ctx.GenSharedLabel();code.jnz(*nan, code.T_NEAR);code.L(*end);@@ -188,23 +189,6 @@ void ForceToDefaultNaN(BlockOfCode& code, FP::FPCR fpcr, Xbyak::Xmm result) {}}-template<size_t fsize>-void ZeroIfNaN(BlockOfCode& code, Xbyak::Xmm result) {- const Xbyak::Xmm nan_mask = xmm0;- if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {- constexpr u32 nan_to_zero = FixupLUT(FpFixup::PosZero,- FpFixup::PosZero);- FCODE(vfixupimmp)(result, result, code.BConst<32>(ptr_b, nan_to_zero), u8(0));- } else if (code.HasHostFeature(HostFeature::AVX)) {- FCODE(vcmpordp)(nan_mask, result, result);- FCODE(vandp)(result, result, nan_mask);- } else {- code.movaps(nan_mask, result);- FCODE(cmpordp)(nan_mask, nan_mask);- code.andps(result, nan_mask);- }-}-template<size_t fsize>void DenormalsAreZero(BlockOfCode& code, FP::FPCR fpcr, std::initializer_list<Xbyak::Xmm> to_daz, Xbyak::Xmm tmp) {if (fpcr.FZ()) {@@ -1330,7 +1314,7 @@ void EmitFPVectorMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(code);const Xbyak::Xmm tmp = ctx.reg_alloc.ScratchXmm(code);- SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), fallback = ctx.GenSharedLabel();MaybeStandardFPSCRValue(code, ctx, fpcr_controlled, [&] {code.movaps(result, xmm_a);@@ -1603,7 +1587,7 @@ static void EmitRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* inconst Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(code, args[1]);const Xbyak::Xmm tmp = ctx.reg_alloc.ScratchXmm(code);- SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), fallback = ctx.GenSharedLabel();MaybeStandardFPSCRValue(code, ctx, fpcr_controlled, [&] {code.movaps(result, GetVectorOf<fsize, false, 0, 2>(code));@@ -1776,7 +1760,7 @@ static void EmitRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* insconst Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(code);const Xbyak::Xmm value = ctx.reg_alloc.ScratchXmm(code);- SharedLabel bad_values = GenSharedLabel(), end = GenSharedLabel();+ SharedLabel bad_values = ctx.GenSharedLabel(), end = ctx.GenSharedLabel();code.movaps(value, operand);@@ -1867,7 +1851,7 @@ static void EmitRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* inconst Xbyak::Xmm tmp = ctx.reg_alloc.ScratchXmm(code);const Xbyak::Xmm mask = ctx.reg_alloc.ScratchXmm(code);- SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();+ SharedLabel end = ctx.GenSharedLabel(), fallback = ctx.GenSharedLabel();MaybeStandardFPSCRValue(code, ctx, fpcr_controlled, [&] {code.vmovaps(result, GetVectorOf<fsize, false, 0, 3>(code));@@ -2004,120 +1988,123 @@ void EmitX64::EmitFPVectorToHalf32(EmitContext& ctx, IR::Inst* inst) {template<size_t fsize, bool unsigned_>void EmitFPVectorToFixed(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {const size_t fbits = inst->GetArg(1).GetU8();- const auto rounding = static_cast<FP::RoundingMode>(inst->GetArg(2).GetU8());+ const auto rounding = FP::RoundingMode(inst->GetArg(2).GetU8());[[maybe_unused]] const bool fpcr_controlled = inst->GetArg(3).GetU1();- if constexpr (fsize != 16) {- if (code.HasHostFeature(HostFeature::SSE41) && rounding != FP::RoundingMode::ToNearest_TieAwayFromZero) {- auto args = ctx.reg_alloc.GetArgumentInfo(inst);-- const Xbyak::Xmm src = ctx.reg_alloc.UseScratchXmm(code, args[0]);+ if (code.HasHostFeature(HostFeature::SSE41) && fsize != 16 && rounding != FP::RoundingMode::ToNearest_TieAwayFromZero) {+ auto args = ctx.reg_alloc.GetArgumentInfo(inst);+ const Xbyak::Xmm src = ctx.reg_alloc.UseScratchXmm(code, args[0]);+ MaybeStandardFPSCRValue(code, ctx, fpcr_controlled, [&] {+ const int round_imm = [&] {+ switch (rounding) {+ case FP::RoundingMode::ToNearest_TieEven:+ default:+ return 0b00;+ case FP::RoundingMode::TowardsPlusInfinity:+ return 0b10;+ case FP::RoundingMode::TowardsMinusInfinity:+ return 0b01;+ case FP::RoundingMode::TowardsZero:+ return 0b11;+ }+ }();+ const auto perform_conversion = [&code, &ctx](const Xbyak::Xmm& src) {+ // MSVC doesn't allow us to use a [&] capture, so we have to do this instead.+ (void)ctx;- MaybeStandardFPSCRValue(code, ctx, fpcr_controlled, [&] {- const int round_imm = [&] {- switch (rounding) {- case FP::RoundingMode::ToNearest_TieEven:- default:- return 0b00;- case FP::RoundingMode::TowardsPlusInfinity:- return 0b10;- case FP::RoundingMode::TowardsMinusInfinity:- return 0b01;- case FP::RoundingMode::TowardsZero:- return 0b11;- }- }();+ if constexpr (fsize == 32) {+ code.cvttps2dq(src, src);+ } else {+ if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {+ code.vcvttpd2qq(src, src);+ } else {+ const Xbyak::Reg64 hi = ctx.reg_alloc.ScratchGpr(code);+ const Xbyak::Reg64 lo = ctx.reg_alloc.ScratchGpr(code);- const auto perform_conversion = [&code, &ctx](const Xbyak::Xmm& src) {- // MSVC doesn't allow us to use a [&] capture, so we have to do this instead.- (void)ctx;+ code.cvttsd2si(lo, src);+ code.punpckhqdq(src, src);+ code.cvttsd2si(hi, src);+ code.movq(src, lo);+ code.pinsrq(src, hi, 1);- if constexpr (fsize == 32) {- code.cvttps2dq(src, src);- } else {- if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {- code.vcvttpd2qq(src, src);- } else {- const Xbyak::Reg64 hi = ctx.reg_alloc.ScratchGpr(code);- const Xbyak::Reg64 lo = ctx.reg_alloc.ScratchGpr(code);-- code.cvttsd2si(lo, src);- code.punpckhqdq(src, src);- code.cvttsd2si(hi, src);- code.movq(src, lo);- code.pinsrq(src, hi, 1);-- ctx.reg_alloc.Release(hi);- ctx.reg_alloc.Release(lo);- }+ ctx.reg_alloc.Release(hi);+ ctx.reg_alloc.Release(lo);}- };-- if (fbits != 0) {- const u64 scale_factor = fsize == 32- ? static_cast<u64>(fbits + 127) << 23- : static_cast<u64>(fbits + 1023) << 52;- FCODE(mulp)(src, GetVectorOf<fsize>(code, scale_factor));}+ };+ if (fbits != 0) {+ const u64 scale_factor = fsize == 32+ ? u64(fbits + 127) << 23+ : u64(fbits + 1023) << 52;+ FCODE(mulp)(src, GetVectorOf<fsize>(code, scale_factor));+ }- FCODE(roundp)(src, src, static_cast<u8>(round_imm));- ZeroIfNaN<fsize>(code, src);+ FCODE(roundp)(src, src, u8(round_imm));+ const Xbyak::Xmm nan_mask = xmm0;+ if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {+ static constexpr u32 nan_to_zero = FixupLUT(FpFixup::PosZero, FpFixup::PosZero);+ FCODE(vfixupimmp)(src, src, code.BConst<32>(ptr_b, nan_to_zero), u8(0));+ } else if (code.HasHostFeature(HostFeature::AVX)) {+ FCODE(vcmpordp)(nan_mask, src, src);+ FCODE(vandp)(src, src, nan_mask);+ } else {+ code.movaps(nan_mask, src);+ FCODE(cmpordp)(nan_mask, nan_mask);+ code.andps(src, nan_mask);+ }- constexpr u64 float_upper_limit_signed = fsize == 32 ? 0x4f000000 : 0x43e0000000000000;- [[maybe_unused]] constexpr u64 float_upper_limit_unsigned = fsize == 32 ? 0x4f800000 : 0x43f0000000000000;+ constexpr u64 float_upper_limit_signed = fsize == 32 ? 0x4f000000 : 0x43e0000000000000;+ [[maybe_unused]] constexpr u64 float_upper_limit_unsigned = fsize == 32 ? 0x4f800000 : 0x43f0000000000000;- if constexpr (unsigned_) {- if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {- // Mask positive values- code.xorps(xmm0, xmm0);- FCODE(vcmpp)(k1, src, xmm0, Cmp::GreaterEqual_OQ);-- // Convert positive values to unsigned integers, write 0 anywhere else- // vcvttp*2u*q already saturates out-of-range values to (0xFFFF...)- if constexpr (fsize == 32) {- code.vcvttps2udq(src | k1 | T_z, src);- } else {- code.vcvttpd2uqq(src | k1 | T_z, src);- }+ if constexpr (unsigned_) {+ if (code.HasHostFeature(HostFeature::AVX512_OrthoFloat)) {+ // Mask positive values+ code.xorps(xmm0, xmm0);+ FCODE(vcmpp)(k1, src, xmm0, Cmp::GreaterEqual_OQ);++ // Convert positive values to unsigned integers, write 0 anywhere else+ // vcvttp*2u*q already saturates out-of-range values to (0xFFFF...)+ if (fsize == 32) {+ code.vcvttps2udq(src | k1 | T_z, src);} else {- // Zero is minimum- code.xorps(xmm0, xmm0);- FCODE(cmplep)(xmm0, src);- FCODE(andp)(src, xmm0);-- // Will we exceed unsigned range?- const Xbyak::Xmm exceed_unsigned = ctx.reg_alloc.ScratchXmm(code);- code.movaps(exceed_unsigned, GetVectorOf<fsize, float_upper_limit_unsigned>(code));- FCODE(cmplep)(exceed_unsigned, src);-- // Will be exceed signed range?- const Xbyak::Xmm tmp = ctx.reg_alloc.ScratchXmm(code);- code.movaps(tmp, GetVectorOf<fsize, float_upper_limit_signed>(code));- code.movaps(xmm0, tmp);- FCODE(cmplep)(xmm0, src);- FCODE(andp)(tmp, xmm0);- FCODE(subp)(src, tmp);- perform_conversion(src);- ICODE(psll)(xmm0, u8(fsize - 1));- FCODE(orp)(src, xmm0);-- // Saturate to max- FCODE(orp)(src, exceed_unsigned);+ code.vcvttpd2uqq(src | k1 | T_z, src);}} else {- using FPT = mcl::unsigned_integer_of_size<fsize>; // WORKAROUND: For issue 678 on MSVC- constexpr u64 integer_max = FPT((std::numeric_limits<std::conditional_t<unsigned_, FPT, std::make_signed_t<FPT>>>::max)());+ // Zero is minimum+ code.xorps(xmm0, xmm0);+ FCODE(cmplep)(xmm0, src);+ FCODE(andp)(src, xmm0);- code.movaps(xmm0, GetVectorOf<fsize, float_upper_limit_signed>(code));+ // Will we exceed unsigned range?+ const Xbyak::Xmm exceed_unsigned = ctx.reg_alloc.ScratchXmm(code);+ code.movaps(exceed_unsigned, GetVectorOf<fsize, float_upper_limit_unsigned>(code));+ FCODE(cmplep)(exceed_unsigned, src);++ // Will be exceed signed range?+ const Xbyak::Xmm tmp = ctx.reg_alloc.ScratchXmm(code);+ code.movaps(tmp, GetVectorOf<fsize, float_upper_limit_signed>(code));+ code.movaps(xmm0, tmp);FCODE(cmplep)(xmm0, src);+ FCODE(andp)(tmp, xmm0);+ FCODE(subp)(src, tmp);perform_conversion(src);- FCODE(blendvp)(src, GetVectorOf<fsize, integer_max>(code));- }- });+ ICODE(psll)(xmm0, u8(fsize - 1));+ FCODE(orp)(src, xmm0);- ctx.reg_alloc.DefineValue(code, inst, src);- return;- }+ // Saturate to max+ FCODE(orp)(src, exceed_unsigned);+ }+ } else {+ using FPT = mcl::unsigned_integer_of_size<fsize>; // WORKAROUND: For issue 678 on MSVC+ constexpr u64 integer_max = FPT((std::numeric_limits<std::conditional_t<unsigned_, FPT, std::make_signed_t<FPT>>>::max)());+ code.movaps(xmm0, GetVectorOf<fsize, float_upper_limit_signed>(code));+ FCODE(cmplep)(xmm0, src);+ perform_conversion(src);+ FCODE(blendvp)(src, GetVectorOf<fsize, integer_max>(code));+ }+ });+ ctx.reg_alloc.DefineValue(code, inst, src);+ return;}using FPT = mcl::unsigned_integer_of_size<fsize>; // WORKAROUND: For issue 678 on MSVC