Skip to content

PS2 games in the browser: static recompilation to WebAssembly with a native runtime

C++
61.2 KB
Raw Blame
#include "vu_jit.hpp"

#include "ps2/vu_ops.hpp"

#include <cstddef>
#include <cstdio>
#include <cstring>
#include <map>
#include <set>

namespace ps2::vu::jit {

const char *const kImportNames[kImports] = {
    "mac", "status", "clip", "force_mac", "efu", "kick", "top", "itop", "interpret",
};

namespace {

constexpr std::uint32_t kMask = 16383u;
constexpr int kAcc = 32;

struct ImportType {
    std::uint8_t params;
    bool result;
};

constexpr ImportType kImportTypes[kImports] = {
    {1, true}, {1, true}, {1, true}, {2, false}, {4, false}, {1, false}, {0, true}, {0, true}, {2, false},
};

enum Local : std::uint32_t {
    kR, kMem, kPend, kPendj, kEnding, kJpc, kPcw, kExit, kUclip, kTmp, kTmp2, kMacv, kStat, kSlot, kT, kSign, kExp, kBi, kBk, kBkNext, kVi0, kI32Locals = kVi0 + 16,
    kBits = kI32Locals, kUv, kV0, kV1, kV2, kV3, kV4, kV5, kShadow0
};
constexpr std::uint32_t kI32Count = kI32Locals - 2;
constexpr std::uint32_t kVecTemps = kShadow0 - kUv;

constexpr std::uint32_t kOne = 0x3F800000u;

std::uint32_t vf_off(unsigned reg) { return static_cast<std::uint32_t>(offsetof(Vu0Regs, vf) + reg * 16u); }
std::uint32_t reg_off(int reg) { return reg == kAcc ? static_cast<std::uint32_t>(offsetof(Vu0Regs, acc)) : vf_off(static_cast<unsigned>(reg)); }
std::uint32_t vi_off(unsigned i) { return static_cast<std::uint32_t>(offsetof(Vu0Regs, vi) + i * 2u); }
constexpr std::uint32_t kOffQ = offsetof(Vu0Regs, q), kOffP = offsetof(Vu0Regs, p), kOffI = offsetof(Vu0Regs, i), kOffR = offsetof(Vu0Regs, r);
constexpr std::uint32_t kOffMac = offsetof(Vu0Regs, mac);
constexpr std::uint32_t kOffMh = offsetof(vuops::FlagPipe, mh), kOffCh = offsetof(vuops::FlagPipe, ch), kOffMacTs = offsetof(vuops::FlagPipe, mac_ts),
                        kOffMacV = offsetof(vuops::FlagPipe, mac_v), kOffStatusV = offsetof(vuops::FlagPipe, status_v), kOffClipTs = offsetof(vuops::FlagPipe, clip_ts),
                        kOffClipV = offsetof(vuops::FlagPipe, clip_v), kOffT = offsetof(vuops::FlagPipe, t), kOffStalls = offsetof(vuops::FlagPipe, stalls), kOffQPending = offsetof(vuops::FlagPipe, q_pending),
                        kOffQValue = offsetof(vuops::FlagPipe, q_value), kOffQReady = offsetof(vuops::FlagPipe, q_ready);
constexpr std::uint32_t kOffStatus = offsetof(Vu0Regs, status), kOffClip = offsetof(Vu0Regs, clip), kOffTpc = offsetof(Vu0Regs, tpc);

bool lane(unsigned dest, int i) { return dest & (8u >> i); }

struct Out {
    std::vector<std::uint8_t> b;
    void byte(std::uint8_t v) { b.push_back(v); }
    void u32(std::uint32_t v) {
        do {
            std::uint8_t c = v & 0x7Fu;
            v >>= 7;
            if (v) c |= 0x80u;
            b.push_back(c);
        } while (v);
    }
    void s32(std::int32_t v) {
        for (;;) {
            const std::uint8_t c = static_cast<std::uint8_t>(v & 0x7F);
            v >>= 7;
            if ((v == 0 && !(c & 0x40)) || (v == -1 && (c & 0x40))) {
                b.push_back(c);
                return;
            }
            b.push_back(c | 0x80u);
        }
    }
    void s64(std::int64_t v) {
        for (;;) {
            const std::uint8_t c = static_cast<std::uint8_t>(v & 0x7F);
            v >>= 7;
            if ((v == 0 && !(c & 0x40)) || (v == -1 && (c & 0x40))) {
                b.push_back(c);
                return;
            }
            b.push_back(c | 0x80u);
        }
    }
    void name(const char *s) {
        u32(static_cast<std::uint32_t>(std::strlen(s)));
        b.insert(b.end(), s, s + std::strlen(s));
    }
    void section(std::uint8_t id, const Out &body) {
        byte(id);
        u32(static_cast<std::uint32_t>(body.b.size()));
        b.insert(b.end(), body.b.begin(), body.b.end());
    }
};

struct Code : Out {
    std::uint32_t depth = 0;
    void simd(std::uint32_t o) {
        byte(0xFD);
        u32(o);
    }
    void i32c(std::uint32_t v) {
        byte(0x41);
        s32(static_cast<std::int32_t>(v));
    }
    void i64c(std::uint64_t v) {
        byte(0x42);
        s64(static_cast<std::int64_t>(v));
    }
    void get(std::uint32_t l) {
        byte(0x20);
        u32(l);
    }
    void set(std::uint32_t l) {
        byte(0x21);
        u32(l);
    }
    void tee(std::uint32_t l) {
        byte(0x22);
        u32(l);
    }
    void load(std::uint32_t off) {
        byte(0x28);
        u32(2);
        u32(off);
    }
    void load8(std::uint32_t off) {
        byte(0x2D);
        u32(0);
        u32(off);
    }
    void load16(std::uint32_t off) {
        byte(0x2F);
        u32(1);
        u32(off);
    }
    void store(std::uint32_t off) {
        byte(0x36);
        u32(2);
        u32(off);
    }
    void store8(std::uint32_t off) {
        byte(0x3A);
        u32(0);
        u32(off);
    }
    void store16(std::uint32_t off) {
        byte(0x3B);
        u32(1);
        u32(off);
    }
    void vload(std::uint32_t off) {
        simd(0x00);
        u32(4);
        u32(off);
    }
    void vstore(std::uint32_t off) {
        simd(0x0B);
        u32(4);
        u32(off);
    }
    void vconst(std::uint32_t a, std::uint32_t b2, std::uint32_t c, std::uint32_t d) {
        simd(0x0C);
        for (std::uint32_t w : {a, b2, c, d})
            for (int k = 0; k < 4; ++k) byte(static_cast<std::uint8_t>(w >> (k * 8)));
    }
    void vsplatc(std::uint32_t v) { vconst(v, v, v, v); }
    void shuffle(const std::uint8_t (&lanes)[16]) {
        simd(0x0D);
        for (std::uint8_t l : lanes) byte(l);
    }
    void call(std::uint32_t f) {
        byte(0x10);
        u32(f);
    }
    void tail(std::uint32_t f) {
        byte(0x12);
        u32(f);
    }
    void op(std::uint8_t o) { byte(o); }
    void open_if() {
        byte(0x04);
        byte(0x40);
        ++depth;
    }
    void end() {
        byte(0x0B);
        --depth;
    }
};

enum : std::uint8_t {
    kOpDrop = 0x1A, kOpSelect = 0x1B, kOpEq = 0x46, kOpNe = 0x47, kOpLtS = 0x48, kOpGtS = 0x4A, kOpLeS = 0x4C, kOpGeS = 0x4E, kOpGeU = 0x4F,
    kOpAdd = 0x6A, kOpSub = 0x6B, kOpMul = 0x6C, kOpAnd = 0x71, kOpOr = 0x72, kOpXor = 0x73, kOpShl = 0x74, kOpShrU = 0x76, kOpExt16 = 0xC1, kOpBr = 0x0C,
    kOpI64Add = 0x7C, kOpI64Or = 0x84, kOpI64Shl = 0x86, kOpI64ShrU = 0x88, kOpF64Sqrt = 0x9F, kOpF64Div = 0xA3, kOpWrap = 0xA7, kOpExtendU = 0xAD,
    kOpI64FromF64 = 0xBD, kOpF64FromI64 = 0xBF,
};

enum : std::uint32_t {
    kVSplat = 0x11, kVExtract = 0x1B, kI32x4Eq = 0x37, kI32x4LtS = 0x39, kI32x4GtS = 0x3B, kI32x4LeS = 0x3D, kI32x4GeS = 0x3F, kF32x4Lt = 0x43,
    kF32x4Ge = 0x46, kVAnd = 0x4E, kVAndnot = 0x4F, kVOr = 0x50, kVBitselect = 0x52, kI32x4Neg = 0xA1, kI32x4Add = 0xAE, kI32x4MinS = 0xB6,
    kF32x4Add = 0xE4, kF32x4Sub = 0xE5, kF32x4Mul = 0xE6, kI32x4TruncF32x4S = 0xF8, kF32x4ConvertI32x4S = 0xFA,
};

enum Kind { Nop, Add, Sub, Madd, Msub, Max, Min, Mul, Abs, Ftoi, Itof, Clip, Opmula, Opmsub };
enum Src { Vec, Bc, Q, I };

struct Up {
    Kind kind = Nop;
    Src src = Vec;
    int target = -1;
    bool flags = false;
    int frac = 0;
    unsigned dest = 0, ft = 0, fs = 0, bc = 0;
};

Up decode_upper(std::uint32_t hi) {
    Up u;
    u.dest = (hi >> 21) & 15u;
    u.ft = (hi >> 16) & 31u;
    u.fs = (hi >> 11) & 31u;
    u.bc = hi & 3u;
    const unsigned fd = (hi >> 6) & 31u;
    const unsigned funct = hi & 63u;
    static const int kFrac[4] = {0, 4, 12, 15};
    const auto set = [&](Kind k, Src s, int t, bool f) {
        u.kind = k;
        u.src = s;
        u.target = t;
        u.flags = f;
    };
    if (funct < 0x3C) {
        switch (funct) {
        case 0x00: case 0x01: case 0x02: case 0x03: set(Add, Bc, static_cast<int>(fd), true); break;
        case 0x04: case 0x05: case 0x06: case 0x07: set(Sub, Bc, static_cast<int>(fd), true); break;
        case 0x08: case 0x09: case 0x0A: case 0x0B: set(Madd, Bc, static_cast<int>(fd), true); break;
        case 0x0C: case 0x0D: case 0x0E: case 0x0F: set(Msub, Bc, static_cast<int>(fd), true); break;
        case 0x10: case 0x11: case 0x12: case 0x13: set(Max, Bc, static_cast<int>(fd), false); break;
        case 0x14: case 0x15: case 0x16: case 0x17: set(Min, Bc, static_cast<int>(fd), false); break;
        case 0x18: case 0x19: case 0x1A: case 0x1B: set(Mul, Bc, static_cast<int>(fd), true); break;
        case 0x1C: set(Mul, Q, static_cast<int>(fd), true); break;
        case 0x1D: set(Max, I, static_cast<int>(fd), false); break;
        case 0x1E: set(Mul, I, static_cast<int>(fd), true); break;
        case 0x1F: set(Min, I, static_cast<int>(fd), false); break;
        case 0x20: set(Add, Q, static_cast<int>(fd), true); break;
        case 0x21: set(Madd, Q, static_cast<int>(fd), true); break;
        case 0x22: set(Add, I, static_cast<int>(fd), true); break;
        case 0x23: set(Madd, I, static_cast<int>(fd), true); break;
        case 0x24: set(Sub, Q, static_cast<int>(fd), true); break;
        case 0x25: set(Msub, Q, static_cast<int>(fd), true); break;
        case 0x26: set(Sub, I, static_cast<int>(fd), true); break;
        case 0x27: set(Msub, I, static_cast<int>(fd), true); break;
        case 0x28: set(Add, Vec, static_cast<int>(fd), true); break;
        case 0x29: set(Madd, Vec, static_cast<int>(fd), true); break;
        case 0x2A: set(Mul, Vec, static_cast<int>(fd), true); break;
        case 0x2B: set(Max, Vec, static_cast<int>(fd), false); break;
        case 0x2C: set(Sub, Vec, static_cast<int>(fd), true); break;
        case 0x2D: set(Msub, Vec, static_cast<int>(fd), true); break;
        case 0x2E: set(Opmsub, Vec, static_cast<int>(fd), true); break;
        case 0x2F: set(Min, Vec, static_cast<int>(fd), false); break;
        default: break;
        }
    } else {
        const unsigned index = ((hi >> 6) & 31u) * 4u + (funct & 3u);
        switch (index) {
        case 0x00: case 0x01: case 0x02: case 0x03: set(Add, Bc, kAcc, true); break;
        case 0x04: case 0x05: case 0x06: case 0x07: set(Sub, Bc, kAcc, true); break;
        case 0x08: case 0x09: case 0x0A: case 0x0B: set(Madd, Bc, kAcc, true); break;
        case 0x0C: case 0x0D: case 0x0E: case 0x0F: set(Msub, Bc, kAcc, true); break;
        case 0x10: case 0x11: case 0x12: case 0x13: set(Itof, Vec, static_cast<int>(u.ft), false); u.frac = kFrac[index & 3u]; break;
        case 0x14: case 0x15: case 0x16: case 0x17: set(Ftoi, Vec, static_cast<int>(u.ft), false); u.frac = kFrac[index & 3u]; break;
        case 0x18: case 0x19: case 0x1A: case 0x1B: set(Mul, Bc, kAcc, true); break;
        case 0x1C: set(Mul, Q, kAcc, true); break;
        case 0x1D: set(Abs, Vec, static_cast<int>(u.ft), false); break;
        case 0x1E: set(Mul, I, kAcc, true); break;
        case 0x1F: set(Clip, Vec, -1, false); break;
        case 0x20: set(Add, Q, kAcc, true); break;
        case 0x21: set(Madd, Q, kAcc, true); break;
        case 0x22: set(Add, I, kAcc, true); break;
        case 0x23: set(Madd, I, kAcc, true); break;
        case 0x24: set(Sub, Q, kAcc, true); break;
        case 0x25: set(Msub, Q, kAcc, true); break;
        case 0x26: set(Sub, I, kAcc, true); break;
        case 0x27: set(Msub, I, kAcc, true); break;
        case 0x28: set(Add, Vec, kAcc, true); break;
        case 0x29: set(Madd, Vec, kAcc, true); break;
        case 0x2A: set(Mul, Vec, kAcc, true); break;
        case 0x2C: set(Sub, Vec, kAcc, true); break;
        case 0x2D: set(Msub, Vec, kAcc, true); break;
        case 0x2E: set(Opmula, Vec, kAcc, true); break;
        default: break;
        }
    }
    if (u.kind == Opmula || u.kind == Opmsub) u.dest = 0xE;
    return u;
}

bool arithmetic(Kind k) { return k == Add || k == Sub || k == Mul || k == Madd || k == Msub; }
bool outer(Kind k) { return k == Opmula || k == Opmsub; }
bool normalized(Kind k) { return arithmetic(k) || outer(k); }

bool active(const Up &u, bool flags_live) {
    if (u.kind == Nop || u.kind == Clip) return false;
    return !(u.dest == 0 || (u.target == 0 && !(u.flags && flags_live)));
}

bool sets_flags(std::uint32_t hi) {
    const Up u = decode_upper(hi);
    return active(u, true) && u.flags;
}

std::vector<int> shadow_reads(std::uint32_t hi) {
    const Up u = decode_upper(hi);
    std::vector<int> regs;
    if (!active(u, true) || !normalized(u.kind)) return regs;
    if ((u.src == Vec || u.src == Bc) && u.ft) regs.push_back(static_cast<int>(u.ft));
    if (u.fs) regs.push_back(static_cast<int>(u.fs));
    if (u.kind == Madd || u.kind == Msub || u.kind == Opmsub) regs.push_back(kAcc);
    return regs;
}

struct LowerInfo {
    int writes = -1;
    bool reads_flags = false, reads_status = false;
    bool branch = false, jump = false;
    std::uint32_t target = 0;
    bool supported = true;
};

LowerInfo analyze_lower(std::uint32_t w, std::uint32_t pc) {
    LowerInfo l;
    const unsigned dest = (w >> 21) & 15u, ft = (w >> 16) & 31u;
    const std::int32_t imm11 = static_cast<std::int32_t>(w << 21) >> 21;
    if ((w >> 25) == 0x40u) {
        const unsigned funct = w & 63u;
        if (funct < 0x3C) {
            l.supported = funct == 0x30 || funct == 0x31 || funct == 0x32 || funct == 0x34 || funct == 0x35;
            return l;
        }
        const unsigned index = ((w >> 6) & 31u) * 4u + (funct & 3u);
        switch (index) {
        case 0x30: case 0x31: case 0x34: case 0x36: case 0x3D: case 0x40: case 0x41: case 0x64:
            if (ft) l.writes = static_cast<int>(ft);
            break;
        case 0x35: case 0x37: case 0x38: case 0x39: case 0x3A: case 0x3B: case 0x3C: case 0x3E: case 0x3F: case 0x42: case 0x43:
        case 0x68: case 0x69: case 0x6C: case 0x70: case 0x71: case 0x72: case 0x73: case 0x74: case 0x75: case 0x76: case 0x78:
        case 0x79: case 0x7A: case 0x7B: case 0x7C: case 0x7D: case 0x7E:
            break;
        default: l.supported = false; break;
        }
        (void)dest;
        return l;
    }
    const std::uint32_t target = (pc + 8u + static_cast<std::uint32_t>(imm11) * 8u) & kMask;
    switch (w >> 25) {
    case 0x00: if (ft) l.writes = static_cast<int>(ft); break;
    case 0x01: case 0x04: case 0x05: case 0x08: case 0x09: case 0x10: case 0x11: case 0x12: case 0x13: case 0x1C: break;
    case 0x14: case 0x15: case 0x16: case 0x17: l.reads_flags = l.reads_status = true; break;
    case 0x18: case 0x1A: case 0x1B: l.reads_flags = true; break;
    case 0x20: case 0x21: case 0x28: case 0x29: case 0x2C: case 0x2D: case 0x2E: case 0x2F: l.branch = true; l.target = target; break;
    case 0x24: case 0x25: l.jump = true; break;
    default: l.supported = false; break;
    }
    return l;
}

class Translator {
public:
    Translator(const std::uint8_t *code, std::uint32_t entry, const Layout &layout) : code_(code), entry_(entry & kMask), layout_(layout) {}

    bool run(std::vector<std::uint8_t> &out);

private:
    const std::uint8_t *code_;
    std::uint32_t entry_;
    Layout layout_;
    Code c_;
    std::vector<std::uint32_t> pcs_;
    std::set<std::uint32_t> in_;
    std::map<std::uint32_t, LowerInfo> lowers_;
    std::map<std::uint32_t, bool> live_in_;
    std::map<int, std::uint32_t> shadow_;
    std::vector<std::uint64_t> shadow_in_, shadow_out_;
    std::map<std::uint32_t, std::uint32_t> case_of_;
    std::uint32_t cases_ = 0, case_index_ = 0;
    std::uint32_t vi_used_ = 0, vi_written_ = 0;
    std::map<std::uint32_t, unsigned> vi_def_;
    std::map<std::uint32_t, int> delay_;
    std::set<std::uint32_t> keep_bk_, set_bi_;
    int delay_mode_ = 0;
    unsigned delay_reg_ = 0;
    bool any_status_ = false, any_flags_ = false, any_clip_ = false;

    std::uint32_t word(std::uint32_t at) const {
        std::uint32_t v;
        std::memcpy(&v, code_ + (at & kMask), 4);
        return v;
    }

    std::vector<std::int64_t> successors(std::uint32_t pc) {
        std::vector<std::int64_t> s{static_cast<std::int64_t>((pc + 8u) & kMask)};
        const std::uint32_t prev = (pc - 8u) & kMask;
        if (in_.count(prev)) {
            const LowerInfo &pl = lowers_[prev];
            if (pl.branch) s.push_back(pl.target);
            if (pl.jump || (word(prev + 4) & 0x40000000u)) s.push_back(-1);
        }
        return s;
    }

    bool out_live(std::uint32_t pc) {
        for (std::int64_t s : successors(pc))
            if (s < 0 || !in_.count(static_cast<std::uint32_t>(s)) || live_in_[static_cast<std::uint32_t>(s)]) return true;
        return false;
    }

    std::vector<std::uint32_t> flow(std::uint32_t pc) {
        std::vector<std::uint32_t> s;
        const std::uint32_t next = (pc + 8u) & kMask;
        if (in_.count(next)) s.push_back(next);
        const std::uint32_t prev = (pc - 8u) & kMask;
        if (in_.count(prev)) {
            const LowerInfo &pl = lowers_[prev];
            if (pl.branch && in_.count(pl.target)) s.push_back(pl.target);
        }
        return s;
    }

    unsigned vi_def(std::uint32_t pc) {
        const std::uint32_t w = word(pc);
        if ((word(pc + 4) & 0x80000000u) || !lowers_[pc].supported || branch_delay_exempt(w)) return 0;
        const unsigned dest = (w >> 21) & 15u, it = (w >> 16) & 15u, is = (w >> 11) & 15u, id = (w >> 6) & 15u;
        if ((w >> 25) == 0x40u) {
            const unsigned funct = w & 63u;
            if (funct < 0x3C) return funct == 0x32 ? it : (funct == 0x30 || funct == 0x31 || funct == 0x34 || funct == 0x35) ? id : 0;
            switch (((w >> 6) & 31u) * 4u + (funct & 3u)) {
            case 0x34: case 0x36: return is;
            case 0x35: case 0x37: case 0x3C: case 0x68: case 0x69: return it;
            case 0x3E: return dest ? it : 0;
            default: return 0;
            }
        }
        switch (w >> 25) {
        case 0x04: return dest ? it : 0;
        case 0x08: case 0x09: return it;
        default: return 0;
        }
    }

    std::uint32_t branch_reads(std::uint32_t pc) {
        if (!lowers_[pc].branch) return 0;
        const std::uint32_t w = word(pc);
        const unsigned it = (w >> 16) & 15u, is = (w >> 11) & 15u;
        switch (w >> 25) {
        case 0x2C: case 0x2D: case 0x2E: case 0x2F: return (1u << is) & ~1u;
        case 0x28: case 0x29: return ((1u << is) | (1u << it)) & ~1u;
        default: return 0;
        }
    }

    int lower_write(std::uint32_t pc) {
        const LowerInfo &li = lowers_[pc];
        return (word(pc + 4) & 0x80000000u) || !li.supported ? -1 : li.writes;
    }

    void fl() { c_.i32c(layout_.flags); }
    void store_t() {
        fl();
        c_.get(kT);
        c_.store(kOffT);
    }
    // FlagPipe::now(): the clip flags count waits too
    void now() {
        c_.get(kT);
        fl();
        c_.load(kOffStalls);
        c_.op(kOpAdd);
    }
    void br(std::uint32_t depth) {
        c_.byte(kOpBr);
        c_.u32(depth);
    }
    std::uint32_t fallback_depth() const { return cases_ - 1 - case_index_ + c_.depth; }
    std::uint32_t loop_depth() const { return cases_ - case_index_ + c_.depth; }
    std::uint32_t ret_depth() const { return cases_ - case_index_ + 1 + c_.depth; }

    void leave(std::uint32_t pc) {
        c_.i32c(pc & kMask);
        c_.set(kExit);
        br(fallback_depth());
    }

    void go(std::uint32_t pc) {
        const auto it = case_of_.find(pc & kMask);
        if (it == case_of_.end()) {
            leave(pc);
            return;
        }
        if (it->second > case_index_) {
            br(it->second - case_index_ - 1 + c_.depth);
            return;
        }
        c_.i32c(it->second);
        c_.set(kPcw);
        br(loop_depth());
    }

    void vfix() {
        c_.tee(kV0);
        c_.vsplatc(0x7FFFFFFFu);
        c_.simd(kVAnd);
        c_.vsplatc(0x7F7FFFFFu);
        c_.simd(kI32x4MinS);
        c_.tee(kV1);
        c_.get(kV1);
        c_.vsplatc(0x00800000u);
        c_.simd(kF32x4Ge);
        c_.simd(kVAnd);
        c_.get(kV0);
        c_.vsplatc(0x80000000u);
        c_.simd(kVAnd);
        c_.simd(kVOr);
    }

    void vkey_of(std::uint32_t local) {
        c_.get(local);
        c_.vsplatc(0x80000000u);
        c_.simd(kVAnd);
        c_.get(local);
        c_.get(local);
        c_.vsplatc(0x7F800000u);
        c_.simd(kVAnd);
        c_.vsplatc(0);
        c_.simd(kI32x4Eq);
        c_.simd(kVBitselect);
        c_.tee(kV4);
        c_.vsplatc(0x7FFFFFFFu);
        c_.simd(kVAnd);
        c_.tee(kV5);
        c_.simd(kI32x4Neg);
        c_.get(kV5);
        c_.get(kV4);
        c_.vsplatc(0);
        c_.simd(kI32x4LtS);
        c_.simd(kVBitselect);
    }

    void raw_vec(unsigned reg) {
        if (!reg) {
            c_.vconst(0, 0, 0, kOne);
            return;
        }
        c_.get(kR);
        c_.vload(vf_off(reg));
    }

    void raw_lane(unsigned reg, unsigned l) {
        if (!reg) {
            c_.i32c(l == 3 ? kOne : 0u);
            return;
        }
        c_.get(kR);
        c_.load(vf_off(reg) + l * 4u);
    }

    void mem_lane(unsigned reg, unsigned l) {
        c_.get(kR);
        c_.load(vf_off(reg) + l * 4u);
    }

    void shadow_vec(unsigned reg) {
        if (!reg) {
            c_.vconst(0, 0, 0, kOne);
            return;
        }
        c_.get(shadow_.at(static_cast<int>(reg)));
    }

    void vi(unsigned i) {
        if (!i) {
            c_.i32c(0);
            return;
        }
        vi_used_ |= 1u << i;
        c_.get(kVi0 + i);
    }

    void branch_vi(unsigned i) {
        if (!i || !delay_mode_ || (delay_mode_ == 1 && i != delay_reg_)) {
            vi(i);
            return;
        }
        c_.get(kBk);
        if (delay_mode_ == 1) return;
        vi(i);
        c_.get(kBi);
        c_.i32c(i);
        c_.op(kOpEq);
        c_.op(kOpSelect);
    }

    template <typename F>
    void set_vi(unsigned i, F value) {
        value();
        if (!i) {
            c_.op(kOpDrop);
            return;
        }
        vi_used_ |= 1u << i;
        vi_written_ |= 1u << i;
        c_.i32c(0xFFFFu);
        c_.op(kOpAnd);
        c_.set(kVi0 + i);
    }

    void write_back_vi() {
        for (unsigned i = 1; i < 16; ++i) {
            if (!(vi_written_ & (1u << i))) continue;
            c_.get(kR);
            c_.get(kVi0 + i);
            c_.store16(vi_off(i));
        }
    }

    template <typename F>
    void qword(F index) {
        c_.get(kMem);
        index();
        c_.i32c(1023);
        c_.op(kOpAnd);
        c_.i32c(4);
        c_.op(kOpShl);
        c_.op(kOpAdd);
    }

    void lanes_copy(unsigned dest, unsigned to) {
        if (dest == 0xF) {
            c_.get(kR);
            c_.get(kTmp);
            c_.vload(0);
            c_.vstore(vf_off(to));
            return;
        }
        for (unsigned l = 0; l < 4; ++l) {
            if (!lane(dest, static_cast<int>(l))) continue;
            c_.get(kR);
            c_.get(kTmp);
            c_.load(l * 4u);
            c_.store(vf_off(to) + l * 4u);
        }
    }

    void lanes_store(unsigned dest, unsigned from) {
        if (dest == 0xF) {
            c_.get(kTmp);
            c_.get(kR);
            c_.vload(vf_off(from));
            c_.vstore(0);
            return;
        }
        for (unsigned l = 0; l < 4; ++l) {
            if (!lane(dest, static_cast<int>(l))) continue;
            c_.get(kTmp);
            mem_lane(from, l);
            c_.store(l * 4u);
        }
    }

    void lanes_fill(unsigned dest, unsigned to, std::uint32_t value_local) {
        for (unsigned l = 0; l < 4; ++l) {
            if (!lane(dest, static_cast<int>(l))) continue;
            c_.get(kR);
            c_.get(value_local);
            c_.store(vf_off(to) + l * 4u);
        }
    }

    void lane_mask(unsigned d) {
        c_.vconst(lane(d, 0) ? ~0u : 0u, lane(d, 1) ? ~0u : 0u, lane(d, 2) ? ~0u : 0u, lane(d, 3) ? ~0u : 0u);
    }

    void flags_inline(unsigned d) {
        const auto bits = [&](unsigned shift) {
            c_.vconst(lane(d, 0) ? 8u << shift : 0u, lane(d, 1) ? 4u << shift : 0u, lane(d, 2) ? 2u << shift : 0u, lane(d, 3) ? 1u << shift : 0u);
        };
        c_.set(kV0);
        c_.get(kV0);
        c_.vsplatc(0x7F800000u);
        c_.simd(kVAnd);
        c_.vsplatc(0);
        c_.simd(kI32x4Eq);
        bits(0);
        c_.simd(kVAnd);
        c_.get(kV0);
        c_.vsplatc(0);
        c_.simd(kI32x4LtS);
        bits(4);
        c_.simd(kVAnd);
        c_.simd(kVOr);
        c_.get(kV0);
        c_.vsplatc(0x7FFFFFFFu);
        c_.simd(kVAnd);
        c_.vsplatc(0x7FFFFFFFu);
        c_.simd(kI32x4Eq);
        bits(12);
        c_.simd(kVAnd);
        c_.simd(kVOr);
        c_.tee(kV1);
        for (std::uint8_t l = 0; l < 4; ++l) {
            if (l) c_.get(kV1);
            c_.simd(kVExtract);
            c_.byte(l);
            if (l) c_.op(kOpOr);
        }
        c_.set(kMacv);
        c_.get(kR);
        c_.get(kMacv);
        c_.store(kOffMac);
        c_.get(kR);
        c_.load(kOffStatus);
        c_.i32c(0xFF0u);
        c_.op(kOpAnd);
        const std::pair<std::uint32_t, std::uint32_t> groups[3] = {{0xFu, 0}, {0xF0u, 1}, {0xF000u, 3}};
        for (const auto &[mask, shift] : groups) {
            c_.get(kMacv);
            c_.i32c(mask);
            c_.op(kOpAnd);
            c_.i32c(0);
            c_.op(kOpNe);
            if (shift) {
                c_.i32c(shift);
                c_.op(kOpShl);
            }
            c_.op(kOpOr);
        }
        c_.tee(kStat);
        c_.get(kStat);
        c_.i32c(0xFu);
        c_.op(kOpAnd);
        c_.i32c(6);
        c_.op(kOpShl);
        c_.op(kOpOr);
        c_.set(kStat);
        c_.get(kR);
        c_.get(kStat);
        c_.store(kOffStatus);
        fl();
        fl();
        c_.load(kOffMh);
        c_.tee(kSlot);
        c_.i32c(1);
        c_.op(kOpAdd);
        c_.store(kOffMh);
        c_.get(kSlot);
        c_.i32c(7);
        c_.op(kOpAnd);
        c_.i32c(2);
        c_.op(kOpShl);
        c_.tee(kSlot);
        c_.get(kT);
        c_.store(layout_.flags + kOffMacTs);
        c_.get(kSlot);
        c_.get(kMacv);
        c_.store(layout_.flags + kOffMacV);
        c_.get(kSlot);
        c_.get(kStat);
        c_.store(layout_.flags + kOffStatusV);
    }

    template <typename V, typename T>
    void ring_clip(V value, T stamp) {
        fl();
        fl();
        c_.load(kOffCh);
        c_.tee(kSlot);
        c_.i32c(1);
        c_.op(kOpAdd);
        c_.store(kOffCh);
        c_.get(kSlot);
        c_.i32c(7);
        c_.op(kOpAnd);
        c_.i32c(2);
        c_.op(kOpShl);
        c_.tee(kSlot);
        stamp();
        c_.store(layout_.flags + kOffClipTs);
        c_.get(kSlot);
        value();
        c_.store(layout_.flags + kOffClipV);
    }

    void take_q() {
        c_.get(kR);
        fl();
        c_.load(kOffQValue);
        c_.store(kOffQ);
        fl();
        c_.i32c(0);
        c_.store8(kOffQPending);
    }

    void flush_q() {
        fl();
        c_.load8(kOffQPending);
        c_.open_if();
        take_q();
        c_.end();
    }

    void sync_q() {
        fl();
        c_.load8(kOffQPending);
        c_.get(kT);
        fl();
        c_.load(kOffQReady);
        c_.op(kOpGeU);
        c_.op(kOpAnd);
        c_.open_if();
        take_q();
        c_.end();
    }

    void to_double(std::uint32_t x) {
        c_.get(x);
        c_.i32c(0x80000000u);
        c_.op(kOpAnd);
        c_.op(kOpExtendU);
        c_.i64c(32);
        c_.op(kOpI64Shl);
        c_.get(x);
        c_.i32c(0x7FFFFFFFu);
        c_.op(kOpAnd);
        c_.op(kOpExtendU);
        c_.i64c(29);
        c_.op(kOpI64Shl);
        c_.i64c(896ull << 52);
        c_.op(kOpI64Add);
        c_.i64c(0);
        c_.get(x);
        c_.i32c(0x7F800000u);
        c_.op(kOpAnd);
        c_.op(kOpSelect);
        c_.op(kOpI64Or);
        c_.op(kOpF64FromI64);
    }

    void from_double() {
        c_.op(kOpI64FromF64);
        c_.tee(kBits);
        c_.i64c(32);
        c_.op(kOpI64ShrU);
        c_.op(kOpWrap);
        c_.i32c(0x80000000u);
        c_.op(kOpAnd);
        c_.set(kSign);
        c_.get(kBits);
        c_.i64c(52);
        c_.op(kOpI64ShrU);
        c_.op(kOpWrap);
        c_.i32c(0x7FFu);
        c_.op(kOpAnd);
        c_.i32c(896);
        c_.op(kOpSub);
        c_.set(kExp);
        c_.get(kSign);
        c_.i32c(0x7FFFFFFFu);
        c_.op(kOpOr);
        c_.get(kSign);
        c_.get(kExp);
        c_.i32c(23);
        c_.op(kOpShl);
        c_.op(kOpOr);
        c_.get(kBits);
        c_.i64c(29);
        c_.op(kOpI64ShrU);
        c_.op(kOpWrap);
        c_.i32c(0x7FFFFFu);
        c_.op(kOpAnd);
        c_.op(kOpOr);
        c_.get(kExp);
        c_.i32c(255);
        c_.op(kOpGtS);
        c_.op(kOpSelect);
        c_.get(kSign);
        c_.get(kExp);
        c_.i32c(0);
        c_.op(kOpGtS);
        c_.op(kOpSelect);
    }

    template <typename F>
    void start_q(F value, std::uint32_t latency) {
        flush_q();
        fl();
        value();
        c_.store(kOffQValue);
        fl();
        c_.get(kT);
        c_.i32c(latency);
        c_.op(kOpAdd);
        c_.store(kOffQReady);
        fl();
        c_.i32c(1);
        c_.store8(kOffQPending);
    }

    void divide_status(std::uint32_t divisor) {
        c_.get(kR);
        c_.get(kR);
        c_.load(kOffStatus);
        c_.i32c(~0x30u);
        c_.op(kOpAnd);
        c_.i32c(0);
        c_.i32c(0x20u);
        c_.get(divisor);
        c_.i32c(0x7F800000u);
        c_.op(kOpAnd);
        c_.op(kOpSelect);
        c_.op(kOpOr);
        c_.store(kOffStatus);
    }

    void clip_value(const Up &u);
    void itof(const Up &u);
    void ftoi(const Up &u);
    void outer_product(const Up &u);
    void upper_pre(const Up &u, bool flags_live);
    void upper_post(const Up &u, bool flags_live);
    bool lower(std::uint32_t w, std::uint32_t pc);
    void fsset(std::uint32_t w);
    void instruction(std::uint32_t pc, std::uint32_t following);
    void prologue();
};

void Translator::clip_value(const Up &u) {
    raw_vec(u.fs);
    c_.set(kV2);
    vkey_of(kV2);
    c_.set(kV3);
    raw_lane(u.ft, 3);
    c_.i32c(0x7FFFFFFFu);
    c_.op(kOpAnd);
    c_.tee(kTmp);
    c_.i32c(0);
    c_.get(kTmp);
    c_.i32c(0x7F800000u);
    c_.op(kOpAnd);
    c_.op(kOpSelect);
    c_.simd(kVSplat);
    c_.set(kV4);
    c_.get(kV3);
    c_.get(kV4);
    c_.simd(kI32x4GtS);
    c_.vconst(1, 4, 16, 0);
    c_.simd(kVAnd);
    c_.get(kV3);
    c_.get(kV4);
    c_.simd(kI32x4Neg);
    c_.simd(kI32x4LtS);
    c_.vconst(2, 8, 32, 0);
    c_.simd(kVAnd);
    c_.simd(kVOr);
    c_.tee(kV5);
    for (std::uint8_t l = 0; l < 3; ++l) {
        if (l) c_.get(kV5);
        c_.simd(kVExtract);
        c_.byte(l);
        if (l) c_.op(kOpOr);
    }
    c_.get(kR);
    c_.load(kOffClip);
    c_.i32c(6);
    c_.op(kOpShl);
    c_.op(kOpOr);
    c_.i32c(0xFFFFFFu);
    c_.op(kOpAnd);
    c_.set(kUclip);
}

void Translator::itof(const Up &u) {
    raw_vec(u.fs);
    c_.tee(kV2);
    c_.simd(kF32x4ConvertI32x4S);
    c_.tee(kV3);
    c_.vsplatc(0x4F000000u);
    c_.simd(kF32x4Ge);
    c_.tee(kV4);
    c_.get(kV3);
    c_.get(kV4);
    c_.simd(kVAndnot);
    c_.simd(kI32x4TruncF32x4S);
    c_.set(kV5);
    c_.get(kV2);
    c_.vsplatc(0);
    c_.simd(kI32x4GeS);
    c_.get(kV5);
    c_.get(kV2);
    c_.simd(kI32x4GtS);
    c_.simd(kVAnd);
    c_.simd(kVOr);
    c_.get(kV2);
    c_.vsplatc(0);
    c_.simd(kI32x4LtS);
    c_.get(kV5);
    c_.get(kV2);
    c_.simd(kI32x4LtS);
    c_.simd(kVAnd);
    c_.simd(kVOr);
    c_.get(kV3);
    c_.simd(kI32x4Add);
    if (u.frac) {
        c_.vsplatc((127u - static_cast<std::uint32_t>(u.frac)) << 23);
        c_.simd(kF32x4Mul);
    }
    c_.set(kUv);
}

void Translator::ftoi(const Up &u) {
    raw_vec(u.fs);
    vfix();
    if (u.frac) {
        c_.vsplatc((127u + static_cast<std::uint32_t>(u.frac)) << 23);
        c_.simd(kF32x4Mul);
    }
    c_.tee(kV2);
    c_.vsplatc(0x4F000000u);
    c_.simd(kF32x4Ge);
    c_.tee(kV3);
    c_.get(kV2);
    c_.vsplatc(0xCF000000u);
    c_.simd(kF32x4Lt);
    c_.tee(kV4);
    c_.simd(kVOr);
    c_.set(kV5);
    c_.get(kV2);
    c_.get(kV5);
    c_.simd(kVAndnot);
    c_.simd(kI32x4TruncF32x4S);
    c_.get(kV5);
    c_.simd(kVAndnot);
    c_.get(kV3);
    c_.vsplatc(0x7FFFFFFFu);
    c_.simd(kVAnd);
    c_.simd(kVOr);
    c_.get(kV4);
    c_.vsplatc(0x80000000u);
    c_.simd(kVAnd);
    c_.simd(kVOr);
    c_.set(kUv);
}

void Translator::outer_product(const Up &u) {
    static const std::uint8_t kYzx[16] = {4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15};
    static const std::uint8_t kZxy[16] = {8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15};
    if (u.kind == Opmsub) c_.get(shadow_.at(kAcc));
    shadow_vec(u.fs);
    shadow_vec(u.fs);
    c_.shuffle(kYzx);
    shadow_vec(u.ft);
    shadow_vec(u.ft);
    c_.shuffle(kZxy);
    c_.simd(kF32x4Mul);
    vfix();
    if (u.kind == Opmsub) {
        c_.simd(kF32x4Sub);
        vfix();
    }
    c_.set(kUv);
}

void Translator::upper_pre(const Up &u, bool flags_live) {
    if (u.kind == Clip) {
        clip_value(u);
        return;
    }
    if (!active(u, flags_live)) return;
    if (outer(u.kind)) {
        outer_product(u);
        return;
    }
    if (arithmetic(u.kind)) {
        if (u.kind == Madd || u.kind == Msub) c_.get(shadow_.at(kAcc));
        shadow_vec(u.fs);
        switch (u.src) {
        case Vec: shadow_vec(u.ft); break;
        case Bc:
            if (u.ft) {
                c_.get(shadow_.at(static_cast<int>(u.ft)));
                c_.simd(kVExtract);
                c_.byte(static_cast<std::uint8_t>(u.bc));
                c_.simd(kVSplat);
            } else {
                c_.vsplatc(u.bc == 3 ? kOne : 0u);
            }
            break;
        case Q:
            c_.get(kR);
            c_.load(kOffQ);
            c_.simd(kVSplat);
            vfix();
            break;
        default:
            c_.get(kR);
            c_.load(kOffI);
            c_.simd(kVSplat);
            vfix();
            break;
        }
        switch (u.kind) {
        case Add: c_.simd(kF32x4Add); vfix(); break;
        case Sub: c_.simd(kF32x4Sub); vfix(); break;
        case Mul: c_.simd(kF32x4Mul); vfix(); break;
        case Madd:
            c_.simd(kF32x4Mul);
            vfix();
            c_.simd(kF32x4Add);
            vfix();
            break;
        default:
            c_.simd(kF32x4Mul);
            vfix();
            c_.simd(kF32x4Sub);
            vfix();
            break;
        }
        c_.set(kUv);
        return;
    }
    if (u.kind == Itof) {
        itof(u);
        return;
    }
    if (u.kind == Ftoi) {
        ftoi(u);
        return;
    }
    raw_vec(u.fs);
    if (u.kind == Abs) {
        c_.vsplatc(0x7FFFFFFFu);
        c_.simd(kVAnd);
        c_.set(kUv);
        return;
    }
    c_.set(kV2);
    switch (u.src) {
    case Vec: raw_vec(u.ft); break;
    case Bc:
        raw_lane(u.ft, u.bc);
        c_.simd(kVSplat);
        break;
    case Q:
        c_.get(kR);
        c_.load(kOffQ);
        c_.simd(kVSplat);
        break;
    default:
        c_.get(kR);
        c_.load(kOffI);
        c_.simd(kVSplat);
        break;
    }
    c_.set(kV3);
    c_.get(kV2);
    c_.get(kV3);
    vkey_of(kV2);
    vkey_of(kV3);
    c_.simd(u.kind == Max ? kI32x4GeS : kI32x4LeS);
    c_.simd(kVBitselect);
    c_.set(kUv);
}

void Translator::upper_post(const Up &u, bool flags_live) {
    if (u.kind == Clip) {
        c_.get(kR);
        c_.get(kUclip);
        c_.store(kOffClip);
        ring_clip([&] { c_.get(kUclip); }, [&] { now(); });
        return;
    }
    if (!active(u, flags_live)) return;
    if (u.target == 0) {
        c_.get(kUv);
        flags_inline(u.dest);
        return;
    }
    const std::uint32_t dst = reg_off(u.target);
    c_.get(kR);
    c_.get(kUv);
    if (u.dest != 0xF) {
        c_.get(kR);
        c_.vload(dst);
        lane_mask(u.dest);
        c_.simd(kVBitselect);
    }
    c_.vstore(dst);
    if (u.flags && flags_live) {
        c_.get(kUv);
        flags_inline(u.dest);
    }
}

bool Translator::lower(std::uint32_t w, std::uint32_t pc) {
    const unsigned dest = (w >> 21) & 15u, it = (w >> 16) & 15u, is = (w >> 11) & 15u, id = (w >> 6) & 15u;
    const unsigned ft = (w >> 16) & 31u, fs = (w >> 11) & 31u;
    const std::int32_t imm11 = static_cast<std::int32_t>(w << 21) >> 21;
    const auto binary = [&](unsigned out, unsigned a, unsigned b, std::uint8_t op) {
        set_vi(out, [&] {
            vi(a);
            vi(b);
            c_.op(op);
        });
    };
    if ((w >> 25) == 0x40u) {
        const unsigned funct = w & 63u;
        if (funct < 0x3C) {
            switch (funct) {
            case 0x30: binary(id, is, it, kOpAdd); break;
            case 0x31: binary(id, is, it, kOpSub); break;
            case 0x32:
                set_vi(it, [&] {
                    vi(is);
                    c_.i32c(static_cast<std::uint32_t>(static_cast<std::int32_t>(((w >> 6) & 31u) << 27) >> 27));
                    c_.op(kOpAdd);
                });
                break;
            case 0x34: binary(id, is, it, kOpAnd); break;
            case 0x35: binary(id, is, it, kOpOr); break;
            default: return false;
            }
            return true;
        }
        const unsigned index = ((w >> 6) & 31u) * 4u + (funct & 3u);
        const unsigned fsf = (w >> 21) & 3u, ftf = (w >> 23) & 3u;
        switch (index) {
        case 0x30:
            if (ft)
                for (unsigned l = 0; l < 4; ++l) {
                    if (!lane(dest, static_cast<int>(l))) continue;
                    c_.get(kR);
                    mem_lane(fs, l);
                    c_.store(vf_off(ft) + l * 4u);
                }
            break;
        case 0x31:
            if (ft) {
                c_.get(kR);
                c_.vload(vf_off(fs));
                c_.set(kV2);
                for (unsigned l = 0; l < 4; ++l) {
                    if (!lane(dest, static_cast<int>(l))) continue;
                    c_.get(kR);
                    c_.get(kV2);
                    c_.simd(kVExtract);
                    c_.byte(static_cast<std::uint8_t>((l + 1) & 3u));
                    c_.store(vf_off(ft) + l * 4u);
                }
            }
            break;
        case 0x34:
            qword([&] { vi(is); });
            c_.set(kTmp);
            if (ft) lanes_copy(dest, ft);
            set_vi(is, [&] {
                vi(is);
                c_.i32c(1);
                c_.op(kOpAdd);
            });
            break;
        case 0x35:
            qword([&] { vi(it); });
            c_.set(kTmp);
            lanes_store(dest, fs);
            set_vi(it, [&] {
                vi(it);
                c_.i32c(1);
                c_.op(kOpAdd);
            });
            break;
        case 0x36:
            set_vi(is, [&] {
                vi(is);
                c_.i32c(1);
                c_.op(kOpSub);
            });
            qword([&] { vi(is); });
            c_.set(kTmp);
            if (ft) lanes_copy(dest, ft);
            break;
        case 0x37:
            set_vi(it, [&] {
                vi(it);
                c_.i32c(1);
                c_.op(kOpSub);
            });
            qword([&] { vi(it); });
            c_.set(kTmp);
            lanes_store(dest, fs);
            break;
        case 0x38:
            mem_lane(fs, fsf);
            c_.set(kTmp);
            mem_lane(ft, ftf);
            c_.set(kTmp2);
            start_q(
                [&] {
                    to_double(kTmp);
                    to_double(kTmp2);
                    c_.op(kOpF64Div);
                    from_double();
                    c_.get(kTmp);
                    c_.get(kTmp2);
                    c_.op(kOpXor);
                    c_.i32c(0x80000000u);
                    c_.op(kOpAnd);
                    c_.i32c(0x7FFFFFFFu);
                    c_.op(kOpOr);
                    c_.get(kTmp2);
                    c_.i32c(0x7F800000u);
                    c_.op(kOpAnd);
                    c_.op(kOpSelect);
                },
                q_delay(code_, kMask, pc, 7));
            divide_status(kTmp2);
            break;
        case 0x39:
            mem_lane(ft, ftf);
            c_.i32c(0x7FFFFFFFu);
            c_.op(kOpAnd);
            c_.set(kTmp2);
            start_q(
                [&] {
                    to_double(kTmp2);
                    c_.op(kOpF64Sqrt);
                    from_double();
                },
                q_delay(code_, kMask, pc, 7));
            break;
        case 0x3A:
            mem_lane(fs, fsf);
            c_.set(kTmp);
            mem_lane(ft, ftf);
            c_.i32c(0x7FFFFFFFu);
            c_.op(kOpAnd);
            c_.set(kTmp2);
            start_q(
                [&] {
                    to_double(kTmp);
                    to_double(kTmp2);
                    c_.op(kOpF64Sqrt);
                    c_.op(kOpF64Div);
                    from_double();
                    c_.get(kTmp);
                    c_.i32c(0x80000000u);
                    c_.op(kOpAnd);
                    c_.i32c(0x7FFFFFFFu);
                    c_.op(kOpOr);
                    c_.get(kTmp2);
                    c_.i32c(0x7F800000u);
                    c_.op(kOpAnd);
                    c_.op(kOpSelect);
                },
                q_delay(code_, kMask, pc, 13));
            divide_status(kTmp2);
            break;
        case 0x3B: flush_q(); break;
        case 0x3C: set_vi(it, [&] { mem_lane(fs, fsf); }); break;
        case 0x3D:
            if (ft) {
                vi(is);
                c_.op(kOpExt16);
                c_.set(kTmp);
                lanes_fill(dest, ft, kTmp);
            }
            break;
        case 0x3E:
            for (unsigned l = 0; l < 4; ++l)
                if (lane(dest, static_cast<int>(l))) {
                    set_vi(it, [&] {
                        qword([&] { vi(is); });
                        c_.load16(l * 4u);
                    });
                    break;
                }
            break;
        case 0x3F:
            qword([&] { vi(is); });
            c_.set(kTmp);
            vi(it);
            c_.set(kTmp2);
            for (unsigned l = 0; l < 4; ++l) {
                if (!lane(dest, static_cast<int>(l))) continue;
                c_.get(kTmp);
                c_.get(kTmp2);
                c_.store(l * 4u);
            }
            break;
        case 0x40:
            c_.get(kR);
            c_.get(kR);
            c_.load(kOffR);
            c_.tee(kTmp);
            c_.i32c(4);
            c_.op(kOpShrU);
            c_.get(kTmp);
            c_.i32c(22);
            c_.op(kOpShrU);
            c_.op(kOpXor);
            c_.i32c(1);
            c_.op(kOpAnd);
            c_.get(kTmp);
            c_.i32c(1);
            c_.op(kOpShl);
            c_.op(kOpOr);
            c_.i32c(0x7FFFFFu);
            c_.op(kOpAnd);
            c_.store(kOffR);
            if (ft) {
                c_.get(kR);
                c_.load(kOffR);
                c_.i32c(kOne);
                c_.op(kOpOr);
                c_.set(kTmp);
                lanes_fill(dest, ft, kTmp);
            }
            break;
        case 0x41:
            if (ft) {
                c_.get(kR);
                c_.load(kOffR);
                c_.i32c(0x7FFFFFu);
                c_.op(kOpAnd);
                c_.i32c(kOne);
                c_.op(kOpOr);
                c_.set(kTmp);
                lanes_fill(dest, ft, kTmp);
            }
            break;
        case 0x42:
            c_.get(kR);
            mem_lane(fs, fsf);
            c_.i32c(0x7FFFFFu);
            c_.op(kOpAnd);
            c_.store(kOffR);
            break;
        case 0x43:
            c_.get(kR);
            c_.get(kR);
            c_.load(kOffR);
            mem_lane(fs, fsf);
            c_.op(kOpXor);
            c_.i32c(0x7FFFFFu);
            c_.op(kOpAnd);
            c_.store(kOffR);
            break;
        case 0x64:
            if (ft) {
                c_.get(kR);
                c_.load(kOffP);
                c_.set(kTmp);
                lanes_fill(dest, ft, kTmp);
            }
            break;
        case 0x68: set_vi(it, [&] { c_.call(kTop); }); break;
        case 0x69: set_vi(it, [&] { c_.call(kItop); }); break;
        case 0x6C:
            vi(is);
            c_.call(kKick);
            break;
        case 0x70: case 0x71: case 0x72: case 0x73: case 0x74: case 0x75: case 0x76: case 0x78: case 0x79: case 0x7A: case 0x7C: case 0x7D: case 0x7E:
            c_.get(kR);
            c_.i32c(index);
            c_.i32c(fs);
            c_.i32c(fsf);
            c_.call(kEfu);
            break;
        case 0x7B: break;
        default: return false;
        }
        return true;
    }
    const std::uint32_t imm15 = ((w >> 10) & 0x7800u) | (w & 0x7FFu);
    const std::uint32_t imm12 = ((w >> 10) & 0x800u) | (w & 0x7FFu);
    const std::uint32_t imm24 = w & 0xFFFFFFu;
    const auto status = [&] {
        store_t();
        fl();
        c_.call(kStatus);
    };
    const auto mac = [&] {
        store_t();
        fl();
        c_.call(kMac);
    };
    const auto clip = [&] {
        store_t();
        fl();
        c_.call(kClip);
    };
    const auto offset = [&](unsigned reg) {
        vi(reg);
        c_.i32c(static_cast<std::uint32_t>(imm11));
        c_.op(kOpAdd);
    };
    switch (w >> 25) {
    case 0x00:
        qword([&] { offset(is); });
        c_.set(kTmp);
        if (ft) lanes_copy(dest, ft);
        break;
    case 0x01:
        qword([&] { offset(it); });
        c_.set(kTmp);
        lanes_store(dest, fs);
        break;
    case 0x04:
        for (unsigned l = 0; l < 4; ++l)
            if (lane(dest, static_cast<int>(l))) {
                set_vi(it, [&] {
                    qword([&] { offset(is); });
                    c_.load16(l * 4u);
                });
                break;
            }
        break;
    case 0x05:
        qword([&] { offset(is); });
        c_.set(kTmp);
        vi(it);
        c_.set(kTmp2);
        for (unsigned l = 0; l < 4; ++l) {
            if (!lane(dest, static_cast<int>(l))) continue;
            c_.get(kTmp);
            c_.get(kTmp2);
            c_.store(l * 4u);
        }
        break;
    case 0x08:
        set_vi(it, [&] {
            vi(is);
            c_.i32c(imm15);
            c_.op(kOpAdd);
        });
        break;
    case 0x09:
        set_vi(it, [&] {
            vi(is);
            c_.i32c(imm15);
            c_.op(kOpSub);
        });
        break;
    case 0x10:
        set_vi(1, [&] {
            clip();
            c_.i32c(0xFFFFFFu);
            c_.op(kOpAnd);
            c_.i32c(imm24);
            c_.op(kOpEq);
        });
        break;
    case 0x11:
        c_.get(kR);
        c_.i32c(imm24);
        c_.store(kOffClip);
        ring_clip([&] { c_.i32c(imm24); },
                  [&] {
                      now();
                      c_.i32c(4);
                      c_.op(kOpSub);
                  });
        break;
    case 0x12:
        set_vi(1, [&] {
            clip();
            c_.i32c(imm24);
            c_.op(kOpAnd);
            c_.i32c(0);
            c_.op(kOpNe);
        });
        break;
    case 0x13:
        set_vi(1, [&] {
            clip();
            c_.i32c(imm24);
            c_.op(kOpOr);
            c_.i32c(0xFFFFFFu);
            c_.op(kOpAnd);
            c_.i32c(0xFFFFFFu);
            c_.op(kOpEq);
        });
        break;
    case 0x14:
        set_vi(it, [&] {
            status();
            c_.i32c(0xFFFu);
            c_.op(kOpAnd);
            c_.i32c(imm12);
            c_.op(kOpEq);
        });
        break;
    case 0x15: break; // fsset(), after the upper instruction
    case 0x16:
        set_vi(it, [&] {
            status();
            c_.i32c(imm12);
            c_.op(kOpAnd);
        });
        break;
    case 0x17:
        set_vi(it, [&] {
            status();
            c_.i32c(0xFFFu);
            c_.op(kOpAnd);
            c_.i32c(imm12);
            c_.op(kOpOr);
        });
        break;
    case 0x18:
        set_vi(it, [&] {
            mac();
            c_.i32c(0xFFFFu);
            c_.op(kOpAnd);
            vi(is);
            c_.op(kOpEq);
        });
        break;
    case 0x1A:
        set_vi(it, [&] {
            mac();
            vi(is);
            c_.op(kOpAnd);
        });
        break;
    case 0x1B:
        set_vi(it, [&] {
            mac();
            c_.i32c(0xFFFFu);
            c_.op(kOpAnd);
            vi(is);
            c_.op(kOpOr);
        });
        break;
    case 0x1C:
        set_vi(it, [&] {
            clip();
            c_.i32c(0xFFFu);
            c_.op(kOpAnd);
        });
        break;
    case 0x20:
        c_.i32c(1);
        c_.set(kPend);
        break;
    case 0x21:
        set_vi(it, [&] { c_.i32c((pc + 16u) / 8u); });
        c_.i32c(1);
        c_.set(kPend);
        break;
    case 0x24:
    case 0x25:
        vi(is);
        c_.i32c(8);
        c_.op(kOpMul);
        c_.i32c(kMask);
        c_.op(kOpAnd);
        c_.set(kJpc);
        if ((w >> 25) == 0x25) set_vi(it, [&] { c_.i32c((pc + 16u) / 8u); });
        c_.i32c(1);
        c_.set(kPendj);
        break;
    case 0x28:
    case 0x29:
        branch_vi(is);
        branch_vi(it);
        c_.op((w >> 25) == 0x28 ? kOpEq : kOpNe);
        c_.set(kPend);
        break;
    case 0x2C: case 0x2D: case 0x2E: case 0x2F: {
        static const std::uint8_t kCompare[4] = {kOpLtS, kOpGtS, kOpLeS, kOpGeS};
        branch_vi(is);
        c_.op(kOpExt16);
        c_.i32c(0);
        c_.op(kCompare[(w >> 25) - 0x2C]);
        c_.set(kPend);
        break;
    }
    default: return false;
    }
    return true;
}

// FSSET waits for the upper instruction of its pair, whose status it replaces
// the sticky flags of (as in the interpreter)
void Translator::fsset(std::uint32_t w) {
    const std::uint32_t imm12 = ((w >> 10) & 0x800u) | (w & 0x7FFu);
    c_.get(kR);
    c_.get(kR);
    c_.load(kOffStatus);
    c_.i32c(0x3Fu);
    c_.op(kOpAnd);
    c_.i32c(imm12 & 0xFC0u);
    c_.op(kOpOr);
    c_.store(kOffStatus);
    fl();
    c_.get(kR);
    c_.call(kForceMac);
}

void Translator::instruction(std::uint32_t pc, std::uint32_t following) {
    const std::uint32_t lo = word(pc), hi = word(pc + 4);
    const Up u = decode_upper(hi);
    const bool flags_live = any_status_ ? true : (any_flags_ && out_live(pc));
    const LowerInfo &li = lowers_[pc];

    c_.get(kT);
    c_.i32c(1);
    c_.op(kOpAdd);
    c_.set(kT);
    if (any_clip_)
        if (const std::uint32_t wait = stall(code_, kMask, pc)) {
            fl();
            fl();
            c_.load(kOffStalls);
            c_.i32c(wait);
            c_.op(kOpAdd);
            c_.store(kOffStalls);
        }
    if (reads_q(hi)) {
        const QRead kind = q_read_kind(code_, kMask, pc);
        if (kind == kQNew) flush_q();
        else if (kind == kQDynamic) sync_q();
    }
    upper_pre(u, flags_live);
    const unsigned def = vi_def_[pc];
    if (keep_bk_.count(pc)) {
        vi(def);
        c_.set(kBkNext);
    }
    const auto delay = delay_.find(pc);
    delay_mode_ = delay == delay_.end() ? 0 : delay->second;
    delay_reg_ = delay_mode_ == 1 ? vi_def_[(pc - 8u) & kMask] : 0;
    bool fsset_after = false;
    if (hi & 0x80000000u) {
        c_.get(kR);
        c_.i32c(lo);
        c_.store(kOffI);
    } else if (!li.supported || !lower(lo, pc)) {
        leave(pc);
    } else {
        fsset_after = (lo >> 25) == 0x15;
    }
    delay_mode_ = 0;
    if (keep_bk_.count(pc)) {
        c_.get(kBkNext);
        c_.set(kBk);
    }
    if (set_bi_.count(pc)) {
        c_.i32c(def);
        c_.set(kBi);
    }
    upper_post(u, flags_live);
    if (fsset_after) fsset(lo);
    const int written = lower_write(pc);
    const std::uint64_t live = shadow_out_[pc >> 3];
    if (written >= 0 && shadow_.count(written) && ((live >> written) & 1u)) {
        c_.get(kR);
        c_.vload(reg_off(written));
        vfix();
        c_.set(shadow_.at(written));
    }
    const bool target_active = active(u, flags_live) && u.target > 0;
    if (target_active && u.target != written && shadow_.count(u.target) && ((live >> u.target) & 1u)) {
        if (!normalized(u.kind)) {
            c_.get(kR);
            c_.vload(reg_off(u.target));
            vfix();
        } else if (u.dest == 0xF) {
            c_.get(kUv);
        } else {
            c_.get(kUv);
            c_.get(shadow_.at(u.target));
            lane_mask(u.dest);
            c_.simd(kVBitselect);
        }
        c_.set(shadow_.at(u.target));
    }
    if (hi & 0x40000000u) {
        c_.i32c(1);
        c_.set(kEnding);
    }

    const std::uint32_t prev = (pc - 8u) & kMask;
    if (in_.count(prev)) {
        const LowerInfo &pl = lowers_[prev];
        if (word(prev + 4) & 0x40000000u) {
            c_.get(kEnding);
            c_.open_if();
            if (pl.branch || pl.jump) {
                if (pl.branch) c_.i32c(pl.target & kMask);
                else c_.get(kJpc);
                c_.i32c((pc + 8u) & kMask);
                c_.get(pl.branch ? kPend : kPendj);
                c_.op(kOpSelect);
            } else {
                c_.i32c((pc + 8u) & kMask);
            }
            c_.set(kExit);
            br(ret_depth());
            c_.end();
        }
        if (pl.branch) {
            c_.get(kPend);
            c_.open_if();
            c_.i32c(0);
            c_.set(kPend);
            go(pl.target);
            c_.end();
        }
        if (pl.jump) {
            c_.get(kPendj);
            c_.open_if();
            c_.i32c(0);
            c_.set(kPendj);
            c_.get(kJpc);
            c_.set(kExit);
            br(fallback_depth());
            c_.end();
        }
    }
    const std::uint32_t next = (pc + 8u) & kMask;
    if (!in_.count(next)) leave(next);
    else if (next != following) go(next);
}

void Translator::prologue() {
    for (const std::uint32_t at : {kOffMacTs, kOffMacTs + 16u, kOffClipTs, kOffClipTs + 16u}) {
        fl();
        c_.vconst(0, 0, 0, 0);
        c_.vstore(at);
    }
    const std::pair<std::uint32_t, std::uint32_t> rings[3] = {{kOffMac, kOffMacV}, {kOffStatus, kOffStatusV}, {kOffClip, kOffClipV}};
    for (const auto &[from, to] : rings) {
        c_.get(kR);
        c_.load(from);
        c_.simd(kVSplat);
        c_.set(kV0);
        fl();
        c_.get(kV0);
        c_.vstore(to);
        fl();
        c_.get(kV0);
        c_.vstore(to + 16u);
    }
    fl();
    c_.i32c(0);
    c_.store(kOffMh);
    fl();
    c_.i32c(0);
    c_.store(kOffCh);
    fl();
    c_.i32c(0);
    c_.store8(kOffQPending);
    fl();
    c_.i32c(0);
    c_.store(kOffStalls);
    c_.i32c(16);
    c_.set(kT);
    for (unsigned i = 1; i < 16; ++i) {
        if (!(vi_used_ & (1u << i))) continue;
        c_.get(kR);
        c_.load16(vi_off(i));
        c_.set(kVi0 + i);
    }
    const std::uint64_t live = shadow_in_[entry_ >> 3];
    for (const auto &[reg, local] : shadow_) {
        if (!((live >> reg) & 1u)) continue;
        c_.get(kR);
        c_.vload(reg_off(reg));
        vfix();
        c_.set(local);
    }
    c_.i32c(case_of_.at(entry_));
    c_.set(kPcw);
}

bool Translator::run(std::vector<std::uint8_t> &out) {
    pcs_ = reachable(code_, kMask, entry_);
    if (pcs_.empty() || branch_in_delay_slot(code_, kMask, entry_)) return false;
    in_.insert(pcs_.begin(), pcs_.end());
    for (std::uint32_t pc : pcs_) {
        LowerInfo l;
        if (!(word(pc + 4) & 0x80000000u)) l = analyze_lower(word(pc), pc);
        if (branch_in_delay_slot(code_, kMask, pc)) l.supported = false;
        any_status_ = any_status_ || l.reads_status;
        any_flags_ = any_flags_ || l.reads_flags;
        any_clip_ = any_clip_ || reads_clip(word(pc), word(pc + 4));
        lowers_[pc] = l;
    }
    for (std::uint32_t pc : pcs_) live_in_[pc] = false;
    for (bool changed = true; changed;) {
        changed = false;
        for (auto it = pcs_.rbegin(); it != pcs_.rend(); ++it) {
            const std::uint32_t pc = *it;
            const bool out = out_live(pc);
            bool reader_near = false;
            for (std::uint32_t k = 1; k <= 3; ++k) {
                const std::uint32_t q = (pc + 8u * k) & kMask;
                if (in_.count(q) && (lowers_[q].reads_flags || lowers_[q].branch || lowers_[q].jump)) reader_near = true;
            }
            const bool kills = sets_flags(word(pc + 4)) && !reader_near;
            const bool v = lowers_[pc].reads_flags || (!kills && out);
            if (v != live_in_[pc]) {
                live_in_[pc] = v;
                changed = true;
            }
        }
    }

    std::set<std::uint32_t> leaders{entry_, pcs_.front()};
    for (std::uint32_t pc : pcs_) {
        if (!in_.count((pc - 8u) & kMask)) leaders.insert(pc);
        const LowerInfo &l = lowers_[pc];
        if (l.branch && in_.count(l.target)) leaders.insert(l.target);
    }
    for (std::uint32_t pc : leaders) case_of_[pc] = cases_++;
    std::map<std::uint32_t, std::vector<std::uint32_t>> preds;
    for (std::uint32_t pc : pcs_) {
        vi_def_[pc] = vi_def(pc);
        for (std::uint32_t s : flow(pc)) preds[s].push_back(pc);
    }
    for (std::uint32_t pc : pcs_) {
        const std::uint32_t reads = branch_reads(pc);
        bool hit = false;
        for (std::uint32_t p : preds[pc]) hit = hit || (vi_def_[p] && ((reads >> vi_def_[p]) & 1u));
        if (!hit) continue;
        const bool dynamic = case_of_.count(pc) != 0;
        delay_[pc] = dynamic ? 2 : 1;
        for (std::uint32_t p : preds[pc]) {
            if (vi_def_[p]) keep_bk_.insert(p);
            if (dynamic) set_bi_.insert(p);
        }
    }

    std::set<int> shadowed;
    for (std::uint32_t pc : pcs_)
        for (int reg : shadow_reads(word(pc + 4))) shadowed.insert(reg);
    std::uint32_t next_local = kShadow0;
    for (int reg : shadowed) shadow_[reg] = next_local++;
    std::vector<std::uint64_t> uses(2048, 0), defs(2048, 0);
    for (std::uint32_t pc : pcs_) {
        const std::uint32_t hi = word(pc + 4), at = pc >> 3;
        for (int reg : shadow_reads(hi)) uses[at] |= 1ull << reg;
        const int written = lower_write(pc);
        if (written >= 0 && shadow_.count(written)) defs[at] |= 1ull << written;
        const Up u = decode_upper(hi);
        if (active(u, true) && u.target > 0 && u.target != written && shadow_.count(u.target)) {
            defs[at] |= 1ull << u.target;
            if (normalized(u.kind) && u.dest != 0xF) uses[at] |= 1ull << u.target;
        }
    }
    shadow_in_.assign(2048, 0);
    shadow_out_.assign(2048, 0);
    for (bool changed = true; changed;) {
        changed = false;
        for (auto it = pcs_.rbegin(); it != pcs_.rend(); ++it) {
            const std::uint32_t at = *it >> 3;
            std::uint64_t o = 0;
            for (std::uint32_t s : flow(*it)) o |= shadow_in_[s >> 3];
            shadow_out_[at] = o;
            const std::uint64_t v = uses[at] | (o & ~defs[at]);
            if (v != shadow_in_[at]) {
                shadow_in_[at] = v;
                changed = true;
            }
        }
    }

    c_.byte(0x02);
    c_.byte(0x40);
    c_.byte(0x03);
    c_.byte(0x40);
    c_.byte(0x02);
    c_.byte(0x40);
    for (std::uint32_t k = 0; k < cases_; ++k) {
        c_.byte(0x02);
        c_.byte(0x40);
    }
    c_.get(kPcw);
    c_.byte(0x0E);
    c_.u32(cases_);
    for (std::uint32_t k = 0; k < cases_; ++k) c_.u32(k);
    c_.u32(cases_);
    for (std::size_t i = 0; i < pcs_.size(); ++i) {
        const std::uint32_t pc = pcs_[i];
        const auto it = case_of_.find(pc);
        if (it != case_of_.end()) {
            c_.byte(0x0B);
            case_index_ = it->second;
        }
        instruction(pc, i + 1 < pcs_.size() ? pcs_[i + 1] : ~0u);
    }
    c_.byte(0x0B);
    write_back_vi();
    c_.get(kR);
    c_.get(kExit);
    c_.tail(kInterpret);
    c_.byte(0x0B);
    c_.byte(0x0B);
    write_back_vi();
    flush_q();
    c_.get(kR);
    c_.get(kExit);
    c_.store(kOffTpc);
    c_.byte(0x0B);

    std::vector<std::uint8_t> body_code = std::move(c_.b);
    c_.b.clear();
    prologue();
    c_.b.insert(c_.b.end(), body_code.begin(), body_code.end());

    Out m;
    m.b = {0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00};
    std::vector<std::pair<std::uint8_t, bool>> types;
    std::vector<std::uint32_t> import_type(kImports);
    for (std::uint32_t i = 0; i < kImports; ++i) {
        const std::pair<std::uint8_t, bool> t{kImportTypes[i].params, kImportTypes[i].result};
        auto found = std::find(types.begin(), types.end(), t);
        if (found == types.end()) {
            types.push_back(t);
            found = types.end() - 1;
        }
        import_type[i] = static_cast<std::uint32_t>(found - types.begin());
    }
    const std::uint32_t run_type = static_cast<std::uint32_t>(types.size());
    Out ts;
    ts.u32(run_type + 1);
    for (const auto &[params, result] : types) {
        ts.byte(0x60);
        ts.u32(params);
        for (std::uint8_t p = 0; p < params; ++p) ts.byte(0x7F);
        ts.u32(result ? 1 : 0);
        if (result) ts.byte(0x7F);
    }
    ts.byte(0x60);
    ts.u32(2);
    ts.byte(0x7F);
    ts.byte(0x7F);
    ts.u32(0);
    m.section(1, ts);

    Out is;
    is.u32(kImports + 1);
    is.name("env");
    is.name("memory");
    is.byte(0x02);
    is.byte(0x03);
    is.u32(1);
    is.u32(65536);
    for (std::uint32_t i = 0; i < kImports; ++i) {
        is.name("env");
        is.name(kImportNames[i]);
        is.byte(0x00);
        is.u32(import_type[i]);
    }
    m.section(2, is);

    Out fs;
    fs.u32(1);
    fs.u32(run_type);
    m.section(3, fs);

    Out es;
    es.u32(1);
    es.name("run");
    es.byte(0x00);
    es.u32(kImports);
    m.section(7, es);

    Out body;
    body.u32(3);
    body.u32(kI32Count);
    body.byte(0x7F);
    body.u32(1);
    body.byte(0x7E);
    body.u32(kVecTemps + static_cast<std::uint32_t>(shadow_.size()));
    body.byte(0x7B);
    body.b.insert(body.b.end(), c_.b.begin(), c_.b.end());
    Out cs;
    cs.u32(1);
    cs.u32(static_cast<std::uint32_t>(body.b.size()));
    cs.b.insert(cs.b.end(), body.b.begin(), body.b.end());
    m.section(10, cs);

    char label[16];
    std::snprintf(label, sizeof label, "vu1_%04X", entry_);
    Out names, functions;
    functions.u32(1);
    functions.u32(kImports);
    functions.name(label);
    names.name("name");
    names.section(1, functions);
    m.section(0, names);

    out = std::move(m.b);
    return true;
}

}

bool translate(const std::uint8_t *code, std::uint32_t entry, const Layout &layout, std::vector<std::uint8_t> &out) {
    Translator t(code, entry, layout);
    return t.run(out);
}

}