Bring PS5 GTA6 to 120FPS with DLSS 5. An advanced Windows video player and capture enhancement tool featuring real-time preview for videos, images, and capture card sources. Combine AI-powered Super Resolution, DLSS5 image enhancement, and Frame Generation to deliver a next-generation visual experience.
C++
#include "veyra/ngx/AmpereMfgUnlock.h"
#include "veyra/Log.h"
#include "compat/RestoreMemory.h"
#include <bcrypt.h>
#include <psapi.h>
#include <algorithm>
#include <array>
#include <cstring>
#include <format>
#include <mutex>
#include <string>
#include <vector>
namespace veyra::ngx {
namespace {
constexpr uint32_t kFatbinMagic = 0xBA55ED50u;
constexpr size_t kOuterHeader = 16;
constexpr uint32_t kPtxKind = 1;
constexpr uint32_t kCubinKind = 2;
constexpr uint32_t kSm89 = 89;
constexpr uint32_t kSm86 = 86;
constexpr uint64_t kCompressedFlag = 0x2000ull;
constexpr size_t kRecordBytes = 48;
constexpr size_t kEntryNameOffset = 0x10; // slot+0x10 -> "main_kernel"
constexpr size_t kDescNameOffset = 0x28; // slot+0x28 -> next record +0 ("dlfg_kernel")
constexpr char kEntryName[] = "main_kernel";
constexpr char kDescriptorName[] = "dlfg_kernel";
constexpr size_t kMaximumFatbinBytes = 16ull << 20;
// Temporal (midpoint) correction, identical technique to the Ada port.
constexpr char kSm89Target[] = ".target sm_89";
constexpr char kSm86Target[] = ".target sm_86";
constexpr char kJoinLabel[] = "$L__BB0_3:";
constexpr char kMidpointBits[] = "0f3F000000";
constexpr char kMulPrefix[] = "mul.ftz.f32 ";
constexpr char kCurrToPrev[] = "%f136";
constexpr char kPrevToCurr[] = "%f134";
constexpr size_t kTemporalMidpoints = 104;
// The Ampere compiler must never see Ada/Blackwell-only constructs; the same
// refusal list dashdogy applies before retargeting.
constexpr const char* kUnsupportedPtxMarkers[] = {
".e4m3", ".e5m2", "wgmma.", "tcgen05.", "mma.sp::ordered_metadata",
"sm_90", "sm_120"};
uint16_t readU16(const uint8_t* p) { uint16_t v; std::memcpy(&v, p, 2); return v; }
uint32_t readU32(const uint8_t* p) { uint32_t v; std::memcpy(&v, p, 4); return v; }
uint64_t readU64(const uint8_t* p) { uint64_t v; std::memcpy(&v, p, 8); return v; }
bool sha256Equals(const uint8_t* bytes, size_t count, const char* expected) {
if (bytes == nullptr || count == 0 || count > ULONG_MAX || expected == nullptr ||
std::strlen(expected) != 64) {
return false;
}
BCRYPT_ALG_HANDLE algorithm = nullptr;
std::array<uint8_t, 32> digest{};
NTSTATUS status = BCryptOpenAlgorithmProvider(&algorithm, BCRYPT_SHA256_ALGORITHM, nullptr, 0);
if (status >= 0) {
status = BCryptHash(algorithm, nullptr, 0, const_cast<PUCHAR>(bytes),
static_cast<ULONG>(count), digest.data(),
static_cast<ULONG>(digest.size()));
}
if (algorithm != nullptr) {
BCryptCloseAlgorithmProvider(algorithm, 0);
}
if (status < 0) {
return false;
}
static constexpr char kDigits[] = "0123456789ABCDEF";
for (size_t index = 0; index < digest.size(); ++index) {
if (expected[index * 2] != kDigits[digest[index] >> 4] ||
expected[index * 2 + 1] != kDigits[digest[index] & 0x0F]) {
return false;
}
}
return true;
}
// Plain LZ4 block format, as emitted by the CUDA fatbin packer.
bool lz4BlockDecompress(const uint8_t* src, size_t srcSize, uint8_t* dst, size_t dstSize) {
size_t in = 0, out = 0;
while (in < srcSize) {
const uint8_t token = src[in++];
size_t literals = token >> 4;
if (literals == 15) {
uint8_t ext = 0;
do {
if (in >= srcSize) return false;
ext = src[in++];
literals += ext;
} while (ext == 0xFF);
}
if (literals > srcSize - in || literals > dstSize - out) return false;
std::memcpy(dst + out, src + in, literals);
in += literals;
out += literals;
if (in == srcSize) break;
if (srcSize - in < 2) return false;
const size_t back = size_t(src[in]) | (size_t(src[in + 1]) << 8);
in += 2;
if (back == 0 || back > out) return false;
size_t match = 4 + (token & 0x0F);
if ((token & 0x0F) == 15) {
uint8_t ext = 0;
do {
if (in >= srcSize) return false;
ext = src[in++];
match += ext;
} while (ext == 0xFF);
}
if (match > dstSize - out) return false;
for (size_t i = 0; i < match; ++i) dst[out + i] = dst[out + i - back];
out += match;
}
return in == srcSize && out == dstSize;
}
bool pointsToCString(const uint8_t* base, size_t imageSize, uint64_t value, const char* expected) {
const auto start = reinterpret_cast<uintptr_t>(base);
if (value < start || value >= start + imageSize) return false;
const size_t len = std::strlen(expected);
if (value + len + 1 > start + imageSize) return false;
return std::memcmp(reinterpret_cast<const void*>(value), expected, len + 1) == 0;
}
// One sm_89 PTX entry inside a fatbin (the upstream FindUniqueSm89Ptx profile).
struct FatbinInfo {
const uint8_t* address = nullptr;
size_t size = 0;
size_t entryOffset = 0;
uint32_t entryHeaderBytes = 0;
uint32_t compressedBytes = 0;
uint64_t rawBytes = 0;
uint64_t flags = 0;
uint32_t entryCount = 0;
bool temporalShape = false;
};
bool parseFatbin(const uint8_t* fat, size_t size, FatbinInfo& info) {
info = FatbinInfo{};
if (fat == nullptr || size < kOuterHeader + 64 || size > kMaximumFatbinBytes) return false;
if (readU32(fat) != kFatbinMagic || readU16(fat + 6) != kOuterHeader) return false;
if (readU64(fat + 8) + kOuterHeader != size) return false;
size_t offset = kOuterHeader;
size_t ptxCount = 0;
while (offset + 64 <= size) {
const uint16_t kind = readU16(fat + offset);
const uint32_t headerBytes = readU32(fat + offset + 4);
const uint64_t payloadBytes = readU64(fat + offset + 8);
if (headerBytes < 64 || payloadBytes == 0 ||
offset + headerBytes + payloadBytes > size) {
return false;
}
const uint32_t arch = readU32(fat + offset + 28);
++info.entryCount;
if (kind == kPtxKind && arch == kSm89) {
++ptxCount;
info.entryOffset = offset;
info.entryHeaderBytes = headerBytes;
info.compressedBytes = readU32(fat + offset + 16);
info.rawBytes = readU64(fat + offset + 56);
info.flags = readU64(fat + offset + 40);
}
offset += headerBytes + payloadBytes;
}
if (offset != size || ptxCount != 1 || info.compressedBytes == 0 ||
info.rawBytes == 0 || (info.flags & kCompressedFlag) == 0) {
return false;
}
info.address = fat;
info.size = size;
return true;
}
// Decompress the single sm_89 PTX entry; returns the raw PTX copy.
bool extractSm89Ptx(const FatbinInfo& info, std::vector<uint8_t>& ptx) {
ptx.assign(size_t(info.rawBytes), uint8_t{0});
const uint8_t* payload = info.address + info.entryOffset + info.entryHeaderBytes;
return lz4BlockDecompress(payload, info.compressedBytes, ptx.data(), ptx.size());
}
// The temporal kernel carries a unique join label and exactly 104 compiled-in
// 0.5 midpoint multiplies; that shape decides which fatbin gets the midpoint
// correction. Returns hasShape plus the midpoint offsets.
bool inspectTemporalShape(const std::vector<uint8_t>& ptx, size_t& joinLabelCount,
std::vector<size_t>& midpoints) {
joinLabelCount = 0;
midpoints.clear();
const char* begin = reinterpret_cast<const char*>(ptx.data());
const size_t n = ptx.size();
const size_t labelLen = sizeof(kJoinLabel) - 1;
for (size_t i = 0; i + labelLen <= n; ++i) {
if (std::memcmp(begin + i, kJoinLabel, labelLen) == 0) ++joinLabelCount;
}
const size_t midLen = sizeof(kMidpointBits) - 1;
const size_t mulLen = sizeof(kMulPrefix) - 1;
for (size_t i = 0; i + midLen < n; ++i) {
if (std::memcmp(begin + i, kMidpointBits, midLen) != 0) continue;
if (begin[i + midLen] != ';') continue;
size_t line = i;
while (line > 0 && begin[line - 1] != '\n') --line;
if (i - line < mulLen) continue;
if (std::memcmp(begin + line, kMulPrefix, mulLen) != 0) continue;
midpoints.push_back(i);
}
return joinLabelCount == 1 && midpoints.size() == kTemporalMidpoints;
}
// Rewrite the temporal PTX exactly like the Ada port: inject the temporal
// parameter right after the unique join label and replace the first 52
// midpoints with 1-t and the second 52 with t.
bool correctTemporalPtx(std::vector<uint8_t>& ptx) {
const char* begin = reinterpret_cast<const char*>(ptx.data());
const size_t n = ptx.size();
const size_t labelLen = sizeof(kJoinLabel) - 1;
size_t label = SIZE_MAX;
for (size_t i = 0; i + labelLen <= n; ++i) {
if (std::memcmp(begin + i, kJoinLabel, labelLen) != 0) continue;
label = i;
break;
}
if (label == SIZE_MAX) return false;
size_t insertion = label + labelLen;
while (insertion < n && begin[insertion] != '\n') ++insertion;
if (insertion >= n) return false;
++insertion;
size_t labelCount = 0;
std::vector<size_t> marks;
if (!inspectTemporalShape(ptx, labelCount, marks)) return false;
if (marks.empty() || marks.front() <= insertion) return false;
static constexpr char kTemporalInput[] =
"ld.param.f32 %f134, [main_kernel_param_0+32];\r\n"
"mov.f32 %f135, 0f3F800000;\r\n"
"sub.ftz.f32 %f136, %f135, %f134;\r\n";
const size_t midLen = sizeof(kMidpointBits) - 1;
std::vector<uint8_t> patched;
patched.reserve(n + sizeof(kTemporalInput));
auto append = [&patched](const void* p, size_t bytes) {
const auto* b = static_cast<const uint8_t*>(p);
patched.insert(patched.end(), b, b + bytes);
};
append(ptx.data(), insertion);
append(kTemporalInput, sizeof(kTemporalInput) - 1);
size_t src = insertion;
const size_t half = marks.size() / 2;
for (size_t i = 0; i < marks.size(); ++i) {
append(ptx.data() + src, marks[i] - src);
const char* scale = (i < half) ? kCurrToPrev : kPrevToCurr;
append(scale, 5);
src = marks[i] + midLen;
}
append(ptx.data() + src, n - src);
ptx.swap(patched);
return true;
}
// Build the single-entry sm_86 fatbin for one source fatbin.
// Layout: outer header (16) + one PTX entry header (104) + padded raw PTX.
bool buildSm86Fatbin(const FatbinInfo& info, std::vector<uint8_t>& out, std::string& why) {
std::vector<uint8_t> ptx;
if (!extractSm89Ptx(info, ptx)) {
why = "LZ4 decompression failed";
return false;
}
if (info.temporalShape) {
if (!correctTemporalPtx(ptx)) {
why = "temporal midpoint correction failed";
return false;
}
}
const std::string_view text(reinterpret_cast<const char*>(ptx.data()), ptx.size());
for (const char* marker : kUnsupportedPtxMarkers) {
if (text.find(marker) != std::string_view::npos) {
why = std::format("PTX contains '{}', which the Ampere compiler cannot consume", marker);
return false;
}
}
const size_t targetOffset = text.find(kSm89Target);
if (targetOffset == std::string_view::npos) {
why = "PTX does not declare .target sm_89";
return false;
}
std::memcpy(ptx.data() + targetOffset, kSm86Target, sizeof(kSm86Target) - 1);
const size_t padded = (ptx.size() + 7) & ~size_t{7};
const size_t finalSize = kOuterHeader + info.entryHeaderBytes + padded;
out.assign(finalSize, 0);
// outer header: same magic/header size, updated payload
std::memcpy(out.data(), info.address, kOuterHeader);
const uint64_t payload = finalSize - kOuterHeader;
std::memcpy(out.data() + 8, &payload, 8);
// entry header: copy and rewrite sizes / arch / flags
uint8_t* entry = out.data() + kOuterHeader;
std::memcpy(entry, info.address + info.entryOffset, info.entryHeaderBytes);
const uint64_t padded64 = padded;
// Match upstream BuildAmpereSm86Fatbin: compressed entries use the padded
// length after expansion; originally raw entries retain an explicit size.
const uint32_t actualPayloadBytes = (info.flags & kCompressedFlag) != 0
? 0u : static_cast<uint32_t>(ptx.size());
const uint64_t zero64 = 0;
const uint32_t sm86 = kSm86;
const uint64_t uncompressed = info.flags & ~kCompressedFlag;
std::memcpy(entry + 8, &padded64, 8);
std::memcpy(entry + 16, &actualPayloadBytes, 4);
std::memcpy(entry + 28, &sm86, 4);
std::memcpy(entry + 40, &uncompressed, 8);
std::memcpy(entry + 56, &zero64, 8);
std::memcpy(out.data() + kOuterHeader + info.entryHeaderBytes, ptx.data(), ptx.size());
return true;
}
// ---- module and section helpers -------------------------------------------
struct ImageLayout {
uint8_t* base = nullptr;
const IMAGE_NT_HEADERS64* nt = nullptr;
const IMAGE_SECTION_HEADER* sections = nullptr;
};
bool openImage(HMODULE module, ImageLayout& image) {
if (module == nullptr) return false;
auto* base = reinterpret_cast<uint8_t*>(module);
const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
if (dos->e_magic != IMAGE_DOS_SIGNATURE) return false;
const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS64*>(base + dos->e_lfanew);
if (nt->Signature != IMAGE_NT_SIGNATURE) return false;
image.base = base;
image.nt = nt;
image.sections = IMAGE_FIRST_SECTION(nt);
return true;
}
struct SlotHit {
uint8_t* slot = nullptr; // address of the fatbin pointer field
const uint8_t* fatbin = nullptr;
size_t fatbinSize = 0;
std::string entryName;
};
// Find every registration-table slot: an 8-byte field inside a read-only,
// non-executable section that points at a fatbin with one sm_89 PTX entry,
// whose record neighbours carry the audited name pointers.
std::vector<SlotHit> findSlotHits(const ImageLayout& image) {
std::vector<SlotHit> hits;
const size_t imageSize = image.nt->OptionalHeader.SizeOfImage;
const auto imageStart = reinterpret_cast<uintptr_t>(image.base);
const auto* section = image.sections;
for (WORD i = 0; i < image.nt->FileHeader.NumberOfSections; ++i, ++section) {
if ((section->Characteristics & IMAGE_SCN_MEM_READ) == 0) continue;
if ((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0) continue;
uint8_t* sec = image.base + section->VirtualAddress;
const size_t size = section->Misc.VirtualSize;
if (size < kDescNameOffset + 8) continue;
for (size_t off = 8; off + kDescNameOffset + 8 <= size; off += 8) {
uint64_t value = 0;
std::memcpy(&value, sec + off, 8);
if (value < imageStart || value >= imageStart + imageSize) continue;
const auto* candidate = reinterpret_cast<const uint8_t*>(value);
if (readU32(candidate) != kFatbinMagic) continue;
uint64_t entryNamePointer = 0, descName = 0;
std::memcpy(&entryNamePointer, sec + off + kEntryNameOffset, 8);
// The record's own name field sits 8 bytes before the slot; the
// following record's first field would read past a run's end.
std::memcpy(&descName, sec + off - 8, 8);
if (!pointsToCString(image.base, imageSize, entryNamePointer, kEntryName)) continue;
if (!pointsToCString(image.base, imageSize, descName, kDescriptorName)) continue;
const size_t total = size_t(readU64(candidate + 8)) + kOuterHeader;
if (total < 1024 || total > kMaximumFatbinBytes) continue;
if (value + total > imageStart + imageSize) continue;
if (value < imageStart) continue;
FatbinInfo info{};
if (!parseFatbin(candidate, total, info)) continue;
uint64_t entryNameAddress = 0;
std::memcpy(&entryNameAddress, sec + off + kEntryNameOffset, 8);
std::string entryName;
if (entryNameAddress >= imageStart &&
entryNameAddress < imageStart + imageSize) {
const char* text = reinterpret_cast<const char*>(entryNameAddress);
const size_t maxLength = imageStart + imageSize - entryNameAddress;
const size_t length = strnlen_s(text, maxLength);
if (length > 0 && length < 96) entryName.assign(text, length);
}
hits.push_back({sec + off, candidate, total, std::move(entryName)});
}
}
return hits;
}
// RIP-relative `lea reg, [rip+disp32]` sites whose target is exactly one of the
// known fatbin addresses.
struct LeaSite {
uint8_t* disp32 = nullptr;
const uint8_t* target = nullptr;
};
std::vector<LeaSite> findLeaSites(const ImageLayout& image,
const std::vector<const uint8_t*>& fatbins) {
std::vector<LeaSite> sites;
const auto* section = image.sections;
for (WORD i = 0; i < image.nt->FileHeader.NumberOfSections; ++i, ++section) {
if ((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) == 0) continue;
uint8_t* begin = image.base + section->VirtualAddress;
const size_t size = section->Misc.VirtualSize;
for (size_t off = 0; off + 7 <= size; ++off) {
const uint8_t first = begin[off];
const uint8_t second = begin[off + 1];
if (!((first == 0x48 || first == 0x4C) && second == 0x8D)) continue;
const uint8_t modrm = begin[off + 2];
if ((modrm & 0xC7) != 0x05) continue;
int32_t disp = 0;
std::memcpy(&disp, begin + off + 3, 4);
const auto target = reinterpret_cast<const uint8_t*>(
reinterpret_cast<uintptr_t>(begin + off) + 7 + intptr_t(disp));
if (std::find(fatbins.begin(), fatbins.end(), target) == fatbins.end()) continue;
sites.push_back({begin + off + 3, target});
}
}
return sites;
}
struct SectionInfo {
const char* name = "";
const uint8_t* begin = nullptr;
size_t size = 0;
};
std::vector<SectionInfo> listedSections(const ImageLayout& image) {
std::vector<SectionInfo> result;
const auto* section = image.sections;
for (WORD i = 0; i < image.nt->FileHeader.NumberOfSections; ++i, ++section) {
if ((section->Characteristics & IMAGE_SCN_MEM_READ) == 0) continue;
result.push_back({reinterpret_cast<const char*>(section->Name),
image.base + section->VirtualAddress,
section->Misc.VirtualSize});
}
return result;
}
// Collect every structurally valid fatbin in the mapped image.
std::vector<FatbinInfo> collectFatbins(const ImageLayout& image) {
std::vector<FatbinInfo> result;
for (const auto& section : listedSections(image)) {
const uint8_t* begin = section.begin;
const size_t size = section.size;
if (size < kOuterHeader) continue;
for (size_t off = 0; off + kOuterHeader <= size; off += 8) {
if (readU32(begin + off) != kFatbinMagic) continue;
const size_t total = size_t(readU64(begin + off + 8)) + kOuterHeader;
if (total < 1024 || total > kMaximumFatbinBytes ||
off + total > size) {
continue;
}
FatbinInfo info{};
if (!parseFatbin(begin + off, total, info)) continue;
bool duplicate = false;
for (const auto& existing : result) {
if (existing.address == info.address) { duplicate = true; break; }
}
if (!duplicate) result.push_back(info);
}
}
return result;
}
bool sectionIsRdata(const SectionInfo& section) {
return std::strncmp(section.name, ".rdata", 6) == 0;
}
bool sectionIsData(const SectionInfo& section) {
return std::strncmp(section.name, ".data", 5) == 0;
}
struct LeaWrite {
uint8_t* disp32 = nullptr;
int32_t original = 0;
};
struct FontWrite {
uint8_t* address = nullptr;
std::vector<uint8_t> original;
DWORD protection = 0;
};
// Adapted from upstream FontPageProtectionMatches/RestoreFontPage/WriteFont.
// Windows may report a privatized WRITECOPY image page as READWRITE. Accept
// that transition only after the working set proves the page is private.
bool writeFontPage(const FontWrite& page, const uint8_t* bytes) {
if (GetEnvironmentVariableW(L"VEYRA_TEST_PATCH_RESTORE_FAIL", nullptr, 0)) return false;
DWORD ignored = 0;
const bool writable = VirtualProtect(page.address, page.original.size(), PAGE_WRITECOPY, &ignored) != FALSE;
if (writable) std::memcpy(page.address, bytes, page.original.size());
bool restored = false;
for (unsigned retry = 0; retry != 2 && !restored; ++retry) {
VirtualProtect(page.address, page.original.size(), page.protection, &ignored);
MEMORY_BASIC_INFORMATION memory{};
if (VirtualQuery(page.address, &memory, sizeof(memory)) != sizeof(memory) ||
memory.State != MEM_COMMIT || memory.Type != MEM_IMAGE) continue;
restored = memory.Protect == page.protection;
if (!restored && page.protection == PAGE_WRITECOPY && memory.Protect == PAGE_READWRITE) {
PSAPI_WORKING_SET_EX_INFORMATION working{};
working.VirtualAddress = page.address;
restored = K32QueryWorkingSetEx(GetCurrentProcess(), &working, sizeof(working)) &&
working.VirtualAttributes.Valid && !working.VirtualAttributes.Shared;
}
}
if (!restored) compat::memoryProtectionUncertain = true;
const bool ok = writable && restored && std::memcmp(page.address, bytes, page.original.size()) == 0;
if (!ok) log::error("ampere-mfg", std::format("font page write failed writable={} restored={} win32={}",
writable, restored, GetLastError()));
return ok;
}
struct GlobalState {
std::mutex mutex;
AmpereMfgUnlock::State state{};
bool installed = false;
std::vector<std::pair<uint8_t*, uint8_t>> archGateWrites;
std::vector<std::pair<uint64_t*, uint64_t>> slotWrites;
std::vector<std::pair<uint8_t*, uint32_t>> sizeWrites;
std::vector<LeaWrite> leaWrites;
std::vector<FontWrite> fontWrites;
void* allocation = nullptr;
};
GlobalState& global() {
static GlobalState instance;
return instance;
}
// Patch the 0x1b0 architecture compares to 0x170 (Ampere) so 30/40/50 series
// all pass; the gate only decides the reported multi-frame ceiling.
bool patchArchGates(uint8_t* base, const IMAGE_NT_HEADERS64* nt,
std::vector<std::pair<uint8_t*, uint8_t>>& sites) {
const auto* section = IMAGE_FIRST_SECTION(nt);
for (WORD i = 0; i < nt->FileHeader.NumberOfSections; ++i, ++section) {
if ((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) == 0) continue;
uint8_t* start = base + section->VirtualAddress;
const size_t size = section->Misc.VirtualSize;
if (size < 6) continue;
for (size_t off = 0; off + 6 <= size; ++off) {
unsigned char* target = nullptr;
if (start[off] == 0x3D && start[off + 1] == 0xB0 && start[off + 2] == 0x01 &&
start[off + 3] == 0x00 && start[off + 4] == 0x00) {
target = start + off + 1;
} else if (start[off] == 0x81 && start[off + 1] >= 0xF8 &&
start[off + 1] <= 0xFF && start[off + 2] == 0xB0 &&
start[off + 3] == 0x01 && start[off + 4] == 0x00 &&
start[off + 5] == 0x00) {
target = start + off + 2;
}
if (target == nullptr) continue;
sites.push_back({target, *target});
const uint8_t replacement=0x70;
if(!compat::restoreMemory(target,&replacement,1))return false;
if (target + 1 - start <= LONG(size)) {
off += 5;
}
}
}
return true;
}
// Allocate the rebuilt set inside an address window that every RIP-relative
// reference can reach. Walks the address space with VirtualQuery and asks for
// a concrete free region, which avoids the VirtualAlloc2 address-requirement
// path that rejects the same request with ERROR_INVALID_PARAMETER on some
// Windows builds (observed locally with 26100).
void* allocateReachable(size_t bytes, const uint8_t* moduleBase) {
const uintptr_t base = reinterpret_cast<uintptr_t>(moduleBase);
constexpr uintptr_t kReach = 0x50000000ull; // 1.25 GiB, well inside disp32
const uintptr_t low = base > kReach ? base - kReach : 0x10000;
const uintptr_t high = base + kReach;
uintptr_t address = (low + 0xFFFF) & ~uintptr_t{0xFFFF};
while (address < high) {
MEMORY_BASIC_INFORMATION memory{};
if (VirtualQuery(reinterpret_cast<void*>(address), &memory, sizeof(memory)) !=
sizeof(memory)) {
break;
}
const auto regionBase = reinterpret_cast<uintptr_t>(memory.BaseAddress);
const uintptr_t regionEnd = regionBase + memory.RegionSize;
if (regionEnd <= address) break;
if (memory.State == MEM_FREE && memory.RegionSize >= bytes) {
uintptr_t candidate = (regionBase + 0xFFFF) & ~uintptr_t{0xFFFF};
if (candidate < base - kReach) candidate = (base - kReach + 0xFFFF) & ~uintptr_t{0xFFFF};
if (candidate + bytes <= regionEnd && candidate + bytes <= high) {
void* result = VirtualAlloc(reinterpret_cast<void*>(candidate), bytes,
MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
if (result != nullptr) return result;
}
}
address = regionEnd;
}
// Fall back to any address; the lea publication step re-validates reach
// and rolls everything back if a displacement would not fit.
return VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
}
// CUDA driver preflight (the intent of dashdogy's ampere_cuda_program
// validation): in a private context on the active GPU, the driver must accept
// every rebuilt fatbin before any pointer is published. On an RTX 30 host this
// compiles the sm_86 programs on the real Ampere device; on other hosts it at
// least proves the PTX/JIT path is legal.
bool preflightPrograms(const std::vector<std::pair<const uint8_t*, size_t>>& programs,
const std::vector<std::string>& entryNames,
uint64_t adapterLuid, std::string& why, size_t& loaded) {
loaded = 0;
HMODULE cuda = LoadLibraryExW(L"nvcuda.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32);
if (cuda == nullptr) {
why = "nvcuda.dll is not available";
return false;
}
// nvcuda.dll is intentionally never unloaded: the CUDA driver does not
// support being freed while the process lives.
const auto init = reinterpret_cast<int (*)(unsigned)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuInit")));
const auto deviceGet = reinterpret_cast<int (*)(int*, int)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuDeviceGet")));
const auto deviceGetCount = reinterpret_cast<int (*)(int*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuDeviceGetCount")));
const auto deviceGetLuid = reinterpret_cast<int (*)(char*, unsigned*, int)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuDeviceGetLuid")));
const auto ctxCreate = reinterpret_cast<int (*)(void**, unsigned, int)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuCtxCreate_v2")));
const auto ctxDestroy = reinterpret_cast<int (*)(void*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuCtxDestroy_v2")));
const auto ctxGetCurrent = reinterpret_cast<int (*)(void**)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuCtxGetCurrent")));
const auto ctxPopCurrent = reinterpret_cast<int (*)(void**)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuCtxPopCurrent_v2")));
const auto moduleLoad = reinterpret_cast<int (*)(void**, const void*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuModuleLoadData")));
const auto moduleGetFunction = reinterpret_cast<int (*)(void**, void*, const char*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuModuleGetFunction")));
const auto functionLoad = reinterpret_cast<int (*)(void*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuFuncLoad")));
const auto moduleUnload = reinterpret_cast<int (*)(void*)>(
reinterpret_cast<void*>(GetProcAddress(cuda, "cuModuleUnload")));
if (init == nullptr || deviceGet == nullptr || deviceGetCount == nullptr ||
deviceGetLuid == nullptr || ctxCreate == nullptr ||
ctxDestroy == nullptr || ctxGetCurrent == nullptr || ctxPopCurrent == nullptr ||
moduleLoad == nullptr || moduleGetFunction == nullptr || moduleUnload == nullptr ||
programs.size() != entryNames.size()) {
why = "nvcuda.dll does not export the required driver API";
return false;
}
if (init(0) != 0) {
why = "cuInit failed";
return false;
}
int device = -1, deviceCount = 0;
const int countResult = deviceGetCount(&deviceCount);
if (countResult != 0) {
why = std::format("cuDeviceGetCount failed result={}", countResult);
return false;
}
unsigned matches = 0;
for (int ordinal = 0; ordinal < deviceCount; ++ordinal) {
int candidate = -1;
const int getResult = deviceGet(&candidate, ordinal);
if (getResult != 0) {
why = std::format("cuDeviceGet failed ordinal={} result={}", ordinal, getResult);
return false;
}
uint64_t luid = 0;
unsigned nodeMask = 0;
const int luidResult = deviceGetLuid(reinterpret_cast<char*>(&luid), &nodeMask, candidate);
if ((adapterLuid == 0 && ordinal == 0) ||
(adapterLuid != 0 && luidResult == 0 && luid == adapterLuid && nodeMask != 0)) {
device = candidate;
++matches;
}
}
if (matches != 1) {
why = std::format("CUDA adapter match failed D3D12 LUID=0x{:X} matches={}", adapterLuid, matches);
return false;
}
log::info("ampere-mfg", std::format("CUDA preflight adapter={} D3D12 LUID=0x{:X}", device, adapterLuid));
void* previousContext = nullptr;
if (ctxGetCurrent(&previousContext) != 0) {
why = "cuCtxGetCurrent failed";
return false;
}
void* context = nullptr;
if (ctxCreate(&context, 0, device) != 0) {
why = "cuCtxCreate failed";
return false;
}
bool ok = true;
for (size_t index = 0; index < programs.size(); ++index) {
void* module = nullptr;
const int result = moduleLoad(&module, programs[index].first);
if (result != 0) {
why = std::format("the CUDA driver rejected rebuilt program {} (cuModuleLoadData={})",
index, result);
ok = false;
break;
}
void* function = nullptr;
const char* entry = entryNames[index].empty() ? "main_kernel" : entryNames[index].c_str();
const int functionResult = module != nullptr
? moduleGetFunction(&function, module, entry) : 200;
const int loadResult = functionResult == 0 && functionLoad != nullptr && function != nullptr
? functionLoad(function) : functionResult == 0 && functionLoad == nullptr ? 0 : 200;
const int unloadResult = module != nullptr ? moduleUnload(module) : 200;
if (functionResult != 0 || loadResult != 0 || unloadResult != 0 || function == nullptr) {
why = std::format("CUDA function preflight failed for program {} entry={} module={} function={} cuModuleGetFunction={} cuFuncLoad={} cuModuleUnload={}",
index, entry, module != nullptr ? 1 : 0, function != nullptr ? 1 : 0,
functionResult, loadResult, unloadResult);
ok = false;
break;
}
++loaded;
}
void* current = nullptr;
const int currentResult = ctxGetCurrent(¤t);
void* popped = nullptr;
const int popResult = currentResult == 0 && current == context ? ctxPopCurrent(&popped) : 201;
const int destroyResult = popResult == 0 && popped == context ? ctxDestroy(context) : 201;
void* restoredContext = nullptr;
const int restoredResult = ctxGetCurrent(&restoredContext);
if (popResult != 0 || popped != context || destroyResult != 0 ||
restoredResult != 0 || restoredContext != previousContext) {
why = std::format("CUDA private context cleanup failed pop={} destroy={} restored={}",
popResult, destroyResult, restoredResult == 0 && restoredContext == previousContext ? 1 : 0);
ok = false;
}
return ok;
}
} // namespace
AmpereMfgUnlock::Scan AmpereMfgUnlock::scan(HMODULE module) {
Scan result{};
ImageLayout image{};
if (!openImage(module, image)) {
result.detail = "not a valid PE image";
return result;
}
result.moduleValid = true;
result.identityMatched = image.nt->OptionalHeader.SizeOfImage == kKnownSizeOfImage &&
image.nt->FileHeader.TimeDateStamp == kKnownTimeDateStamp;
// Architecture gate sites (0x1b0 compares, two encodings).
{
const auto* section = image.sections;
for (WORD i = 0; i < image.nt->FileHeader.NumberOfSections; ++i, ++section) {
if ((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) == 0) continue;
const uint8_t* start = image.base + section->VirtualAddress;
const size_t size = section->Misc.VirtualSize;
for (size_t off = 0; off + 6 <= size; ++off) {
if (start[off] == 0x3D && start[off + 1] == 0xB0 && start[off + 2] == 0x01 &&
start[off + 3] == 0x00 && start[off + 4] == 0x00) {
++result.archGateSites;
} else if (start[off] == 0x81 && start[off + 1] >= 0xF8 &&
start[off + 1] <= 0xFF && start[off + 2] == 0xB0 &&
start[off + 3] == 0x01 && start[off + 4] == 0x00 &&
start[off + 5] == 0x00) {
++result.archGateSites;
}
}
}
}
const auto fatbins = collectFatbins(image);
const auto hits = findSlotHits(image);
result.slotPointers = hits.size();
// Debug aid: per-section fatbin counts (kept in the detail string).
std::string perSection;
for (const auto& section : listedSections(image)) {
size_t count = 0;
for (size_t off = 0; off + kOuterHeader <= section.size; off += 8) {
if (readU32(section.begin + off) != kFatbinMagic) continue;
const size_t total = size_t(readU64(section.begin + off + 8)) + kOuterHeader;
if (total < 1024 || total > kMaximumFatbinBytes ||
off + total > section.size) {
continue;
}
FatbinInfo info{};
if (parseFatbin(section.begin + off, total, info)) ++count;
}
perSection += std::format(" [{}:{}]", section.name, count);
}
perSection += std::format(" total={}", fatbins.size());
// Validate the eight runs of 25 consecutive 48-byte-strided slots.
{
size_t index = 0;
std::vector<const uint8_t*> seen;
while (index < hits.size()) {
size_t run = 1;
while (index + run < hits.size() &&
hits[index + run].slot == hits[index + run - 1].slot + kRecordBytes) {
++run;
}
if (run == kExpectedSlotsPerRun) {
++result.slotRuns;
for (size_t k = 0; k < run; ++k) {
const auto* fb = hits[index + k].fatbin;
if (std::find(seen.begin(), seen.end(), fb) == seen.end()) seen.push_back(fb);
}
}
index += run;
}
result.programFatbins = seen.size();
}
// Classify the remaining fatbins: .rdata network programs vs .data aux.
std::vector<const uint8_t*> programAddresses;
for (const auto& hit : hits) {
if (std::find(programAddresses.begin(), programAddresses.end(), hit.fatbin) ==
programAddresses.end()) {
programAddresses.push_back(hit.fatbin);
}
}
std::vector<const uint8_t*> allAddresses;
for (const auto& fb : fatbins) allAddresses.push_back(fb.address);
for (const auto& section : listedSections(image)) {
const auto begin = reinterpret_cast<uintptr_t>(section.begin);
const auto end = begin + section.size;
for (const auto& fb : fatbins) {
const auto address = reinterpret_cast<uintptr_t>(fb.address);
if (address < begin || address >= end) continue;
const bool isProgram =
std::find(programAddresses.begin(), programAddresses.end(), fb.address) !=
programAddresses.end();
if (isProgram) continue;
if (sectionIsRdata(section)) {
++result.networkFatbins;
} else if (std::strncmp(section.name, ".data", 5) == 0) {
++result.auxFatbins;
}
}
}
// RIP-relative references.
{
const auto sites = findLeaSites(image, allAddresses);
result.leaSites = sites.size();
// Each fatbin outside the registration table must be referenced once.
size_t referencedOnce = 0;
for (const auto& fb : fatbins) {
const bool isProgram =
std::find(programAddresses.begin(), programAddresses.end(), fb.address) !=
programAddresses.end();
if (isProgram) continue;
const size_t count = std::count_if(
sites.begin(), sites.end(),
[&](const LeaSite& site) { return site.target == fb.address; });
if (count == 1) ++referencedOnce;
}
result.detail = std::format(
"runs={} slotPointers={} programFatbins={} networkFatbins={} auxFatbins={} "
"leaSites={} uniquelyReferenced={} gateSites={}",
result.slotRuns, result.slotPointers, result.programFatbins,
result.networkFatbins, result.auxFatbins, sites.size(), referencedOnce,
result.archGateSites);
result.detail += perSection;
}
// The temporal slot must be unique among the program fatbins.
{
size_t temporalCount = 0;
for (const auto& address : programAddresses) {
FatbinInfo info{};
bool found = false;
for (const auto& fb : fatbins) {
if (fb.address == address) { info = fb; found = true; break; }
}
if (!found) continue;
std::vector<uint8_t> ptx;
if (!extractSm89Ptx(info, ptx)) continue;
size_t labelCount = 0;
std::vector<size_t> midpoints;
if (inspectTemporalShape(ptx, labelCount, midpoints)) ++temporalCount;
}
result.temporalSlotUnique = temporalCount == 1;
}
return result;
}
bool AmpereMfgUnlock::adapterIsAmpere(uint32_t vendorId, uint32_t deviceId) {
if (vendorId != 0x10DE) return false;
// Ampere GA10x device-id window (RTX 30 series, desktop and mobile).
// Turing tops out below 0x2200 and Ada starts at 0x2680, so this window
// cannot catch a different generation.
return deviceId >= 0x2200 && deviceId < 0x2680;
}
AmpereMfgUnlock::State AmpereMfgUnlock::apply(HMODULE module, bool retargetArchGates, uint64_t adapterLuid) {
auto& g = global();
std::lock_guard lock(g.mutex);
State state{};
if (module == nullptr) {
state.detail = L"no runtime module handle";
g.state = state;
return state;
}
state.moduleFound = true;
if (g.installed) {
state = g.state;
return state;
}
ImageLayout image{};
if (!openImage(module, image)) {
state.detail = L"runtime module is not a valid PE image";
g.state = state;
return state;
}
state.identityVerified = image.nt->OptionalHeader.SizeOfImage == kKnownSizeOfImage &&
image.nt->FileHeader.TimeDateStamp == kKnownTimeDateStamp;
if (!state.identityVerified) {
state.detail = L"runtime identity (SizeOfImage/TimeDateStamp) does not match the audited 310.7 build";
g.state = state;
return state;
}
auto fatbins = collectFatbins(image);
const auto hits = findSlotHits(image);
// Registration tables: eight runs of 25 consecutive 48-byte-strided slots.
size_t slotRuns = 0;
std::vector<const uint8_t*> programAddresses;
{
size_t index = 0;
while (index < hits.size()) {
size_t run = 1;
while (index + run < hits.size() &&
hits[index + run].slot == hits[index + run - 1].slot + kRecordBytes) {
++run;
}
if (run == kExpectedSlotsPerRun) {
++slotRuns;
for (size_t k = 0; k < run; ++k) {
const auto* fb = hits[index + k].fatbin;
if (std::find(programAddresses.begin(), programAddresses.end(), fb) ==
programAddresses.end()) {
programAddresses.push_back(fb);
}
}
}
index += run;
}
}
if (slotRuns != kExpectedSlotRuns ||
hits.size() != kExpectedSlotRuns * kExpectedSlotsPerRun ||
programAddresses.size() != kExpectedProgramFatbins) {
state.detail = std::format(
L"registration tables do not match the audited layout (runs={} pointers={} programs={})",
slotRuns, hits.size(), programAddresses.size());
g.state = state;
return state;
}
const auto sections = listedSections(image);
auto classify = [&](const uint8_t* address) -> int {
if (std::find(programAddresses.begin(), programAddresses.end(), address) !=
programAddresses.end()) {
return 0; // program slot
}
const auto value = reinterpret_cast<uintptr_t>(address);
for (const auto& section : sections) {
const auto begin = reinterpret_cast<uintptr_t>(section.begin);
if (value < begin || value >= begin + section.size) continue;
if (sectionIsRdata(section)) return 1;
if (sectionIsData(section)) return 2;
}
return -1;
};
size_t networkCount = 0, auxCount = 0, temporalCount = 0;
for (auto& fb : fatbins) {
const int cls = classify(fb.address);
if (cls == 1) ++networkCount;
else if (cls == 2) ++auxCount;
std::vector<uint8_t> ptx;
size_t labelCount = 0;
std::vector<size_t> midpoints;
fb.temporalShape = extractSm89Ptx(fb, ptx) &&
inspectTemporalShape(ptx, labelCount, midpoints);
if (fb.temporalShape) ++temporalCount;
}
if (fatbins.size() != kExpectedProgramFatbins + kExpectedRdataNetworkFatbins +
kExpectedAuxFatbins ||
networkCount != kExpectedRdataNetworkFatbins ||
auxCount != kExpectedAuxFatbins) {
state.detail = std::format(
L"fatbin inventory does not match the audited layout (total={} network={} aux={})",
fatbins.size(), networkCount, auxCount);
g.state = state;
return state;
}
if (temporalCount != 1) {
state.detail = std::format(L"expected exactly one temporal program, found {}", temporalCount);
g.state = state;
return state;
}
{
bool temporalInPrograms = false;
for (const auto& fb : fatbins) {
if (!fb.temporalShape) continue;
temporalInPrograms = std::find(programAddresses.begin(), programAddresses.end(),
fb.address) != programAddresses.end();
}
if (!temporalInPrograms) {
state.detail = L"the temporal program is not one of the registration slots";
g.state = state;
return state;
}
}
std::vector<const uint8_t*> allAddresses;
allAddresses.reserve(fatbins.size());
for (const auto& fb : fatbins) allAddresses.push_back(fb.address);
const auto leaSites = findLeaSites(image, allAddresses);
{
size_t leaOnNonPrograms = 0;
for (const auto& site : leaSites) {
if (classify(site.target) != 0) ++leaOnNonPrograms;
}
if (leaOnNonPrograms != kExpectedLeaSites) {
state.detail = std::format(
L"expected {} RIP-relative fatbin references, found {}",
kExpectedLeaSites, leaOnNonPrograms);
g.state = state;
return state;
}
}
state.slotRuns = slotRuns;
state.slotPointers = hits.size();
state.programFatbins = programAddresses.size();
state.networkFatbins = networkCount;
state.auxFatbins = auxCount;
state.leaSites = leaSites.size();
// Match upstream's 25 registered programs plus the exact auxiliary font.
// DL4RT network and other auxiliary programs remain provider-owned.
struct Replacement {
const uint8_t* original = nullptr;
std::vector<uint8_t> data;
};
std::vector<Replacement> replacements;
replacements.reserve(programAddresses.size());
for (const auto* address : programAddresses) {
const auto fbIt = std::find_if(fatbins.begin(), fatbins.end(),
[&](const FatbinInfo& fb) { return fb.address == address; });
if (fbIt == fatbins.end()) {
state.detail = L"registered kernel fatbin disappeared from the audited inventory";
g.state = state;
return state;
}
const auto& fb = *fbIt;
std::vector<uint8_t> data;
std::string why;
if (!buildSm86Fatbin(fb, data, why)) {
state.detail = std::format(
L"sm_86 rebuild refused for the fatbin at RVA 0x{:X}: {}",
reinterpret_cast<uintptr_t>(fb.address) - reinterpret_cast<uintptr_t>(image.base),
std::wstring(why.begin(), why.end()));
g.state = state;
return state;
}
replacements.push_back({fb.address, std::move(data)});
}
if (replacements.size() != programAddresses.size()) {
state.detail = L"rebuilt program count does not match registration slots";
g.state = state;
return state;
}
// The provider stores the CUDA kernel entry name in each registration
// record. Preserve that contract for cuModuleGetFunction preflight;
// assuming a generic name lets malformed PTX pass the cheap load check.
std::vector<std::string> entryNames;
entryNames.reserve(programAddresses.size());
for (const auto* address : programAddresses) {
const auto hit = std::find_if(hits.begin(), hits.end(),
[&](const SlotHit& slot) { return slot.fatbin == address; });
if (hit == hits.end() || hit->entryName.empty()) {
state.detail = L"a registered kernel is missing its CUDA entry name";
g.state = state;
return state;
}
entryNames.push_back(hit->entryName);
}
// Ported from upstream ampere_font_program.inl PrepareFont (MIT, pinned
// commit in the header). Rebuild PTX from the mapped image; no embedded
// native payload is copied into Veyra's source or written to disk.
constexpr size_t fontBytes = 6752;
constexpr char fontHash[] = "C6CC4129606AF4A3931C98BF7F7405EB9F5469F6522DA914FE7F3E5B39A224A1";
const FatbinInfo* font = nullptr;
for (const auto& fb : fatbins) {
if (classify(fb.address) != 2 || fb.size != fontBytes ||
!sha256Equals(fb.address, fb.size, fontHash)) continue;
if (font) {
state.detail = L"auxiliary font identity is not unique";
g.state = state;
return state;
}
font = &fb;
}
std::vector<uint8_t> fontData;
std::string fontWhy;
if (!font || !buildSm86Fatbin(*font, fontData, fontWhy) || fontData.size() > fontBytes) {
state.detail = std::format(L"auxiliary font rebuild refused: {}",
std::wstring(fontWhy.begin(), fontWhy.end()));
g.state = state;
return state;
}
// The driver must accept the whole rebuilt program set before anything is
// published; a rejected program is a refusal, never a half-installed set.
size_t preflightLoaded = 0;
{
std::vector<std::pair<const uint8_t*, size_t>> programData;
programData.reserve(replacements.size());
for (const auto& replacement : replacements) {
programData.push_back({replacement.data.data(), replacement.data.size()});
}
programData.push_back({fontData.data(), fontData.size()});
entryNames.push_back("cuda_font_kernel");
std::string preflightWhy;
if (!preflightPrograms(programData, entryNames, adapterLuid, preflightWhy, preflightLoaded)) {
state.detail = std::format(L"CUDA preflight refused the rebuilt program set: {}",
std::wstring(preflightWhy.begin(), preflightWhy.end()));
g.state = state;
return state;
}
}
size_t total = 0;
std::vector<size_t> offsets;
offsets.reserve(replacements.size());
for (const auto& replacement : replacements) {
const size_t aligned = (replacement.data.size() + 63) & ~size_t{63};
offsets.push_back(total);
total += aligned;
}
void* block = allocateReachable(total, image.base);
if (block == nullptr) {
state.detail = L"no reachable allocation window for the rebuilt fatbins (VirtualAlloc2 within 1.5 GiB failed)";
g.state = state;
return state;
}
g.allocation = block;
auto* cursor = static_cast<uint8_t*>(block);
for (size_t i = 0; i < replacements.size(); ++i) {
std::memcpy(cursor + offsets[i], replacements[i].data.data(),
replacements[i].data.size());
}
const auto newAddressOf = [&](const uint8_t* original) -> const uint8_t* {
for (size_t i = 0; i < replacements.size(); ++i) {
if (replacements[i].original == original) return cursor + offsets[i];
}
return nullptr;
};
auto rollbackAll = [&]() {
bool restored = true;
for (auto it = g.fontWrites.rbegin(); it != g.fontWrites.rend(); ++it) {
restored = writeFontPage(*it, it->original.data()) && restored;
}
for (auto it = g.leaWrites.rbegin(); it != g.leaWrites.rend(); ++it) {
restored = compat::restoreMemory(it->disp32, &it->original, 4) && restored;
}
for (auto it = g.slotWrites.rbegin(); it != g.slotWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 8) && restored;
}
for (auto it = g.sizeWrites.rbegin(); it != g.sizeWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 4) && restored;
}
for (auto it = g.archGateWrites.rbegin(); it != g.archGateWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 1) && restored;
}
if (!restored) { log::error("ampere-mfg", "rollback incomplete; retaining kernel storage and records"); return; }
g.leaWrites.clear();
g.fontWrites.clear();
g.slotWrites.clear();
g.sizeWrites.clear();
g.archGateWrites.clear();
if (g.allocation != nullptr) {
VirtualFree(g.allocation, 0, MEM_RELEASE);
g.allocation = nullptr;
}
};
// 1. architecture compares (0x1b0 -> 0x170). Skipped when the caller makes
// the provider see Blackwell through the NVAPI spoof: then the provider's
// own compare must stay byte-identical so that the reported id matches it.
if (retargetArchGates) {
const bool written=patchArchGates(image.base, image.nt, g.archGateWrites);
if (!written || g.archGateWrites.size() < kMinArchGateSites ||
g.archGateWrites.size() > kMaxArchGateSites) {
const size_t found = g.archGateWrites.size();
rollbackAll();
state.detail = std::format(L"arch-gate site count {} outside the audited range {}-{}",
found, kMinArchGateSites, kMaxArchGateSites);
g.state = state;
return state;
}
state.archGateSites = g.archGateWrites.size();
} else {
log::info("ampere-mfg", "arch-gate retarget skipped: the provider is being told it runs on Blackwell");
}
state.archGatesPatched = true;
// 2. Publish pointer AND supplied byte length, as upstream PatchProvider
// does at descriptor offsets +8/+16. Rebuilt PTX has a different size.
size_t redirectedSlots = 0;
for (const auto& hit : hits) {
const uint8_t* target = newAddressOf(hit.fatbin);
// Non-kernel records intentionally keep their original provider-owned
// network/auxiliary pointer and descriptor length.
if (target == nullptr) continue;
uint64_t original = 0;
std::memcpy(&original, hit.slot, 8);
g.slotWrites.push_back({reinterpret_cast<uint64_t*>(hit.slot), original});
const uint64_t replacement = reinterpret_cast<uint64_t>(target);
const auto size = static_cast<uint32_t>(readU64(target + 8) + kOuterHeader);
g.sizeWrites.push_back({hit.slot + 8, readU32(hit.slot + 8)});
if (!compat::restoreMemory(hit.slot,&replacement,8) ||
!compat::restoreMemory(hit.slot + 8, &size, 4)) {
rollbackAll();
state.archGatesPatched = false;
state.detail = L"a registration slot page is not writable; runtime left untouched";
g.state = state;
return state;
}
++redirectedSlots;
}
// 3. RIP-relative references
for (const auto& site : leaSites) {
const uint8_t* target = newAddressOf(site.target);
if (target == nullptr) continue;
int32_t original = 0;
std::memcpy(&original, site.disp32, 4);
const int64_t displacement = intptr_t(target) -
(reinterpret_cast<intptr_t>(site.disp32) + 4);
if (displacement < INT32_MIN || displacement > INT32_MAX) {
rollbackAll();
state.archGatesPatched = false;
state.detail = L"a rebuilt fatbin is out of reach of its RIP-relative reference; runtime left untouched";
g.state = state;
return state;
}
const int32_t replacement = static_cast<int32_t>(displacement);
g.leaWrites.push_back({site.disp32, original});
if(!compat::restoreMemory(site.disp32,&replacement,4)){
rollbackAll();state.archGatesPatched=false;
state.detail=L"fatbin code reference publication failed";
g.state=state;return state;
}
}
// Preserve the font's address and original supplied byte length. Record
// each page separately so adjacent globals retain their page protections.
fontData.resize(fontBytes, 0);
SYSTEM_INFO system{};
GetSystemInfo(&system);
for (size_t offset = 0; offset < fontBytes;) {
auto* address = const_cast<uint8_t*>(font->address) + offset;
const size_t bytes = std::min(fontBytes - offset,
size_t(system.dwPageSize) - reinterpret_cast<uintptr_t>(address) % system.dwPageSize);
DWORD protection = 0;
const bool queried = protected_pointer::QueryProtection(reinterpret_cast<uintptr_t>(address), protection, bytes);
if (queried && (protection == PAGE_READONLY || protection == PAGE_READWRITE || protection == PAGE_WRITECOPY))
g.fontWrites.push_back({address, std::vector<uint8_t>(address, address + bytes), protection});
else {
rollbackAll();
state.detail = L"auxiliary font page protection is not supported";
state.archGatesPatched = false;
g.state = state;
return state;
}
if (!writeFontPage(g.fontWrites.back(), fontData.data() + offset)) {
rollbackAll();
state.detail = L"auxiliary font publication failed";
state.archGatesPatched = false;
g.state = state;
return state;
}
offset += bytes;
}
g.installed = true;
state.fatbinsRedirected = true;
state.applied = state.archGatesPatched && state.fatbinsRedirected;
state.detail = std::format(
L"runs={} slots={} fatbins={} font=1 lea={} gates={} preflight={}/{} (in-memory only)",
slotRuns, redirectedSlots, replacements.size(), g.leaWrites.size(), state.archGateSites,
preflightLoaded, replacements.size() + 1);
g.state = state;
veyra::log::info("ampere-mfg",
std::format("RTX 30 sm_86 unlock: {} runs / "
"{} redirected kernel slots, {} fatbins redirected, {} lea references, {} arch gates, identityVerified={}",
slotRuns, redirectedSlots, replacements.size(), g.leaWrites.size(),
state.archGateSites, state.identityVerified ? 1 : 0));
return state;
}
bool AmpereMfgUnlock::release() {
auto& g = global();
std::lock_guard lock(g.mutex);
if (!g.installed && g.archGateWrites.empty() && g.slotWrites.empty() && g.sizeWrites.empty() &&
g.leaWrites.empty() && g.fontWrites.empty() && g.allocation == nullptr) {
return true;
}
bool restored = true;
for (auto it = g.fontWrites.rbegin(); it != g.fontWrites.rend(); ++it) {
restored = writeFontPage(*it, it->original.data()) && restored;
}
for (auto it = g.leaWrites.rbegin(); it != g.leaWrites.rend(); ++it) {
restored = compat::restoreMemory(it->disp32, &it->original, 4) && restored;
}
for (auto it = g.slotWrites.rbegin(); it != g.slotWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 8) && restored;
}
for (auto it = g.sizeWrites.rbegin(); it != g.sizeWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 4) && restored;
}
for (auto it = g.archGateWrites.rbegin(); it != g.archGateWrites.rend(); ++it) {
restored = compat::restoreMemory(it->first, &it->second, 1) && restored;
}
if (!restored) { log::error("ampere-mfg", "rollback incomplete; retaining kernel storage and records"); return false; }
g.leaWrites.clear();
g.fontWrites.clear();
g.slotWrites.clear();
g.sizeWrites.clear();
g.archGateWrites.clear();
if (g.allocation != nullptr) {
VirtualFree(g.allocation, 0, MEM_RELEASE);
g.allocation = nullptr;
}
g.installed = false;
g.state = State{};
veyra::log::info("ampere-mfg", "RTX 30 sm_86 unlock rolled back (runtime image restored)");
return true;
}
AmpereMfgUnlock::State AmpereMfgUnlock::snapshot() {
auto& g = global();
std::lock_guard lock(g.mutex);
return g.state;
}
bool AmpereMfgUnlock::applied() {
auto& g = global();
std::lock_guard lock(g.mutex);
return g.installed;
}
} // namespace veyra::ngx