Skip to content

Bring PS5 GTA6 to 120FPS with DLSS 5. An advanced Windows video player and capture enhancement tool featuring real-time preview for videos, images, and capture card sources. Combine AI-powered Super Resolution, DLSS5 image enhancement, and Frame Generation to deliver a next-generation visual experience.

C++
12.2 KB
Raw Blame
#include "veyra/beauty/FaceTracker.h"

#include <algorithm>
#include <cmath>
#include <cstring>
#include <format>

#include "veyra/Log.h"
#include "veyra/RuntimePaths.h"

namespace veyra::beauty {
namespace {
constexpr const wchar_t* kModelNames[] = {L"face_detection_yunet_2023mar.onnx", L"face_detection_yunet_2026may.onnx"};
constexpr UINT kTrackConstants = 16;
constexpr UINT kDescriptorsPerSlot = 3;     // t0 (unused), u0 faces, u1 candidates
constexpr UINT64 kFaceBytes = 64;

pipeline::ComPtr<ID3D12Resource> makeBuffer(ID3D12Device* device, UINT64 bytes, D3D12_HEAP_TYPE heap, D3D12_RESOURCE_FLAGS flags,
                                             D3D12_RESOURCE_STATES state)
{
    D3D12_HEAP_PROPERTIES hp{};
    hp.Type = heap;
    D3D12_RESOURCE_DESC d{};
    d.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
    d.Width = bytes; d.Height = 1; d.DepthOrArraySize = 1; d.MipLevels = 1;
    d.SampleDesc.Count = 1; d.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; d.Flags = flags;
    pipeline::ComPtr<ID3D12Resource> r;
    if (FAILED(device->CreateCommittedResource(&hp, D3D12_HEAP_FLAG_NONE, &d, state, nullptr, IID_PPV_ARGS(&r)))) return nullptr;
    return r;
}
uint32_t bitsOf(float f) { uint32_t u; std::memcpy(&u, &f, 4); return u; }
} // namespace

std::filesystem::path locateFaceModel()
{
    wchar_t exe[32768]{};
    const DWORD length = GetModuleFileNameW(nullptr, exe, DWORD(std::size(exe)));
    const auto exeDirectory = length && length < std::size(exe) ? std::filesystem::path(exe).parent_path() : std::filesystem::current_path();
    const auto root = runtime::applicationRoot();
    const std::filesystem::path folders[] = {root / "runtime" / "models", root / "runtime_local" / "models", exeDirectory / "models"};
    std::error_code ec;
    for (const auto& folder : folders)
        for (const auto* name : kModelNames)
            if (std::filesystem::is_regular_file(folder / name, ec)) return folder / name;
    return {};
}

bool FaceTracker::initialize(ID3D12Device* device, const std::filesystem::path& model, uint32_t ringSlots, std::string& error)
{
    shutdown();
    if (!device || !ringSlots || model.empty()) { error = model.empty() ? "face model file not found" : "invalid face tracker configuration"; return false; }
    YuNetConfig config;
    config.ringSlots = ringSlots;
    config.readback = false;              // the tracking pass consumes the candidates on the GPU
    config.scoreThreshold = keepThreshold_;
    if (!detector_.initialize(device, model, config, error)) { shutdown(); return false; }
    std::vector<uint8_t> cs;
    if (!pipeline::loadShaderBytes("BeautyTrack.dxil", cs) ||
        !track_.create(device, cs, ringSlots * kDescriptorsPerSlot, 1, 2, kTrackConstants)) {
        error = "face tracking pass creation failed"; shutdown(); return false;
    }
    faces_ = makeBuffer(device, kFaceEntries * kFaceBytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS,
                        D3D12_RESOURCE_STATE_COMMON);
    shapes_ = makeBuffer(device, FaceMeshGpu::kShapeBytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS,
                         D3D12_RESOURCE_STATE_COMMON);
    if (!faces_ || !shapes_) { error = "face buffer allocation failed"; shutdown(); return false; }
    D3D12_DESCRIPTOR_HEAP_DESC sd{};
    sd.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV; sd.NumDescriptors = 1; sd.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_NONE;
    if (FAILED(device->CreateDescriptorHeap(&sd, IID_PPV_ARGS(&staging_)))) { error = "face tracker descriptor heap failed"; shutdown(); return false; }
    // SRVs go through a non-visible heap (DescriptorStager's driver workaround); UAVs are written directly.
    D3D12_SHADER_RESOURCE_VIEW_DESC none{};
    none.Format = DXGI_FORMAT_R32_TYPELESS;
    none.ViewDimension = D3D12_SRV_DIMENSION_BUFFER;
    none.Shader4ComponentMapping = D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING;
    none.Buffer.NumElements = 1;
    none.Buffer.Flags = D3D12_BUFFER_SRV_FLAG_RAW;
    device->CreateShaderResourceView(nullptr, &none, staging_->GetCPUDescriptorHandleForHeapStart());
    const auto at = [&](UINT index) { auto h = track_.heap->GetCPUDescriptorHandleForHeapStart(); h.ptr += SIZE_T(index) * track_.increment; return h; };
    for (uint32_t s = 0; s < ringSlots; ++s) {
        device->CopyDescriptorsSimple(1, at(s * kDescriptorsPerSlot), staging_->GetCPUDescriptorHandleForHeapStart(),
                                      D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV);
        D3D12_UNORDERED_ACCESS_VIEW_DESC faces{};
        faces.Format = DXGI_FORMAT_UNKNOWN;
        faces.ViewDimension = D3D12_UAV_DIMENSION_BUFFER;
        faces.Buffer.NumElements = kFaceEntries;
        faces.Buffer.StructureByteStride = UINT(kFaceBytes);
        device->CreateUnorderedAccessView(faces_.Get(), nullptr, &faces, at(s * kDescriptorsPerSlot + 1));
        D3D12_UNORDERED_ACCESS_VIEW_DESC raw{};
        raw.Format = DXGI_FORMAT_R32_TYPELESS;
        raw.ViewDimension = D3D12_UAV_DIMENSION_BUFFER;
        raw.Buffer.NumElements = UINT(detector_.candidateBufferBytes() / 4);
        raw.Buffer.Flags = D3D12_BUFFER_UAV_FLAG_RAW;
        device->CreateUnorderedAccessView(detector_.candidateBuffer(s), nullptr, &raw, at(s * kDescriptorsPerSlot + 2));
        readbacks_.push_back(makeBuffer(device, kFaceBytes, D3D12_HEAP_TYPE_READBACK, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST));
        if (!readbacks_.back()) { error = "face status readback allocation failed"; shutdown(); return false; }
    }
    pending_.assign(ringSlots, false);
    slots_ = ringSlots;
    ready_ = true;
    // The face mesh is optional: without it every face keeps the oval from its box.
    std::string meshError;
    meshReady_ = mesh_.initialize(device, locateFaceMeshModel(), faces_.Get(), shapes_.Get(), ringSlots, meshError);
    shapesClear_ = makeBuffer(device, FaceMeshGpu::kShapeBytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COMMON);
    meshReady_ = meshReady_ && shapesClear_;
    if (!meshReady_) log::warn("beauty-mesh", "face mesh unavailable (" + meshError + "); faces keep the oval from their box");
    log::info("beauty-detect", std::format("face tracker ready: {}; model {}; slots {}", detector_.modelDescription(),
                                           model.filename().string(), ringSlots));
    return true;
}

void FaceTracker::shutdown()
{
    mesh_.shutdown();
    shapes_.Reset();
    shapesClear_.Reset();
    meshReady_ = meshCleared_ = false;
    detector_.shutdown();
    track_ = {};
    staging_.Reset();
    faces_.Reset();
    readbacks_.clear();
    pending_.clear();
    slots_ = 0;
    ready_ = haveFrame_ = detectedNow_ = false;
    lastFrameId_ = detections_ = 0;
    lastPts_ = 0.0;
    faceCount_ = candidates_ = 0;
}

void FaceTracker::configure(const engine::BeautyDetection& detection, float minConfidence)
{
    detection_ = detection;
    threshold_ = std::clamp(minConfidence, 0.3f, 0.95f);
    // Hysteresis: a tracked face may dip a little below the score that starts one.
    keepThreshold_ = std::max(0.3f, threshold_ - 0.15f);
    detector_.setScoreThreshold(keepThreshold_);
}

bool FaceTracker::record(ID3D12GraphicsCommandList* list, pipeline::StateTracker& tracker, const Frame& f)
{
    detectedNow_ = false;
    if (!ready_ || !list || f.slot >= slots_ || !f.source || !f.width || !f.height) return false;
    // A repeated frame (paused tuning, a redraw) keeps its faces: no detection, no time.
    const bool sameFrame = haveFrame_ && !f.reset && f.frameId == lastFrameId_;
    double dt = 0.0;
    if (haveFrame_ && !f.reset && !sameFrame) dt = std::clamp(f.ptsMs - lastPts_, 0.0, 250.0);
    // Every new frame is detected (about 0.3 ms at the balanced size on an RTX 5070):
    // the faces are this frame's, so the region never trails the picture.
    const bool detect = !sameFrame;
    const double scale = double(detection_.inputLongSide()) / double(std::max(f.width, f.height));
    const uint32_t width = std::clamp(uint32_t(std::lround(f.width * scale)), 32u, 640u);
    const uint32_t height = std::clamp(uint32_t(std::lround(f.height * scale)), 32u, 640u);
    if (detect) {
        YuNetGpu::Source source;
        source.texture = f.source; source.width = f.width; source.height = f.height; source.hdr = f.hdr;
        if (!detector_.record(list, f.slot, source, width, height)) return false;
        ++detections_;
        detectedNow_ = true;
    }
    uint32_t c[kTrackConstants]{};
    c[0] = width; c[1] = height; c[2] = detector_.candidateCapacity();
    c[3] = (detect ? 1u : 0u) | (f.reset ? 2u : 0u) | (f.still ? 4u : 0u);
    c[4] = bitsOf(float(dt));
    // One Euro smoothing (shaders/BeautyTrack.hlsl): the stability sets the cutoff of a
    // face at rest (3 -> 0.3 Hz) and how fast it rises with speed (20 -> 6 Hz per face
    // width per second); tuned in docs/BEAUTY_IMPLEMENTATION_PLAN_2026-10-17.md.
    const float stability = std::clamp(detection_.stability, 0.0f, 1.0f);
    c[5] = bitsOf(3.0f + (0.3f - 3.0f) * stability);
    c[6] = bitsOf(0.3f);
    c[7] = bitsOf(120.0f);      // a missed face keeps moving this long
    c[8] = bitsOf(120.0f);      // then fades out over this long
    c[9] = detection_.maxFaces;
    c[10] = bitsOf(keepThreshold_);
    c[11] = bitsOf(20.0f + (6.0f - 20.0f) * stability);
    c[12] = bitsOf(4.0f);       // speed estimate cutoff, Hz
    c[13] = bitsOf(80.0f);      // a new face fades in over this long
    c[14] = bitsOf(threshold_);
    tracker.transition(list, faces_.Get(), D3D12_RESOURCE_STATE_UNORDERED_ACCESS);
    const auto gpu = [&](UINT index) { return track_.heap->GetGPUDescriptorHandleForHeapStart().ptr + UINT64(index) * track_.increment; };
    track_.bind(list, reinterpret_cast<const float*>(c), gpu(f.slot * kDescriptorsPerSlot), gpu(f.slot * kDescriptorsPerSlot + 1));
    list->Dispatch(1, 1, 1);
    tracker.uavBarrier(list, faces_.Get());
    tracker.transition(list, faces_.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE);
    list->CopyBufferRegion(readbacks_[f.slot].Get(), 0, faces_.Get(), 0, kFaceBytes);
    tracker.transition(list, faces_.Get(), D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
    pending_[f.slot] = true;
    // The outlines of this frame's tracks, smoothed like the tracks themselves. While the
    // status of the last frames reports no face, the mesh's ~45 dispatches would find
    // only empty slots: its outlines are cleared once instead, and a face that appears
    // meanwhile keeps the oval from its box until the mesh runs again (a few frames,
    // during its fade-in). Resets and pictures always run it.
    const bool meshWanted = f.reset || f.still || faceCount_ > 0 || detections_ <= slots_;
    if (meshReady_ && detect && !meshWanted && !meshCleared_) {
        tracker.transition(list, shapes_.Get(), D3D12_RESOURCE_STATE_COPY_DEST);
        tracker.transition(list, shapesClear_.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE);
        list->CopyBufferRegion(shapes_.Get(), 0, shapesClear_.Get(), 0, FaceMeshGpu::kShapeBytes);
        tracker.transition(list, shapes_.Get(), D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
        meshCleared_ = true;
    }
    if (meshReady_ && detect && meshWanted) {
        meshCleared_ = false;
        FaceMeshGpu::Frame m;
        m.source = f.source; m.width = f.width; m.height = f.height; m.hdr = f.hdr; m.slot = f.slot;
        m.maxFaces = detection_.maxFaces; m.dtMs = float(dt); m.reset = f.reset; m.still = f.still;
        m.restCutoff = 3.0f + (0.3f - 3.0f) * stability;
        m.speedFactor = 20.0f + (6.0f - 20.0f) * stability;
        m.speedCutoff = 4.0f;
        if (!mesh_.record(list, tracker, m)) log::warn("beauty-mesh", "face mesh could not be recorded for this frame");
    }
    if (!sameFrame) { lastPts_ = f.ptsMs; lastFrameId_ = f.frameId; }
    haveFrame_ = true;
    return true;
}

void FaceTracker::collect(uint32_t slot)
{
    if (!ready_ || slot >= slots_ || !pending_[slot]) return;
    void* mapped = nullptr;
    D3D12_RANGE range{0, SIZE_T(kFaceBytes)};
    if (SUCCEEDED(readbacks_[slot]->Map(0, &range, &mapped)) && mapped) {
        uint32_t meta[4];
        std::memcpy(meta, static_cast<const uint8_t*>(mapped) + 48, sizeof(meta));
        faceCount_ = std::min(meta[0], 8u);
        candidates_ = meta[3];
        D3D12_RANGE none{0, 0};
        readbacks_[slot]->Unmap(0, &none);
    }
    pending_[slot] = false;
}
} // namespace veyra::beauty