Bring PS5 GTA6 to 120FPS with DLSS 5. An advanced Windows video player and capture enhancement tool featuring real-time preview for videos, images, and capture card sources. Combine AI-powered Super Resolution, DLSS5 image enhancement, and Frame Generation to deliver a next-generation visual experience.
C++
// MPEG chain stage 3: the compressed-payload decoder must drive real H.264
// elementary streams through both backends (D3D12VA and software) and produce
// frames the capture ingress can consume. The physical capture card on this
// machine has no H.264/HEVC format, so the payloads are the Annex-B form of a
// local H.264 clip; the decoder under test is the exact one the capture worker
// uses, driven packet by packet like a live stream.
#include "veyra/source/CaptureCompressedDecoder.h"
#include "veyra/gfx/D3D12DeviceContext.h"
#include "veyra/media/FFmpegVideoDecoder.h"
#include <iostream>
#include <algorithm>
#include <map>
#include <string>
#include <vector>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavcodec/bsf.h>
#include <libavformat/avformat.h>
#include <libavutil/frame.h>
#include <libavutil/adler32.h>
}
using namespace veyra;
namespace {
struct ElementaryStream {
int width = 0, height = 0;
source::CaptureCodec codec = source::CaptureCodec::None;
std::vector<std::vector<uint8_t>> accessUnits;
std::vector<int64_t> pts;
};
bool loadAnnexB(const char* path, ElementaryStream& stream, std::string& error)
{
AVFormatContext* format = nullptr;
if (avformat_open_input(&format, path, nullptr, nullptr) < 0) {
error = std::string("cannot open ") + path;
return false;
}
if (avformat_find_stream_info(format, nullptr) < 0) {
avformat_close_input(&format);
error = "find_stream_info failed";
return false;
}
const int videoIndex = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (videoIndex < 0) {
avformat_close_input(&format);
error = "no video stream";
return false;
}
AVCodecParameters* parameters = format->streams[videoIndex]->codecpar;
const char* filterName = nullptr;
if (parameters->codec_id == AV_CODEC_ID_H264) {
stream.codec = source::CaptureCodec::H264;
filterName = "h264_mp4toannexb";
} else if (parameters->codec_id == AV_CODEC_ID_HEVC) {
stream.codec = source::CaptureCodec::Hevc;
filterName = "hevc_mp4toannexb";
} else {
avformat_close_input(&format);
error = "test input is not H.264/HEVC";
return false;
}
stream.width = parameters->width;
stream.height = parameters->height;
const AVBitStreamFilter* filter = av_bsf_get_by_name(filterName);
AVBSFContext* bsf = nullptr;
if (filter == nullptr || av_bsf_alloc(filter, &bsf) < 0) {
avformat_close_input(&format);
error = "bitstream filter unavailable";
return false;
}
if (avcodec_parameters_copy(bsf->par_in, parameters) < 0) {
av_bsf_free(&bsf);
avformat_close_input(&format);
error = "parameter copy failed";
return false;
}
bsf->time_base_in = format->streams[videoIndex]->time_base;
if (av_bsf_init(bsf) < 0) {
av_bsf_free(&bsf);
avformat_close_input(&format);
error = "bitstream filter init failed";
return false;
}
AVPacket* packet = av_packet_alloc();
AVPacket* filtered = av_packet_alloc();
while (packet != nullptr && filtered != nullptr && av_read_frame(format, packet) >= 0) {
if (packet->stream_index == videoIndex && av_bsf_send_packet(bsf, packet) >= 0) {
while (av_bsf_receive_packet(bsf, filtered) >= 0) {
stream.accessUnits.emplace_back(filtered->data, filtered->data + filtered->size);
stream.pts.push_back(av_rescale_q(filtered->pts,bsf->time_base_out,AVRational{1,10000000}));
av_packet_unref(filtered);
}
}
av_packet_unref(packet);
}
av_packet_free(&packet);
av_packet_free(&filtered);
av_bsf_free(&bsf);
avformat_close_input(&format);
if (stream.accessUnits.empty()) {
error = "no access units extracted";
return false;
}
return true;
}
struct DecodeRun {
uint64_t frames = 0;
uint64_t failures = 0;
bool sawHardwareFrame = false;
std::string firstError;
std::map<int64_t,uint32_t> pixels;
bool recovered=false;
};
DecodeRun decodeAll(ElementaryStream& stream, ID3D12Device* device, ID3D12CommandQueue* queue,bool injectLoss=false)
{
DecodeRun run;
source::CaptureCompressedDecoder decoder;
if (!decoder.open(stream.codec, unsigned(stream.width), unsigned(stream.height), nullptr, 0, device, queue)) {
run.firstError = decoder.lastError().empty() ? "decoder open failed" : decoder.lastError();
++run.failures;
return run;
}
AVFrame* target = av_frame_alloc();
target->format = AV_PIX_FMT_NV12;
target->width = stream.width;
target->height = stream.height;
if (av_frame_get_buffer(target, 32) < 0) {
av_frame_free(&target);
run.firstError = "target allocation failed";
++run.failures;
return run;
}
size_t inputIndex=0;
auto displayPts=stream.pts;std::sort(displayPts.begin(),displayPts.end());
bool awaitingKey=false;
for (const auto& unit : stream.accessUnits) {
if(injectLoss&&inputIndex==7){decoder.recoverAtKeyframe();awaitingKey=true;}
if(injectLoss&&inputIndex>=7&&inputIndex<10){++inputIndex;continue;}
AVFrame* out = nullptr;
bool hardware = false;
const bool produced = decoder.decode(unit.data(), unit.size(), stream.pts[inputIndex++], target, &out, hardware);
if (produced) {
++run.frames;
if(!injectLoss&&(run.frames>displayPts.size()||out->pts!=displayPts[run.frames-1])){++run.failures;run.firstError="decoded PTS was lost or assigned from current packet";}
if(awaitingKey){
if(!(out->flags&AV_FRAME_FLAG_KEY)){++run.failures;run.firstError="recovery exposed a dependent frame before keyframe";}
awaitingKey=false;run.recovered=true;
}
if(!hardware){uint32_t checksum=1;for(int plane=0;plane<2;++plane)for(int y=0;y<(plane?out->height/2:out->height);++y)
checksum=av_adler32_update(checksum,out->data[plane]+ptrdiff_t(y)*out->linesize[plane],out->width);
run.pixels[out->pts]=checksum;}
if (hardware) {
run.sawHardwareFrame = true;
if (out == nullptr || out->format != AV_PIX_FMT_D3D12 || out->data[0] == nullptr) {
run.firstError = "hardware frame is not a D3D12 surface";
++run.failures;
}
} else if (out == nullptr || out != target) {
run.firstError = "software frame did not land in the target";
++run.failures;
}
} else if (!decoder.waitingForInput()) {
if (run.firstError.empty()) run.firstError = decoder.lastError();
++run.failures;
}
AVFrame* remaining=nullptr;bool remainingHardware=false;
if(decoder.decode(nullptr,0,0,target,&remaining,remainingHardware)||!decoder.waitingForInput()){
++run.failures;run.firstError="receive-only drain returned stale output or an error";
}
}
av_frame_free(&target);
return run;
}
bool packetRetryTest(const char* path,gfx::D3D12DeviceContext& device) {
AVFormatContext* format=nullptr;
if(avformat_open_input(&format,path,nullptr,nullptr)<0)return false;
if(avformat_find_stream_info(format,nullptr)<0){avformat_close_input(&format);return false;}
int index=av_find_best_stream(format,AVMEDIA_TYPE_VIDEO,-1,-1,nullptr,0);
if(index<0){avformat_close_input(&format);return false;}
auto* stream=format->streams[index];
media::FFmpegVideoDecoder reference,batched;
bool ok=reference.openSoftware(stream->codecpar,stream->time_base.num,stream->time_base.den)&&batched.openSoftware(stream->codecpar,stream->time_base.num,stream->time_base.den);
std::vector<int64_t> expected,actual;
auto drain=[](auto& decoder,auto& pts){while(auto* f=decoder.receiveFrame())pts.push_back(f->pts);};
AVPacket* packet=av_packet_alloc();size_t count=0;
while(ok&&av_read_frame(format,packet)>=0){
if(packet->stream_index==index){
ok=reference.sendPacket(packet)&&batched.sendPacket(packet);++count;
drain(reference,expected);
if(count%8==0)drain(batched,actual); // deliberately forces send EAGAIN
}
av_packet_unref(packet);
}
ok=ok&&reference.sendPacket(nullptr)&&batched.sendPacket(nullptr);
drain(reference,expected);drain(batched,actual);
ok=ok&&!expected.empty()&&expected==actual&&batched.stats().framesSubmitted==count&&batched.stats().packetRetries>0;
batched.flushBuffers();
ok=ok&&!batched.receiveFrame()&&batched.receiveStatus()==media::DecodeReceiveStatus::NeedInput;
std::cout<<(ok?"PASS ":"FAIL ")<<"EAGAIN resend, output order, accepted count and flush frames="<<actual.size()<<" retries="<<batched.stats().packetRetries<<'\n';
media::FFmpegVideoDecoder interop;
for(int pass=0;ok&&pass<3;++pass){
ok=av_seek_frame(format,index,0,AVSEEK_FLAG_BACKWARD)>=0&&interop.openD3D11VA(stream->codecpar,stream->time_base.num,stream->time_base.den,device.adapter().luid,device.device());
std::vector<int64_t> imported;
auto receive=[&]{while(auto* frame=interop.receiveFrame()){ok=ok&&interop.hardwareFrameImportable();imported.push_back(frame->pts);}};
while(ok&&av_read_frame(format,packet)>=0){
if(packet->stream_index==index){ok=interop.sendPacket(packet);receive();}
av_packet_unref(packet);
}
ok=ok&&interop.sendPacket(nullptr);receive();
ok=ok&&imported==expected;interop.close();interop.close();
}
std::cout<<(ok?"PASS ":"FAIL ")<<"D3D11VA repeated open/decode/close, import and PTS\n";
av_packet_free(&packet);avformat_close_input(&format);
return ok;
}
// Exercise the real codec's hard-error/reset boundary used by Xbox. This is
// a local H.264 replay, not a real Xbox/RX9000 driver reproduction.
bool hardErrorRecovery(ElementaryStream& stream, ID3D12Device* device, ID3D12CommandQueue* queue) {
if(stream.codec!=source::CaptureCodec::H264)return true;
source::CaptureCompressedDecoder decoder;
decoder.setStreamProfile(true);
if(!decoder.open(stream.codec,unsigned(stream.width),unsigned(stream.height),nullptr,0,device,queue))return false;
AVFrame* target=av_frame_alloc();
if(!target)return false;
target->format=AV_PIX_FMT_NV12;target->width=stream.width;target->height=stream.height;
if(av_frame_get_buffer(target,32)<0){av_frame_free(&target);return false;}
AVFrame* output=nullptr;bool hardware=false;size_t at=0;unsigned warm=0;
for(;at<stream.accessUnits.size()&&warm<8;++at){
const auto& unit=stream.accessUnits[at];
if(decoder.decode(unit.data(),unit.size(),stream.pts[at],target,&output,hardware))++warm;
else if(!decoder.waitingForInput()){av_frame_free(&target);return false;}
}
if(warm!=8||at==0){av_frame_free(&target);return false;}
const auto isIdr=[](const std::vector<uint8_t>& unit){
for(size_t i=0;i+3<unit.size();++i)
if(unit[i]==0&&unit[i+1]==0&&unit[i+2]==1&&(unit[i+3]&31)==5)return true;
return false;
};
int64_t recoveryPts=AV_NOPTS_VALUE;
for(size_t next=at;next<stream.accessUnits.size();++next)
if(isIdr(stream.accessUnits[next])){recoveryPts=stream.pts[next];break;}
if(recoveryPts==AV_NOPTS_VALUE){
std::cout<<"FAIL recovery fixture has no future IDR\n";
av_frame_free(&target);return false;
}
const uint8_t invalid[]={0,0,0,1,0x80}; // forbidden_zero_bit, no valid slice
const bool badProduced=decoder.decode(invalid,sizeof(invalid),stream.pts[at-1]+1,target,&output,hardware);
bool ok=warm==8&&!badProduced&&!decoder.waitingForInput();
decoder.recoverAtKeyframe();
ok=ok&&!decoder.decode(nullptr,0,0,target,&output,hardware)&&decoder.waitingForInput();
bool restored=false;
// Feed dependent packets too: reset must withhold them rather than expose
// stale/broken references. The next real IDR restores original timestamps.
for(;ok&&at<stream.accessUnits.size()&&!restored;++at){
const auto& unit=stream.accessUnits[at];
const bool produced=decoder.decode(unit.data(),unit.size(),stream.pts[at],target,&output,hardware);
if(produced){
// A B-frame stream can emit the IDR after later packets. Its PTS
// must belong to the first new IDR, not to the packet sent today.
restored=output&&(output->flags&AV_FRAME_FLAG_KEY)&&output->pts==recoveryPts;
ok=restored;
}else ok=decoder.waitingForInput();
}
std::cout<<((ok&&restored)?"PASS ":"FAIL ")<<"hard H.264 error -> flush -> keyframe recovery backend="
<<decoder.backendName()<<" warm="<<warm<<" restored="<<restored<<" expectedPts="<<recoveryPts<<'\n';
av_frame_free(&target);return ok&&restored;
}
// Hold an old hardware frame across context replacement, as the player's
// mailbox/graph can do, and verify actual reopen + software fallback output.
bool decoderReplacement(ElementaryStream& stream, gfx::D3D12DeviceContext& device) {
source::CaptureCompressedDecoder decoder;
decoder.setStreamProfile(true);
AVFrame* target=av_frame_alloc();
if(!target)return false;
target->format=AV_PIX_FMT_NV12;target->width=stream.width;target->height=stream.height;
if(av_frame_get_buffer(target,32)<0){av_frame_free(&target);return false;}
AVFrame* output=nullptr;AVFrame* held=nullptr;bool hardware=false;size_t at=0;
bool ok=decoder.open(stream.codec,unsigned(stream.width),unsigned(stream.height),nullptr,0,device.device(),device.directQueue());
const auto isIdr=[](const std::vector<uint8_t>& unit){
for(size_t i=0;i+3<unit.size();++i)
if(unit[i]==0&&unit[i+1]==0&&unit[i+2]==1&&(unit[i+3]&31)==5)return true;
return false;
};
const auto prime=[&](unsigned count,bool seekIdr,bool expectHardware){
if(seekIdr)while(at<stream.accessUnits.size()&&!isIdr(stream.accessUnits[at]))++at;
if(at>=stream.accessUnits.size())return false;
const int64_t keyPts=stream.pts[at];unsigned pictures=0;
for(;at<stream.accessUnits.size()&&pictures<count;++at){
const auto& unit=stream.accessUnits[at];
if(decoder.decode(unit.data(),unit.size(),stream.pts[at],target,&output,hardware)){
if(hardware!=expectHardware)return false;
if(seekIdr&&pictures==0&&(!(output->flags&AV_FRAME_FLAG_KEY)||output->pts!=keyPts))return false;
++pictures;
}else if(!decoder.waitingForInput())return false;
}
return pictures==count;
};
ok=ok&&prime(8,false,true);
if(ok)held=av_frame_clone(output);
ok=ok&&held&&held->format==AV_PIX_FMT_D3D12;
unsigned reopened=0,software=0;
// The CPU Xbox suite verifies the bounded error decisions. Here verify
// their resource/codec effects; a malformed AU can legitimately become
// NeedInput under NONKEY discard, so it cannot fake consecutive errors.
for(unsigned phase=0;ok&&phase<2;++phase){
decoder.close();
ok=decoder.open(stream.codec,unsigned(stream.width),unsigned(stream.height),nullptr,0,
phase==0?device.device():nullptr,phase==0?device.directQueue():nullptr);
if(phase==0)++reopened;else ++software;
ok=ok&&prime(1,true,phase==0);
}
ok=ok&&reopened==1&&software==1&&held&&held->data[0]&&held->buf[0];
std::cout<<(ok?"PASS ":"FAIL ")<<"decoder replacement with a leased old D3D12 frame; hardwareReopens="
<<reopened<<" softwareFallbacks="<<software<<'\n';
decoder.close();av_frame_free(&held);av_frame_free(&target);return ok;
}
} // namespace
int main(int argc, char** argv)
{
const char* path = argc > 1 ? argv[1] : "loop/local/fixed_clips/test_h264_1080p.mp4";
ElementaryStream stream;
std::string error;
if (!loadAnnexB(path, stream, error)) {
std::cout << "SKIP " << error << " (path=" << path << ")\n";
return 0; // missing corpus is not a product failure; the gate reports it
}
std::cout << "loaded accessUnits=" << stream.accessUnits.size() << " " << stream.width << "x" << stream.height << '\n';
gfx::D3D12DeviceContext context;
gfx::DeviceContextDesc desc;
Status status;
if (!context.initialize(desc, status)) {
std::cout << "SKIP no D3D12 device available\n";
return 0;
}
int failures = 0;
if(!packetRetryTest(path,context))++failures;
const auto hardware = decodeAll(stream, context.device(), context.directQueue());
const auto software = decodeAll(stream, nullptr, nullptr);
const auto recovery = decodeAll(stream, nullptr, nullptr,true);
std::cout << "hardware frames=" << hardware.frames << " failures=" << hardware.failures
<< " sawD3D12=" << hardware.sawHardwareFrame << " error=" << hardware.firstError << '\n';
std::cout << "software frames=" << software.frames << " failures=" << software.failures
<< " error=" << software.firstError << '\n';
const auto check = [&failures](bool pass, const char* name) {
std::cout << (pass ? "PASS " : "FAIL ") << name << '\n';
if (!pass) ++failures;
};
check(hardware.failures == 0 && hardware.frames > 0, "hardware decode produces frames without errors");
check(hardware.sawHardwareFrame, "hardware backend returns importable D3D12 frames");
check(software.failures == 0 && software.frames > 0, "software decode produces frames without errors");
bool samePixels=true;for(const auto& [pts,pixels]:recovery.pixels){auto found=software.pixels.find(pts);samePixels=samePixels&&found!=software.pixels.end()&&found->second==pixels;}
check(recovery.recovered&&recovery.failures==0&&samePixels,"packet loss resumes at keyframe with reference-identical pixels and PTS");
check(hardErrorRecovery(stream,nullptr,nullptr),"Xbox hard-error boundary recovers on the software codec");
check(hardErrorRecovery(stream,context.device(),context.directQueue()),"Xbox hard-error boundary recovers on D3D12VA");
check(decoderReplacement(stream,context),"Xbox decoder replacement preserves output PTS and old-frame lifetime");
// B-frame reordering may emit a slightly different count on the software
// path; the hardware and software backends must stay within one frame.
const uint64_t difference = hardware.frames > software.frames ? hardware.frames - software.frames : software.frames - hardware.frames;
check(difference <= 1, "hardware and software frame counts agree");
std::cout << (failures == 0 ? "ALL PASS" : "FAILURES") << " " << failures << '\n';
return failures == 0 ? 0 : 1;
}