DLSS 5 Neural Rendering for Images & Video with Frame Generation, RTX Video Super Resolution, HDR, and Color Grading
TypeScript ai-upscalingdlssdlss-5dlss5dlssgframe-generationframe-interpolationimage-enhancementimage-upscalingneural-renderingnvidianvidia-dlss-5nvidia-rtxrtx-videortx-video-super-resolutionvideo-enhancementvideo-upscaling
#include "reshade_video.hpp"
#include "infrastructure/log.hpp"
#include "infrastructure/scratch.hpp"
#include "media/cuda_video.hpp"
#include "media/native_rgb.hpp"
#include "media/video_delivery.hpp"
#include "native_engines.hpp"
#include "reshade_native.hpp"
#include <QElapsedTimer>
#include <algorithm>
#include <cmath>
#include <memory>
#include <optional>
namespace ve {
bool renderReShadeVideo(const RenderRequest& request, const MediaInfo& source,
const Settings& settings, const PipelineEstimate& estimate,
const QString& destination, JobControl& control,
const ProgressCallback& progress) {
const bool sole = estimate.stages == QStringList{"reshade"};
const bool rrBefore = estimate.stages == QStringList{"dlss_ray_reconstruction", "reshade"};
const bool rrAfter = estimate.stages == QStringList{"reshade", "dlss_ray_reconstruction"};
if (request.preview || (!sole && !rrBefore && !rrAfter) ||
((source.width | source.height | estimate.width | estimate.height) & 1)) {
return false;
}
const bool nvenc = settings.codec.contains("NVIDIA NVENC") ||
(settings.ffmpeg_device != "cpu" && QStringList{"H.264", "H.265", "AV1"}.contains(settings.codec));
if (!nvenc || source.range == "pc" ||
(source.hdr ? source.primaries != "bt2020" || source.colorSpace != "bt2020nc"
: !QStringList{"bt709", "unknown", "unspecified", ""}.contains(source.primaries) ||
!QStringList{"bt709", "unknown", "unspecified", ""}.contains(source.colorSpace))) {
return false;
}
if (!source.hdr && !QStringList{"bt709", "gamma22", "bt470m", "iec61966-2-1", "srgb", "unknown", "unspecified", ""}.contains(source.transfer)) {
return false;
}
const int ordinal = gpuOrdinal(settings.ai_gpu_uuid);
const auto devices = detectGpus();
const auto selected = std::ranges::find_if(devices, [ordinal](const Gpu& gpu) { return gpu.index == ordinal; });
if (selected == devices.end() ||
(settings.video_gpu_uuid != "auto" && settings.video_gpu_uuid != selected->uuid) ||
(settings.ffmpeg_device != "auto" && settings.ffmpeg_device != "cpu" &&
settings.ffmpeg_device.mid(7).compare(selected->uuid.mid(4), Qt::CaseInsensitive) != 0)) {
return false;
}
ScratchDirectory temporary(QStringLiteral("reshade-cuda-render-XXXXXX"));
if (!temporary.isValid()) {
throw std::runtime_error("Cannot create the ReShade GPU render directory.");
}
const auto video = temporary.filePath("video.mp4");
const auto codec = settings.codec.startsWith("H.264") ? QString("h264_nvenc")
: settings.codec.startsWith("H.265") ? QString("hevc_nvenc") : QString("av1_nvenc");
const double duration = request.seconds > 0
? std::min(request.seconds, source.duration - request.startSeconds)
: source.duration - request.startSeconds;
qint64 count = 0;
double actualDuration = 0.0;
double origin = 0.0;
QElapsedTimer timer;
timer.start();
try {
CudaVideoDecoder decoder(request.input, ordinal, request.startSeconds, control);
auto frame = decoder.read();
if (!frame) {
throw std::runtime_error("Source contains no decodable video frames.");
}
origin = decoder.timeOrigin();
auto colors = source;
colors.width = estimate.width;
colors.height = estimate.height;
colors.hdr = estimate.hdr;
if (rrBefore || rrAfter || source.precisionBits() > 8 || estimate.hdr) {
colors.pixelFormat = "p010le";
}
ReShadeEngine::initialize(ordinal);
const auto caps = ReShadeEngine::capabilities(ordinal);
if (!caps.cuda_interop_available) {
throw CudaUnavailable("ReShade D3D12-CUDA interop is unavailable on the selected adapter.");
}
if (rrBefore || rrAfter) {
NativeEngines::instance().prepare(ordinal, {"dlss_ray_reconstruction"});
}
const uint32_t transfer = reshadeColorTransfer(source);
std::unique_ptr<NativeEngines::RayReconstruction> rrSession;
if (rrBefore || rrAfter) {
rrSession = std::make_unique<NativeEngines::RayReconstruction>(
QSize(source.width, source.height), settings.rr_optical_flow_quality, false, true, transfer);
}
ReShadeSession rsSession(QSize(source.width, source.height), settings, source, false, true, true);
CudaVideoEncoder encoder(video, codec, settings, colors, decoder.timeBase(), decoder.device(), ordinal, control);
const bool p010 = encoder.pixelFormat() == VideoPixelFormat::P010;
CudaFramePool pool(decoder.device(), QSize(estimate.width, estimate.height), p010, source.hdr);
if (sole) {
info("reshade", "NVDEC -> ReShade (D3D12 + AMD FidelityFX Optical Flow 1.1.2) -> " + codec + " (owned device surfaces)");
} else if (rrBefore) {
info("reshade", "NVDEC -> DLSS Ray Reconstruction -> ReShade -> " + codec + " (owned device surfaces)");
} else {
info("reshade", "NVDEC -> ReShade -> DLSS Ray Reconstruction -> " + codec + " (owned device surfaces)");
}
qint64 previous = -1;
while (frame) {
control.checkpoint();
const double seconds = frame->pts * double(frame->timeBase.numerator) / frame->timeBase.denominator;
if (request.seconds > 0 && seconds >= duration - 1e-9) {
break;
}
if (frame->pts <= previous) {
throw std::runtime_error("Video has non-increasing presentation timestamps.");
}
const float delta = previous >= 0
? float((frame->pts - previous) * 1000.0 * frame->timeBase.numerator / frame->timeBase.denominator)
: (source.fps > 0 ? float(1000.0 / source.fps) : 33.333f);
const bool gap = count > 0 && delta > std::max(250.0f, source.fps > 0 ? float(4000.0 / source.fps) : 250.0f);
previous = frame->pts;
frame->rotation = uint32_t(source.clockwiseRotation());
if (frame->colorMatrix != (source.hdr ? 2u : 1u) || frame->colorRange != 0) {
throw CudaUnavailable("ReShade device delivery requires matching 709/2020 limited-range planes.");
}
const qint64 ptsNs = qint64(std::llround(seconds * 1000000000.0));
const qint64 durNs = std::max<qint64>(1, qint64(std::llround(double(delta) * 1000000.0)));
const bool reset = (count == 0 || gap);
CudaVideoFrame output;
if (sole) {
output = rsSession.process(*frame, pool, p010, reset, count);
} else if (rrBefore) {
auto rgb = rrSession->processRgb(*frame, reset, quint32(count), std::clamp(delta, 0.01f, 1000.0f));
output = rsSession.process(rgb, pool, p010, reset, count, ptsNs, durNs);
output.pts = frame->pts;
output.duration = frame->duration;
output.timeBase = frame->timeBase;
} else {
auto rgb = rsSession.processRgb(*frame, reset, count);
output = rrSession->process(rgb, pool, p010, reset, quint32(count), std::clamp(delta, 0.01f, 1000.0f));
output.pts = frame->pts;
output.duration = frame->duration;
output.timeBase = frame->timeBase;
}
control.checkpoint();
encoder.write(output);
++count;
actualDuration = std::max(actualDuration, seconds + (frame->duration > 0
? frame->duration * double(frame->timeBase.numerator) / frame->timeBase.denominator
: (source.fps > 0 ? 1.0 / source.fps : 1.0 / 30.0)));
if (progress) {
progress({std::min(0.98, duration > 0 ? seconds / duration : 0.1),
QT_TRANSLATE_NOOP("App", "Processing ReShade video"),
count, source.frames, timer.elapsed() > 0 ? count * 1000.0 / timer.elapsed() : 0.0});
}
frame = decoder.read();
}
encoder.finish();
info("reshade", "Device delivery counters: " + rsSession.diagnostics());
if (rrSession) {
info("dlssrr", "Combined RR+ReShade device counters: " + rrSession->diagnostics());
}
} catch (const CudaUnavailable& error) {
if (count) {
throw;
}
control.checkpoint();
info("reshade", "CUDA delivery unavailable; using host composition: " + QString::fromUtf8(error.what()));
return false;
}
if (count <= 0) {
throw std::runtime_error("ReShade produced no video frames.");
}
control.checkpoint();
if (request.seconds > 0) {
actualDuration = std::min(actualDuration, request.seconds);
}
info("reshade", reshadeReport(settings, true));
muxDelivery(video, destination, request, source, settings, actualDuration, control,
request.preview ? std::nullopt : std::optional(origin));
return true;
}
} // namespace ve