DLSS 5 Neural Rendering for Images & Video with Frame Generation, RTX Video Super Resolution, HDR, and Color Grading
TypeScript ai-upscalingdlssdlss-5dlss5dlssgframe-generationframe-interpolationimage-enhancementimage-upscalingneural-renderingnvidianvidia-dlss-5nvidia-rtxrtx-videortx-video-super-resolutionvideo-enhancementvideo-upscaling
#include "image_codecs.hpp"
#include "resolution.hpp"
#include "infrastructure/paths.hpp"
#include <QColorSpace>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QImageReader>
#include <QLibrary>
#include <QRgba64>
#include <QSaveFile>
#include <QSet>
#include <cstdio>
#include <cstdlib>
#include <jpeglib.h>
#include <libraw/libraw.h>
#include <windows.h>
#define LIBHEIF_STATIC_BUILD
#include <algorithm>
#include <cstring>
#include <libheif/heif.h>
#include <memory>
namespace ve {
namespace {
void rawCheck(int result) {
if (result != LIBRAW_SUCCESS)
throw std::runtime_error(libraw_strerror(result));
}
void openRaw(LibRaw& decoder, const QString& path) {
rawCheck(decoder.open_file(path.toStdWString().c_str()));
}
int rawProgress(void* context, enum LibRaw_progress, int, int) {
return static_cast<JobControl*>(context)->cancelled() ? 1 : 0;
}
class HeifLibrary {
public:
HeifLibrary() {
const auto directory = Paths::instance().at("bin/runtime/image-codecs");
const auto names = QDir(directory).entryList({"libheif*.dll"}, QDir::Files);
if (names.isEmpty())
throw std::runtime_error("The bundled HEIF image decoder is missing.");
const auto path = QDir::toNativeSeparators(QDir(directory).filePath(names.front())).toStdWString();
library_ = LoadLibraryExW(path.c_str(), nullptr,
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
if (!library_)
throw std::runtime_error("Cannot load the bundled HEIF codec.");
}
template <class Function> Function resolve(const char* name) {
const auto symbol = GetProcAddress(library_, name);
if (!symbol)
throw std::runtime_error("HEIF decoder is missing export " + std::string(name));
return reinterpret_cast<Function>(symbol);
}
static HeifLibrary& instance() {
static HeifLibrary library;
return library;
}
private:
HMODULE library_ = nullptr;
};
void heifCheck(const heif_error& error) {
if (error.code != heif_error_Ok)
throw std::runtime_error(error.message ? error.message : "HEIF decoding failed.");
}
} // namespace
QImage decodeControlMask(const QString& path, JobControl& control) {
configureImageDecoding();
control.checkpoint();
QImage image;
if (isRawImage(path))
image = decodeRawImage(path, control);
else if (isHeifImage(path))
image = decodeHeifImage(path, control);
else {
QImageReader reader(path);
if (reader.size().isValid()) requireResolution(reader.size(), "Control mask");
reader.setAutoTransform(true);
image = reader.read();
if (image.isNull())
throw std::runtime_error(reader.errorString().toStdString());
}
const bool grayscale = image.format() == QImage::Format_Grayscale8 ||
image.format() == QImage::Format_Grayscale16 ||
image.format() == QImage::Format_Mono || image.format() == QImage::Format_MonoLSB;
const auto rgba = image.convertToFormat(QImage::Format_RGBA64);
requireResolution(image.size(), "Control mask");
requireImageAllocation(rgba, "Control mask conversion");
QImage result(image.size(), QImage::Format_RGBA32FPx4);
requireImageAllocation(result, "Control mask allocation");
// Read the unassociated 16-bit channels directly to avoid rounding through
// Qt's premultiplied float conversion. Grayscale broadcasts to all four.
for (int y = 0; y < result.height(); ++y) {
control.checkpoint();
auto* row = reinterpret_cast<float*>(result.scanLine(y));
const auto* source = reinterpret_cast<const QRgba64*>(rgba.constScanLine(y));
for (int x = 0; x < result.width(); ++x) {
row[x * 4] = source[x].red() / 65535.f;
row[x * 4 + 1] = grayscale ? row[x * 4] : source[x].green() / 65535.f;
row[x * 4 + 2] = grayscale ? row[x * 4] : source[x].blue() / 65535.f;
row[x * 4 + 3] = grayscale ? row[x * 4] : source[x].alpha() / 65535.f;
}
}
return result;
}
QImage jpegOverWhite(const QImage& image) {
const auto source=image.convertToFormat(QImage::Format_RGBA8888);
requireImageAllocation(source, "JPEG input conversion");
QImage result(image.size(), QImage::Format_RGB888);
requireImageAllocation(result, "JPEG image allocation");
for (int y=0;y<source.height();++y) {
const auto* from=source.constScanLine(y);
auto* to=result.scanLine(y);
for (int x=0;x<source.width();++x) {
const unsigned alpha=from[x*4+3];
for (int c=0;c<3;++c)
to[x*3+c]=uchar((unsigned(from[x*4+c])*alpha+255*(255-alpha)+127)/255);
}
}
result.setColorSpace(image.colorSpace());
result.setDotsPerMeterX(image.dotsPerMeterX());
result.setDotsPerMeterY(image.dotsPerMeterY());
return result;
}
void encodeJpeg(const QString& path, const QImage& image, int quality) {
struct Compression {
jpeg_compress_struct compressor{};
jpeg_error_mgr error{};
unsigned char* buffer = nullptr;
unsigned long length = 0;
bool created = false;
~Compression() {
if (created)
jpeg_destroy_compress(&compressor);
std::free(buffer);
}
} state;
state.compressor.err = jpeg_std_error(&state.error);
state.error.error_exit = [](j_common_ptr info) {
char message[JMSG_LENGTH_MAX]{};
info->err->format_message(info, message);
throw std::runtime_error(message);
};
jpeg_create_compress(&state.compressor);
state.created = true;
jpeg_mem_dest(&state.compressor, &state.buffer, &state.length);
state.compressor.image_width = JDIMENSION(image.width());
state.compressor.image_height = JDIMENSION(image.height());
state.compressor.input_components = 3;
state.compressor.in_color_space = JCS_RGB;
jpeg_set_defaults(&state.compressor);
for (int i = 0; i < 3; ++i) {
state.compressor.comp_info[i].h_samp_factor = 1;
state.compressor.comp_info[i].v_samp_factor = 1;
}
jpeg_set_quality(&state.compressor, quality, TRUE);
jpeg_start_compress(&state.compressor, TRUE);
const auto rgb = image.convertToFormat(QImage::Format_RGB888);
while (state.compressor.next_scanline < state.compressor.image_height) {
auto* row = const_cast<JSAMPLE*>(rgb.constScanLine(int(state.compressor.next_scanline)));
jpeg_write_scanlines(&state.compressor, &row, 1);
}
jpeg_finish_compress(&state.compressor);
QSaveFile output(path);
if (!output.open(QIODevice::WriteOnly) ||
output.write(reinterpret_cast<const char*>(state.buffer), qsizetype(state.length)) !=
qsizetype(state.length) ||
!output.commit())
throw std::runtime_error(output.errorString().toStdString());
}
bool isRawImage(const QString& path) {
static const QSet<QString> extensions{
"3fr", "arw", "bay", "cap", "cr2", "cr3", "dcr", "dcs", "dng", "drf", "eip", "erf", "fff",
"gpr", "iiq", "k25", "kdc", "mdc", "mef", "mos", "mrw", "nef", "nrw", "obm", "orf", "pef",
"ptx", "pxn", "r3d", "raf", "raw", "rw2", "rwl", "rwz", "sr2", "srf", "srw", "x3f"};
return extensions.contains(QFileInfo(path).suffix().toLower());
}
bool isHeifImage(const QString& path) {
return QStringList{"heic", "heif"}.contains(QFileInfo(path).suffix().toLower());
}
QSize rawImageSize(const QString& path) {
LibRaw decoder;
openRaw(decoder, path);
rawCheck(decoder.adjust_sizes_info_only());
return {decoder.imgdata.sizes.iwidth, decoder.imgdata.sizes.iheight};
}
QImage decodeRawImage(const QString& path, JobControl& control) {
control.checkpoint();
LibRaw decoder;
decoder.imgdata.rawparams.max_raw_memory_mb = imageDecoderBudgetMiB;
decoder.set_progress_handler(rawProgress, &control);
decoder.imgdata.params.use_camera_wb = 1;
decoder.imgdata.params.output_color = 1;
decoder.imgdata.params.output_bps = 16;
openRaw(decoder, path);
requireResolution({decoder.imgdata.sizes.width, decoder.imgdata.sizes.height}, "RAW image");
const auto unpacked = decoder.unpack();
control.checkpoint();
rawCheck(unpacked);
const auto processed = decoder.dcraw_process();
control.checkpoint();
rawCheck(processed);
int width=0,height=0,colors=0,bits=0;
decoder.get_mem_image_format(&width,&height,&colors,&bits);
if (bits != 16 || (colors != 3 && colors != 4))
throw std::runtime_error("LibRaw did not return a supported RGB image.");
requireResolution({width,height}, "Decoded RAW image");
QImage result(width,height,QImage::Format_RGBA64);
requireImageAllocation(result, "RAW image allocation");
// Export directly into owned rows, avoiding a second full RGB image and
// the legacy processed-image record's 32-bit data_size field.
rawCheck(decoder.copy_mem_image(result.bits(),int(result.bytesPerLine()),0));
for (int y = 0; y < result.height(); ++y) {
control.checkpoint();
auto* row = reinterpret_cast<QRgba64*>(result.scanLine(y));
for (int x = result.width()-1; x >= 0; --x) {
const auto* source = reinterpret_cast<const quint16*>(result.constScanLine(y)) + x*colors;
row[x] = QRgba64::fromRgba64(source[0], source[1], source[2], 65535);
}
}
result.setColorSpace(QColorSpace::SRgb);
return result;
}
QImage decodeHeifImage(const QString& path, JobControl& control) {
control.checkpoint();
auto& library = HeifLibrary::instance();
#define HEIF_FUNCTION(name) const auto name = library.resolve<decltype(&::name)>(#name)
HEIF_FUNCTION(heif_context_alloc);
HEIF_FUNCTION(heif_context_get_security_limits);
HEIF_FUNCTION(heif_context_free);
HEIF_FUNCTION(heif_context_read_from_memory_without_copy);
HEIF_FUNCTION(heif_context_get_primary_image_handle);
HEIF_FUNCTION(heif_image_handle_release);
HEIF_FUNCTION(heif_image_handle_get_width);
HEIF_FUNCTION(heif_image_handle_get_height);
HEIF_FUNCTION(heif_decode_image);
HEIF_FUNCTION(heif_image_release);
HEIF_FUNCTION(heif_image_get_width);
HEIF_FUNCTION(heif_image_get_height);
HEIF_FUNCTION(heif_image_get_plane_readonly);
HEIF_FUNCTION(heif_image_get_bits_per_pixel_range);
HEIF_FUNCTION(heif_image_handle_get_raw_color_profile_size);
HEIF_FUNCTION(heif_image_handle_get_raw_color_profile);
#undef HEIF_FUNCTION
QFile file(path);
if (!file.open(QIODevice::ReadOnly))
throw std::runtime_error(file.errorString().toStdString());
const auto bytes = file.readAll();
std::unique_ptr<heif_context, decltype(heif_context_free)> context(heif_context_alloc(),
heif_context_free);
if (!context)
throw std::bad_alloc{};
auto* limits = heif_context_get_security_limits(context.get());
limits->max_image_size_pixels = maximumImagePixels;
limits->max_memory_block_size = quint64(imageDecoderBudgetMiB) * 1024 * 1024;
heifCheck(heif_context_read_from_memory_without_copy(context.get(), bytes.constData(),
size_t(bytes.size()), nullptr));
heif_image_handle* rawHandle = nullptr;
heifCheck(heif_context_get_primary_image_handle(context.get(), &rawHandle));
std::unique_ptr<heif_image_handle, decltype(heif_image_handle_release)> handle(rawHandle,
heif_image_handle_release);
requireResolution({heif_image_handle_get_width(handle.get()),heif_image_handle_get_height(handle.get())}, "HEIF image");
heif_image* rawImage = nullptr;
heifCheck(heif_decode_image(handle.get(), &rawImage, heif_colorspace_RGB,
heif_chroma_interleaved_RRGGBBAA_LE, nullptr));
std::unique_ptr<heif_image, decltype(heif_image_release)> image(rawImage, heif_image_release);
const int width = heif_image_get_width(image.get(), heif_channel_interleaved),
height = heif_image_get_height(image.get(), heif_channel_interleaved);
requireResolution({width,height}, "Decoded HEIF image");
int stride = 0;
const auto* data = heif_image_get_plane_readonly(image.get(), heif_channel_interleaved, &stride);
const int depth = heif_image_get_bits_per_pixel_range(image.get(), heif_channel_interleaved);
if (!data || width < 1 || height < 1 || depth < 1 || depth > 16 || stride < width * 8)
throw std::runtime_error("HEIF decoder returned invalid image dimensions or precision.");
QImage result(width, height, QImage::Format_RGBA64);
requireImageAllocation(result, "HEIF image allocation");
const auto maximum = double((1u << depth) - 1);
for (int y = 0; y < height; ++y) {
control.checkpoint();
auto* to = reinterpret_cast<QRgba64*>(result.scanLine(y));
const auto* from = reinterpret_cast<const quint16*>(data + qsizetype(y) * stride);
for (int x = 0; x < width; ++x) {
const auto convert = [&](int channel) {
return quint16(std::clamp(std::lround(from[x * 4 + channel] * 65535. / maximum), 0l, 65535l));
};
to[x] = QRgba64::fromRgba64(convert(0), convert(1), convert(2), convert(3));
}
}
const auto profileBytes = heif_image_handle_get_raw_color_profile_size(handle.get());
if (profileBytes && profileBytes < 16 * 1024 * 1024) {
QByteArray profile(qsizetype(profileBytes), Qt::Uninitialized);
heifCheck(heif_image_handle_get_raw_color_profile(handle.get(), profile.data()));
result.setColorSpace(QColorSpace::fromIccProfile(profile));
}
if (!result.colorSpace().isValid())
result.setColorSpace(QColorSpace::SRgb);
auto converted=result.convertedToColorSpace(QColorSpace::SRgb);
requireImageAllocation(converted, "HEIF color conversion");
return converted;
}
} // namespace ve