DLSS 5 Neural Rendering for Images & Video with Frame Generation, RTX Video Super Resolution, HDR, and Color Grading
TypeScript ai-upscalingdlssdlss-5dlss5dlssgframe-generationframe-interpolationimage-enhancementimage-upscalingneural-renderingnvidianvidia-dlss-5nvidia-rtxrtx-videortx-video-super-resolutionvideo-enhancementvideo-upscaling
#include "image_metadata.hpp"
#include "infrastructure/paths.hpp"
#include "infrastructure/process.hpp"
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QRegularExpression>
#include <QSaveFile>
#include "infrastructure/scratch.hpp"
#include <QtEndian>
#include <algorithm>
#include <array>
#include <limits>
#include <vector>
#include <zlib.h>
namespace ve {
namespace {
class Tiff {
public:
explicit Tiff(QByteArray bytes) : bytes_(std::move(bytes)), little_(bytes_.startsWith("II")) {
if (bytes_.size() < 8 || (!little_ && !bytes_.startsWith("MM")) || read16(2) != 42)
throw std::runtime_error("Unsupported EXIF metadata layout.");
}
quint16 read16(qsizetype at) const {
bounds(at, 2);
return little_ ? qFromLittleEndian<quint16>(bytes_.constData() + at)
: qFromBigEndian<quint16>(bytes_.constData() + at);
}
quint32 read32(qsizetype at) const {
bounds(at, 4);
return little_ ? qFromLittleEndian<quint32>(bytes_.constData() + at)
: qFromBigEndian<quint32>(bytes_.constData() + at);
}
void put16(qsizetype at, quint16 value) {
bounds(at, 2);
if (little_)
qToLittleEndian(value, bytes_.data() + at);
else
qToBigEndian(value, bytes_.data() + at);
}
void put32(qsizetype at, quint32 value) {
bounds(at, 4);
if (little_)
qToLittleEndian(value, bytes_.data() + at);
else
qToBigEndian(value, bytes_.data() + at);
}
[[nodiscard]] QByteArray bytes() const { return bytes_; }
QString description() const {
const auto root = read32(4);
const auto count = read16(root);
for (int i = 0; i < count; ++i) {
const auto entry = root + 2 + i * 12;
if (read16(entry) != 270 || read16(entry + 2) != 2)
continue;
const auto length = read32(entry + 4);
if (!length || length > 1024 * 1024)
return {};
const auto offset = length <= 4 ? entry + 8 : read32(entry + 8);
bounds(offset, length);
auto text = bytes_.mid(offset, length);
if (text.endsWith('\0'))
text.chop(1);
return QString::fromUtf8(text);
}
return {};
}
QByteArray withDescription(const QString& value) {
const auto root = read32(4);
const auto count = read16(root);
bounds(root + 2, qsizetype(count) * 12 + 4);
std::vector<QByteArray> entries;
entries.reserve(size_t(count) + 1);
for (int i = 0; i < count; ++i) {
const auto at = root + 2 + i * 12;
if (read16(at) == 270)
continue;
entries.push_back(bytes_.mid(at, 12));
}
QByteArray description = value.toUtf8();
description.append('\0');
if (quint64(bytes_.size()) + quint64(description.size()) + quint64(entries.size()) * 12 + 18 >
std::numeric_limits<quint32>::max() ||
entries.size() >= 65535)
throw std::runtime_error("EXIF metadata is too large.");
if (bytes_.size() % 2)
bytes_.append('\0');
const auto newRoot = bytes_.size();
const auto textAt = newRoot + 2 + qsizetype(entries.size() + 1) * 12 + 4;
const auto next = read32(root + 2 + count * 12);
bytes_.resize(textAt + description.size());
put32(4, quint32(newRoot));
put16(newRoot, quint16(entries.size() + 1));
QByteArray descriptionEntry(12, '\0');
if (little_) {
qToLittleEndian(quint16(270), descriptionEntry.data());
qToLittleEndian(quint16(2), descriptionEntry.data() + 2);
qToLittleEndian(quint32(description.size()), descriptionEntry.data() + 4);
qToLittleEndian(quint32(textAt), descriptionEntry.data() + 8);
} else {
qToBigEndian(quint16(270), descriptionEntry.data());
qToBigEndian(quint16(2), descriptionEntry.data() + 2);
qToBigEndian(quint32(description.size()), descriptionEntry.data() + 4);
qToBigEndian(quint32(textAt), descriptionEntry.data() + 8);
}
if (description.size() <= 4) {
descriptionEntry.replace(8, 4, description.leftJustified(4, '\0'));
}
entries.push_back(descriptionEntry);
std::ranges::sort(entries, [this](const QByteArray& a, const QByteArray& b) {
return (little_ ? qFromLittleEndian<quint16>(a.constData())
: qFromBigEndian<quint16>(a.constData())) <
(little_ ? qFromLittleEndian<quint16>(b.constData())
: qFromBigEndian<quint16>(b.constData()));
});
for (qsizetype i = 0; i < qsizetype(entries.size()); ++i) {
bytes_.replace(newRoot + 2 + i * 12, 12, entries[size_t(i)]);
if ((little_ ? qFromLittleEndian<quint16>(entries[size_t(i)].constData())
: qFromBigEndian<quint16>(entries[size_t(i)].constData())) == 274) {
put16(newRoot + 2 + i * 12 + 2, 3);
put32(newRoot + 2 + i * 12 + 4, 1);
put32(newRoot + 2 + i * 12 + 8, 0);
put16(newRoot + 2 + i * 12 + 8, 1);
}
}
put32(textAt - 4, next);
bytes_.replace(textAt, description.size(), description);
return bytes_;
}
private:
void bounds(qsizetype at, qsizetype size) const {
if (at < 0 || size < 0 || at > bytes_.size() || size > bytes_.size() - at)
throw std::runtime_error("Invalid EXIF metadata offset.");
}
QByteArray bytes_;
bool little_;
};
QByteArray segment(quint8 marker, const QByteArray& payload) {
if (payload.size() > 65533)
throw std::runtime_error("Image metadata exceeds the JPEG segment limit.");
QByteArray bytes(4, Qt::Uninitialized);
bytes[0] = '\xff';
bytes[1] = char(marker);
qToBigEndian(quint16(payload.size() + 2), bytes.data() + 2);
return bytes + payload;
}
QByteArray exv(const ImageMetadata& metadata) {
QByteArray bytes = QByteArray::fromHex("ff014578697632");
if (!metadata.exif.isEmpty())
bytes += segment(0xe1, QByteArray("Exif\0\0", 6) + metadata.exif);
if (!metadata.xmp.isEmpty())
bytes += segment(0xe1, QByteArray("http://ns.adobe.com/xap/1.0/\0", 29) + metadata.xmp);
bytes += QByteArray::fromHex("ffd9");
return bytes;
}
QByteArray pngChunk(const QByteArray& name, const QByteArray& data) {
QByteArray bytes(4, Qt::Uninitialized);
qToBigEndian(quint32(data.size()), bytes.data());
bytes += name;
bytes += data;
const auto crc =
quint32(crc32(0, reinterpret_cast<const Bytef*>(bytes.constData() + 4), uInt(bytes.size() - 4)));
QByteArray tail(4, Qt::Uninitialized);
qToBigEndian(crc, tail.data());
return bytes + tail;
}
void save(const QString& path, const QByteArray& data) {
QSaveFile file(path);
if (!file.open(QIODevice::WriteOnly) || file.write(data) != data.size() || !file.commit())
throw std::runtime_error(file.errorString().toStdString());
}
void insertMetadata(const QString& path, qsizetype prefixBytes, const QByteArray& chunks, JobControl& control) {
if (chunks.isEmpty()) return;
QFile input(path);
QSaveFile output(path);
if (!input.open(QIODevice::ReadOnly) || !output.open(QIODevice::WriteOnly))
throw std::runtime_error("Cannot open image for metadata insertion.");
const auto prefix=input.read(prefixBytes);
if (prefix.size()!=prefixBytes || output.write(prefix)!=prefix.size() || output.write(chunks)!=chunks.size())
throw std::runtime_error("Cannot write image metadata.");
while (!input.atEnd()) {
control.checkpoint();
const auto block=input.read(8*1024*1024);
if (block.isEmpty() || output.write(block)!=block.size())
throw std::runtime_error("Cannot copy image data while inserting metadata.");
}
input.close();
control.checkpoint();
if (!output.commit()) throw std::runtime_error(output.errorString().toStdString());
}
} // namespace
QString mergeRenderDescription(const QString& existing, const QString& note) {
if (existing.isEmpty())
return note;
static const QRegularExpression own(
"(?m)^(?:Visual Enhancer|DLSS 5 Visual Enhancer):\\r?\\nDLSS 5 Neural Rendering Settings - NR Style "
"- (?:Style [0-2]|Default|Natural|Cinematic),[^\\r\\n]+$");
auto merged = existing;
if (own.match(existing).hasMatch()) {
merged.replace(own, note);
return merged;
}
return existing + "\n\n" + note;
}
ImageMetadata readImageMetadata(const QString& source, JobControl& control) {
ImageMetadata metadata;
if (source.isEmpty())
return metadata;
if (QFileInfo(source).suffix().compare("png", Qt::CaseInsensitive) == 0) {
QFile input(source);
if (input.open(QIODevice::ReadOnly)) {
const auto bytes = input.readAll();
for (qsizetype at = 8; at + 12 <= bytes.size();) {
const auto length = qFromBigEndian<quint32>(bytes.constData() + at);
if (length > quint64(bytes.size() - at - 12))
break;
const auto name = bytes.mid(at + 4, 4), data = bytes.mid(at + 8, length);
if (name == "eXIf")
metadata.exif = data;
if (name == "iTXt" && data.startsWith("XML:com.adobe.xmp")) {
const auto separator = data.indexOf('\0');
if (separator >= 0 && separator + 5 <= data.size() && data[separator + 1] == 0)
metadata.xmp = data.mid(separator + 5);
}
at += 12 + length;
}
if (!metadata.exif.isEmpty())
return metadata;
}
}
const auto data =
runProcess(Paths::instance().at("bin/exiv2/exiv2.exe"), {"-ea-", source}, &control, 30000).output;
if (data.size() > 16 * 1024 * 1024)
throw std::runtime_error("Image metadata exceeds the supported size.");
for (qsizetype at = 7; at + 4 <= data.size();) {
if (quint8(data[at]) != 255)
break;
const auto marker = quint8(data[at + 1]);
if (marker == 0xd9)
break;
const auto length = qFromBigEndian<quint16>(data.constData() + at + 2);
if (length < 2 || at + 2 + length > data.size())
break;
const auto payload = data.mid(at + 4, length - 2);
if (marker == 0xe1) {
if (payload.startsWith(QByteArray("Exif\0\0", 6)))
metadata.exif = payload.mid(6);
else if (payload.startsWith(QByteArray("http://ns.adobe.com/xap/1.0/\0", 29)))
metadata.xmp = payload.mid(29);
}
at += 2 + length;
}
return metadata;
}
void addImageDescription(ImageMetadata& metadata, const QString& note) {
auto bytes = metadata.exif;
if (bytes.isEmpty())
bytes = QByteArray::fromHex("49492a0008000000000000000000");
Tiff tiff(bytes);
metadata.exif = tiff.withDescription(mergeRenderDescription(tiff.description(), note));
}
void normalizeImageOrientation(ImageMetadata& metadata) {
if (metadata.exif.isEmpty())
return;
Tiff tiff(metadata.exif);
const auto root = tiff.read32(4);
const auto count = tiff.read16(root);
for (int index = 0; index < count; ++index) {
const auto entry = root + 2 + index * 12;
if (tiff.read16(entry) != 274)
continue;
tiff.put16(entry + 2, 3);
tiff.put32(entry + 4, 1);
tiff.put32(entry + 8, 0);
tiff.put16(entry + 8, 1);
}
metadata.exif = tiff.bytes();
}
void writeImageMetadata(const QString& destination, const ImageMetadata& metadata, JobControl& control) {
control.checkpoint();
const auto extension = QFileInfo(destination).suffix().toLower();
if ((extension == "jpg" || extension == "jpeg") && !metadata.icc.isEmpty()) {
QFile input(destination);
if (!input.open(QIODevice::ReadOnly))
throw std::runtime_error(input.errorString().toStdString());
const auto original = input.read(2);
input.close();
if (!original.startsWith(QByteArray::fromHex("ffd8")))
throw std::runtime_error("Invalid JPEG image.");
const auto count = (metadata.icc.size() + 65518) / 65519;
if (count > 255)
throw std::runtime_error("ICC profile is too large.");
QByteArray chunks;
for (qsizetype index = 0; index < count; ++index) {
auto payload = QByteArray("ICC_PROFILE\0", 12);
payload.append(char(index + 1));
payload.append(char(count));
payload += metadata.icc.mid(index * 65519, 65519);
chunks += segment(0xe2, payload);
}
insertMetadata(destination,2,chunks,control);
}
if (QFileInfo(destination).suffix().compare("png", Qt::CaseInsensitive) == 0) {
QFile input(destination);
if (!input.open(QIODevice::ReadOnly))
throw std::runtime_error(input.errorString().toStdString());
const auto original = input.read(33);
input.close();
if (!original.startsWith(QByteArray::fromHex("89504e470d0a1a0a")) || original.size() < 33)
throw std::runtime_error("Invalid PNG image.");
QByteArray chunks;
if (!metadata.exif.isEmpty())
chunks += pngChunk("eXIf", metadata.exif);
if (!metadata.xmp.isEmpty())
chunks += pngChunk("iTXt", QByteArray("XML:com.adobe.xmp\0\0\0\0\0", 22) + metadata.xmp);
insertMetadata(destination,33,chunks,control);
return;
}
if (metadata.exif.isEmpty() && metadata.xmp.isEmpty())
return;
ScratchDirectory temporary(QStringLiteral("metadata-XXXXXX"));
if (!temporary.isValid())
throw std::runtime_error("Cannot create the metadata directory.");
save(temporary.filePath(QFileInfo(destination).completeBaseName() + ".exv"), exv(metadata));
const auto result = runProcess(Paths::instance().at("bin/exiv2/exiv2.exe"),
{"-ieix", "-l", temporary.path(), destination}, &control, 30000);
if (result.exitCode)
throw std::runtime_error(QString::fromUtf8(result.error).toStdString());
}
} // namespace ve