DLSS 5 Neural Rendering for Images & Video with Frame Generation, RTX Video Super Resolution, HDR, and Color Grading
TypeScript ai-upscalingdlssdlss-5dlss5dlssgframe-generationframe-interpolationimage-enhancementimage-upscalingneural-renderingnvidianvidia-dlss-5nvidia-rtxrtx-videortx-video-super-resolutionvideo-enhancementvideo-upscaling
#include "preview_player.hpp"
#include "media/timeline.hpp"
#include "infrastructure/paths.hpp"
#include "infrastructure/process.hpp"
#include "media/media.hpp"
#include "media/packet_stream.hpp"
#include "processing/preview_cache.hpp"
#include <QVideoFrameFormat>
#include <QSet>
#include <QScopeGuard>
#include <algorithm>
#include <cmath>
#include <cstring>
namespace ve {
namespace { QSet<PreviewPlayer*> players; }
PreviewPlayer::PreviewPlayer(QObject* parent)
: QObject(parent), thread_([this](std::stop_token stop) { decode(stop); }) {
players.insert(this);
timer_.setInterval(5);
connect(&timer_, &QTimer::timeout, this, &PreviewPlayer::tick);
timer_.start();
}
PreviewPlayer::~PreviewPlayer() {
close();
if (thread_.joinable()) thread_.join();
players.remove(this);
}
QStringList PreviewPlayer::processingFilesInUse() {
QStringList paths;
for (auto* player : players) {
if (!player->closed_ && player->source_.isLocalFile()) paths.append(player->source_.toLocalFile());
std::lock_guard lock(player->mutex_);
if (!player->decodingPath_.isEmpty()) paths.append(player->decodingPath_);
}
return paths;
}
void PreviewPlayer::setState(int state) {
if (state_ != state) {
state_ = state;
emit playbackStateChanged(state_);
}
}
void PreviewPlayer::setStatus(int status) {
if (status_ != status) {
status_ = status;
emit mediaStatusChanged(status_);
}
}
void PreviewPlayer::setSource(const QUrl& source) {
if (source == source_)
return;
source_ = source;
if (sink_)
sink_->setVideoFrame({});
position_ = duration_ = 0;
fps_ = 0;
completedLoops_ = 0;
error_.clear();
emit durationChanged(0);
emit positionChanged(0);
emit videoFpsChanged();
emit errorStringChanged();
setState(2);
setStatus(source.isEmpty() ? 0 : 1);
restart();
emit sourceChanged();
}
void PreviewPlayer::setVideoOutput(QObject* output) {
output_ = output;
sink_ = output ? qvariant_cast<QVideoSink*>(output->property("videoSink")) : nullptr;
if (!sink_)
sink_ = qobject_cast<QVideoSink*>(output);
emit videoOutputChanged();
}
void PreviewPlayer::restart() {
std::lock_guard lock(mutex_);
if (control_)
control_->cancel();
++generation_;
queue_.clear();
primed_ = finished_ = false;
request_ = Request{source_.toLocalFile(), position_, generation_};
condition_.notify_all();
}
void PreviewPlayer::seek(int position) {
position_ = std::clamp(position, 0, std::max(0, duration_ - 1));
clockPosition_ = position_;
clock_.restart();
emit positionChanged(position_);
restart();
}
void PreviewPlayer::play() {
if (closed_ || source_.isEmpty())
return;
if (status_ == 6)
seek(0);
clockPosition_ = position_;
clock_.restart();
setState(1);
}
void PreviewPlayer::pause() {
setState(2);
}
void PreviewPlayer::setPlaybackRate(double rate) {
if (!std::isfinite(rate) || rate <= 0 || rate == rate_)
return;
rate_ = rate;
clockPosition_ = position_;
clock_.restart();
emit playbackRateChanged();
}
void PreviewPlayer::stop() {
pause();
seek(0);
setState(0);
}
void PreviewPlayer::close() {
if (closed_)
return;
closed_ = true;
timer_.stop();
thread_.request_stop();
{
std::lock_guard lock(mutex_);
if (control_)
control_->cancel();
queue_.clear();
}
condition_.notify_all();
}
void PreviewPlayer::tick() {
if (closed_)
return;
std::optional<Frame> latest;
bool finished = false;
const int desired = state_ == 1 ? clockPosition_ + int(clock_.elapsed() * rate_) : position_;
{
std::lock_guard lock(mutex_);
while (!queue_.empty() && (!primed_ || (state_ == 1 && queue_.front().position <= desired))) {
latest = std::move(queue_.front());
queue_.pop_front();
primed_ = true;
}
finished = finished_ && queue_.empty();
condition_.notify_all();
}
if (latest && latest->generation == generation_) {
if (sink_)
sink_->setVideoFrame(latest->video);
if (position_ != latest->position) {
position_ = latest->position;
emit positionChanged(position_);
}
if (status_ == 1)
setStatus(2);
}
if (finished && state_ == 1) {
if (loops_ == -1 || ++completedLoops_ < loops_) {
seek(0);
play();
} else {
setState(0);
setStatus(6);
}
}
}
void PreviewPlayer::decode(std::stop_token stop) {
QByteArray sourceIdentity;
MediaInfo sourceInfo;
while (!stop.stop_requested()) {
Request request;
std::shared_ptr<JobControl> control;
{
std::unique_lock lock(mutex_);
condition_.wait(lock, stop, [this] { return request_.has_value(); });
if (stop.stop_requested())
return;
request = *request_;
request_.reset();
control = std::make_shared<JobControl>();
control_ = control;
decodingPath_ = request.path;
}
const auto releasePath = qScopeGuard([this] { std::lock_guard lock(mutex_); decodingPath_.clear(); });
if (request.path.isEmpty())
continue;
const auto generation = request.generation;
try {
const auto identity = previewFileIdentity(request.path);
if (identity != sourceIdentity) {
auto info = probeMedia(request.path, control.get());
control->checkpoint();
sourceInfo = std::move(info);
sourceIdentity = identity;
}
const auto info = sourceInfo;
const int width = (info.width + 1) / 2 * 2, height = (info.height + 1) / 2 * 2;
const double fps = info.fps > 0 ? info.fps : 30;
QMetaObject::invokeMethod(
this,
[this, generation, info, fps] {
if (generation != generation_)
return;
duration_ = int(info.duration * 1000);
fps_ = fps;
emit durationChanged(duration_);
emit videoFpsChanged();
setStatus(2);
},
Qt::QueuedConnection);
QVideoFrameFormat format(QSize(width, height), QVideoFrameFormat::Format_P010);
format.setViewport(QRect(0, 0, info.width, info.height));
format.setColorRange(QVideoFrameFormat::ColorRange_Video);
format.setColorSpace(info.primaries == "bt2020" ? QVideoFrameFormat::ColorSpace_BT2020
: QVideoFrameFormat::ColorSpace_BT709);
format.setColorTransfer(info.transfer == "smpte2084" ? QVideoFrameFormat::ColorTransfer_ST2084
: info.transfer == "arib-std-b67"
? QVideoFrameFormat::ColorTransfer_STD_B67
: QVideoFrameFormat::ColorTransfer_BT709);
const auto matrix = info.primaries == "bt2020" ? QString("bt2020nc") : QString("bt709");
const auto parked = quantizePlayhead(request.position, fps, info.duration);
const double seekSeconds = std::max(0., parked.seconds - .002);
PipeProcess decoder(Paths::instance().ffmpeg(),
{"-v",
"error",
"-ss",
QString::number(seekSeconds, 'f', 9),
"-copyts", "-start_at_zero",
"-i",
request.path,
"-an",
"-vf",
"scale=in_range=auto:out_range=tv:out_color_matrix=" + matrix +
",pad=" + QString::number(width) + ':' + QString::number(height),
"-fps_mode",
"passthrough",
"-enc_time_base:v",
"demux",
"-c:v",
"rawvideo",
"-f",
"nut",
"-write_index",
"0",
"-flush_packets",
"1",
"-pix_fmt",
"p010le",
"pipe:1"},PipeProcess::OutputMode::Backpressured);
FramePacketReader packets(decoder, *control, RawPixelFormat::P010);
while (!stop.stop_requested()) {
control->checkpoint();
const auto packet = packets.readPixels();
if (!packet)
break;
const auto& bytes = packet->pixels;
QVideoFrame video(format);
if (!video.map(QVideoFrame::WriteOnly))
throw std::runtime_error("Cannot allocate the preview display frame.");
qsizetype offset = 0;
for (int plane = 0; plane < 2; ++plane) {
const int rows = plane == 0 ? height : height / 2;
for (int y = 0; y < rows; ++y)
std::memcpy(video.bits(plane) + qsizetype(y) * video.bytesPerLine(plane),
bytes.constData() + offset + qsizetype(y) * width * 2, size_t(width) * 2);
offset += qsizetype(rows) * width * 2;
}
video.unmap();
const int position = int(std::lround(packet->seconds() * 1000));
video.setStartTime(qint64(position) * 1000);
video.setEndTime(qint64(position) * 1000 +
(packet->duration > 0
? packet->duration * 1000000ll * packet->timeBase.numerator /
packet->timeBase.denominator
: qint64(1000000. / fps)));
{
std::unique_lock lock(mutex_);
condition_.wait(lock, stop, [&] { return queue_.size() < 3 || control->cancelled(); });
control->checkpoint();
if (stop.stop_requested())
return;
if (generation != generation_)
break;
queue_.push_back({std::move(video), position, generation});
}
}
{
std::lock_guard lock(mutex_);
if (generation == generation_)
finished_ = true;
}
} catch (const Cancelled&) {
} catch (const std::exception& exception) {
const auto message = QString::fromUtf8(exception.what());
QMetaObject::invokeMethod(
this,
[this, generation, message] {
if (generation != generation_)
return;
error_ = message;
emit errorStringChanged();
emit errorOccurred(1, message);
setStatus(7);
},
Qt::QueuedConnection);
}
}
}
} // namespace ve