Skip to content

DLSS 5 Neural Rendering for Images & Video with Frame Generation, RTX Video Super Resolution, HDR, and Color Grading

Add files via upload

Merserkc95c050ead79
diff --git a/app.py b/app.py
index 9f6ed43..eeb3640 100644
--- a/app.py
+++ b/app.py
@@ -352,10 +352,9 @@ def build_app() -> gr.Blocks:
# are covered by cleanup_old_caches() at startup in main().
delete_cache=(CACHE_SWEEP_INTERVAL_SECONDS, CACHE_MAX_AGE_SECONDS),
) as demo:
- # One runtime source of truth for the global Neural Rendering processing
- # path. The visible control lives in Settings, while render/start events
- # consume this State so each queued operation snapshots the selected mode.
- processing_engine_state = gr.State(value=settings.nr_gpu_mode)
+ # Image and Live have no output-codec choice and remain GPU-resident.
+ # Video processors derive CUDA versus host staging from their codec.
+ processing_engine_state = gr.State(value=True)
nr_mask_state = gr.State(value=None)
with gr.Tabs(selected="neural-rendering", elem_id="main-tabs"):
# Keep every tab tree mounted from first paint. Stateful File/Gallery/Video
@@ -370,9 +369,7 @@ def build_app() -> gr.Blocks:
with gr.Tab("Live", id="live", render_children=True) as live_root_tab:
live_tab = build_live_tab(settings, processing_engine_state, nr_mask_state)
with gr.Tab("Settings", id="settings", render_children=True) as settings_root_tab:
- settings_tab = build_settings_tab(
- settings, ai_gpu_choices, video_gpu_choices, processing_engine_state
- )
+ settings_tab = build_settings_tab(settings, ai_gpu_choices, video_gpu_choices)
with gr.Tab("About", id="about", render_children=True) as about_root_tab:
build_about_tab()
diff --git a/src/about/ui.py b/src/about/ui.py
index 3e3a6ee..68f985a 100644
--- a/src/about/ui.py
+++ b/src/about/ui.py
@@ -6,7 +6,7 @@ def build_about_tab():
"""
<section class="about-details" aria-labelledby="about-name">
<h2 id="about-name">DLSS 5 Visual Enhancer</h2>
- <p class="about-version">Version v8.0</p>
+ <p class="about-version">Version v9.0</p>
<div class="about-links">
<p>
<a href="https://github.com/Merserk/dlss5-visual-enhancer"
diff --git a/src/core/app_log.py b/src/core/app_log.py
index 6add7ee..7994a36 100644
--- a/src/core/app_log.py
+++ b/src/core/app_log.py
@@ -7,6 +7,7 @@ small ``.err`` file only on failure (see :func:`fail`).
"""
from __future__ import annotations
+import atexit
import io
import re
import threading
@@ -24,6 +25,21 @@ MAX_TAIL = 500
RETENTION_DAYS = 14
+def _close_handle() -> None:
+ """Flush and close the process log without trying to write another event."""
+ global _handle
+ with _lock:
+ handle, _handle = _handle, None
+ if handle is not None:
+ try:
+ handle.close()
+ except OSError:
+ pass
+
+
+atexit.register(_close_handle)
+
+
def _compact(text: object, limit: int) -> str:
single = re.sub(r"\s+", " ", str(text or "")).strip()
return single if len(single) <= limit else single[:limit] + "..."
@@ -87,11 +103,16 @@ def _write(level: str, tag: str, msg: str) -> None:
handle = _handle
if handle is None:
try:
- handle = open(init_session(), "a", encoding="utf-8", errors="replace")
+ # init_session owns creation of the one process-wide handle. Opening
+ # the returned path again here replaced that handle without closing
+ # it whenever the first log call initialized the session.
+ init_session()
except OSError:
return
with _lock:
- _handle = handle
+ handle = _handle
+ if handle is None:
+ return
try:
with _lock:
handle.write(line)
diff --git a/src/core/batch_ui.py b/src/core/batch_ui.py
index 6616c92..c8431f1 100644
--- a/src/core/batch_ui.py
+++ b/src/core/batch_ui.py
@@ -842,3 +842,30 @@ def bind_batch_ui(
show_progress="hidden",
api_visibility="private",
)
+
+ def refresh_realtime_preview_after(trigger):
+ """Regenerate this tab's preview after a successful programmatic update."""
+ reset_invalidated = trigger.success(
+ invalidate_auto_preview,
+ inputs=[tab.job_state, tab.sources],
+ outputs=auto_token,
+ queue=False,
+ show_progress="hidden",
+ trigger_mode="always_last",
+ api_visibility="private",
+ )
+ return reset_invalidated.then(
+ automatic_preview,
+ inputs=[
+ tab.job_state, auto_token, tab.input_path, tab.output_path,
+ *tab.preview_inputs,
+ ],
+ outputs=[display_media, tab.status, direct_save, archive_button, archive_download],
+ concurrency_limit=1,
+ concurrency_id="neural-realtime-preview",
+ trigger_mode="always_last",
+ show_progress="hidden",
+ api_visibility="private",
+ )
+
+ return refresh_realtime_preview_after
diff --git a/src/core/ffmpeg/__init__.py b/src/core/ffmpeg/__init__.py
index 0c01cbb..7af9557 100644
--- a/src/core/ffmpeg/__init__.py
+++ b/src/core/ffmpeg/__init__.py
@@ -1,7 +1,7 @@
from .codecs import (
AUTO_BITRATE_DIVISORS, CODEC_CHOICES, ENCODING_QUALITIES, HDR_ALLOWED_CODECS,
_base_codec, _is_hdr_allowed_codec, _is_nvenc_codec, _normalize_codec,
- calculate_auto_bitrate_kbps, hdr_mode_supported, resolve_encoding_quality,
+ calculate_auto_bitrate_kbps, container_for_codec, hdr_mode_supported, resolve_encoding_quality,
validate_codec_container,
)
from .encoder import probe_nvenc_codecs, resolve_video_gpu, start_encoder
@@ -17,7 +17,7 @@ from .probe import preview_frame_count, probe_video
__all__ = [
"AUTO_BITRATE_DIVISORS", "CODEC_CHOICES", "DEFAULT_PREVIEW_ENCODING",
"ENCODING_QUALITIES", "HDR_ALLOWED_CODECS", "PREVIEW_ENCODING_CHOICES",
- "RawVideoPacketMuxer", "calculate_auto_bitrate_kbps", "final_mux", "hdr_mode_supported",
+ "RawVideoPacketMuxer", "calculate_auto_bitrate_kbps", "container_for_codec", "final_mux", "hdr_mode_supported",
"is_browser_playable", "is_user_playable_request", "make_browser_preview",
"normalize_preview_encoding", "prepare_mux_comment", "preview_frame_count", "probe_nvenc_codecs",
"probe_video", "resolve_encoding_quality", "resolve_final_preview",
diff --git a/src/core/ffmpeg/codecs.py b/src/core/ffmpeg/codecs.py
index e15648f..8adec29 100644
--- a/src/core/ffmpeg/codecs.py
+++ b/src/core/ffmpeg/codecs.py
@@ -45,6 +45,16 @@ _NVENC_ENCODERS = {
"AV1 (NVIDIA NVENC)": "av1_nvenc",
}
+_AUTOMATIC_CONTAINERS = {
+ "H.264": "MP4",
+ "H.264 (NVIDIA NVENC)": "MP4",
+ "H.265": "MKV",
+ "H.265 (NVIDIA NVENC)": "MKV",
+ "AV1": "MKV",
+ "AV1 (NVIDIA NVENC)": "MKV",
+ "ProRes Proxy": "MOV",
+}
+
# Keep AUTO_BITRATE_DIVISORS keyed by base codec; "HEVC" alias preserved for compat.
AUTO_BITRATE_DIVISORS = {
"H.264": 165_888,
@@ -90,6 +100,15 @@ def hdr_mode_supported(codec: str) -> bool:
return _is_hdr_allowed_codec(codec)
+def container_for_codec(codec: str) -> str:
+ """Return the single application container selected for a video codec."""
+ normalized = _normalize_codec(codec)
+ try:
+ return _AUTOMATIC_CONTAINERS[normalized]
+ except KeyError as exc:
+ raise ValueError(f"Unknown video codec: {codec!r}.") from exc
+
+
def _hdr_color_args(metadata: dict | None) -> list[str]:
"""Return ffmpeg color flags copying input colorspace when possible.
diff --git a/src/core/ffmpeg/probe.py b/src/core/ffmpeg/probe.py
index 6b33c63..01bf469 100644
--- a/src/core/ffmpeg/probe.py
+++ b/src/core/ffmpeg/probe.py
@@ -79,9 +79,35 @@ def _positive_count(value: object) -> int:
return 0
+def _sampled_packet_timeline_is_cfr(
+ path: str | os.PathLike[str], time_base: Fraction,
+ controller: JobController | None,
+) -> bool:
+ """Reject VFR timelines that container-level average/nominal rates hide.
+
+ Packet PTS values are sorted to remove codec reordering. Reading 128 packet
+ headers is bounded and does not decode or transfer frame pixels.
+ """
+ data = _run_json([
+ str(FFPROBE), "-v", "error", "-select_streams", "v:0",
+ "-read_intervals", "%+#128", "-show_packets", "-show_entries",
+ "packet=pts", "-of", "json", str(path),
+ ], controller=controller)
+ timestamps = sorted({int(packet["pts"]) for packet in data.get("packets") or []
+ if packet.get("pts") not in {None, "N/A"}})
+ deltas = [b - a for a, b in zip(timestamps, timestamps[1:]) if b > a]
+ if len(deltas) < 2:
+ return True
+ # CFR timestamp quantization may alternate adjacent deltas by one time-base
+ # tick. A wider spread is an actual cadence change.
+ tolerance_ticks = max(1, int(round(float(Fraction(1, 1000000) / time_base))))
+ return max(deltas) - min(deltas) <= tolerance_ticks
+
+
def probe_video(
path: str | os.PathLike[str], *, count_mode: str = "exact",
strict_decode: bool = False, controller: JobController | None = None,
+ inspect_timestamps: bool = False,
) -> dict:
"""Probe video metadata, optionally counting decoded frames or packets.
@@ -108,7 +134,7 @@ def probe_video(
"-select_streams",
"v:0",
"-show_entries",
- "stream=index,codec_name,width,height,avg_frame_rate,r_frame_rate,time_base,duration,nb_frames,nb_read_frames,nb_read_packets,color_range,color_primaries,color_transfer,color_space:stream_tags=rotate:stream_side_data=rotation",
+ "stream=index,codec_name,pix_fmt,bits_per_raw_sample,width,height,avg_frame_rate,r_frame_rate,time_base,duration,nb_frames,nb_read_frames,nb_read_packets,color_range,color_primaries,color_transfer,color_space:stream_tags=rotate:stream_side_data=rotation",
"-show_entries",
"format=duration,format_name",
"-of",
@@ -153,10 +179,22 @@ def probe_video(
if nominal_rate_text != "0/0"
else rate
)
+ time_base = Fraction(stream.get("time_base") or "1/1000")
+ cfr = rate == nominal_rate
+ if cfr and inspect_timestamps:
+ cfr = _sampled_packet_timeline_is_cfr(path, time_base, controller)
transfer = stream.get("color_transfer") or "unknown"
primaries = stream.get("color_primaries") or "unknown"
color_space = stream.get("color_space") or "unknown"
color_range = stream.get("color_range") or "unknown"
+ pixel_format = stream.get("pix_fmt") or "unknown"
+ try:
+ depth = max(component.bits for component in av.VideoFormat(pixel_format).components)
+ except (ValueError, TypeError):
+ try:
+ depth = int(stream.get("bits_per_raw_sample") or 8)
+ except (TypeError, ValueError):
+ depth = 8
format_duration = _positive_float((data.get("format") or {}).get("duration"))
stream_duration = _positive_float(stream.get("duration"))
return {
@@ -170,12 +208,14 @@ def probe_video(
"fps": float(rate),
"rate": rate,
"nominal_rate": nominal_rate,
- "cfr": rate == nominal_rate,
- "time_base": Fraction(stream.get("time_base") or "1/1000"),
+ "cfr": cfr,
+ "time_base": time_base,
"duration": format_duration or stream_duration,
"video_duration": stream_duration or format_duration,
"video_stream_duration": stream_duration,
"codec": stream.get("codec_name") or "unknown",
+ "pixel_format": pixel_format,
+ "depth": depth,
"format": (data.get("format") or {}).get("format_name") or "unknown",
"color_transfer": transfer,
"color_primaries": primaries,
diff --git a/src/core/neural_bridge.py b/src/core/neural_bridge.py
index 87fc5bf..8c32d81 100644
--- a/src/core/neural_bridge.py
+++ b/src/core/neural_bridge.py
@@ -21,6 +21,7 @@ from typing import Any, Callable
import numpy as np
from .paths import DLSSNR_BRIDGE, DLSSNR_DIR
+from .ngx_runtime import NGX_RUNTIME_LOCK
BRIDGE_ABI_VERSION = 6
@@ -603,7 +604,8 @@ class NeuralBridgeManager:
"""One serialized bridge instance shared by every logical render session."""
def __init__(self) -> None:
- self._lock = threading.RLock()
+ # NGX is process-wide even when feature APIs use different backends.
+ self._lock = NGX_RUNTIME_LOCK
self._library: Any | None = None
self._initialized_ordinal: int | None = None
self._cuda_driver: _CudaDriver | None = None
@@ -927,8 +929,22 @@ class NeuralBridgeManager:
def initialize(self, gpu: dict[str, Any], *, require_cuda: bool) -> dict[str, Any]:
with self._lock:
self._guard_poison()
- self._load()
ordinal = int(gpu.get("cuda_ordinal", gpu.get("index", 0)))
+ if self._initialized_ordinal is None:
+ # The NVIDIA D3D12 NGX core keeps the first feature search path
+ # for the lifetime of the process. If Neural Rendering starts
+ # first, a later DLSSG initialization succeeds superficially
+ # but reports Frame Generation as unavailable. Prime DLSSG on
+ # the same adapter before Feature 18; failure remains isolated
+ # so systems without the optional DLSSG runtime can still use
+ # Neural Rendering.
+ try:
+ from ..frame_interpolation.native import initialize_bridge
+
+ initialize_bridge(ordinal)
+ except Exception:
+ pass
+ self._load()
if self._initialized_ordinal is None:
assert self._library is not None
error = ctypes.create_string_buffer(4096)
diff --git a/src/core/ngx_runtime.py b/src/core/ngx_runtime.py
new file mode 100644
index 0000000..f384102
--- /dev/null
+++ b/src/core/ngx_runtime.py
@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+"""Process-wide coordination for NVIDIA NGX feature evaluations.
+
+RTX Video, Neural Rendering, and Frame Interpolation load NGX into the same
+long-lived Python process. NVIDIA's feature runtimes are not safe to evaluate
+concurrently, so all native entry points share this re-entrant lock. Normal
+session teardown must not unload NGX or call its core shutdown routine.
+"""
+
+import threading
+
+
+NGX_RUNTIME_LOCK = threading.RLock()
diff --git a/src/frame_interpolation/capabilities.py b/src/frame_interpolation/capabilities.py
index 5c37636..927e2b6 100644
--- a/src/frame_interpolation/capabilities.py
+++ b/src/frame_interpolation/capabilities.py
@@ -1,29 +1,26 @@
from __future__ import annotations
-import json
+import ctypes
import os
import subprocess
from functools import lru_cache
from pathlib import Path
-from ..core.paths import DLSSG_DIR
from ..core.gpu_selection import detect_gpu
from .models import FrameInterpolationCapabilities
+from .native import BRIDGE, BRIDGE_ABI_VERSION, RUNTIME_DIR, initialize_bridge
-
-RUNTIME_DIR = DLSSG_DIR
DLSSG_RUNTIME = RUNTIME_DIR / "nvngx_dlssg.dll"
-DLSSG_WORKER = RUNTIME_DIR / "dlssg-worker.exe"
+
+
def _hags_enabled() -> bool:
if os.name != "nt":
return False
try:
import winreg
-
with winreg.OpenKey(
- winreg.HKEY_LOCAL_MACHINE,
- r"SYSTEM\CurrentControlSet\Control\GraphicsDrivers",
- ) as key:
+ winreg.HKEY_LOCAL_MACHINE,
+ r"SYSTEM\CurrentControlSet\Control\GraphicsDrivers") as key:
value, _kind = winreg.QueryValueEx(key, "HwSchMode")
return int(value) == 2
except (OSError, ValueError):
@@ -36,125 +33,63 @@ def _authenticode_status(path: Path) -> str:
escaped = str(path.resolve()).replace("'", "''")
try:
process = subprocess.run(
- [
- "powershell.exe",
- "-NoProfile",
- "-NonInteractive",
- "-Command",
- "Import-Module \"$env:WINDIR\\System32\\WindowsPowerShell\\v1.0\\Modules\\"
- "Microsoft.PowerShell.Security\\Microsoft.PowerShell.Security.psd1\"; "
- f"[string](Get-AuthenticodeSignature -LiteralPath '{escaped}').Status",
- ],
- capture_output=True,
- text=True,
- encoding="utf-8",
- errors="replace",
- timeout=15,
- )
+ ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command",
+ "Import-Module \"$env:WINDIR\\System32\\WindowsPowerShell\\v1.0\\Modules\\"
+ "Microsoft.PowerShell.Security\\Microsoft.PowerShell.Security.psd1\"; "
+ f"[string](Get-AuthenticodeSignature -LiteralPath '{escaped}').Status"],
+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
except (OSError, subprocess.TimeoutExpired):
return "Unavailable"
return process.stdout.strip() if process.returncode == 0 else "Unavailable"
-def _probe_worker() -> dict:
- if not DLSSG_WORKER.is_file():
- raise RuntimeError(f"DLSSG worker is missing: {DLSSG_WORKER}")
- command = [str(DLSSG_WORKER), "--probe"]
- process = subprocess.run(
- command,
- cwd=str(RUNTIME_DIR),
- capture_output=True,
- text=True,
- encoding="utf-8",
- errors="replace",
- timeout=45,
- )
- if process.returncode:
- detail = process.stderr.strip() or process.stdout.strip()
- raise RuntimeError(detail or f"DLSSG capability probe exited with {process.returncode}.")
- lines = [line for line in process.stdout.splitlines() if line.strip()]
- if not lines:
- raise RuntimeError("DLSSG capability probe returned no result.")
- return json.loads(lines[-1])
+def _nvof_present() -> bool:
+ try:
+ ctypes.WinDLL("nvofapi64.dll")
+ return True
+ except (AttributeError, OSError):
+ return False
@lru_cache(maxsize=32)
def probe_frame_interpolation_capabilities(
- ai_gpu_uuid: str = "auto",
-) -> FrameInterpolationCapabilities:
+ ai_gpu_uuid: str = "auto") -> FrameInterpolationCapabilities:
gpu = detect_gpu(ai_gpu_uuid)
gpu_name = str(gpu.get("display_name") or gpu.get("name") or "NVIDIA RTX GPU")
driver = str(gpu.get("driver") or "unknown")
- hags = _hags_enabled()
- if not DLSSG_RUNTIME.is_file():
- return FrameInterpolationCapabilities(
- available=False,
- gpu=gpu_name,
- driver=driver,
- hags_enabled=hags,
- native_generated_frame_max=0,
- native_multiplier=1,
- cascade_available=False,
- runtime_version="missing",
- worker_version="missing",
- signature_status="Missing",
- detail=f"NVIDIA DLSSG runtime is missing: {DLSSG_RUNTIME}",
- gpu_uuid=str(gpu.get("uuid") or ai_gpu_uuid),
- )
- if not DLSSG_WORKER.is_file():
- return FrameInterpolationCapabilities(
- available=False,
- gpu=gpu_name,
- driver=driver,
- hags_enabled=hags,
- native_generated_frame_max=0,
- native_multiplier=1,
- cascade_available=False,
- runtime_version="unknown",
- worker_version="missing",
- signature_status=_authenticode_status(DLSSG_RUNTIME),
- detail=f"Direct D3D12 DLSSG worker is missing: {DLSSG_WORKER}",
- gpu_uuid=str(gpu.get("uuid") or ai_gpu_uuid),
- )
-
- signature_status = _authenticode_status(DLSSG_RUNTIME)
- try:
- probed = _probe_worker()
- generated_max = max(0, int(probed.get("multi_frame_count_max", 0)))
- multiplier = generated_max + 1 if generated_max else 1
- available = bool(probed.get("available", False)) and generated_max >= 1
- detail = str(probed.get("detail") or "")
- except Exception as exc:
- generated_max = 0
- multiplier = 1
- available = False
- detail = str(exc)
- probed = {}
-
- diagnostic_notes: list[str] = []
+ hags, nvof = _hags_enabled(), _nvof_present()
+ signature_status = _authenticode_status(DLSSG_RUNTIME) if DLSSG_RUNTIME.is_file() else "Missing"
+ missing = [path for path in (BRIDGE, DLSSG_RUNTIME) if not path.is_file()]
+ status: dict = {}
+ detail = ""
+ if missing:
+ detail = "Frame Interpolation runtime is missing: " + ", ".join(map(str, missing))
+ else:
+ try:
+ status = initialize_bridge(int(gpu["cuda_ordinal"]))
+ except Exception as exc:
+ detail = str(exc)
+ generated_max = max(0, int(status.get("multi_frame_count_max", 0)))
+ available = bool(status.get("available")) and generated_max >= 1 and nvof
+ notes: list[str] = []
+ if not nvof:
+ notes.append("NVIDIA Optical Flow is unavailable.")
if not hags:
- diagnostic_notes.append("HAGS is disabled; the native runtime may reject Frame Generation.")
+ notes.append("HAGS is disabled; DLSS Frame Generation may be rejected by the runtime.")
if signature_status not in {"Valid", "Unavailable"}:
- diagnostic_notes.append(
- f"Authenticode status is {signature_status}; this is diagnostic only and is not blocked."
- )
- if diagnostic_notes:
- detail = " ".join([part for part in [detail, *diagnostic_notes] if part])
-
+ notes.append(f"DLSSG Authenticode status is {signature_status} (diagnostic only).")
+ detail = " ".join(part for part in (detail, *notes) if part)
return FrameInterpolationCapabilities(
- available=available,
- gpu=gpu_name,
- driver=driver,
- hags_enabled=hags,
+ available=available, gpu=gpu_name, driver=driver, hags_enabled=hags,
native_generated_frame_max=generated_max,
- native_multiplier=multiplier,
- cascade_available=available and multiplier >= 2,
- runtime_version=str(probed.get("runtime_version") or "unknown"),
- worker_version=str(probed.get("worker_version") or "unknown"),
- signature_status=signature_status,
- detail=detail,
- gpu_uuid=str(gpu.get("uuid") or ai_gpu_uuid),
- )
+ native_multiplier=generated_max + 1 if generated_max else 1,
+ cascade_available=available and generated_max >= 1,
+ runtime_version=str(status.get("runtime_version") or "unknown"),
+ bridge_version=str(status.get("bridge_version") or "missing"),
+ bridge_abi_version=int(status.get("abi_version") or BRIDGE_ABI_VERSION),
+ nvof_available=nvof, cuda_interop=bool(status.get("cuda_interop")),
+ signature_status=signature_status, detail=detail,
+ gpu_uuid=str(gpu.get("uuid") or ai_gpu_uuid))
def clear_capability_cache() -> None:
diff --git a/src/frame_interpolation/guides.py b/src/frame_interpolation/guides.py
deleted file mode 100644
index b66135d..0000000
--- a/src/frame_interpolation/guides.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-import cv2
-import numpy as np
-
-
-@dataclass(slots=True)
-class Guide:
- motion: np.ndarray
- reset: bool
- scene_score: float
- duplicate: bool
- confidence: float
-
-
-class DLSSGGuideGenerator:
- """CUDA-free guide estimation; it never creates an image frame."""
-
- def __init__(self, width: int, height: int, flow_width: int = 640) -> None:
- self.width = width
- self.height = height
- scale = min(1.0, flow_width / max(1, width))
- self.flow_width = max(64, int(round(width * scale / 2) * 2))
- self.flow_height = max(64, int(round(height * scale / 2) * 2))
- self.previous: np.ndarray | None = None
- self.zero = np.zeros((height, width, 2), dtype=np.float16)
- self.flow = cv2.DISOpticalFlow_create(cv2.DISOPTICAL_FLOW_PRESET_MEDIUM)
- self.flow.setUseSpatialPropagation(True)
- self.flow.setFinestScale(1)
-
- def _gray(self, rgba: np.ndarray) -> np.ndarray:
- gray = cv2.cvtColor(rgba, cv2.COLOR_RGBA2GRAY)
- return cv2.resize(gray, (self.flow_width, self.flow_height), interpolation=cv2.INTER_AREA)
-
- def process(self, rgba: np.ndarray, *, force_reset: bool = False) -> Guide:
- current = self._gray(rgba)
- if self.previous is None:
- guide = Guide(self.zero, True, 1.0, False, 0.0)
- else:
- difference = cv2.absdiff(current, self.previous)
- score = float(np.mean(difference)) / 255.0
- duplicate = score < 0.0005
- reset = force_reset or score > 0.24
- if reset or duplicate:
- vectors = self.zero
- confidence = 1.0 if duplicate else 0.0
- else:
- calculated = self.flow.calc(current, self.previous, None)
- calculated = cv2.resize(
- calculated, (self.width, self.height), interpolation=cv2.INTER_LINEAR
- )
- calculated[..., 0] *= self.width / self.flow_width
- calculated[..., 1] *= self.height / self.flow_height
- # Reducing an H×W×2 boolean array along its tiny last axis is
- # expensive. These two channel checks produce the same mask.
- finite = np.isfinite(calculated[..., 0])
- np.logical_and(finite, np.isfinite(calculated[..., 1]), out=finite)
- confidence = float(np.mean(finite))
- if confidence < 1.0:
- calculated[~finite] = 0
- reset = confidence < 0.98
- vectors = self.zero if reset else np.ascontiguousarray(calculated.astype(np.float16))
- guide = Guide(vectors, reset, score, duplicate, confidence)
- self.previous = current
- return guide
diff --git a/src/frame_interpolation/models.py b/src/frame_interpolation/models.py
index 1cccdce..3ff0b04 100644
--- a/src/frame_interpolation/models.py
+++ b/src/frame_interpolation/models.py
@@ -47,7 +47,7 @@ class FrameInterpolationOptions:
video_gpu_uuid: str = "auto"
target_fps: str = "60"
engine: str = "Auto"
- codec: str = "H.264"
+ codec: str = "H.264 (NVIDIA NVENC)"
container: str = "MP4"
quality: str = "Auto (Default)"
hdr_mode: bool = False
@@ -73,7 +73,10 @@ class FrameInterpolationCapabilities:
native_multiplier: int
cascade_available: bool
runtime_version: str
- worker_version: str
+ bridge_version: str
+ bridge_abi_version: int
+ nvof_available: bool
+ cuda_interop: bool
signature_status: str
detail: str = ""
gpu_uuid: str = "auto"
@@ -106,6 +109,16 @@ class FrameInterpolationResult:
scene_cuts: int
elapsed_seconds: float
timings: dict[str, float] = field(default_factory=dict)
+ bridge_version: str = ""
+ bridge_abi_version: int = 0
+ nvof_available: bool = False
+ memory_path: str = ""
+ decode_backend: str = ""
+ encode_backend: str = ""
+ upload_bytes: int = 0
+ download_bytes: int = 0
+ surface_pool_pressure: dict[str, int] = field(default_factory=dict)
+ diagnostics: dict = field(default_factory=dict)
@dataclass(slots=True)
diff --git a/src/frame_interpolation/native.py b/src/frame_interpolation/native.py
index ff4df37..9e3ef14 100644
--- a/src/frame_interpolation/native.py
+++ b/src/frame_interpolation/native.py
@@ -1,224 +1,474 @@
from __future__ import annotations
-import collections
-import struct
-import subprocess
+"""Persistent in-process CUDA/D3D12/NVOF/DLSSG bridge."""
+
+import ctypes
+import gc
import threading
-from fractions import Fraction
+from pathlib import Path
+from typing import Any
+import av
import numpy as np
-from ..core import app_log
from ..core.jobs import Cancelled, JobController
-from .capabilities import DLSSG_WORKER, RUNTIME_DIR
+from ..core.neural_bridge import _DLPackPlane
+from ..core.ngx_runtime import NGX_RUNTIME_LOCK
+from ..core.paths import RUNTIME
+RUNTIME_DIR = RUNTIME / "dlssg"
+BRIDGE = RUNTIME_DIR / "neuroframe_engine.dll"
+BRIDGE_ABI_VERSION = 1
+MEMORY_HOST, MEMORY_CUDA = 1, 2
+FORMAT_RGBA8, FORMAT_NV12, FORMAT_P010 = 1, 4, 5
+FLAG_FORCE_RESET, FLAG_DETECT_CUT = 1, 2
+MAX_GENERATED = 4
-SETUP_MAGIC = 0x31534746
-SETUP_OUT_MAGIC = 0x31524746
-FRAME_MAGIC = 0x31464746
-FRAME_OUT_MAGIC = 0x314F4746
+class FrameDescriptorV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("memory_type", ctypes.c_uint32), ("pixel_format", ctypes.c_uint32),
+ ("width", ctypes.c_uint32), ("height", ctypes.c_uint32),
+ ("planes", ctypes.c_uint64 * 4), ("strides", ctypes.c_uint32 * 4),
+ ("color_matrix", ctypes.c_uint32), ("color_range", ctypes.c_uint32),
+ ("color_primaries", ctypes.c_uint32), ("color_transfer", ctypes.c_uint32),
+ ("rotation", ctypes.c_uint32), ("reserved", ctypes.c_uint32 * 3),
+ ]
-def _read_into_exact(stream, destination) -> None:
- view = memoryview(destination).cast("B")
- offset = 0
- while offset < len(view):
- count = stream.readinto(view[offset:])
- if not count:
- raise RuntimeError("The direct DLSSG worker closed its output unexpectedly.")
- offset += count
+ @classmethod
+ def empty(cls) -> "FrameDescriptorV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
-def _read_exact(stream, size: int) -> bytes:
- data = bytearray(size)
- _read_into_exact(stream, data)
- return bytes(data)
+class SessionDescriptorV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("width", ctypes.c_uint32), ("height", ctypes.c_uint32),
+ ("generated_count", ctypes.c_uint32), ("hdr", ctypes.c_uint32),
+ ("surface_pool_size", ctypes.c_uint32), ("reserved", ctypes.c_uint32 * 5),
+ ]
-def _write_all(stream, source) -> None:
- view = memoryview(source).cast("B")
- offset = 0
- while offset < len(view):
- count = stream.write(view[offset:])
- if not count:
- raise RuntimeError("The direct DLSSG worker closed its input unexpectedly.")
- offset += count
+class FrameResultV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("generated_count", ctypes.c_uint32), ("reset", ctypes.c_uint32),
+ ("scene_cut", ctypes.c_uint32), ("duplicate", ctypes.c_uint32),
+ ("interpolation_disabled", ctypes.c_uint32), ("nvof_active", ctypes.c_uint32),
+ ("upload_bytes", ctypes.c_uint64), ("download_bytes", ctypes.c_uint64),
+ ("surface_handles", ctypes.c_uint64 * MAX_GENERATED),
+ ("scene_score", ctypes.c_double), ("input_milliseconds", ctypes.c_double),
+ ("optical_flow_milliseconds", ctypes.c_double),
+ ("ngx_milliseconds", ctypes.c_double), ("output_milliseconds", ctypes.c_double),
+ ("total_milliseconds", ctypes.c_double),
+ ]
+ @classmethod
+ def empty(cls) -> "FrameResultV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
-class DirectDLSSGSession:
- """One direct D3D12 NGX DLSSG history and binary stream."""
-
- def __init__(
- self,
- width: int,
- height: int,
- frame_count: int,
- generated_count: int,
- controller: JobController,
- ) -> None:
- if not DLSSG_WORKER.is_file():
- raise RuntimeError(f"Direct DLSSG worker is missing: {DLSSG_WORKER}")
- self.width = int(width)
- self.height = int(height)
- self.frame_bytes = self.width * self.height * 4
- self.generated_count = int(generated_count)
- self.controller = controller
- self.logs: collections.deque[str] = collections.deque(maxlen=300)
- command = [str(DLSSG_WORKER), "--serve"]
- self.process = subprocess.Popen(
- command,
- cwd=str(RUNTIME_DIR),
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- bufsize=0,
- creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
- )
- controller.register(self.process)
- assert self.process.stdin is not None
- assert self.process.stdout is not None
- assert self.process.stderr is not None
- self._log_thread = threading.Thread(target=self._read_logs, daemon=True)
- self._log_thread.start()
- self.closed = False
- self._next_index = 0
+
+class BridgeStatusV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("flags", ctypes.c_uint32), ("gpu_ordinal", ctypes.c_int32),
+ ("multi_frame_count_max", ctypes.c_uint32), ("active_sessions", ctypes.c_uint32),
+ ("bridge_version", ctypes.c_char * 32), ("runtime_version", ctypes.c_char * 32),
+ ("gpu_name", ctypes.c_char * 128), ("last_error", ctypes.c_char * 512),
+ ]
+
+ @classmethod
+ def empty(cls) -> "BridgeStatusV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
+
+
+class DLSSGBridgeError(RuntimeError):
+ pass
+
+
+class DLSSGBridgePoisonedError(DLSSGBridgeError):
+ pass
+
+
+def _decode(value: Any) -> str:
+ return bytes(value).split(b"\0", 1)[0].decode("utf-8", "replace")
+
+
+class _BridgeManager:
+ def __init__(self) -> None:
+ self.library: Any | None = None
+ self.gpu_ordinal: int | None = None
+ self.poisoned_reason = ""
+ self.ffmpeg_devices: dict[int, tuple[Any, ctypes.c_void_p]] = {}
+
+ def _ensure_ffmpeg_device(self, ordinal: int) -> None:
+ if ordinal in self.ffmpeg_devices:
+ return
+ av_root = Path(av.__file__).resolve().parent.parent
+ candidates = sorted((av_root / "av.libs").glob("avutil-*.dll"))
+ if not candidates:
+ raise DLSSGBridgeError("PyAV's CUDA runtime is missing.")
+ avutil = ctypes.CDLL(str(candidates[0]))
+ avutil.av_hwdevice_find_type_by_name.argtypes = [ctypes.c_char_p]
+ avutil.av_hwdevice_find_type_by_name.restype = ctypes.c_int
+ avutil.av_dict_set.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p,
+ ctypes.c_char_p, ctypes.c_int]
+ avutil.av_dict_set.restype = ctypes.c_int
+ avutil.av_dict_free.argtypes = [ctypes.POINTER(ctypes.c_void_p)]
+ avutil.av_hwdevice_ctx_create.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int,
+ ctypes.c_char_p, ctypes.c_void_p, ctypes.c_int]
+ avutil.av_hwdevice_ctx_create.restype = ctypes.c_int
+ kind = int(avutil.av_hwdevice_find_type_by_name(b"cuda"))
+ options, reference = ctypes.c_void_p(), ctypes.c_void_p()
+ if kind < 0 or avutil.av_dict_set(ctypes.byref(options), b"primary_ctx", b"1", 0) < 0:
+ raise DLSSGBridgeError("Could not prepare FFmpeg's CUDA primary context.")
+ try:
+ result = int(avutil.av_hwdevice_ctx_create(
+ ctypes.byref(reference), kind, str(int(ordinal)).encode("ascii"), options, 0))
+ finally:
+ avutil.av_dict_free(ctypes.byref(options))
+ if result < 0 or not reference.value:
+ raise DLSSGBridgeError(f"FFmpeg CUDA primary-context creation failed ({result}).")
+ self.ffmpeg_devices[ordinal] = (avutil, reference)
+
+ def _load(self) -> Any:
+ if self.library is not None:
+ return self.library
+ if not BRIDGE.is_file():
+ raise DLSSGBridgeError(f"Frame Interpolation bridge is missing: {BRIDGE}")
try:
- self._setup(frame_count)
+ library = ctypes.WinDLL(str(BRIDGE))
+ except OSError as exc:
+ raise DLSSGBridgeError(f"Could not load neuroframe_engine.dll: {exc}") from exc
+ library.fi_abi_version.restype = ctypes.c_uint32
+ library.fi_version.restype = ctypes.c_char_p
+ library.fi_init.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_int]
+ library.fi_init.restype = ctypes.c_int
+ library.fi_get_status_v1.argtypes = [ctypes.POINTER(BridgeStatusV1)]
+ library.fi_get_status_v1.restype = ctypes.c_int
+ library.fi_session_create.argtypes = [ctypes.POINTER(SessionDescriptorV1), ctypes.c_void_p, ctypes.c_int]
+ library.fi_session_create.restype = ctypes.c_void_p
+ library.fi_session_release.argtypes = [ctypes.c_void_p]
+ library.fi_process_frame_v1.argtypes = [
+ ctypes.c_void_p, ctypes.POINTER(FrameDescriptorV1), ctypes.c_uint32,
+ ctypes.c_uint32, ctypes.POINTER(FrameResultV1), ctypes.c_void_p, ctypes.c_int,
+ ]
+ library.fi_process_frame_v1.restype = ctypes.c_int
+ library.fi_surface_frame_desc.argtypes = [ctypes.c_void_p, ctypes.POINTER(FrameDescriptorV1)]
+ library.fi_surface_frame_desc.restype = ctypes.c_int
+ library.fi_surface_retain.argtypes = [ctypes.c_void_p]
+ library.fi_surface_release.argtypes = [ctypes.c_void_p]
+ library.fi_surface_copy_to_host.argtypes = [
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p,
+ ctypes.c_uint32, ctypes.c_void_p, ctypes.c_int,
+ ]
+ library.fi_surface_copy_to_host.restype = ctypes.c_int
+ library.fi_session_diagnostics.argtypes = [ctypes.c_void_p,
+ ctypes.POINTER(ctypes.c_uint64), ctypes.c_uint32]
+ library.fi_session_diagnostics.restype = ctypes.c_int
+ if int(library.fi_abi_version()) != BRIDGE_ABI_VERSION:
+ raise DLSSGBridgeError("Frame Interpolation bridge ABI does not match this application.")
+ self.library = library
+ return library
+
+ def guard(self) -> None:
+ if self.poisoned_reason:
+ raise DLSSGBridgePoisonedError(
+ f"The DLSSG engine is poisoned: {self.poisoned_reason}. Restart the application.")
+
+ def initialize(self, ordinal: int) -> dict[str, Any]:
+ with NGX_RUNTIME_LOCK:
+ self.guard()
+ if self.gpu_ordinal is not None and self.gpu_ordinal != int(ordinal):
+ raise DLSSGBridgeError(
+ "Frame Interpolation is already bound to another AI GPU. Restart before changing it.")
+ self._ensure_ffmpeg_device(int(ordinal))
+ library = self._load()
+ error = ctypes.create_string_buffer(4096)
+ if not library.fi_init(int(ordinal), str(RUNTIME_DIR), error, len(error)):
+ detail = error.value.decode("utf-8", "replace") or "unknown initialization failure"
+ if "poison" in detail.casefold() or "access violation" in detail.casefold():
+ self.poisoned_reason = detail
+ self.guard()
+ raise DLSSGBridgeError(f"DLSSG bridge initialization failed: {detail}")
+ self.gpu_ordinal = int(ordinal)
+ return self.status()
+
+ def status(self) -> dict[str, Any]:
+ record = BridgeStatusV1.empty()
+ if not self._load().fi_get_status_v1(ctypes.byref(record)):
+ raise DLSSGBridgeError("DLSSG bridge returned an invalid status record.")
+ return {
+ "abi_version": int(record.abi_version),
+ "bridge_version": _decode(record.bridge_version),
+ "runtime_version": _decode(record.runtime_version),
+ "initialized": bool(record.flags & 1), "poisoned": bool(record.flags & 2),
+ "available": bool(record.flags & 4), "cuda_interop": bool(record.flags & 8),
+ "gpu_ordinal": int(record.gpu_ordinal), "gpu_name": _decode(record.gpu_name),
+ "multi_frame_count_max": int(record.multi_frame_count_max),
+ "active_sessions": int(record.active_sessions), "last_error": _decode(record.last_error),
+ }
+
+ def call(self, label: str, function, keepalive: tuple[Any, ...], timeout: float) -> Any:
+ self.guard()
+ done = threading.Event()
+ result: list[Any] = []
+ failure: list[BaseException] = []
+
+ def invoke() -> None:
+ try:
+ with NGX_RUNTIME_LOCK:
+ result.append(function())
+ except BaseException as exc:
+ failure.append(exc)
+ finally:
+ done.set()
+
+ thread = threading.Thread(target=invoke, name=f"dlssg-{label}", daemon=True)
+ thread.start()
+ if not done.wait(timeout):
+ _ = keepalive
+ self.poisoned_reason = f"{label} exceeded {timeout:g} seconds"
+ self.guard()
+ if failure:
+ self.poisoned_reason = f"native exception during {label}: {failure[0]}"
+ self.guard()
+ return result[0]
+
+
+_MANAGER = _BridgeManager()
+
+
+def initialize_bridge(ordinal: int) -> dict[str, Any]:
+ return _MANAGER.initialize(ordinal)
+
+
+class DLSSGCudaSurface:
+ def __init__(self, library: Any, handle: int, descriptor: FrameDescriptorV1, ordinal: int) -> None:
+ self.library, self.handle, self.descriptor = library, int(handle), descriptor
+ self.ordinal, self.closed = int(ordinal), False
+
+ def retain(self) -> None:
+ self.library.fi_surface_retain(ctypes.c_void_p(self.handle))
+
+ def release(self) -> None:
+ self.library.fi_surface_release(ctypes.c_void_p(self.handle))
+
+ def close(self) -> None:
+ if not self.closed:
+ self.release()
+ self.closed = True
+
+ def to_av_frame(self) -> av.VideoFrame:
+ descriptor = self.descriptor
+ bits = 16 if descriptor.pixel_format == FORMAT_P010 else 8
+ item_size = bits // 8
+ y = _DLPackPlane(
+ pointer=int(descriptor.planes[0]), shape=(int(descriptor.height), int(descriptor.width)),
+ strides=(int(descriptor.strides[0]) // item_size, 1), bits=bits,
+ device_id=self.ordinal, retain=self.retain, release=self.release)
+ uv = _DLPackPlane(
+ pointer=int(descriptor.planes[1]),
+ shape=((int(descriptor.height) + 1) // 2, (int(descriptor.width) + 1) // 2, 2),
+ strides=(int(descriptor.strides[1]) // item_size, 2, 1), bits=bits,
+ device_id=self.ordinal, retain=self.retain, release=self.release)
+ try:
+ frame = av.VideoFrame.from_dlpack(
+ [y, uv], format="p010le" if bits == 16 else "nv12",
+ width=int(descriptor.width), height=int(descriptor.height),
+ device_id=self.ordinal, primary_ctx=True)
except BaseException:
- self.abort()
self.close()
raise
+ self.close()
+ return frame
- def _setup(self, frame_count: int) -> None:
- _write_all(
- self.process.stdin,
- struct.pack(
- "<5I",
- SETUP_MAGIC,
- self.width,
- self.height,
- max(1, int(frame_count)),
- self.generated_count,
- )
- )
- self.process.stdin.flush()
- magic, status, maximum, _reserved = struct.unpack(
- "<4I", _read_exact(self.process.stdout, struct.calcsize("<4I"))
- )
- if magic != SETUP_OUT_MAGIC or status:
- self.close()
- app_log.error("dlssg", f"DLSSG session creation failed (status {status})", self.log_text()[-500:])
- raise RuntimeError(
- f"DLSSG session creation failed (status {status}); runtime maximum is "
- f"{maximum + 1}×.\n{self.log_text()}"
- )
- if self.generated_count > maximum:
- self.close()
- raise RuntimeError(
- f"DLSSG worker rejected {self.generated_count} generated frame(s); "
- f"MultiFrameCountMax is {maximum}."
- )
-
- def _read_logs(self) -> None:
- assert self.process.stderr is not None
- for raw in iter(self.process.stderr.readline, b""):
- self.logs.append(raw.decode("utf-8", "replace").rstrip())
-
- def log_text(self) -> str:
- return "\n".join(self.logs)
-
- def process_frame(
- self,
- rgba: np.ndarray,
- motion: np.ndarray,
- timestamp: Fraction,
- *,
- reset: bool,
- ) -> list[np.ndarray]:
+ def to_host_frame(self) -> av.VideoFrame:
+ pixel_format = "p010le" if self.descriptor.pixel_format == FORMAT_P010 else "nv12"
+ frame = av.VideoFrame(int(self.descriptor.width), int(self.descriptor.height), pixel_format)
+ error = ctypes.create_string_buffer(2048)
+ if not self.library.fi_surface_copy_to_host(
+ ctypes.c_void_p(self.handle), ctypes.c_void_p(int(frame.planes[0].buffer_ptr)),
+ int(frame.planes[0].line_size), ctypes.c_void_p(int(frame.planes[1].buffer_ptr)),
+ int(frame.planes[1].line_size), error, len(error)):
+ raise DLSSGBridgeError(error.value.decode("utf-8", "replace") or "CUDA download failed")
+ self.close()
+ return frame
+
+
+class DirectDLSSGSession:
+ """One persistent interpolation history; all frame pixels stay in-process."""
+
+ def __init__(self, width: int, height: int, generated_count: int,
+ controller: JobController, gpu_ordinal: int, *, hdr: bool = False,
+ timeout: float = 180.0) -> None:
+ self.width, self.height = int(width), int(height)
+ self.generated_count, self.controller = int(generated_count), controller
+ self.ordinal, self.timeout, self.hdr = int(gpu_ordinal), float(timeout), bool(hdr)
+ self.library = _MANAGER._load()
+ self.status = _MANAGER.initialize(self.ordinal)
+ descriptor = SessionDescriptorV1()
+ descriptor.struct_size, descriptor.abi_version = ctypes.sizeof(descriptor), BRIDGE_ABI_VERSION
+ descriptor.width, descriptor.height = self.width, self.height
+ descriptor.generated_count, descriptor.hdr = self.generated_count, int(self.hdr)
+ descriptor.surface_pool_size = 8
+ error = ctypes.create_string_buffer(4096)
+ handle = _MANAGER.call(
+ "create", lambda: self.library.fi_session_create(ctypes.byref(descriptor), error, len(error)),
+ (descriptor, error), self.timeout)
+ if not handle:
+ raise DLSSGBridgeError(error.value.decode("utf-8", "replace") or "DLSSG session creation failed")
+ self.handle, self.closed = int(handle), False
+ self.completed_frames = self.scene_cuts = self.duplicates = 0
+ self.upload_bytes = self.download_bytes = 0
+ self.stage_timings = {"input_ms": 0.0, "optical_flow_ms": 0.0,
+ "ngx_ms": 0.0, "output_ms": 0.0, "total_ms": 0.0}
+
+ def _source_descriptor(self, frame: Any, *, color_matrix: int, color_range: int,
+ color_primaries: int, color_transfer: int,
+ rotation: int) -> tuple[FrameDescriptorV1, tuple[Any, ...]]:
+ descriptor = FrameDescriptorV1.empty()
+ descriptor.color_matrix, descriptor.color_range = int(color_matrix), int(color_range)
+ descriptor.color_primaries, descriptor.color_transfer = int(color_primaries), int(color_transfer)
+ descriptor.rotation = int(rotation) % 360
+ if isinstance(frame, np.ndarray):
+ pixels = np.ascontiguousarray(frame, dtype=np.uint8)
+ if pixels.shape != (self.height, self.width, 4):
+ raise ValueError(f"Host DLSSG frame has unexpected shape {pixels.shape}.")
+ descriptor.memory_type, descriptor.pixel_format = MEMORY_HOST, FORMAT_RGBA8
+ descriptor.width, descriptor.height = self.width, self.height
+ descriptor.planes[0], descriptor.strides[0] = int(pixels.ctypes.data), int(pixels.strides[0])
+ descriptor.rotation = 0
+ return descriptor, (pixels,)
+ if frame.format.name != "cuda":
+ if frame.format.name not in {"p010", "p010le"} or len(frame.planes) < 2:
+ raise ValueError("Host DLSSG video input must be RGBA8 or P010.")
+ descriptor.memory_type, descriptor.pixel_format = MEMORY_HOST, FORMAT_P010
+ descriptor.width, descriptor.height = int(frame.width), int(frame.height)
+ descriptor.planes[0], descriptor.planes[1] = (
+ int(frame.planes[0].buffer_ptr), int(frame.planes[1].buffer_ptr))
+ descriptor.strides[0], descriptor.strides[1] = (
+ int(frame.planes[0].line_size), int(frame.planes[1].line_size))
+ descriptor.rotation = 0
+ return descriptor, (frame,)
+ if len(frame.planes) < 2:
+ raise ValueError("CUDA DLSSG input must be a CUDA NV12/P010 PyAV frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ if sw_format not in {"nv12", "p010", "p010le"}:
+ raise ValueError(f"Unsupported CUDA interpolation format: {sw_format or 'unknown'}.")
+ descriptor.memory_type = MEMORY_CUDA
+ descriptor.pixel_format = FORMAT_P010 if sw_format != "nv12" else FORMAT_NV12
+ descriptor.width, descriptor.height = int(frame.width), int(frame.height)
+ descriptor.planes[0], descriptor.planes[1] = (int(frame.planes[0].buffer_ptr),
+ int(frame.planes[1].buffer_ptr))
+ descriptor.strides[0], descriptor.strides[1] = (int(frame.planes[0].line_size),
+ int(frame.planes[1].line_size))
+ return descriptor, (frame,)
+
+ def process_frame(self, frame: Any, *, force_reset: bool = False,
+ detect_scene_cut: bool = True, output_cuda: bool = True,
+ output_p010: bool = False, color_matrix: int = 1,
+ color_range: int = 0, color_primaries: int = 1,
+ color_transfer: int = 0, rotation: int = 0) -> tuple[list[av.VideoFrame], dict[str, Any]]:
if self.closed:
- raise RuntimeError("DLSSG session is closed.")
+ raise DLSSGBridgeError("DLSSG session is closed.")
if self.controller.cancel.is_set():
raise Cancelled("Frame interpolation was cancelled.")
- color = np.ascontiguousarray(rgba, dtype=np.uint8)
- vectors = np.ascontiguousarray(motion, dtype=np.float16)
- if color.shape != (self.height, self.width, 4):
- raise ValueError(f"DLSSG color frame has unexpected shape {color.shape}.")
- if vectors.shape != (self.height, self.width, 2):
- raise ValueError(f"DLSSG motion field has unexpected shape {vectors.shape}.")
- assert self.process.stdin is not None
- assert self.process.stdout is not None
- _write_all(
- self.process.stdin,
- struct.pack(
- "<4I2q",
- FRAME_MAGIC,
- self._next_index,
- int(reset),
- 0,
- timestamp.numerator,
- timestamp.denominator,
- )
- )
- _write_all(self.process.stdin, color)
- _write_all(self.process.stdin, vectors)
- self.process.stdin.flush()
- self._next_index += 1
- magic, status, generated, disabled = struct.unpack(
- "<4I", _read_exact(self.process.stdout, struct.calcsize("<4I"))
- )
- if magic != FRAME_OUT_MAGIC or status:
- app_log.error("dlssg", f"DLSSG evaluation failed at frame {self._next_index - 1} (status {status})", self.log_text()[-500:])
- raise RuntimeError(
- f"DLSSG evaluation failed at input frame {self._next_index - 1} "
- f"(status {status}).\n{self.log_text()}"
- )
- if generated > self.generated_count or (disabled and generated):
- raise RuntimeError("DLSSG worker returned an invalid generated-frame count.")
- if disabled:
- return []
- frames = []
- for _ in range(generated):
- # Each returned array owns its storage until every downstream user
- # releases it. Reading into it avoids three full-frame copies.
- frame = np.empty((self.height, self.width, 4), dtype=np.uint8)
- _read_into_exact(self.process.stdout, frame)
- frames.append(frame)
- return frames
+ source, keepalive = self._source_descriptor(
+ frame, color_matrix=color_matrix, color_range=color_range,
+ color_primaries=color_primaries, color_transfer=color_transfer, rotation=rotation)
+ result = FrameResultV1.empty()
+ error = ctypes.create_string_buffer(4096)
+ flags = int(force_reset) * FLAG_FORCE_RESET | int(detect_scene_cut) * FLAG_DETECT_CUT
+ output_format = FORMAT_P010 if output_p010 else FORMAT_NV12
+ ok = _MANAGER.call(
+ "process", lambda: self.library.fi_process_frame_v1(
+ ctypes.c_void_p(self.handle), ctypes.byref(source), output_format, flags,
+ ctypes.byref(result), error, len(error)),
+ (*keepalive, source, result, error), self.timeout)
+ if not ok:
+ detail = error.value.decode("utf-8", "replace") or "unknown native failure"
+ if "access violation" in detail.casefold() or "poison" in detail.casefold():
+ _MANAGER.poisoned_reason = detail
+ raise DLSSGBridgeError(f"DLSSG evaluation failed: {detail}")
+ frames: list[av.VideoFrame] = []
+ try:
+ for index in range(int(result.generated_count)):
+ handle = int(result.surface_handles[index])
+ descriptor = FrameDescriptorV1.empty()
+ if not handle or not self.library.fi_surface_frame_desc(
+ ctypes.c_void_p(handle), ctypes.byref(descriptor)):
+ raise DLSSGBridgeError("DLSSG returned an invalid output surface.")
+ surface = DLSSGCudaSurface(self.library, handle, descriptor, self.ordinal)
+ frames.append(surface.to_av_frame() if output_cuda else surface.to_host_frame())
+ except BaseException:
+ for index in range(len(frames), int(result.generated_count)):
+ handle = int(result.surface_handles[index])
+ if handle:
+ self.library.fi_surface_release(ctypes.c_void_p(handle))
+ raise
+ self.completed_frames += 1
+ self.scene_cuts += int(result.scene_cut)
+ self.duplicates += int(result.duplicate)
+ self.upload_bytes += int(result.upload_bytes)
+ self.download_bytes += int(result.download_bytes)
+ detail = {
+ "reset": bool(result.reset), "scene_cut": bool(result.scene_cut),
+ "duplicate": bool(result.duplicate), "scene_score": float(result.scene_score),
+ "interpolation_disabled": bool(result.interpolation_disabled),
+ "nvof_active": bool(result.nvof_active),
+ "input_ms": float(result.input_milliseconds),
+ "optical_flow_ms": float(result.optical_flow_milliseconds),
+ "ngx_ms": float(result.ngx_milliseconds),
+ "output_ms": float(result.output_milliseconds),
+ "total_ms": float(result.total_milliseconds),
+ "upload_bytes": int(result.upload_bytes), "download_bytes": int(result.download_bytes),
+ }
+ for key in self.stage_timings:
+ self.stage_timings[key] += detail[key]
+ return frames, detail
- def abort(self) -> None:
- if self.process.poll() is None:
- try:
- self.process.terminate()
- except OSError:
- pass
+ def diagnostics(self) -> dict[str, Any]:
+ values = (ctypes.c_uint64 * 8)()
+ if self.closed or not self.library.fi_session_diagnostics(
+ ctypes.c_void_p(self.handle), values, len(values)):
+ return {}
+ return {
+ "processed_frames": int(values[0]), "scene_cuts": int(values[1]),
+ "duplicates": int(values[2]), "upload_bytes": int(values[3]),
+ "download_bytes": int(values[4]), "surface_pool_waits": int(values[5]),
+ "surface_pool_allocated": int(values[6]), "surface_pool_capacity": int(values[7]),
+ "nvof_mode": "SLOW", "timings_ms": dict(self.stage_timings),
+ }
- def close(self) -> None:
- if getattr(self, "closed", False):
+ def close(self, *, abort: bool = False) -> None:
+ if self.closed:
return
self.closed = True
- process = self.process
- try:
- if process.stdin and not process.stdin.closed:
- process.stdin.close()
- except OSError:
- pass
- try:
- process.wait(timeout=20)
- except subprocess.TimeoutExpired:
- process.terminate()
- try:
- process.wait(timeout=5)
- except subprocess.TimeoutExpired:
- process.kill()
- process.wait(timeout=5)
- self.controller.unregister(process)
- self._log_thread.join(timeout=1)
- for stream in (process.stdout, process.stderr):
- if stream is not None:
- stream.close()
+ if _MANAGER.poisoned_reason:
+ return
+ # Drop Python/PyAV references before releasing the session. The native
+ # bridge also defers destruction until every DLPack surface is returned,
+ # which makes cancellation and encoder exceptions safe.
+ gc.collect()
+ _MANAGER.call("release", lambda: self.library.fi_session_release(
+ ctypes.c_void_p(self.handle)), (self,), 10.0 if abort else self.timeout)
+
+ def abort(self) -> None:
+ self.close(abort=True)
def __enter__(self) -> "DirectDLSSGSession":
return self
- def __exit__(self, *_args) -> None:
- self.close()
+ def __exit__(self, exc_type, *_args) -> None:
+ self.close(abort=exc_type is not None)
diff --git a/src/frame_interpolation/pipeline.py b/src/frame_interpolation/pipeline.py
deleted file mode 100644
index 61746f1..0000000
--- a/src/frame_interpolation/pipeline.py
+++ /dev/null
@@ -1,218 +0,0 @@
-"""Ordered CPU/GPU overlap with bounded admission and a single queue owner."""
-from __future__ import annotations
-
-import time
-from collections import deque
-from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
-from typing import Callable, Iterable
-
-from ..core.jobs import Cancelled, JobController
-
-BUFFER_LIMIT_BYTES = 1 << 30
-
-
-class PipeWriter:
- """A subprocess pipe cannot seek, even when its Python wrapper has seek()."""
-
- def __init__(self, stream):
- self.stream = stream
-
- def write(self, data):
- view = memoryview(data)
- offset = 0
- while offset < len(view):
- count = self.stream.write(view[offset:])
- if not count:
- raise BrokenPipeError("Video encoder closed its input.")
- offset += count
- return offset
-
- def flush(self):
- self.stream.flush()
-
-
-def run_pipeline(
- source: Iterable,
- stages: list,
- writer,
- controller: JobController,
- frame_bytes: int,
- timings: dict[str, float],
- progress: Callable[[int], None] | None = None,
- *,
- buffer_limit_bytes: int = BUFFER_LIMIT_BYTES,
-) -> dict:
- """Coordinate four single-owner workers; workers never wait on queue puts.
-
- One credit represents one packed RGBA or FP16 motion array. Reserve all
- possible native outputs before evaluation, plus motion storage before guide
- preparation. Conservative accounting counts shared arrays more than once.
- Existing per-stage history, guide scratch space and codec buffers are not
- additional queue storage and are not included in this limit.
- """
- if frame_bytes <= 0 or buffer_limit_bytes <= 0:
- raise ValueError("Frame size and buffer limit must be positive.")
- started = time.perf_counter()
- iterator = iter(source)
- max_generated = max((stage.generated_count for stage in stages), default=0)
- capacity = buffer_limit_bytes // frame_bytes
- decoded = 0
- processed = 0
- used = peak = 0
-
- def timed(key, function, *args):
- before = time.perf_counter()
- try:
- return function(*args)
- finally:
- timings[key] = timings.get(key, 0.0) + time.perf_counter() - before
-
- # Even one input/output transaction may exceed the queue budget for enormous
- # frames. Keep the sequential path available without allocating extra queues.
- if capacity < max_generated + 3:
- try:
- for frame in iterator:
- if controller.cancel.is_set():
- raise Cancelled("Frame interpolation stopped by user.")
- items = [frame]
- for index, stage in enumerate(stages):
- next_items = []
- for item in items:
- prepared = timed(f"guide_{index + 1}_seconds", stage.prepare, item)
- next_items.extend(timed(f"native_{index + 1}_seconds", stage.evaluate, prepared))
- items = next_items
- for item in items:
- timed("encode_write_seconds", writer.push, item)
- decoded += 1
- if progress:
- progress(decoded)
- timed("encode_write_seconds", writer.finish)
- except BaseException:
- controller.terminate_processes()
- raise
- finally:
- if hasattr(iterator, "close"):
- iterator.close()
- timings["pipeline_seconds"] = time.perf_counter() - started
- return {"mode": "sequential", "buffer_limit_bytes": buffer_limit_bytes,
- "peak_buffer_bytes": 0, "decoded_frames": decoded}
-
- edges = [deque() for _ in range(len(stages) + 1)]
- prepared_queues = [deque() for _ in stages]
- edge_capacity = max(4, max_generated + 1)
- pools = {name: ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"FI-{name}")
- for name in ("decode", "guide", "native", "encode")}
- pending = {}
- ended = False
-
- def reserve(credits):
- nonlocal used, peak
- used += credits
- if used > capacity:
- raise RuntimeError("Frame interpolation buffer reservation exceeded its limit.")
- peak = max(peak, used)
-
- def consume(items):
- for item in items:
- if controller.cancel.is_set():
- raise Cancelled("Frame interpolation stopped by user.")
- writer.push(item)
-
- try:
- while True:
- if controller.cancel.is_set():
- raise Cancelled("Frame interpolation stopped by user.")
- # Harvest before dispatch. Results own their arrays until consumed;
- # neither futures nor workers can block while publishing a result.
- for name, (future, index, credits) in list(pending.items()):
- if not future.done():
- continue
- result = future.result()
- del pending[name]
- if name == "decode":
- if result is None:
- ended = True
- used -= 1
- else:
- edges[0].append(result)
- decoded += 1
- elif name == "guide":
- prepared_queues[index].append(result)
- elif name == "native":
- if len(result) > credits + 1:
- raise RuntimeError("DLSSG produced more frames than reserved.")
- edges[index + 1].extend(result)
- used -= 2 + credits - len(result)
- if index == 0:
- processed += 1
- if progress:
- progress(processed)
- else:
- used -= credits
- if not stages and progress:
- processed += credits
- progress(processed)
- # Do not keep the last harvested frame/result alive in locals.
- result = None
- future = None
-
- if "encode" not in pending and edges[-1]:
- items = [edges[-1].popleft() for _ in range(min(edge_capacity, len(edges[-1])))]
- pending["encode"] = (pools["encode"].submit(timed, "encode_write_seconds", consume, items),
- -1, len(items))
- items = None
-
- if "native" not in pending:
- for index in reversed(range(len(stages))):
- count = stages[index].generated_count
- if (prepared_queues[index] and used + count <= capacity
- and len(edges[index + 1]) + count + 1 <= edge_capacity):
- reserve(count)
- prepared = prepared_queues[index].popleft()
- pending["native"] = (pools["native"].submit(
- timed, f"native_{index + 1}_seconds", stages[index].evaluate, prepared), index, count)
- prepared = None
- break
-
- if "guide" not in pending:
- for index in reversed(range(len(stages))):
- # Retain native output headroom even if the GPU is idle.
- if (edges[index] and len(prepared_queues[index]) < 2
- and used + 1 <= capacity - max_generated):
- reserve(1)
- item = edges[index].popleft()
- pending["guide"] = (pools["guide"].submit(
- timed, f"guide_{index + 1}_seconds", stages[index].prepare, item), index, 1)
- item = None
- break
-
- if (not ended and "decode" not in pending and len(edges[0]) < edge_capacity
- and used + 1 <= capacity - max_generated - 1):
- reserve(1)
- pending["decode"] = (pools["decode"].submit(timed, "decode_seconds", next, iterator, None), -1, 1)
-
- if not pending:
- if ended and not any(edges) and not any(prepared_queues):
- break
- raise RuntimeError("Frame interpolation could not drain its bounded pipeline.")
- before_wait = time.perf_counter()
- wait([value[0] for value in pending.values()], timeout=0.05, return_when=FIRST_COMPLETED)
- timings["coordinator_wait_seconds"] = timings.get("coordinator_wait_seconds", 0.0) + time.perf_counter() - before_wait
-
- pools["encode"].submit(timed, "encode_write_seconds", writer.finish).result()
- except BaseException:
- # Closing worker/encoder processes releases any OS pipe I/O in flight.
- # Do not mark the shared batch controller cancelled for a failed item.
- controller.terminate_processes()
- raise
- finally:
- for pool in pools.values():
- pool.shutdown(wait=True, cancel_futures=True)
- pending.clear()
- for edge in [*edges, *prepared_queues]:
- edge.clear()
- if hasattr(iterator, "close"):
- iterator.close()
- timings["pipeline_seconds"] = time.perf_counter() - started
- return {"mode": "overlapped", "buffer_limit_bytes": buffer_limit_bytes,
- "peak_buffer_bytes": peak * frame_bytes, "decoded_frames": decoded}
diff --git a/src/frame_interpolation/preview.py b/src/frame_interpolation/preview.py
index bb9013a..13eb8d4 100644
--- a/src/frame_interpolation/preview.py
+++ b/src/frame_interpolation/preview.py
@@ -42,7 +42,10 @@ def frame_interpolation_capability_text() -> str:
detail = f"\n{capabilities.detail}" if capabilities.detail else ""
return (
f"{state} — GPU: {capabilities.gpu} | Driver: {capabilities.driver} | "
- f"HAGS: {hags}\nNative maximum: {native} | Experimental cascade: {cascade} | "
+ f"HAGS: {hags}\nNative maximum: {native} | Cascade: {cascade} | "
+ f"NVOF: {'SLOW/available' if capabilities.nvof_available else 'unavailable'}\n"
+ f"Bridge: {capabilities.bridge_version} (ABI {capabilities.bridge_abi_version}) | "
+ f"CUDA interop: {'ready' if capabilities.cuda_interop else 'unavailable'} | "
f"DLSSG runtime: {capabilities.runtime_version}{detail}"
)
@@ -56,7 +59,7 @@ def describe_frame_interpolation_plan(
if not selected:
return "Choose a video to preflight its DLSSG path and temporal precision."
try:
- metadata = probe_video(selected, count_mode="metadata")
+ metadata = probe_video(selected, count_mode="metadata", inspect_timestamps=True)
ai_gpu_uuid, _video_gpu_uuid = processing_gpu_settings()
capabilities = probe_frame_interpolation_capabilities(ai_gpu_uuid)
plan = choose_interpolation_plan(
diff --git a/src/frame_interpolation/processor.py b/src/frame_interpolation/processor.py
index 62cff9d..1881141 100644
--- a/src/frame_interpolation/processor.py
+++ b/src/frame_interpolation/processor.py
@@ -1,140 +1,249 @@
from __future__ import annotations
+import gc
import math
import os
-import shutil
-import subprocess
+import queue
+import tempfile
import threading
import time
from contextlib import nullcontext, suppress
from dataclasses import dataclass, replace
from fractions import Fraction
from pathlib import Path
-from typing import Callable
+from typing import Any, Callable
import av
import numpy as np
+from av.codec.hwaccel import HWAccel
from ..core import app_log, ffmpeg
+from ..core.disk_paths import OutputFile, prepare_output_dir
from ..core.gpu_selection import resolve_runtime_ai_gpu
from ..core.jobs import Cancelled, active_job
from ..core.naming import output_filename, validate_rename
-from ..core.disk_paths import OutputFile, prepare_output_dir
from ..core.paths import JOBS, OUTPUTS
from ..core.runtime import prepare_runtime, rotate_frame
+from ..upscale.video.cuda_transfer import CudaTransferPool
from .capabilities import probe_frame_interpolation_capabilities
-from .guides import DLSSGGuideGenerator, Guide
from .models import FrameInterpolationOptions, FrameInterpolationResult
from .native import DirectDLSSGSession
from .scheduler import choose_interpolation_plan, output_frame_count
-from .pipeline import run_pipeline, PipeWriter
_BATCH_CONTEXT = threading.local()
+_END = object()
+_NVENC = {
+ "H.264 (NVIDIA NVENC)": "h264_nvenc",
+ "H.265 (NVIDIA NVENC)": "hevc_nvenc",
+ "AV1 (NVIDIA NVENC)": "av1_nvenc",
+}
+
-@dataclass(frozen=True, slots=True)
+@dataclass(slots=True)
class TimedFrame:
- rgba: np.ndarray
+ bridge_frame: Any
+ encode_frame: av.VideoFrame
timestamp: Fraction
segment: int
provenance: str
source_index: int | None = None
-@dataclass(frozen=True, slots=True)
-class PreparedFrame:
- frame: TimedFrame
- previous_timestamp: Fraction | None
- guide: Guide
- reset: bool
+@dataclass(slots=True)
+class _Encoder:
+ name: str
+ display: str
+ pixel_format: str
+ cuda_input: bool
+ options: dict[str, str]
+ bitrate: int
+
+
+def _is_nvenc(codec: str) -> bool:
+ return ffmpeg._normalize_codec(codec) in _NVENC
+
+
+def automatic_cuda_path(codec: str, rotation: int | float = 0) -> bool:
+ """Select the fastest valid transport without a user-facing mode switch."""
+ return _is_nvenc(codec) and not bool(rotation)
+
+
+def _codec_available(name: str) -> bool:
+ try:
+ av.codec.Codec(name, "w")
+ return True
+ except Exception:
+ return False
+
+
+def _encoder(options: FrameInterpolationOptions, codec: str, width: int, height: int,
+ rate: Fraction, hdr: bool, cuda_input: bool, ordinal: int) -> _Encoder:
+ normalized = ffmpeg._normalize_codec(codec)
+ quality = ffmpeg.resolve_encoding_quality(
+ options.quality, codec, width, height, float(rate), hdr_mode=hdr)
+ bitrate = int(quality.get("target_bitrate_kbps") or 0) * 1000
+ if normalized in _NVENC:
+ name = _NVENC[normalized]
+ values = {"preset": "p6", "rc": "vbr", "gpu": str(ordinal),
+ "rc-lookahead": "0", "surfaces": "8", "delay": "3"}
+ if name != "av1_nvenc":
+ values["tune"] = "hq"
+ if quality["mode"] == "constant-quality":
+ values["cq"] = "0"
+ bitrate = 0
+ return _Encoder(name, name, "cuda" if cuda_input else ("p010le" if hdr else "nv12"),
+ cuda_input, values, bitrate)
+ common = {"crf": "0"} if quality["mode"] == "constant-quality" else {}
+ if normalized == "H.264":
+ return _Encoder("libx264", "libx264", "yuv420p", False,
+ {"preset": "slow", **common}, bitrate)
+ if normalized in {"H.265", "HEVC"}:
+ return _Encoder("libx265", "libx265", "yuv420p10le" if hdr else "yuv420p",
+ False, {"preset": "slow", **common}, bitrate)
+ if normalized == "AV1":
+ name = "libsvtav1" if _codec_available("libsvtav1") else "libaom-av1"
+ if not _codec_available(name):
+ raise RuntimeError("No in-process CPU AV1 encoder is available.")
+ values = ({"preset": "6"} if name == "libsvtav1" else {"cpu-used": "4"})
+ values.update(common)
+ return _Encoder(name, name, "yuv420p10le" if hdr else "yuv420p", False, values, bitrate)
+ if normalized == "ProRes Proxy":
+ values = {"profile": "0"}
+ if quality.get("bits_per_mb") is not None:
+ values["bits_per_mb"] = str(int(quality["bits_per_mb"]))
+ return _Encoder("prores_ks", "prores_ks (Proxy)", "yuv422p10le", False, values, 0)
+ raise ValueError(f"Unsupported in-process codec: {codec}.")
+
+
+def _matrix_code(metadata: dict[str, Any]) -> int:
+ value = str(metadata.get("color_space") or "").casefold()
+ if value in {"bt2020nc", "bt2020c"}:
+ return 2
+ if value in {"bt470bg", "smpte170m", "smpte240m", "fcc"}:
+ return 0
+ return 1
+
+
+def _range_code(metadata: dict[str, Any]) -> int:
+ return int(str(metadata.get("color_range") or "").casefold() in {"pc", "jpeg", "full"})
+
+
+def _primaries_code(metadata: dict[str, Any]) -> int:
+ value = str(metadata.get("color_primaries") or "").casefold()
+ return 2 if value == "bt2020" else 0 if value in {"bt470bg", "bt470m"} else 1
+
+
+def _transfer_code(metadata: dict[str, Any]) -> int:
+ return {"iec61966-2-1": 1, "srgb": 1, "gamma22": 2, "bt470m": 2,
+ "gamma28": 3, "bt470bg": 3, "linear": 4}.get(
+ str(metadata.get("color_transfer") or "").casefold(), 0)
+
+
+def _set_color_properties(context: Any, metadata: dict[str, Any], hdr: bool) -> None:
+ primaries = str(metadata.get("color_primaries") or "").casefold()
+ transfer = str(metadata.get("color_transfer") or "").casefold()
+ matrix = str(metadata.get("color_space") or "").casefold()
+ context.color_primaries = {"bt2020": 9, "smpte432": 12}.get(primaries, 1)
+ context.color_trc = {"smpte2084": 16, "arib-std-b67": 18, "iec61966-2-1": 13}.get(
+ transfer, 1)
+ context.colorspace = {"bt2020nc": 9, "bt2020c": 10, "smpte170m": 6,
+ "bt470bg": 5}.get(matrix, 1)
+ context.color_range = 2 if _range_code(metadata) else 1
+ if hdr and primaries in {"", "unknown"}:
+ context.color_primaries, context.color_trc, context.colorspace = 9, 16, 9
+
+
+def _put(target: queue.Queue, item: Any, stop: threading.Event,
+ controller: Any, failures: list[BaseException]) -> None:
+ while not stop.is_set():
+ if controller.cancel.is_set():
+ stop.set()
+ raise Cancelled("Frame interpolation stopped by user.")
+ if failures:
+ stop.set()
+ raise failures[0]
+ try:
+ target.put(item, timeout=0.05)
+ return
+ except queue.Full:
+ continue
+ raise failures[0] if failures else Cancelled("Frame interpolation stopped by user.")
+
+
+def _get(source: queue.Queue, stop: threading.Event,
+ controller: Any, failures: list[BaseException]) -> Any:
+ while True:
+ if controller.cancel.is_set():
+ stop.set()
+ raise Cancelled("Frame interpolation stopped by user.")
+ if failures:
+ stop.set()
+ raise failures[0]
+ try:
+ return source.get(timeout=0.05)
+ except queue.Empty:
+ if stop.is_set():
+ raise failures[0] if failures else Cancelled("Frame interpolation stopped by user.")
+
+
+def _validate_decoded_frame(frame: Any) -> Any:
+ if bool(getattr(frame, "is_corrupt", False)):
+ raise RuntimeError("The video decoder returned a corrupt frame.")
+ return frame
class DLSSGStage:
- def __init__(
- self,
- session: DirectDLSSGSession,
- width: int,
- height: int,
- generated_count: int,
- *,
- detect_source_cuts: bool,
- ) -> None:
- self.session = session
- self.guides = DLSSGGuideGenerator(width, height)
- self.generated_count = generated_count
+ def __init__(self, session: DirectDLSSGSession, generated_count: int, *,
+ detect_source_cuts: bool, cuda_output: bool, output_p010: bool,
+ colors: dict[str, int]) -> None:
+ self.session, self.generated_count = session, int(generated_count)
+ self.detect_source_cuts, self.cuda_output = detect_source_cuts, cuda_output
+ self.output_p010, self.colors = output_p010, colors
self.previous: TimedFrame | None = None
- self.detect_source_cuts = detect_source_cuts
- self.scene_cuts = 0
- self.duplicates = 0
+ self.scene_cuts = self.duplicates = 0
- def prepare(self, frame: TimedFrame) -> PreparedFrame:
- # Only the guide worker advances this history. Capture all temporal
- # decisions here so native evaluation can overlap the next preparation.
+ def push(self, frame: TimedFrame) -> list[TimedFrame]:
previous = self.previous
- force_reset = self.previous is not None and frame.segment != self.previous.segment
- guide = self.guides.process(frame.rgba, force_reset=force_reset)
- if previous is not None and self.detect_source_cuts and guide.reset and not force_reset:
+ force_reset = previous is not None and frame.segment != previous.segment
+ bridge_frame = frame.bridge_frame
+ if (isinstance(bridge_frame, av.VideoFrame)
+ and bridge_frame.format.name not in {"cuda", "p010", "p010le"}):
+ bridge_frame = np.ascontiguousarray(bridge_frame.to_ndarray(format="rgba"))
+ generated, detail = self.session.process_frame(
+ bridge_frame, force_reset=force_reset,
+ detect_scene_cut=self.detect_source_cuts, output_cuda=self.cuda_output,
+ output_p010=self.output_p010, **self.colors)
+ if previous is not None and detail["scene_cut"] and self.detect_source_cuts:
frame = replace(frame, segment=previous.segment + 1)
- force_reset = True
self.scene_cuts += 1
- if previous is not None and guide.duplicate:
- self.duplicates += 1
- self.previous = frame
- return PreparedFrame(frame, previous.timestamp if previous else None, guide,
- previous is None or force_reset or guide.reset)
-
- def evaluate(self, prepared: PreparedFrame) -> list[TimedFrame]:
- frame = prepared.frame
- generated = self.session.process_frame(
- frame.rgba,
- prepared.guide.motion,
- frame.timestamp,
- reset=prepared.reset,
- )
+ self.duplicates += int(previous is not None and detail["duplicate"])
output: list[TimedFrame] = []
- if not prepared.reset and prepared.previous_timestamp is not None:
- interval = frame.timestamp - prepared.previous_timestamp
- for index, rgba in enumerate(generated, start=1):
- output.append(
- TimedFrame(
- rgba,
- prepared.previous_timestamp
- + interval * Fraction(index, self.generated_count + 1),
- frame.segment,
- "DLSSG",
- None,
- )
- )
+ if previous is not None and not detail["reset"]:
+ interval = frame.timestamp - previous.timestamp
+ for index, pixels in enumerate(generated, start=1):
+ output.append(TimedFrame(
+ pixels, pixels,
+ previous.timestamp + interval * Fraction(index, self.generated_count + 1),
+ frame.segment, "DLSSG"))
+ self.previous = frame
output.append(frame)
return output
- def push(self, frame: TimedFrame) -> list[TimedFrame]:
- return self.evaluate(self.prepare(frame))
-
class NearestTimestampWriter:
- def __init__(self, nut, stream, target_rate: Fraction, output_count: int) -> None:
- self.nut = nut
- self.stream = stream
- self.target_rate = target_rate
- self.output_count = output_count
+ def __init__(self, target_rate: Fraction, output_count: int,
+ emit: Callable[[TimedFrame, int], None]) -> None:
+ self.target_rate, self.output_count, self.emit = target_rate, int(output_count), emit
self.next_index = 0
self.previous: TimedFrame | None = None
self.tie_late = False
- self.copied = 0
- self.generated = 0
+ self.copied = self.generated = 0
self.max_error = Fraction(0)
self.selected_real_ids: set[int] = set()
def _write(self, frame: TimedFrame, ideal: Fraction) -> None:
- packet = av.Packet(memoryview(frame.rgba).cast("B"))
- packet.stream = self.stream
- packet.pts = packet.dts = self.next_index
- packet.duration = 1
- packet.time_base = Fraction(1, 1) / self.target_rate
- packet.is_keyframe = True
- self.nut.mux(packet)
- error = abs(frame.timestamp - ideal)
- self.max_error = max(self.max_error, error)
+ self.emit(frame, self.next_index)
+ self.max_error = max(self.max_error, abs(frame.timestamp - ideal))
if frame.provenance == "DLSSG":
self.generated += 1
else:
@@ -153,43 +262,20 @@ class NearestTimestampWriter:
if ideal < midpoint:
self._write(self.previous, ideal)
elif ideal == midpoint:
- selected = current if self.tie_late else self.previous
+ self._write(current if self.tie_late else self.previous, ideal)
self.tie_late = not self.tie_late
- self._write(selected, ideal)
else:
break
self.previous = current
def finish(self) -> None:
if self.previous is None:
- raise RuntimeError("The input video contains no decodable frames.")
+ raise ValueError("The input video contains no decodable frames.")
while self.next_index < self.output_count:
- ideal = Fraction(self.next_index, 1) / self.target_rate
- # A finite video has no future endpoint beyond its final real frame.
- # Extend that endpoint over the final frame-duration without asking a
- # synthesizer to extrapolate beyond known motion.
- endpoint = TimedFrame(
- self.previous.rgba,
- ideal,
- self.previous.segment,
- "Source",
- self.previous.source_index,
- )
- self._write(endpoint, ideal)
-
-def _json_safe(value):
- if isinstance(value, Fraction):
- return str(value)
- if isinstance(value, dict):
- return {key: _json_safe(item) for key, item in value.items()}
- if isinstance(value, (list, tuple)):
- return [_json_safe(item) for item in value]
- return value
-
-
-def _duration_fraction(
- metadata: dict, source_rate: Fraction, frames: int, *, cfr: bool
-) -> Fraction:
+ self._write(self.previous, Fraction(self.next_index, 1) / self.target_rate)
+
+
+def _duration_fraction(metadata: dict, source_rate: Fraction, frames: int, *, cfr: bool) -> Fraction:
if cfr and frames > 0:
return Fraction(frames, 1) / source_rate
return Fraction(str(metadata["duration"]))
@@ -199,343 +285,462 @@ def _validate(options: FrameInterpolationOptions) -> None:
_ = options.target_rate
validate_rename(options.rename_mode, options.custom_suffix)
ffmpeg.validate_codec_container(options.codec, options.container)
- if getattr(options, "hdr_mode", False) and not ffmpeg.hdr_mode_supported(options.codec):
- raise ValueError(
- f"HDR Mode is only available for H.265, H.265 (NVIDIA NVENC), AV1, AV1 (NVIDIA NVENC) and ProRes Proxy; current codec is {options.codec!r}."
- )
- if options.preview_seconds is not None:
- if not math.isfinite(float(options.preview_seconds)) or float(options.preview_seconds) <= 0:
- raise ValueError("Preview duration must be positive.")
+ if options.hdr_mode and not ffmpeg.hdr_mode_supported(options.codec):
+ raise ValueError(f"HDR Mode is unavailable for {options.codec!r}.")
+ if options.preview_seconds is not None and (
+ not math.isfinite(float(options.preview_seconds)) or float(options.preview_seconds) <= 0):
+ raise ValueError("Preview duration must be positive.")
+
def interpolate_video(
- input_path: str | os.PathLike[str],
- options: FrameInterpolationOptions | None = None,
- progress: Callable[[float, str], None] | None = None,
- *, output_dir: str | os.PathLike[str] | None = None, controller=None,
+ input_path: str | os.PathLike[str], options: FrameInterpolationOptions | None = None,
+ progress: Callable[[float, str], None] | None = None, *,
+ output_dir: str | os.PathLike[str] | None = None, controller=None,
) -> FrameInterpolationResult:
- options = options or FrameInterpolationOptions()
+ operation_started = time.perf_counter()
+ options = replace(options) if options is not None else FrameInterpolationOptions()
+ options.container = ffmpeg.container_for_codec(options.codec)
_validate(options)
source = Path(input_path).resolve()
if not source.is_file():
raise FileNotFoundError(source)
- prepared_runtime = prepare_runtime()
- ai_gpu = resolve_runtime_ai_gpu(
- prepared_runtime.gpus, prepared_runtime.runtime_bundle, options.ai_gpu_uuid
- )
+ runtime_started = time.perf_counter()
+ runtime = prepare_runtime()
+ runtime_prepare_seconds = time.perf_counter() - runtime_started
+ ai_gpu = resolve_runtime_ai_gpu(runtime.gpus, runtime.runtime_bundle, options.ai_gpu_uuid)
+ capability_started = time.perf_counter()
capabilities = probe_frame_interpolation_capabilities(options.ai_gpu_uuid)
+ capability_probe_seconds = time.perf_counter() - capability_started
if not capabilities.available:
- raise RuntimeError(
- "Direct NVIDIA DLSS Frame Generation is unavailable. " + capabilities.detail
- )
+ raise RuntimeError("Direct NVIDIA DLSS Frame Generation is unavailable. " + capabilities.detail)
prepared_controller = getattr(_BATCH_CONTEXT, "controller", None)
job_context = nullcontext(prepared_controller) if prepared_controller is not None else active_job(controller)
with job_context as controller:
assert controller is not None
started = time.perf_counter()
- def _report_progress(value: float, desc: str) -> None:
- if progress is None:
- return
- try:
- v = float(value)
- v = 0.0 if v < 0 else (1.0 if v > 1 else v)
+ def update(value: float, message: str) -> None:
+ if controller.cancel.is_set():
+ raise Cancelled("Frame interpolation stopped by user.")
+ if progress:
+ value = max(0.0, min(1.0, float(value)))
elapsed = time.perf_counter() - started
- if 0.01 < v < 0.99 and elapsed > 0.5:
- eta = elapsed * (1 - v) / max(v, 1e-6)
- desc = f"{desc} - Time Remaining: {eta:.1f}s"
- progress(v, desc)
- except Exception:
- try:
- progress(value, desc)
- except Exception:
- pass
-
- timings: dict[str, float] = {}
- output: Path | None = None
- job_dir: Path | None = None
- sessions: list[DirectDLSSGSession] = []
- encoder = None
- encoder_thread = None
- nut = None
+ if 0.01 < value < 0.99 and elapsed > .5:
+ message += f" - Time Remaining: {elapsed * (1-value) / max(value,1e-6):.1f}s"
+ progress(value, message)
+
output_file: OutputFile | None = None
- performance = {}
+ decoded_container: Any | None = None
+ encoded_container: Any | None = None
+ decode_thread: threading.Thread | None = None
+ encode_thread: threading.Thread | None = None
+ sessions: list[DirectDLSSGSession] = []
+ transfer_pool: CudaTransferPool | None = None
+ job: tempfile.TemporaryDirectory[str] | None = None
+ stop, decode_stop = threading.Event(), threading.Event()
+ failures: list[BaseException] = []
+ timings: dict[str, float] = {
+ "preflight_seconds": time.perf_counter() - operation_started,
+ "runtime_prepare_seconds": runtime_prepare_seconds,
+ "capability_probe_seconds": capability_probe_seconds,
+ "session_initialization_seconds": 0.0,
+ "probe_seconds": 0.0, "decode_seconds": 0.0, "software_prepare_seconds": 0.0,
+ "bridge_input_seconds": 0.0, "optical_flow_seconds": 0.0,
+ "dlssg_seconds": 0.0, "bridge_output_seconds": 0.0,
+ "encode_seconds": 0.0, "cross_gpu_seconds": 0.0,
+ }
try:
- probe_started = time.perf_counter()
- metadata = ffmpeg.probe_video(source, count_mode="metadata")
- is_preview = options.preview_seconds is not None
- compat_preview = is_preview and bool(getattr(options, "preview_compat", True))
- effective_hdr = (
- bool(getattr(options, "hdr_mode", False))
- and (not is_preview or not compat_preview)
- and ffmpeg.hdr_mode_supported(options.codec)
- )
- if metadata["hdr"] and not effective_hdr:
- raise ValueError(
- "Frame Interpolation v1 accepts SDR video only. Enable HDR Mode (H.265/AV1/ProRes, 10-bit, copies input colorspace) to keep HDR, or convert PQ/HLG to SDR."
- )
- hdr_metadata = None
- if effective_hdr:
- hdr_metadata = {
- "color_space": metadata.get("color_space", "unknown"),
- "color_primaries": metadata.get("color_primaries", "unknown"),
- "color_transfer": metadata.get("color_transfer", "unknown"),
- "hdr": bool(metadata.get("hdr", False)),
- }
+ probe_start = time.perf_counter()
+ metadata = ffmpeg.probe_video(
+ source, count_mode="metadata", inspect_timestamps=True)
source_rate = Fraction(metadata["rate"])
cfr = bool(metadata.get("cfr", True))
- frames = int(metadata["frames"])
- if frames <= 0:
- frames = int(ffmpeg.probe_video(source, count_mode="exact")["frames"])
- full_duration = _duration_fraction(
- metadata, source_rate, frames, cfr=cfr
- )
- duration = full_duration
- if options.preview_seconds is not None:
+ source_frames = int(metadata["frames"])
+ if source_frames <= 0:
+ source_frames = int(ffmpeg.probe_video(source, count_mode="exact")["frames"])
+ duration = _duration_fraction(metadata, source_rate, source_frames, cfr=cfr)
+ preview = options.preview_seconds is not None
+ compat_preview = preview and bool(options.preview_compat)
+ effective_hdr = bool(options.hdr_mode and not compat_preview)
+ if metadata["hdr"] and not effective_hdr:
+ raise ValueError("HDR input requires Frame Interpolation HDR Mode with a 10-bit codec.")
+ if preview:
duration = min(duration, Fraction(str(options.preview_seconds)))
- frames = min(frames, ffmpeg.preview_frame_count(source, float(duration)))
- duration = min(duration, Fraction(frames, 1) / source_rate)
+ source_frames = min(source_frames, ffmpeg.preview_frame_count(source, float(duration)))
+ duration = min(duration, Fraction(source_frames, 1) / source_rate)
plan = choose_interpolation_plan(
- source_rate,
- options.target_rate,
- options.engine,
- capabilities.native_multiplier,
- cfr=cfr,
- )
+ source_rate, options.target_rate, options.engine,
+ capabilities.native_multiplier, cfr=cfr)
output_count = output_frame_count(duration, options.target_rate)
- timings["probe_seconds"] = time.perf_counter() - probe_started
- _report_progress(0.01, f"{plan.path}: {source_rate} → {options.target_rate} FPS")
+ timings["probe_seconds"] = time.perf_counter() - probe_start
+ update(.01, f"{plan.path}: {source_rate} → {options.target_rate} FPS")
+
+ selected_codec = "H.264 (NVIDIA NVENC)" if compat_preview else options.codec
+ video_gpu = ffmpeg.resolve_video_gpu(
+ runtime.gpus, options.video_gpu_uuid, selected_codec,
+ int(metadata["width"]), int(metadata["height"]))
+ if _is_nvenc(selected_codec) and video_gpu is None:
+ raise RuntimeError("The selected NVENC codec has no available encoding GPU.")
+ ai_ordinal = int(ai_gpu["cuda_ordinal"])
+ encode_ordinal = int(video_gpu["cuda_ordinal"]) if video_gpu is not None else ai_ordinal
+ # The processing path is automatic, matching Upscale: an NVENC
+ # output keeps decode/interpolation/encode on CUDA, while a CPU
+ # codec uses the in-process host-staging ABI.
+ want_cuda = automatic_cuda_path(selected_codec, metadata["rotation"])
+ decode_device = HWAccel(
+ "cuda", device=str(ai_ordinal), allow_software_fallback=True,
+ options={"primary_ctx": "1"}, is_hw_owned=True) if want_cuda else None
+ decoded_container = av.open(str(source), hwaccel=decode_device)
+ input_stream = decoded_container.streams.video[0]
+ input_stream.thread_type = "AUTO"
+ decoder = iter(decoded_container.decode(input_stream))
+ tick = time.perf_counter()
+ try:
+ first_frame = _validate_decoded_frame(next(decoder))
+ except StopIteration as exc:
+ raise ValueError("The input contains no decodable video frames.") from exc
+ timings["decode_seconds"] += time.perf_counter() - tick
+ cuda_route = bool(want_cuda and first_frame.format.name == "cuda")
+ decode_backend = "nvdec" if cuda_route else "software"
+ output_p010 = bool(effective_hdr)
+ encoder = _encoder(options, selected_codec, int(metadata["width"]),
+ int(metadata["height"]), options.target_rate,
+ effective_hdr, cuda_route, encode_ordinal)
destination = prepare_output_dir(output_dir, default=OUTPUTS)
- JOBS.mkdir(exist_ok=True)
- app_log.info("frame-interp", f"start src={source.name}")
- stamp = time.strftime("%Y%m%d-%H%M%S") + f"-{time.time_ns() % 1_000_000:06d}"
- job_dir = JOBS / f"{source.stem}-DLSSFG-{stamp}-{os.getpid()}"
- job_dir.mkdir(parents=True, exist_ok=False)
+ stamp = time.strftime("%Y%m%d-%H%M%S") + f"-{time.time_ns()%1_000_000:06d}"
extension = {"MP4": ".mp4", "MKV": ".mkv", "MOV": ".mov"}[options.container]
- auto_stem = f"{source.stem}_{'DLSSFG_PREVIEW' if options.preview_seconds else 'DLSSFG'}_{stamp}"
output = destination / output_filename(
- source, extension, options.rename_mode, options.custom_suffix, auto_stem
- )
+ source, extension, options.rename_mode, options.custom_suffix,
+ f"{source.stem}_{'DLSSFG_PREVIEW' if preview else 'DLSSFG'}_{stamp}")
output_file = OutputFile(output)
- # NUT preserves the exact rational frame clock between PyAV and FFmpeg;
- # Matroska's millisecond timestamp scale rounds 60000/1001 and short previews.
- temp_video = job_dir / "interpolated-video.nut"
- selected_codec = "H.264" if compat_preview else options.codec
- video_gpu = ffmpeg.resolve_video_gpu(
- prepared_runtime.gpus,
- options.video_gpu_uuid,
- selected_codec,
- int(metadata["width"]),
- int(metadata["height"]),
- )
- setup_started = time.perf_counter()
- encoder, encoder_thread, encoder_logs, selected_encoder, quality = ffmpeg.start_encoder(
- temp_video,
- selected_codec,
- options.quality,
- controller,
- int(metadata["width"]),
- int(metadata["height"]),
- float(options.target_rate),
- None if video_gpu is None else int(video_gpu["cuda_ordinal"]),
- video_gpu is not None,
- hdr_mode=effective_hdr,
- hdr_metadata=hdr_metadata,
- )
- assert encoder.stdin is not None
- if plan.path == "Native DLSSG":
- sessions.append(
- DirectDLSSGSession(
- int(metadata["width"]), int(metadata["height"]), frames,
- plan.generated_per_interval, controller,
- )
- )
- elif plan.path == "Cascade":
- for stage_index in range(plan.cascade_stages):
- expected = max(1, (frames - 1) * (1 << stage_index) + 1)
- sessions.append(
- DirectDLSSGSession(
- int(metadata["width"]), int(metadata["height"]), expected, 1, controller
- )
- )
- stages = [
- DLSSGStage(
- session,
- int(metadata["width"]),
- int(metadata["height"]),
- plan.generated_per_interval if plan.path == "Native DLSSG" else 1,
- detect_source_cuts=index == 0,
- )
- for index, session in enumerate(sessions)
- ]
- nut = av.open(PipeWriter(encoder.stdin), mode="w", format="nut")
- raw_stream = nut.add_stream("rawvideo", rate=options.target_rate)
- raw_stream.width = int(metadata["width"])
- raw_stream.height = int(metadata["height"])
- raw_stream.pix_fmt = "rgba"
- raw_stream.codec_context.codec_tag = "RGBA"
- raw_stream.time_base = Fraction(1, 1) / options.target_rate
- nut.start_encoding()
- writer = NearestTimestampWriter(nut, raw_stream, options.target_rate, output_count)
- timings["encoder_and_worker_setup_seconds"] = time.perf_counter() - setup_started
-
- decoded = 0
- discontinuities = 0
- nominal_interval = Fraction(1, 1) / source_rate
-
- def decode_frames():
- nonlocal decoded, discontinuities
- segment = 0
- last_timestamp = None
- with av.open(str(source)) as container:
- stream = container.streams.video[0]
- stream.thread_type = "AUTO"
- for frame in container.decode(stream):
- if decoded >= frames:
- break
- if controller.cancel.is_set():
- raise Cancelled("Frame interpolation stopped by user.")
- timestamp = (
- Fraction(frame.pts) * Fraction(stream.time_base)
- if frame.pts is not None and stream.time_base is not None
- else Fraction(decoded, 1) / source_rate
- )
- if last_timestamp is not None and (
- timestamp <= last_timestamp or timestamp - last_timestamp > nominal_interval * 2
- ):
- segment += 1
- discontinuities += 1
- last_timestamp = timestamp
- rgba = rotate_frame(frame.to_ndarray(format="rgba"), int(metadata["rotation"]))
- item = TimedFrame(np.ascontiguousarray(rgba), timestamp, segment, "Source", decoded)
- decoded += 1
- yield item
-
- def pipeline_progress(processed):
- if processed % max(1, frames // 100) == 0:
- _report_progress(0.04 + 0.82 * processed / frames,
- f"DLSSG source frame {processed}/{frames}")
-
- performance = run_pipeline(
- decode_frames(), stages, writer, controller,
- int(metadata["width"]) * int(metadata["height"]) * 4,
- timings, pipeline_progress,
- )
- nut.close()
- nut = None
- encoder.stdin.close()
- shutdown_started = time.perf_counter()
+ JOBS.mkdir(exist_ok=True)
+ job = tempfile.TemporaryDirectory(prefix="dlssg-bridge-", dir=JOBS)
+ try:
+ temp_video = Path(job.name) / ("encoded.mov" if options.container == "MOV" else
+ "encoded.mkv" if options.container == "MKV" else "encoded.mp4")
+ encoded_container = av.open(str(temp_video), mode="w")
+ encode_device = HWAccel(
+ "cuda", device=str(encode_ordinal), options={"primary_ctx": "1"},
+ is_hw_owned=True) if cuda_route else None
+ output_stream = encoded_container.add_stream(
+ encoder.name, rate=options.target_rate, hwaccel=encode_device)
+ output_stream.width, output_stream.height = int(metadata["width"]), int(metadata["height"])
+ output_stream.pix_fmt = encoder.pixel_format
+ if cuda_route:
+ output_stream.codec_context.sw_format = "p010le" if output_p010 else "nv12"
+ output_tb = Fraction(1, 1) / options.target_rate
+ output_stream.time_base = output_tb
+ output_stream.codec_context.time_base = output_tb
+ output_stream.codec_context.options = encoder.options
+ output_stream.codec_context.bit_rate = encoder.bitrate
+ _set_color_properties(output_stream.codec_context, metadata, effective_hdr)
+ output_stream.codec_context.open()
+
+ generated_count = plan.generated_per_interval if plan.path == "Native DLSSG" else 1
+ session_count = 1 if plan.path == "Native DLSSG" else plan.cascade_stages
+ initialization_start = time.perf_counter()
+ for _ in range(session_count):
+ sessions.append(DirectDLSSGSession(
+ int(metadata["width"]), int(metadata["height"]), generated_count,
+ controller, ai_ordinal, hdr=effective_hdr))
+ timings["session_initialization_seconds"] = time.perf_counter() - initialization_start
+ colors = {"color_matrix": _matrix_code(metadata), "color_range": _range_code(metadata),
+ "color_primaries": _primaries_code(metadata),
+ "color_transfer": _transfer_code(metadata),
+ "rotation": int(metadata["rotation"])}
+ stages = [DLSSGStage(
+ session, generated_count, detect_source_cuts=index == 0,
+ cuda_output=cuda_route, output_p010=output_p010, colors=colors)
+ for index, session in enumerate(sessions)]
+ if cuda_route and encode_ordinal != ai_ordinal:
+ transfer_pool = CudaTransferPool(ai_ordinal, encode_ordinal, controller)
+
+ decode_queue: queue.Queue = queue.Queue(maxsize=4)
+ encode_queue: queue.Queue = queue.Queue(maxsize=4)
+
+ def decode_worker() -> None:
+ try:
+ _put(decode_queue, first_frame, stop, controller, failures)
+ while not stop.is_set() and not decode_stop.is_set():
+ frame_start = time.perf_counter()
+ try:
+ frame = next(decoder)
+ except StopIteration:
+ timings["decode_seconds"] += time.perf_counter() - frame_start
+ break
+ timings["decode_seconds"] += time.perf_counter() - frame_start
+ _validate_decoded_frame(frame)
+ _put(decode_queue, frame, stop, controller, failures)
+ except BaseException as exc:
+ failures.append(exc); stop.set()
+ finally:
+ if not decode_stop.is_set() and not stop.is_set():
+ with suppress(BaseException):
+ _put(decode_queue, _END, stop, controller, failures)
+
+ encoded_frames = [0]
+
+ def encode_worker() -> None:
+ try:
+ while not stop.is_set():
+ item = _get(encode_queue, stop, controller, failures)
+ if item is _END:
+ break
+ frame, pts = item
+ encode_start = time.perf_counter()
+ if not cuda_route and frame.format.name != encoder.pixel_format:
+ frame = frame.reformat(format=encoder.pixel_format)
+ frame.pts, frame.time_base, frame.duration = pts, output_tb, 1
+ for packet in output_stream.encode(frame):
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - encode_start
+ encoded_frames[0] += 1
+ del frame, item
+ if not failures and not controller.cancel.is_set():
+ encode_start = time.perf_counter()
+ for packet in output_stream.encode():
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - encode_start
+ except BaseException as exc:
+ failures.append(exc); stop.set()
+
+ decode_thread = threading.Thread(target=decode_worker, name="dlssg-decoder", daemon=True)
+ encode_thread = threading.Thread(target=encode_worker, name="dlssg-encoder", daemon=True)
+ decode_thread.start(); encode_thread.start()
+ pipeline_start = time.perf_counter()
+ decoded = discontinuities = 0
+ last_timestamp: Fraction | None = None
+ origin_timestamp: Fraction | None = None
+ source_segment = 0
+ nominal = Fraction(1, 1) / source_rate
+ source_dimensions = (int(first_frame.width), int(first_frame.height))
+
+ def emit(selected: TimedFrame, index: int) -> None:
+ frame = selected.encode_frame
+ if transfer_pool is not None:
+ transfer_start = time.perf_counter()
+ frame, _detail = transfer_pool.transfer(frame)
+ timings["cross_gpu_seconds"] += time.perf_counter() - transfer_start
+ _put(encode_queue, (frame, index), stop, controller, failures)
+
+ writer = NearestTimestampWriter(options.target_rate, output_count, emit)
+ last_update = 0.0
+ while decoded < source_frames:
+ frame = _get(decode_queue, stop, controller, failures)
+ if frame is _END:
+ break
+ if (int(frame.width), int(frame.height)) != source_dimensions:
+ raise ValueError("Source dimensions changed during decoding.")
+ raw_timestamp = (Fraction(frame.pts) * Fraction(frame.time_base)
+ if frame.pts is not None and frame.time_base is not None
+ else Fraction(decoded, 1) / source_rate)
+ if origin_timestamp is None:
+ origin_timestamp = raw_timestamp
+ timestamp = raw_timestamp - origin_timestamp
+ if last_timestamp is not None and (
+ timestamp <= last_timestamp or timestamp - last_timestamp > nominal * 2):
+ source_segment += 1
+ discontinuities += 1
+ last_timestamp = timestamp
+ if cuda_route:
+ bridge_frame, encode_frame = frame, frame
+ else:
+ prepare_start = time.perf_counter()
+ if effective_hdr:
+ if metadata["rotation"]:
+ rgb10 = rotate_frame(
+ frame.to_ndarray(format="gbrp10le"),
+ int(metadata["rotation"]),
+ )
+ oriented = av.VideoFrame.from_ndarray(
+ np.ascontiguousarray(rgb10), format="gbrp10le")
+ bridge_frame = oriented.reformat(format="p010le")
+ encode_frame = bridge_frame
+ else:
+ bridge_frame = frame.reformat(format="p010le")
+ encode_frame = frame
+ else:
+ rgba = rotate_frame(frame.to_ndarray(format="rgba"), int(metadata["rotation"]))
+ rgba = np.ascontiguousarray(rgba)
+ encode_frame = (frame if not metadata["rotation"]
+ else av.VideoFrame.from_ndarray(rgba, format="rgba"))
+ bridge_frame = rgba
+ timings["software_prepare_seconds"] += time.perf_counter() - prepare_start
+ items = [TimedFrame(bridge_frame, encode_frame, timestamp, source_segment,
+ "Source", decoded)]
+ for stage in stages:
+ next_items: list[TimedFrame] = []
+ for item in items:
+ native_start = time.perf_counter()
+ produced = stage.push(item)
+ native_elapsed = time.perf_counter() - native_start
+ next_items.extend(produced)
+ timings["bridge_total_seconds"] = timings.get("bridge_total_seconds", 0.0) + native_elapsed
+ items = next_items
+ for item in items:
+ writer.push(item)
+ decoded += 1
+ del frame, items
+ now = time.perf_counter()
+ if now - last_update > .2:
+ update(.04 + .82 * decoded / max(1, source_frames),
+ f"DLSSG GPU pipeline {decoded}/{source_frames}; "
+ f"{writer.next_index/max(.01,now-pipeline_start):.1f} output fps")
+ last_update = now
+ if decoded != source_frames:
+ raise RuntimeError(f"Decoded {decoded} source frames; expected {source_frames}.")
+ writer.finish()
+ _put(encode_queue, _END, threading.Event(), controller, failures)
+ decode_stop.set()
+ # A truncated preview intentionally stops before decoder EOF.
+ # Free one or more queue slots so a producer already blocked in
+ # _put can publish its current frame, observe decode_stop, and exit.
+ while True:
+ try:
+ decode_queue.get_nowait()
+ except queue.Empty:
+ break
+ decode_thread.join(timeout=30); encode_thread.join(timeout=300)
+ if decode_thread.is_alive() or encode_thread.is_alive():
+ raise RuntimeError("The bounded Frame Interpolation pipeline did not drain.")
+ if failures:
+ raise failures[0]
+ if encoded_frames[0] != output_count:
+ raise RuntimeError(f"Encoder accepted {encoded_frames[0]} frames; expected {output_count}.")
+ timings["pipeline_seconds"] = time.perf_counter() - pipeline_start
+ encoded_container.close(); encoded_container = None
+ decoded_container.close(); decoded_container = None
+ gc.collect()
+
+ session_diagnostics = [session.diagnostics() for session in sessions]
+ for session in sessions:
+ timings["bridge_input_seconds"] += session.stage_timings["input_ms"] / 1000
+ timings["optical_flow_seconds"] += session.stage_timings["optical_flow_ms"] / 1000
+ timings["dlssg_seconds"] += session.stage_timings["ngx_ms"] / 1000
+ timings["bridge_output_seconds"] += session.stage_timings["output_ms"] / 1000
+ session.close()
+ sessions.clear()
+ transfer_diagnostics = transfer_pool.diagnostics() if transfer_pool else {}
+ if transfer_pool:
+ transfer_pool.close(); transfer_pool = None
+ update(.9, "Muxing original audio, subtitles, chapters, and metadata")
+ mux_start = time.perf_counter()
+ ffmpeg.final_mux(
+ temp_video, source, output_file.temporary, options.container, controller,
+ preserve_supported_subtitles=True,
+ source_time_origin=metadata.get("origin", 0))
+ timings["mux_seconds"] = time.perf_counter() - mux_start
+ update(.96, "Verifying exact frame count, rational FPS, dimensions, and HDR")
+ verify_start = time.perf_counter()
+ verified = ffmpeg.probe_video(output_file.temporary, count_mode="packets", controller=controller)
+ if int(verified["frames"]) != output_count:
+ verified = ffmpeg.probe_video(output_file.temporary, count_mode="exact", controller=controller)
+ if int(verified["frames"]) != output_count or Fraction(verified["rate"]) != options.target_rate:
+ raise RuntimeError(
+ f"Output verification found {verified['frames']} frames at {verified['rate']} FPS; "
+ f"expected {output_count} at {options.target_rate}.")
+ if (int(verified["width"]), int(verified["height"])) != (
+ int(metadata["width"]), int(metadata["height"])):
+ raise RuntimeError("Output dimensions changed during interpolation.")
+ if effective_hdr and (not verified.get("hdr") or int(verified.get("depth", 0)) < 10):
+ raise RuntimeError("Saved interpolation failed HDR/10-bit verification.")
+ timings["verification_seconds"] = time.perf_counter() - verify_start
+ elapsed = time.perf_counter() - started
+ timings["total_seconds"] = elapsed
+ timings["steady_state_fps"] = output_count / max(timings["pipeline_seconds"], 1e-9)
+ stage_rates = [
+ output_count / max(timings["decode_seconds"], 1e-9),
+ output_count / max(timings.get("bridge_total_seconds", 0.0), 1e-9),
+ output_count / max(timings["encode_seconds"], 1e-9),
+ ]
+ timings["slowest_stage_ceiling_fps"] = min(stage_rates)
+ timings["ceiling_utilization"] = timings["steady_state_fps"] / max(min(stage_rates), 1e-9)
+ upload_bytes = sum(int(item.get("upload_bytes", 0)) for item in session_diagnostics)
+ download_bytes = sum(int(item.get("download_bytes", 0)) for item in session_diagnostics)
+ if transfer_diagnostics:
+ upload_bytes += int(transfer_diagnostics.get("pinned_upload_bytes", 0))
+ download_bytes += int(transfer_diagnostics.get("pinned_download_bytes", 0))
+ if cuda_route and encode_ordinal == ai_ordinal:
+ memory_path = "cuda_nvdec_shared_d3d12_dlssg_dlpack_nvenc"
+ if upload_bytes or download_bytes:
+ raise RuntimeError("Same-GPU CUDA route unexpectedly transferred full-frame host pixels.")
+ elif cuda_route:
+ memory_path = str(transfer_diagnostics.get("memory_path") or "cuda_cross_gpu")
+ else:
+ memory_path = "host_staging_inprocess_dlssg_encoder"
+ pool_pressure = {
+ "capacity": 8,
+ "allocated": max((int(item.get("surface_pool_allocated", 0))
+ for item in session_diagnostics), default=0),
+ "waits": sum(int(item.get("surface_pool_waits", 0))
+ for item in session_diagnostics),
+ }
+ scene_cuts = stages[0].scene_cuts if stages else 0
+ duplicate_intervals = sum(stage.duplicates for stage in stages)
+ # A target cadence such as 24 -> 60 does not necessarily land
+ # on every real source timestamp; those frames are scheduling
+ # alternatives, not decoder drops. Exact decoded/encoded counts
+ # have already been asserted above, so the pipeline dropped none.
+ unselected_source_frames = max(0, decoded - len(writer.selected_real_ids))
+ dropped = 0
+ diagnostics = {
+ "bridge": session_diagnostics, "cross_gpu_transfer": transfer_diagnostics,
+ "adapters": {"ai": ai_gpu, "decode": ai_gpu if cuda_route else "CPU",
+ "encode": video_gpu if _is_nvenc(selected_codec) else "CPU"},
+ "discontinuities": discontinuities, "duplicate_intervals": duplicate_intervals,
+ "unselected_source_frames": unselected_source_frames,
+ }
+ output_file.publish()
+ app_log.info("frame-interp", f"done src={source.name} out={output.name} "
+ f"frames={output_count} elapsed={elapsed:.2f}s memory={memory_path}")
+ update(1.0, "Frame interpolation complete — in-process GPU pipeline confirmed")
+ return FrameInterpolationResult(
+ output_path=str(output.resolve()), report_path=str(app_log.session_path()),
+ selected_path=plan.path, native_multiplier=plan.native_multiplier,
+ cascade_stages=plan.cascade_stages, copied_frames=writer.copied,
+ generated_frames=writer.generated, dropped_frames=dropped,
+ output_frames=output_count,
+ maximum_temporal_approximation_seconds=float(writer.max_error),
+ scene_cuts=scene_cuts, elapsed_seconds=elapsed, timings=timings,
+ bridge_version=capabilities.bridge_version,
+ bridge_abi_version=capabilities.bridge_abi_version,
+ nvof_available=capabilities.nvof_available, memory_path=memory_path,
+ decode_backend=decode_backend, encode_backend=encoder.display,
+ upload_bytes=upload_bytes, download_bytes=download_bytes,
+ surface_pool_pressure=pool_pressure, diagnostics=diagnostics)
+ finally:
+ # On a failure the output container may still own the temporary
+ # file. The outer handler closes it, then performs a second,
+ # reliable cleanup attempt without masking the original error.
+ with suppress(PermissionError):
+ job.cleanup()
+ except BaseException as exc:
+ stop.set(); decode_stop.set()
+ for thread in (decode_thread, encode_thread):
+ if thread is not None:
+ thread.join(timeout=2)
for session in sessions:
- session.close()
- sessions.clear()
- timings["worker_shutdown_seconds"] = time.perf_counter() - shutdown_started
- drain_started = time.perf_counter()
- encoder_code = encoder.wait(timeout=180)
- encoder_thread.join(timeout=2)
- controller.unregister(encoder)
- if encoder_code:
- raise RuntimeError("Video encoder failed:\n" + "\n".join(encoder_logs[-40:]))
- timings["encoder_drain_seconds"] = time.perf_counter() - drain_started
- _report_progress(0.9, "Muxing original audio, subtitles, chapters, and metadata")
- mux_started = time.perf_counter()
- ffmpeg.final_mux(
- temp_video,
- source,
- output_file.temporary,
- options.container,
- controller,
- preserve_supported_subtitles=True,
- )
- timings["mux_seconds"] = time.perf_counter() - mux_started
- verification_started = time.perf_counter()
- _report_progress(0.96, "Verifying saved output")
- verified = ffmpeg.probe_video(output_file.temporary, count_mode="packets", controller=controller)
- if int(verified["frames"]) != output_count:
- verified = ffmpeg.probe_video(output_file.temporary, count_mode="exact", controller=controller)
- if int(verified["frames"]) != output_count:
- raise RuntimeError(
- f"Output verification found {verified['frames']} frames; expected {output_count}."
- )
- if Fraction(verified["rate"]) != options.target_rate:
- raise RuntimeError(
- f"Output verification found {verified['rate']} FPS; expected {options.target_rate}."
- )
- scene_cuts = sum(stage.scene_cuts for stage in stages)
- duplicate_intervals = sum(stage.duplicates for stage in stages)
- dropped_frames = max(0, decoded - len(writer.selected_real_ids))
- timings["verification_seconds"] = time.perf_counter() - verification_started
- elapsed = time.perf_counter() - started
- timings["total_seconds"] = elapsed
- performance.update(source_frames_per_second=decoded / elapsed,
- output_frames_per_second=output_count / elapsed)
- report_path = app_log.session_path()
- if controller.cancel.is_set():
- app_log.info("frame-interp", f"cancelled src={source.name}")
- raise Cancelled("Frame interpolation stopped by user.")
- app_log.info(
- "frame-interp",
- f"done src={source.name} out={output.name} frames={output_count} "
- f"elapsed={elapsed:.0f}s",
- )
- output_file.publish()
- _report_progress(1.0, "Frame interpolation complete")
- return FrameInterpolationResult(
- output_path=str(output.resolve()),
- report_path=str(report_path),
- selected_path=plan.path,
- native_multiplier=plan.native_multiplier,
- cascade_stages=plan.cascade_stages,
- copied_frames=writer.copied,
- generated_frames=writer.generated,
- dropped_frames=dropped_frames,
- output_frames=output_count,
- maximum_temporal_approximation_seconds=float(writer.max_error),
- scene_cuts=scene_cuts,
- elapsed_seconds=elapsed,
- timings=timings,
- )
- except Exception as exc:
- controller.terminate_processes()
+ with suppress(Exception):
+ session.close(abort=True)
+ if transfer_pool is not None:
+ with suppress(Exception):
+ transfer_pool.close(abort=True)
+ if encoded_container is not None:
+ with suppress(Exception):
+ encoded_container.close()
+ if decoded_container is not None:
+ with suppress(Exception):
+ decoded_container.close()
+ if job is not None:
+ with suppress(Exception):
+ job.cleanup()
if output_file is not None:
output_file.cleanup(rollback=True)
- if controller.cancel.is_set():
+ if controller.cancel.is_set() or isinstance(exc, Cancelled):
raise Cancelled("Frame interpolation stopped by user.") from exc
- if isinstance(exc, Cancelled):
- app_log.info("frame-interp", f"cancelled src={source.name}")
- raise
- tails: dict[str, object] = {}
- with suppress(Exception):
- tails["worker"] = [session.log_text() for session in sessions]
- if "encoder_logs" in locals() and encoder_logs:
- tails["ffmpeg"] = list(encoder_logs)[-40:]
- failure_path = app_log.fail("frame-interp", f"DLSSFG_{source.stem}", exc, tails or None)
+ failure_path = app_log.fail("frame-interp", f"DLSSFG_{source.stem}", exc)
raise RuntimeError(f"{exc}\nDetails: {failure_path.resolve()}") from exc
finally:
if output_file is not None:
output_file.cleanup()
- if encoder is not None and encoder.poll() is None:
- encoder.terminate()
- try:
- encoder.wait(timeout=5)
- except subprocess.TimeoutExpired:
- encoder.kill()
- encoder.wait(timeout=5)
- if nut is not None:
- try:
- nut.close()
- except (OSError, ValueError):
- pass
- for session in sessions:
- session.close()
- if encoder is not None:
- if encoder_thread is not None:
- encoder_thread.join(timeout=2)
- for stream in (encoder.stdin, encoder.stderr):
- if stream is not None and not stream.closed:
- try:
- stream.close()
- except OSError:
- pass
- controller.unregister(encoder)
- if job_dir and job_dir.exists():
- shutil.rmtree(job_dir, ignore_errors=True)
diff --git a/src/frame_interpolation/ui.py b/src/frame_interpolation/ui.py
index 5d2e7ed..6075ce1 100644
--- a/src/frame_interpolation/ui.py
+++ b/src/frame_interpolation/ui.py
@@ -10,7 +10,7 @@ from ..core.batch_ui import (
build_path_controls, build_save_controls,
)
-from ..core.ffmpeg import hdr_mode_supported
+from ..core.ffmpeg import container_for_codec, hdr_mode_supported
from ..core.ffmpeg.preview import normalize_preview_encoding, resolve_final_preview
from ..core.naming import RENAME_MODES
from ..settings.models import CODEC_CHOICES, CONTAINER_CHOICES, QUALITY_CHOICES, UISettings, coerce_hdr_mode
@@ -27,6 +27,10 @@ def hdr_mode_update(codec: str):
return gr.update(interactive=True)
+def codec_settings_update(codec: str):
+ return gr.update(value=container_for_codec(codec)), hdr_mode_update(codec)
+
+
def rename_suffix_update(mode: str):
return gr.update(interactive=mode == "Custom")
@@ -79,7 +83,12 @@ def render_frame_interpolation_batch(
f"{value.elapsed_seconds:.1f}s; processing {value.output_frames / max(value.elapsed_seconds, 0.001):.1f} frames/s; "
f"native {value.native_multiplier}×; cascade stages {value.cascade_stages}; "
f"copied {value.copied_frames}, DLSSG {value.generated_frames}, "
- f"cuts {value.scene_cuts}; report: {value.report_path}"
+ f"cuts {value.scene_cuts}; {value.decode_backend} → {value.encode_backend}; "
+ f"memory {value.memory_path}; transfers {value.upload_bytes}/{value.download_bytes} bytes; "
+ f"pool {value.surface_pool_pressure.get('allocated', 0)}/"
+ f"{value.surface_pool_pressure.get('capacity', 0)} "
+ f"(waits {value.surface_pool_pressure.get('waits', 0)}); "
+ f"bridge {value.bridge_version}/ABI {value.bridge_abi_version}; report: {value.report_path}"
)
ordered.append(
(item.index, [Path(item.input_path).name, "Complete", Path(value.output_path).name, details])
@@ -200,7 +209,9 @@ def build_frame_interpolation_tab(settings: UISettings) -> FrameInterpolationTab
label="Video codec",
)
container = gr.Dropdown(
- CONTAINER_CHOICES, value=settings.frame_interpolation_container, label="Container"
+ CONTAINER_CHOICES,
+ value=container_for_codec(settings.frame_interpolation_codec),
+ label="Container (automatic)", interactive=False,
)
with gr.Row():
rename_mode = gr.Radio(
@@ -227,9 +238,13 @@ def build_frame_interpolation_tab(settings: UISettings) -> FrameInterpolationTab
label="Batch results", wrap=True,
)
tab = FrameInterpolationTab(
- sources, input_preview, input_actions, select_source, clear_source, target_fps, engine, quality, codec, container, rename_mode,
- custom_suffix, hdr_mode, preview, render, stop, reset, output_video, save_download, zip_button, zip_download, status, results
- )
+ sources=sources, input_preview=input_preview, input_actions=input_actions,
+ select_source=select_source, clear_source=clear_source, target_fps=target_fps,
+ engine=engine, quality=quality, codec=codec,
+ container=container, rename_mode=rename_mode, custom_suffix=custom_suffix,
+ hdr_mode=hdr_mode, preview=preview, render=render, stop=stop, reset=reset,
+ output_video=output_video, save_download=save_download, zip_button=zip_button,
+ zip_download=zip_download, status=status, results=results)
tab.input_path, tab.output_path = input_path, output_path
bind_frame_interpolation_events(tab)
return tab
@@ -241,5 +256,8 @@ def bind_frame_interpolation_events(tab: FrameInterpolationTab) -> None:
preview_mode=update_frame_interpolation_preview_mode, archive_prefix="DLSSFG_VIDEO_BATCH",
preview_actions=[(tab.preview, preview_frame_interpolation)],
)
- tab.codec.change(hdr_mode_update, inputs=tab.codec, outputs=tab.hdr_mode, queue=False)
+ tab.codec.change(
+ codec_settings_update, inputs=tab.codec,
+ outputs=[tab.container, tab.hdr_mode], queue=False,
+ )
tab.rename_mode.change(rename_suffix_update, inputs=tab.rename_mode, outputs=tab.custom_suffix, queue=False)
diff --git a/src/neural_rendering/image/ui.py b/src/neural_rendering/image/ui.py
index 7b9c967..dbcbdf1 100644
--- a/src/neural_rendering/image/ui.py
+++ b/src/neural_rendering/image/ui.py
@@ -298,6 +298,7 @@ class ImageTab:
input_path: object = None
output_path: object = None
job_state: object = None
+ refresh_realtime_preview_after: object = None
@property
def render_inputs(self) -> list[object]:
@@ -381,7 +382,7 @@ def build_image_tab(settings: UISettings, gpu_mode_state: object, mask_state: ob
def bind_image_events(tab: ImageTab) -> None:
- bind_batch_ui(
+ tab.refresh_realtime_preview_after = bind_batch_ui(
tab, render_image_batch, kind="image", preview_mode=preview_input_images,
archive_prefix="DLSS5_IMAGE_BATCH",
preview_actions=[(tab.preview, preview_rendered_image)],
diff --git a/src/neural_rendering/video/processor.py b/src/neural_rendering/video/processor.py
index 50f4d28..bac58e9 100644
--- a/src/neural_rendering/video/processor.py
+++ b/src/neural_rendering/video/processor.py
@@ -34,6 +34,11 @@ from .sizing import resolve_native_settings, resolve_output_size, resolve_upscal
validate_codec_container = ffmpeg.validate_codec_container
_BATCH_CONTEXT = threading.local()
+
+def automatic_gpu_mode(codec: str) -> bool:
+ """Use CUDA transport exactly when the selected encoder is NVENC."""
+ return ffmpeg._is_nvenc_codec(codec)
+
def _validate_preview_options(
options: ConversionOptions,
) -> tuple[float | None, int | None]:
@@ -68,6 +73,7 @@ def convert_video(
*, output_dir: str | os.PathLike[str] | None = None, controller=None,
) -> ConversionResult:
options = replace(options) if options is not None else ConversionOptions()
+ options.container = ffmpeg.container_for_codec(options.codec)
source = Path(input_path).resolve()
if not source.is_file():
raise FileNotFoundError(source)
@@ -78,6 +84,10 @@ def convert_video(
# Compat previews use the forced H.264 SDR 8-bit path; user-encoded previews
# (Preview Encoding Auto-playable / Disabled) preserve the HDR choice.
compat_preview = is_preview and bool(getattr(options, "preview_compat", True))
+ # Processing-engine selection is codec-driven. Keep the legacy option in
+ # the data model for preset/API compatibility, but never let an old saved
+ # value override the automatic NVENC/CPU routing decision.
+ options.nr_gpu_mode = automatic_gpu_mode(options.codec)
hdr_requested = bool(options.preserve_hdr)
if compat_preview:
hdr_requested = False
@@ -94,7 +104,7 @@ def convert_video(
with job_context as controller:
assert controller is not None
- if options.nr_gpu_mode and ffmpeg._is_nvenc_codec(options.codec):
+ if options.nr_gpu_mode:
from .cuda_pipeline import convert_video_cuda_nvenc
return convert_video_cuda_nvenc(
diff --git a/src/neural_rendering/video/ui.py b/src/neural_rendering/video/ui.py
index d4a3965..15dd809 100644
--- a/src/neural_rendering/video/ui.py
+++ b/src/neural_rendering/video/ui.py
@@ -9,7 +9,7 @@ from ...core.batch_ui import (
BATCH_HEADERS, bind_batch_ui, build_media_clear_button, build_media_select_button,
build_path_controls, build_save_controls,
)
-from ...core.ffmpeg import hdr_mode_supported
+from ...core.ffmpeg import container_for_codec, hdr_mode_supported
from ...core.ffmpeg.preview import normalize_preview_encoding, resolve_final_preview
from ...core.naming import RENAME_MODES
from ...core.runtime import NR_STYLES, UPSCALING_MODES
@@ -286,6 +286,7 @@ class VideoTab:
input_path: object = None
output_path: object = None
job_state: object = None
+ refresh_realtime_preview_after: object = None
@property
def render_inputs(self) -> list[object]:
@@ -343,7 +344,10 @@ def build_video_tab(settings: UISettings, gpu_mode_state: object, mask_state: ob
codec = gr.Dropdown(
CODEC_CHOICES, value=settings.codec, label="Video codec",
)
- container = gr.Dropdown(CONTAINER_CHOICES, value=settings.container, label="Container")
+ container = gr.Dropdown(
+ CONTAINER_CHOICES, value=container_for_codec(settings.codec),
+ label="Container (automatic)", interactive=False,
+ )
with gr.Row():
rename_mode = gr.Radio(
RENAME_MODES, value=settings.video_rename_mode, label="Rename",
@@ -379,7 +383,7 @@ def build_video_tab(settings: UISettings, gpu_mode_state: object, mask_state: ob
def bind_video_events(tab: VideoTab) -> None:
- bind_batch_ui(
+ tab.refresh_realtime_preview_after = bind_batch_ui(
tab, render_video_batch, kind="video", preview_mode=update_video_preview_mode,
archive_prefix="DLSS5_VIDEO_BATCH",
preview_actions=[(tab.preview_frame, preview_one_frame), (tab.preview, preview_video)],
@@ -387,4 +391,10 @@ def bind_video_events(tab: VideoTab) -> None:
realtime_components=tab.neural,
)
tab.rename_mode.change(rename_suffix_update, inputs=tab.rename_mode, outputs=tab.custom_suffix, queue=False)
- tab.codec.change(hdr_mode_update, inputs=tab.codec, outputs=tab.hdr_mode, queue=False)
+ tab.codec.change(
+ lambda selected: (
+ gr.update(value=container_for_codec(selected)),
+ hdr_mode_update(selected),
+ ),
+ inputs=tab.codec, outputs=[tab.container, tab.hdr_mode], queue=False,
+ )
diff --git a/src/settings/models.py b/src/settings/models.py
index 1649da0..04b42e6 100644
--- a/src/settings/models.py
+++ b/src/settings/models.py
@@ -16,7 +16,7 @@ CONTAINER_CHOICES = ("MP4", "MKV", "MOV")
IMAGE_FORMAT_CHOICES = ("PNG", "JPEG", "WebP", "AVIF", "TIFF")
CONFIG_SECTION = "Settings"
PRESET_FORMAT = "dlss5-visual-enhancer-settings-preset"
-PRESET_SCHEMA_VERSION = 6
+PRESET_SCHEMA_VERSION = 7
MAX_PRESET_BYTES = 1024 * 1024
AUTOMATIC_MASK_CHOICES = ("Off", "On")
@@ -75,10 +75,12 @@ class UISettings:
nr_gpu_mode: bool = True
frame_interpolation_target_fps: str = "60"
frame_interpolation_engine: str = "Auto"
- frame_interpolation_codec: str = "H.264"
+ # Factory default only. Existing saved selections are loaded unchanged.
+ frame_interpolation_codec: str = "H.264 (NVIDIA NVENC)"
frame_interpolation_container: str = "MP4"
frame_interpolation_quality: str = "Auto (Default)"
frame_interpolation_hdr_mode: bool = False
+ frame_interpolation_gpu_mode: bool = True
frame_interpolation_rename_mode: str = "Auto"
frame_interpolation_custom_suffix: str = "_Frame_Interpolation"
preview_encoding: str = "Auto"
@@ -109,7 +111,7 @@ class UISettings:
upscale_hdr_peak_luminance: int = 1000
upscale_hdr_precision: str = "Packed 10-bit"
upscale_codec: str = "H.265 (NVIDIA NVENC)"
- upscale_container: str = "MP4"
+ upscale_container: str = "MKV"
upscale_quality: str = "Auto (Default)"
upscale_rename_mode: str = "Auto"
upscale_custom_suffix: str = "_Upscale"
@@ -169,6 +171,8 @@ def _validate(settings: UISettings) -> UISettings:
raise ValueError("HDR Mode must be a boolean value.")
if not isinstance(settings.frame_interpolation_hdr_mode, bool):
raise ValueError("Frame Interpolation HDR Mode must be a boolean value.")
+ if not isinstance(settings.frame_interpolation_gpu_mode, bool):
+ raise ValueError("Frame Interpolation GPU mode must be a boolean value.")
if not isinstance(settings.full_size_image_previews, bool):
raise ValueError("Full size quality preview must be a boolean value.")
# Migrate old codec names before validation
diff --git a/src/settings/presets.py b/src/settings/presets.py
index c2a29d5..921f074 100644
--- a/src/settings/presets.py
+++ b/src/settings/presets.py
@@ -103,7 +103,7 @@ def _coerce_preset_value(field_name: str, value: Any, current: UISettings) -> An
def import_settings_preset(
path: str | os.PathLike[str], current: UISettings
) -> tuple[str, UISettings]:
- """Load and migrate schema-v1 through v6 presets, then validate atomically."""
+ """Load and migrate all supported preset schemas, then validate atomically."""
preset_path = Path(path)
if preset_path.suffix.casefold() != ".json":
raise ValueError("Choose a JSON preset file.")
@@ -157,6 +157,8 @@ def import_settings_preset(
changes["nr_passes"] = DEFAULT_SETTINGS.nr_passes
if version < 6:
changes["shimmer_suppression"] = DEFAULT_SETTINGS.shimmer_suppression
+ if version < 7:
+ changes["frame_interpolation_gpu_mode"] = DEFAULT_SETTINGS.frame_interpolation_gpu_mode
# NR Preset was removed entirely (non-functional). Old preset files still
# carry it; ignore so imports from previous builds keep working.
# (Unknown keys are already filtered above; this covers any edge case where
diff --git a/src/settings/storage.py b/src/settings/storage.py
index 61af7c0..aab7e8a 100644
--- a/src/settings/storage.py
+++ b/src/settings/storage.py
@@ -215,6 +215,8 @@ def load_settings(path: str | os.PathLike[str]) -> UISettings:
DEFAULT_SETTINGS.frame_interpolation_quality,
),
frame_interpolation_hdr_mode=fi_hdr_mode_value(),
+ frame_interpolation_gpu_mode=boolean(
+ "frame_interpolation_gpu_mode", DEFAULT_SETTINGS.frame_interpolation_gpu_mode),
frame_interpolation_rename_mode=frame_interpolation_rename_mode,
frame_interpolation_custom_suffix=frame_interpolation_custom_suffix,
preview_encoding=choice(
@@ -309,6 +311,7 @@ def save_settings(path: str | os.PathLike[str], settings: UISettings) -> None:
"frame_interpolation_container": settings.frame_interpolation_container,
"frame_interpolation_quality": settings.frame_interpolation_quality,
"frame_interpolation_hdr_mode": str(settings.frame_interpolation_hdr_mode).lower(),
+ "frame_interpolation_gpu_mode": str(settings.frame_interpolation_gpu_mode).lower(),
"frame_interpolation_rename_mode": settings.frame_interpolation_rename_mode,
"frame_interpolation_custom_suffix": settings.frame_interpolation_custom_suffix,
"preview_encoding": settings.preview_encoding,
diff --git a/src/settings/ui.py b/src/settings/ui.py
index 1fb83d6..a777f13 100644
--- a/src/settings/ui.py
+++ b/src/settings/ui.py
@@ -4,7 +4,7 @@ from dataclasses import dataclass, replace
import gradio as gr
-from ..core.ffmpeg import hdr_mode_supported, probe_nvenc_codecs
+from ..core.ffmpeg import container_for_codec, hdr_mode_supported, probe_nvenc_codecs
from ..core.gpu_selection import gpu_choice_label
from ..core.paths import CONFIG_PATH
from ..core.runtime import UPSCALING_MODES, prepare_runtime
@@ -24,22 +24,6 @@ _SKIN_STRUCTURE_INDEX = 5
_AUTOMATIC_MASK_INDEX = 7
_NR_COLOR_INDEX = 8
_TONE_PRESERVATION_INDEX = 9
-PROCESSING_ENGINE_CHOICES = ("VRAM", "RAM")
-
-
-def processing_engine_choice(enabled: bool) -> str:
- """Map the stored bool path to its visible VRAM/RAM label."""
- return "VRAM" if enabled else "RAM"
-
-
-def parse_processing_engine(value: object) -> bool:
- """Map the visible VRAM/RAM selection back to its stored bool path."""
- if value not in PROCESSING_ENGINE_CHOICES:
- choices = ", ".join(PROCESSING_ENGINE_CHOICES)
- raise ValueError(f"Processing Engine Path must be one of: {choices}.")
- return value == "VRAM"
-
-
def automatic_mask_for_skin_input(
skin_structure_strength: float, automatic_mask: str,
) -> str:
@@ -174,6 +158,7 @@ def persist_video_settings(
) -> tuple:
with _CONFIG_LOCK:
current = SETTINGS_STATE.current or load_settings(CONFIG_PATH)
+ container = container_for_codec(codec)
coerced_hdr = coerce_hdr_mode(codec, hdr_mode)
settings = replace(
current,
@@ -285,25 +270,6 @@ def apply_detail_only_settings() -> tuple:
return (*shared, *temporal, *temporal)
-def persist_nr_gpu_mode(selection: object) -> bool:
- """Persist the global Processing Engine Path switch and update Live effects."""
- if isinstance(selection, bool):
- enabled = selection
- else:
- try:
- enabled = parse_processing_engine(selection)
- except ValueError as exc:
- raise gr.Error(str(exc)) from exc
- with _CONFIG_LOCK:
- current = SETTINGS_STATE.current or load_settings(CONFIG_PATH)
- settings = replace(current, nr_gpu_mode=enabled)
- if settings != current:
- save_settings(CONFIG_PATH, settings)
- SETTINGS_STATE.current = settings
- _notify_live_effects(settings)
- return enabled
-
-
def persist_frame_interpolation_settings(
target_fps: str,
engine: str,
@@ -316,6 +282,7 @@ def persist_frame_interpolation_settings(
) -> None:
with _CONFIG_LOCK:
current = SETTINGS_STATE.current or load_settings(CONFIG_PATH)
+ container = container_for_codec(codec)
coerced_hdr = coerce_hdr_mode(codec, hdr_mode)
settings = replace(
current,
@@ -335,6 +302,7 @@ def persist_frame_interpolation_settings(
def persist_upscale_settings(*values) -> None:
chosen = dict(zip(SETTING_FIELDS, values))
+ chosen["container"] = container_for_codec(chosen["codec"])
if not hdr_mode_supported(chosen["codec"]):
chosen["hdr_enabled"] = False
options = UpscaleOptions(**chosen)
@@ -393,7 +361,7 @@ def _settings_component_values(settings: UISettings) -> tuple:
interactive=settings.image_rename_mode == "Custom",
),
settings.codec,
- settings.container,
+ container_for_codec(settings.codec),
settings.quality,
gr.update(value=settings.hdr_mode, interactive=hdr_mode_supported(settings.codec)),
settings.video_rename_mode,
@@ -404,7 +372,7 @@ def _settings_component_values(settings: UISettings) -> tuple:
settings.frame_interpolation_target_fps,
settings.frame_interpolation_engine,
settings.frame_interpolation_codec,
- settings.frame_interpolation_container,
+ container_for_codec(settings.frame_interpolation_codec),
settings.frame_interpolation_quality,
gr.update(value=settings.frame_interpolation_hdr_mode, interactive=hdr_mode_supported(settings.frame_interpolation_codec)),
settings.frame_interpolation_rename_mode,
@@ -416,11 +384,10 @@ def _settings_component_values(settings: UISettings) -> tuple:
settings.video_gpu_uuid,
settings.preview_encoding,
settings.full_size_image_previews,
- processing_engine_choice(settings.nr_gpu_mode),
- settings.nr_gpu_mode,
*temporal,
settings.upscale_mode,
- *(getattr(settings, "upscale_" + name) for name in SETTING_FIELDS),
+ *(container_for_codec(settings.upscale_codec) if name == "container"
+ else getattr(settings, "upscale_" + name) for name in SETTING_FIELDS),
*(getattr(settings, "upscale_image_" + name) for name in IMAGE_UPSCALE_FIELDS),
)
@@ -454,7 +421,19 @@ def _normalize_gpu_settings(settings: UISettings, prepared) -> tuple[UISettings,
if video_uuid not in video_values:
warnings.append("Saved Video Processing GPU is unavailable; using Auto.")
video_uuid = "auto"
- return replace(settings, ai_gpu_uuid=ai_uuid, video_gpu_uuid=video_uuid), " ".join(warnings)
+ # Old presets may contain manual path flags. They are retained by the
+ # storage schema for compatibility but normalized so they cannot affect
+ # the now-automatic runtime choice.
+ return replace(
+ settings,
+ ai_gpu_uuid=ai_uuid,
+ video_gpu_uuid=video_uuid,
+ nr_gpu_mode=True,
+ frame_interpolation_gpu_mode=True,
+ container=container_for_codec(settings.codec),
+ frame_interpolation_container=container_for_codec(settings.frame_interpolation_codec),
+ upscale_container=container_for_codec(settings.upscale_codec),
+ ), " ".join(warnings)
def persist_preview_encoding(preview_encoding: str) -> None:
@@ -580,8 +559,6 @@ class SettingsTab:
video_gpu_selector: object
preview_encoding_selector: object
full_size_image_previews: object
- gpu_mode: object
- processing_engine_state: object
preset_name: object
preset_export: object
preset_import: object
@@ -607,7 +584,6 @@ def build_settings_tab(
settings: UISettings,
ai_gpu_choices: list[tuple[str, str]],
video_gpu_choices: list[tuple[str, str]],
- processing_engine_state: object,
) -> SettingsTab:
gr.Markdown("## GPU Selection")
with gr.Row():
@@ -621,13 +597,6 @@ def build_settings_tab(
value=settings.video_gpu_uuid,
label="Video Processing GPU",
)
- gr.Markdown("## Processing Engine Path")
- gpu_mode = gr.Radio(
- choices=list(PROCESSING_ENGINE_CHOICES),
- value=processing_engine_choice(settings.nr_gpu_mode),
- label="Processing Engine Path",
- show_label=False,
- )
gr.Markdown("## Preview Encoding")
preview_encoding_selector = gr.Radio(
choices=list(PREVIEW_ENCODING_CHOICES),
@@ -664,7 +633,7 @@ def build_settings_tab(
preset_status = gr.Markdown("", elem_id="preset-status")
return SettingsTab(
ai_gpu_selector, video_gpu_selector, preview_encoding_selector, full_size_image_previews,
- gpu_mode, processing_engine_state, preset_name, preset_export, preset_import, preset_status
+ preset_name, preset_export, preset_import, preset_status
)
@@ -695,8 +664,6 @@ def settings_component_outputs(image_tab, video_tab, frame_tab, settings_tab, li
settings_tab.video_gpu_selector,
settings_tab.preview_encoding_selector,
settings_tab.full_size_image_previews,
- settings_tab.gpu_mode,
- settings_tab.processing_engine_state,
*live_tab.neural,
upscale_tab.mode,
*upscale_tab.video.settings_inputs,
@@ -811,15 +778,16 @@ def bind_settings_events(settings_tab, image_tab, video_tab, frame_tab, live_tab
inputs=settings_tab.full_size_image_previews,
queue=False,
)
- settings_tab.gpu_mode.input(
- persist_nr_gpu_mode,
- inputs=settings_tab.gpu_mode,
- outputs=settings_tab.processing_engine_state,
- queue=False,
- )
reset_outputs = [*outputs, image_tab.status, video_tab.status, frame_tab.status, upscale_tab.video.status, upscale_tab.image.status]
- image_tab.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
- video_tab.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
+ image_reset = image_tab.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
+ video_reset = video_tab.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
+ # Reset writes every shared Neural Rendering control programmatically, which
+ # does not emit the direct input/release events used by automatic previews.
+ # Refresh both workflows so a prepared preview in the hidden mode cannot
+ # remain rendered with the pre-reset settings.
+ for reset_event in (image_reset, video_reset):
+ image_tab.refresh_realtime_preview_after(reset_event)
+ video_tab.refresh_realtime_preview_after(reset_event)
frame_tab.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
upscale_tab.video.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
upscale_tab.image.reset.click(reset_saved_settings, outputs=reset_outputs, queue=False)
diff --git a/src/upscale/image/batch.py b/src/upscale/image/batch.py
index 9a91303..f3976bd 100644
--- a/src/upscale/image/batch.py
+++ b/src/upscale/image/batch.py
@@ -20,9 +20,10 @@ def upscale_images(input_paths, options=None, progress=None, *, output_dir=None,
if not paths:
raise ValueError("Choose at least one image.")
controller = controller or JobController()
- reporter = BatchProgress(paths, on_item_update, progress)
- successes, failures = [], []
- try:
+ reporter = BatchProgress(paths, on_item_update, progress)
+ successes, failures = [], []
+ session_cache = {}
+ try:
destination = prepare_output_dir(output_dir)
with active_job(controller):
if controller.cancel.is_set():
@@ -34,8 +35,9 @@ def upscale_images(input_paths, options=None, progress=None, *, output_dir=None,
reporter.advance(i)
try:
result = upscale_image(path, options, lambda v, m, i=i: reporter.advance(i, v, m),
- output_dir=destination, controller=controller, _owns_slot=True,
- _capabilities=caps, generate_previews=generate_previews)
+ output_dir=destination, controller=controller, _owns_slot=True,
+ _capabilities=caps, generate_previews=generate_previews,
+ _session_cache=session_cache)
except Exception as exc:
cancelled = controller.cancel.is_set() or isinstance(exc, Cancelled)
failures.append(ImageUpscaleFailure(i, str(path), str(exc), cancelled))
@@ -57,6 +59,10 @@ def upscale_images(input_paths, options=None, progress=None, *, output_dir=None,
result = ImageUpscaleBatchResult(successes, failures, controller.cancel.is_set(), manifest)
reporter.finish(cancelled=result.cancelled, manifest_path=manifest)
return result
- except BaseException as exc:
- reporter.finish(cancelled=controller.cancel.is_set(), error=str(exc))
- raise
+ except BaseException as exc:
+ reporter.finish(cancelled=controller.cancel.is_set(), error=str(exc))
+ raise
+ finally:
+ for session in session_cache.values():
+ if not session.closed:
+ session.close(abort=controller.cancel.is_set())
diff --git a/src/upscale/image/processor.py b/src/upscale/image/processor.py
index 557b765..92086af 100644
--- a/src/upscale/image/processor.py
+++ b/src/upscale/image/processor.py
@@ -71,11 +71,12 @@ def preview_upscale_image(input_path, options=None, progress=None, *, controller
caps = probe_capabilities(options.ai_gpu_uuid, controller=controller)
update(.24, f"RTX VSR: {width}×{height} → {ow}×{oh}")
with RTXVideoSession(
- width, height, ow, oh, options.native_options(), 1, caps, controller
+ width, height, ow, oh, options.native_options(), 1, caps, controller,
+ image_srgb=True,
) as session:
- processed = worker_to_srgb(
- session.process_frame(srgb_to_worker(decoded.rgba)), ow, oh, decoded.alpha
- )
+ processed = np.frombuffer(
+ session.process_frame(np.ascontiguousarray(decoded.rgba)), dtype=np.uint8
+ ).reshape(oh, ow, 4).copy()
if session.completed_frames != 1:
raise RuntimeError("RTX VSR did not process exactly one image.")
last_results = tuple(session.last_results)
@@ -102,17 +103,20 @@ def preview_upscale_image(input_path, options=None, progress=None, *, controller
def upscale_image(input_path, options=None, progress=None, *, output_dir=None, controller=None,
- generate_previews=True, _owns_slot=False, _capabilities=None):
+ generate_previews=True, _owns_slot=False, _capabilities=None,
+ _session_cache=None):
options = replace(options) if options else ImageUpscaleOptions()
options.validate()
source = Path(input_path).resolve()
if not source.is_file():
raise FileNotFoundError(source)
with nullcontext(controller) if _owns_slot else active_job(controller) as controller:
- return _process(source, options, progress, output_dir, controller, generate_previews, _capabilities)
+ return _process(source, options, progress, output_dir, controller, generate_previews,
+ _capabilities, _session_cache)
-def _process(source, options, progress, output_dir, controller, generate_previews, capabilities):
+def _process(source, options, progress, output_dir, controller, generate_previews, capabilities,
+ session_cache):
started = time.monotonic()
stamp = time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:8]
report_path = app_log.session_path()
@@ -137,9 +141,25 @@ def _process(source, options, progress, output_dir, controller, generate_preview
f"{source.stem}_RTXIMAGE_{stamp}")
destination_file = OutputFile(output)
update(.15, f"RTX VSR: {width}×{height} → {ow}×{oh}")
- with RTXVideoSession(width, height, ow, oh, options.native_options(), 1, caps, controller) as session:
- processed = worker_to_srgb(session.process_frame(srgb_to_worker(decoded.rgba)), ow, oh, decoded.alpha)
- if session.completed_frames != 1:
+ key = (width, height, ow, oh, int(options.vsr_quality), str(caps.gpu.get("uuid", "")))
+ if session_cache is None:
+ session_context = RTXVideoSession(
+ width, height, ow, oh, options.native_options(), 1, caps, controller,
+ image_srgb=True)
+ else:
+ session = session_cache.get(key)
+ if session is None or session.closed:
+ session = RTXVideoSession(
+ width, height, ow, oh, options.native_options(), 1, caps, controller,
+ image_srgb=True)
+ session_cache[key] = session
+ session_context = nullcontext(session)
+ with session_context as session:
+ before = session.completed_frames
+ processed = np.frombuffer(
+ session.process_frame(np.ascontiguousarray(decoded.rgba)), dtype=np.uint8
+ ).reshape(oh, ow, 4).copy()
+ if session.completed_frames != before + 1:
raise RuntimeError("RTX VSR did not process exactly one image.")
update(.80, "Saving image")
export = ImageConversionOptions(output_format=options.output_format, quality=int(options.quality),
diff --git a/src/upscale/video/cuda_pipeline.py b/src/upscale/video/cuda_pipeline.py
new file mode 100644
index 0000000..c40ca56
--- /dev/null
+++ b/src/upscale/video/cuda_pipeline.py
@@ -0,0 +1,613 @@
+from __future__ import annotations
+
+"""Bounded NVDEC -> RTX Video CUDA -> same/cross-adapter NVENC pipeline."""
+
+import gc
+import math
+import os
+import queue
+import shutil
+import tempfile
+import threading
+import time
+import uuid
+from contextlib import suppress
+from fractions import Fraction
+from pathlib import Path
+from typing import Any, Callable
+
+import av
+from av.codec.hwaccel import HWAccel
+
+from ...core import app_log, ffmpeg
+from ...core.disk_paths import OutputFile, prepare_output_dir
+from ...core.gpu_detection import detect_gpus
+from ...core.gpu_selection import resolve_ai_gpu
+from ...core.jobs import Cancelled
+from ...core.naming import output_filename
+from ...core.paths import JOBS
+from .media import inspect_video, packed_bytes
+from .models import UpscaleCapabilities, UpscaleOptions, UpscaleResult, output_size
+from .native import RTXVideoSession, probe_capabilities
+from .cuda_transfer import CudaTransferPool
+
+
+_NVENC_CODEC = {
+ "H.264 (NVIDIA NVENC)": "h264_nvenc",
+ "H.265 (NVIDIA NVENC)": "hevc_nvenc",
+ "AV1 (NVIDIA NVENC)": "av1_nvenc",
+}
+_END = object()
+
+
+def is_nvenc(codec: str) -> bool:
+ return ffmpeg._normalize_codec(codec) in _NVENC_CODEC
+
+
+def _sdr_fallback(metadata: dict[str, Any]) -> str:
+ height = int(metadata["stream"]["height"])
+ return "bt470bg" if height == 576 else "smpte170m" if height <= 576 else "bt709"
+
+
+def memory_route(codec: str, ai_gpu: dict[str, Any], video_gpu: dict[str, Any] | None) -> str:
+ """Return the observable pixel-memory route for an explicit codec/GPU choice."""
+ if not is_nvenc(codec):
+ return "gpu_to_pinned_host"
+ if video_gpu is None:
+ raise RuntimeError("An NVIDIA encoder was selected but no NVENC adapter is available.")
+ return ("cuda_zero_copy" if int(video_gpu["cuda_ordinal"]) == int(ai_gpu["cuda_ordinal"])
+ else "cuda_cross_gpu_peer_or_pinned")
+
+
+def normalize_frame_timing(
+ *, frame_pts: int | None, frame_duration: int | None,
+ frame_time_base: Fraction | None, stream_time_base: Fraction,
+ origin_pts: int, delivered: int, default_duration: int,
+ last_pts: int | None,
+) -> tuple[int, int, bool]:
+ """Normalize one CFR/VFR frame without dropping, duplicating, or re-timing it."""
+ frame_tb = frame_time_base or stream_time_base
+ fallback = frame_pts is None
+ pts = (delivered * default_duration if fallback else
+ round(Fraction(int(frame_pts)) * frame_tb / stream_time_base) - origin_pts)
+ if last_pts is not None and pts <= last_pts:
+ raise ValueError("Source timestamps are not strictly increasing; refusing to drop or duplicate frames.")
+ duration = max(
+ 1, round(Fraction(int(frame_duration or default_duration)) * frame_tb / stream_time_base))
+ return pts, duration, fallback
+
+
+def _matrix_code(metadata: dict[str, Any]) -> int:
+ value = str(metadata["stream"].get("color_space") or "").casefold()
+ if value in {"", "unknown", "unspecified", "reserved"}:
+ pixel_format = str(metadata["stream"].get("pix_fmt") or "yuv420p")
+ value = "gbr" if av.VideoFormat(pixel_format).is_rgb else _sdr_fallback(metadata)
+ if value in {"bt2020nc", "bt2020c"}:
+ return 2
+ if value in {"bt470bg", "smpte170m", "smpte240m", "fcc"}:
+ return 0
+ return 1
+
+
+def _primaries_code(metadata: dict[str, Any]) -> int:
+ value = str(metadata["stream"].get("color_primaries") or "").casefold()
+ if value in {"", "unknown", "unspecified", "reserved"}:
+ value = _sdr_fallback(metadata)
+ if value == "bt2020":
+ return 2
+ if value in {"bt470bg", "bt470m"}:
+ return 0
+ if value in {"smpte170m", "smpte240m", "smpte431", "smpte432"}:
+ return 3
+ return 1
+
+
+def _transfer_code(metadata: dict[str, Any]) -> int:
+ value = str(metadata["stream"].get("color_transfer") or "").casefold()
+ if value in {"iec61966-2-1", "srgb"}:
+ return 1
+ if value in {"gamma22", "bt470m"}:
+ return 2
+ if value in {"gamma28", "bt470bg"}:
+ return 3
+ if value == "linear":
+ return 4
+ return 0
+
+
+def _range_code(metadata: dict[str, Any]) -> int:
+ stream = metadata["stream"]
+ tagged_full = str(stream.get("color_range") or "").casefold() in {"pc", "jpeg", "full"}
+ pixel_format = str(stream.get("pix_fmt") or "yuv420p")
+ return int(tagged_full or av.VideoFormat(pixel_format).is_rgb)
+
+
+def _chroma_location_code(metadata: dict[str, Any]) -> int:
+ return {"left": 0, "center": 1, "topleft": 2, "top": 3,
+ "bottomleft": 4, "bottom": 5}.get(
+ str(metadata["stream"].get("chroma_location") or "left").casefold(), 0)
+
+
+def _set_color_properties(context: Any, hdr: bool) -> None:
+ context.color_primaries = 9 if hdr else 1
+ context.color_trc = 16 if hdr else 1
+ context.colorspace = 9 if hdr else 1
+ context.color_range = 1 # MPEG/limited range
+
+
+class _SoftwareNormalizer:
+ """In-process fallback matching the prior color/rotation/SAR boundary."""
+
+ def __init__(self, frame: Any, metadata: dict[str, Any], width: int, height: int, ten_bit: bool) -> None:
+ stream = metadata["stream"]
+ sd = int(stream["height"]) <= 576
+ fallback = "bt470bg" if int(stream["height"]) == 576 else "smpte170m" if sd else "bt709"
+ matrix = str(stream.get("color_space") or fallback)
+ primaries = str(stream.get("color_primaries") or fallback)
+ transfer = str(stream.get("color_transfer") or "bt709")
+ color_range = "pc" if _range_code(metadata) else "tv"
+ precision = "yuv444p10le" if ten_bit else "yuv444p"
+ final_format = "gbrp10le" if ten_bit else "rgba"
+ graph = av.filter.Graph()
+ source = graph.add_buffer(template=frame)
+ colors = graph.add(
+ "colorspace",
+ f"ispace={matrix}:iprimaries={primaries}:itrc={transfer}:irange={color_range}:"
+ f"all=bt709:trc=gamma22:range=pc:format={precision}",
+ )
+ source.link_to(colors)
+ node = colors
+ rotation = int(metadata["rotation"])
+ if rotation == 90:
+ rotated = graph.add("transpose", "cclock")
+ node.link_to(rotated); node = rotated
+ elif rotation == 270:
+ rotated = graph.add("transpose", "clock")
+ node.link_to(rotated); node = rotated
+ elif rotation == 180:
+ horizontal, vertical = graph.add("hflip"), graph.add("vflip")
+ node.link_to(horizontal); horizontal.link_to(vertical); node = vertical
+ scaled = graph.add("scale", f"{width}:{height}:flags=lanczos")
+ square = graph.add("setsar", "1")
+ formatted = graph.add("format", final_format)
+ sink = graph.add("buffersink")
+ node.link_to(scaled); scaled.link_to(square); square.link_to(formatted); formatted.link_to(sink)
+ graph.configure()
+ self.graph, self.source, self.sink = graph, source, sink
+
+ def convert(self, frame: Any):
+ self.source.push(frame)
+ return packed_bytes(self.sink.pull())
+
+
+def _put_bounded(target: queue.Queue, item: Any, stop: threading.Event,
+ controller: Any, failures: list[BaseException]) -> bool:
+ while not stop.is_set():
+ if controller.cancel.is_set():
+ stop.set()
+ return False
+ if failures:
+ stop.set()
+ return False
+ try:
+ target.put(item, timeout=0.05)
+ return True
+ except queue.Full:
+ continue
+ return False
+
+
+def _get_bounded(source: queue.Queue, stop: threading.Event, controller: Any,
+ failures: list[BaseException]) -> Any:
+ while True:
+ if controller.cancel.is_set():
+ stop.set()
+ raise Cancelled("Upscale stopped by user.")
+ if failures:
+ stop.set()
+ raise failures[0]
+ try:
+ return source.get(timeout=0.05)
+ except queue.Empty:
+ if stop.is_set():
+ if failures:
+ raise failures[0]
+ return _END
+
+
+def convert_video_cuda_nvenc(
+ source: Path,
+ options: UpscaleOptions,
+ *,
+ controller: Any,
+ progress: Callable[[float, str], None] | None,
+ output_dir: str | os.PathLike[str] | None,
+ metadata: dict[str, Any] | None = None,
+ capabilities: UpscaleCapabilities | None = None,
+ video_gpu: dict[str, Any] | None = None,
+) -> UpscaleResult:
+ """Run the production CUDA route, including explicit cross-GPU NVENC."""
+ started = time.perf_counter()
+ report_path = app_log.session_path()
+ metadata = metadata or inspect_video(source, controller, reject_hdr=True)
+ width, height = int(metadata["width"]), int(metadata["height"])
+ output_width, output_height, _ = output_size(width, height, options)
+ gpus = detect_gpus()
+ ai_gpu = capabilities.gpu if capabilities is not None else resolve_ai_gpu(gpus, options.ai_gpu_uuid)
+ ai_ordinal = int(ai_gpu["cuda_ordinal"])
+ requested_video_uuid = str(ai_gpu["uuid"]) if options.video_gpu_uuid == "auto" else options.video_gpu_uuid
+ video_gpu = video_gpu or ffmpeg.resolve_video_gpu(
+ gpus, requested_video_uuid, options.codec, output_width, output_height)
+ if video_gpu is None:
+ raise RuntimeError("The CUDA RTX Video route requires an available NVENC adapter.")
+ encode_ordinal = int(video_gpu["cuda_ordinal"])
+ codec_name = _NVENC_CODEC[ffmpeg._normalize_codec(options.codec)]
+ input_format = 2 if int(metadata["depth"]) > 8 else 1
+ output_p010 = bool(options.hdr_enabled or (input_format == 2 and ffmpeg.hdr_mode_supported(options.codec)))
+ quality = ffmpeg.resolve_encoding_quality(
+ options.quality, options.codec, output_width, output_height,
+ float(metadata["rate"]), hdr_mode=output_p010,
+ )
+ preview = options.preview_frames is not None or options.preview_seconds is not None
+ stamp = time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:8]
+ extension = {"MP4": ".mp4", "MKV": ".mkv", "MOV": ".mov"}[options.container]
+ destination = prepare_output_dir(output_dir)
+ kind = "RTXVIDEO_PREVIEW" if preview else "RTXVIDEO"
+ output = destination / output_filename(
+ source, extension, "Auto" if preview else options.rename_mode,
+ options.custom_suffix, f"{source.stem}_{kind}_{stamp}",
+ )
+ destination_file = OutputFile(output)
+ JOBS.mkdir(exist_ok=True)
+ session: RTXVideoSession | None = None
+ decoded_container: Any | None = None
+ encoded_container: Any | None = None
+ delivered = 0
+ decode_backends: set[str] = set()
+ timings = {
+ "decode_seconds": 0.0, "software_prepare_seconds": 0.0,
+ "bridge_input_seconds": 0.0, "ngx_seconds": 0.0,
+ "bridge_output_seconds": 0.0, "encode_seconds": 0.0,
+ }
+ failures: list[BaseException] = []
+ stop = threading.Event()
+ decode_stop = threading.Event()
+ decode_thread: threading.Thread | None = None
+ encode_thread: threading.Thread | None = None
+ transfer_pool: CudaTransferPool | None = None
+
+ def update(value: float, message: str) -> None:
+ if controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ if progress:
+ progress(max(0.0, min(1.0, value)), message)
+
+ try:
+ update(0.01, f"Preparing CUDA decode and encode on {ai_gpu.get('name', 'NVIDIA GPU')}")
+ job = tempfile.TemporaryDirectory(prefix="rtx-video-cuda-", dir=JOBS)
+ try:
+ job_dir = Path(job.name)
+ temp_video = job_dir / {"MP4": "encoded.mp4", "MKV": "encoded.mkv", "MOV": "encoded.mov"}[options.container]
+ decode_device = HWAccel(
+ "cuda", device=str(ai_ordinal), allow_software_fallback=True,
+ options={"primary_ctx": "1"}, is_hw_owned=True,
+ )
+ encode_device = HWAccel(
+ "cuda", device=str(encode_ordinal), options={"primary_ctx": "1"}, is_hw_owned=True,
+ )
+ decoded_container = av.open(str(source), hwaccel=decode_device)
+ input_stream = decoded_container.streams.video[0]
+ input_stream.thread_type = "AUTO"
+ decoder = iter(decoded_container.decode(input_stream))
+ decode_start = time.perf_counter()
+ try:
+ first_frame = next(decoder)
+ except StopIteration as exc:
+ raise ValueError("The input contains no decodable video frames.") from exc
+ timings["decode_seconds"] += time.perf_counter() - decode_start
+
+ encoded_container = av.open(str(temp_video), mode="w")
+ output_stream = encoded_container.add_stream(codec_name, rate=metadata["rate"], hwaccel=encode_device)
+ output_stream.width, output_stream.height = output_width, output_height
+ output_stream.pix_fmt = "cuda"
+ output_stream.codec_context.sw_format = "p010le" if output_p010 else "nv12"
+ output_stream.time_base = input_stream.time_base
+ output_stream.codec_context.time_base = input_stream.time_base
+ encoder_options = {
+ "preset": "p6", "rc": "vbr", "gpu": str(encode_ordinal),
+ # Keep the production encoder's normal B-frame structure. Bound
+ # submission/output buffering to the bridge's eight externally
+ # owned surfaces without changing p6/HQ/rate-control decisions.
+ "rc-lookahead": "0", "surfaces": "8", "delay": "3",
+ }
+ if codec_name != "av1_nvenc":
+ encoder_options["tune"] = "hq"
+ if quality["mode"] == "constant-quality":
+ encoder_options["cq"] = "0"
+ output_stream.codec_context.bit_rate = 0
+ else:
+ output_stream.codec_context.bit_rate = int(quality["target_bitrate_kbps"]) * 1000
+ output_stream.codec_context.options = encoder_options
+ _set_color_properties(output_stream.codec_context, bool(options.hdr_enabled))
+ # Force decoder and encoder CUDA contexts to exist before NGX.
+ output_stream.codec_context.open()
+ capabilities = capabilities or probe_capabilities(options.ai_gpu_uuid, controller=controller)
+ session = RTXVideoSession(
+ width, height, output_width, output_height, options, input_format,
+ capabilities, controller,
+ )
+ if encode_ordinal != ai_ordinal:
+ transfer_pool = CudaTransferPool(ai_ordinal, encode_ordinal, controller)
+
+ decode_queue: queue.Queue = queue.Queue(maxsize=4)
+ encode_queue: queue.Queue = queue.Queue(maxsize=4)
+
+ def decode_worker() -> None:
+ def put_decode(item: Any) -> bool:
+ while not stop.is_set() and not decode_stop.is_set():
+ if controller.cancel.is_set() or failures:
+ stop.set()
+ return False
+ try:
+ decode_queue.put(item, timeout=0.05)
+ return True
+ except queue.Full:
+ continue
+ return False
+
+ try:
+ if not put_decode(first_frame):
+ return
+ while not stop.is_set() and not decode_stop.is_set():
+ tick = time.perf_counter()
+ try:
+ frame = next(decoder)
+ except StopIteration:
+ timings["decode_seconds"] += time.perf_counter() - tick
+ break
+ timings["decode_seconds"] += time.perf_counter() - tick
+ if frame.is_corrupt:
+ raise RuntimeError("The video decoder returned a corrupt frame.")
+ if not put_decode(frame):
+ return
+ except BaseException as exc:
+ failures.append(exc); stop.set()
+ finally:
+ if not decode_stop.is_set():
+ put_decode(_END)
+
+ encoded_frames = [0]
+
+ def encode_worker() -> None:
+ try:
+ while not stop.is_set():
+ item = _get_bounded(encode_queue, stop, controller, failures)
+ if item is _END:
+ break
+ tick = time.perf_counter()
+ for packet in output_stream.encode(item):
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - tick
+ encoded_frames[0] += 1
+ del item
+ if not failures and not controller.cancel.is_set():
+ tick = time.perf_counter()
+ for packet in output_stream.encode():
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - tick
+ except BaseException as exc:
+ failures.append(exc); stop.set()
+
+ decode_thread = threading.Thread(target=decode_worker, name="rtx-video-decoder", daemon=True)
+ encode_thread = threading.Thread(target=encode_worker, name="rtx-video-encoder", daemon=True)
+ pipeline_started = time.perf_counter()
+ decode_thread.start(); encode_thread.start()
+
+ stream_tb = input_stream.time_base or Fraction(1, max(1, round(float(metadata["rate"]))))
+ default_duration = max(1, round(Fraction(1, 1) / metadata["rate"] / stream_tb))
+ origin_pts = round(Fraction(str(metadata["origin"])) / stream_tb)
+ first_time: Fraction | None = None
+ preview_pts_origin: int | None = None
+ last_pts: int | None = None
+ stopped_early = False
+ timestamp_fallbacks = 0
+ software_normalizer: _SoftwareNormalizer | None = None
+ estimated = int(metadata["frames"] or max(1, math.ceil(metadata["duration"] * float(metadata["rate"]))))
+ if options.preview_frames is not None:
+ estimated = min(estimated, int(options.preview_frames))
+ elif options.preview_seconds is not None:
+ estimated = min(estimated, max(1, math.ceil(options.preview_seconds * float(metadata["rate"]))))
+ last_update = 0.0
+ while True:
+ frame = _get_bounded(decode_queue, stop, controller, failures)
+ if frame is _END:
+ break
+ pts, duration, used_fallback = normalize_frame_timing(
+ frame_pts=frame.pts, frame_duration=frame.duration,
+ frame_time_base=frame.time_base, stream_time_base=stream_tb,
+ origin_pts=origin_pts, delivered=delivered,
+ default_duration=default_duration, last_pts=last_pts,
+ )
+ frame_tb = frame.time_base or stream_tb
+ timestamp_fallbacks += int(used_fallback)
+ timestamp = Fraction(pts) * stream_tb
+ if first_time is None:
+ first_time = timestamp
+ if ((options.preview_frames is not None and delivered >= int(options.preview_frames)) or
+ (options.preview_seconds is not None and delivered and
+ float(timestamp - first_time) >= float(options.preview_seconds))):
+ stopped_early = True
+ decode_stop.set()
+ del frame
+ break
+ if frame.format.name == "cuda":
+ decode_backends.add("nvdec")
+ processed, detail = session.process_cuda_frame(
+ frame, color_matrix=_matrix_code(metadata), color_range=_range_code(metadata),
+ color_primaries=_primaries_code(metadata), color_transfer=_transfer_code(metadata),
+ chroma_location=_chroma_location_code(metadata),
+ rotation=int(metadata["rotation"]), output_p010=output_p010,
+ )
+ else:
+ decode_backends.add("software")
+ tick = time.perf_counter()
+ if software_normalizer is None:
+ software_normalizer = _SoftwareNormalizer(frame, metadata, width, height, input_format == 2)
+ packed = software_normalizer.convert(frame)
+ timings["software_prepare_seconds"] += time.perf_counter() - tick
+ processed, detail = session.process_host_to_cuda_frame(
+ packed, output_p010=output_p010, pts=pts,
+ time_base=stream_tb, duration=duration,
+ )
+ if transfer_pool is not None:
+ transferred, transfer_detail = transfer_pool.transfer(processed)
+ del processed
+ processed = transferred
+ timings["cross_gpu_seconds"] = (
+ timings.get("cross_gpu_seconds", 0.0) +
+ float(transfer_detail["copy_ms"]) / 1000.0
+ )
+ timings["bridge_input_seconds"] += detail["input_ms"] / 1000.0
+ timings["ngx_seconds"] += detail["ngx_ms"] / 1000.0
+ timings["bridge_output_seconds"] += detail["output_ms"] / 1000.0
+ if preview:
+ if preview_pts_origin is None:
+ preview_pts_origin = pts
+ processed.pts = pts - preview_pts_origin
+ else:
+ processed.pts = pts
+ processed.time_base = stream_tb
+ processed.duration = duration
+ if not _put_bounded(encode_queue, processed, stop, controller, failures):
+ raise failures[0] if failures else Cancelled("Upscale stopped by user.")
+ delivered += 1
+ last_pts = pts
+ del frame, processed
+ now = time.perf_counter()
+ if now - last_update > 0.2:
+ update(
+ min(0.87, 0.05 + 0.82 * delivered / max(1, estimated)),
+ f"RTX Video CUDA: {width}×{height} → {output_width}×{output_height}; "
+ f"{delivered} frames; {delivered / max(0.01, now-started):.1f} fps",
+ )
+ last_update = now
+ if not delivered:
+ raise ValueError("The input contains no decodable video frames.")
+ _put_bounded(encode_queue, _END, threading.Event(), controller, failures)
+ decode_thread.join(timeout=30)
+ encode_thread.join(timeout=180)
+ if decode_thread.is_alive() or encode_thread.is_alive():
+ raise RuntimeError("The bounded RTX Video pipeline did not drain cleanly.")
+ timings["pipeline_seconds"] = time.perf_counter() - pipeline_started
+ if failures:
+ raise failures[0]
+ if encoded_frames[0] != delivered:
+ raise RuntimeError(f"NVENC accepted {encoded_frames[0]} frames instead of {delivered}.")
+ encoded_container.close(); encoded_container = None
+ decoded_container.close(); decoded_container = None
+ gc.collect()
+ transfer_diagnostics = transfer_pool.diagnostics() if transfer_pool is not None else {}
+ if transfer_pool is not None:
+ transfer_pool.close()
+ transfer_pool = None
+ session_status = session.structured_status(
+ decode_backend="+".join(sorted(decode_backends)) or "unknown",
+ encode_backend=codec_name,
+ )
+ session.close()
+ if session.completed_frames != delivered:
+ raise RuntimeError("RTX Video bridge completion does not match frame accounting.")
+ gc.collect()
+
+ if not preview and (not metadata["frames"] or delivered != metadata["frames"]):
+ exact = ffmpeg.probe_video(source, count_mode="exact", strict_decode=True, controller=controller)
+ if int(exact["frames"]) != delivered:
+ raise RuntimeError(f"Source has {exact['frames']} frames but only {delivered} were processed.")
+ update(0.90, "Muxing original audio, subtitles, chapters, and metadata")
+ mux_started = time.perf_counter()
+ ffmpeg.final_mux(
+ temp_video, source, destination_file.temporary, options.container, controller,
+ preserve_supported_subtitles=True, source_time_origin=metadata["origin"],
+ )
+ timings["final_mux_seconds"] = time.perf_counter() - mux_started
+ update(0.96, "Verifying output frames, resolution, and HDR signaling")
+ verified = ffmpeg.probe_video(destination_file.temporary, count_mode="packets", controller=controller)
+ if int(verified["frames"]) != delivered:
+ verified = ffmpeg.probe_video(
+ destination_file.temporary, count_mode="exact", strict_decode=True, controller=controller)
+ if int(verified["frames"]) != delivered or (int(verified["width"]), int(verified["height"])) != (output_width, output_height):
+ raise RuntimeError("Saved output does not match processed frame count or dimensions.")
+ saved = inspect_video(destination_file.temporary, controller)
+ if options.hdr_enabled and (
+ saved["depth"] < 10 or not saved["hdr"] or
+ saved["stream"].get("color_primaries") != "bt2020" or
+ saved["stream"].get("color_space") != "bt2020nc"
+ ):
+ raise RuntimeError("Saved file failed HDR color and bit-depth verification.")
+ if not options.hdr_enabled and saved["hdr"]:
+ raise RuntimeError("SDR output unexpectedly contains HDR signaling.")
+ if controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ elapsed = time.perf_counter() - started
+ stage_per_frame = {
+ "decode_ms": timings["decode_seconds"] * 1000 / delivered,
+ "bridge_ms": (timings["bridge_input_seconds"] + timings["ngx_seconds"] + timings["bridge_output_seconds"]) * 1000 / delivered,
+ "encode_ms": timings["encode_seconds"] * 1000 / delivered,
+ }
+ slowest_ms = max(stage_per_frame.values())
+ ceiling_fps = 1000.0 / slowest_ms if slowest_ms > 0 else 0.0
+ throughput = delivered / max(timings["pipeline_seconds"], 1e-9)
+ timings.update(
+ steady_state_fps=throughput, slowest_stage_ceiling_fps=ceiling_fps,
+ ceiling_utilization=throughput / ceiling_fps if ceiling_fps else 0.0,
+ timestamp_fallbacks=timestamp_fallbacks,
+ )
+ session_status["timings"] = dict(timings)
+ session_status["adapters"] = {"ai": ai_gpu, "decode": ai_gpu, "encode": video_gpu}
+ if transfer_diagnostics:
+ session_status["cross_gpu_transfer"] = transfer_diagnostics
+ session_status["memory_path"] = transfer_diagnostics["memory_path"]
+ session_status["upload_bytes"] = (
+ int(session_status.get("upload_bytes", 0)) +
+ int(transfer_diagnostics["pinned_upload_bytes"])
+ )
+ session_status["download_bytes"] = (
+ int(session_status.get("download_bytes", 0)) +
+ int(transfer_diagnostics["pinned_download_bytes"])
+ )
+ destination_file.publish()
+ app_log.info(
+ "upscale-cuda", f"done src={source.name} out={output.name} frames={delivered} "
+ f"elapsed={elapsed:.2f}s fps={delivered / max(elapsed, 1e-9):.1f} memory={session_status['memory_path']}",
+ )
+ update(1.0, "Complete — CUDA-resident RTX Video confirmed")
+ return UpscaleResult(
+ str(output), report_path, delivered, output_width, output_height,
+ options.hdr_enabled, elapsed,
+ bridge_version=str(capabilities.bridge_version),
+ memory_path=str(session_status["memory_path"]),
+ decode_backend=str(session_status["decode_backend"]),
+ encode_backend=codec_name, timings=timings, bridge_status=session_status,
+ )
+ finally:
+ job.cleanup()
+ except BaseException:
+ stop.set()
+ decode_stop.set()
+ for thread in (decode_thread, encode_thread):
+ if thread is not None:
+ thread.join(timeout=2)
+ if session is not None and not session.closed:
+ with suppress(Exception):
+ session.close(abort=True)
+ if transfer_pool is not None and not transfer_pool.closed:
+ with suppress(Exception):
+ transfer_pool.close(abort=True)
+ if encoded_container is not None:
+ with suppress(Exception):
+ encoded_container.close()
+ if decoded_container is not None:
+ with suppress(Exception):
+ decoded_container.close()
+ destination_file.cleanup(rollback=True)
+ raise
diff --git a/src/upscale/video/cuda_transfer.py b/src/upscale/video/cuda_transfer.py
new file mode 100644
index 0000000..8673fff
--- /dev/null
+++ b/src/upscale/video/cuda_transfer.py
@@ -0,0 +1,413 @@
+from __future__ import annotations
+
+"""Cross-adapter CUDA frame transfer with DLPack-owned surface lifetimes."""
+
+import ctypes
+import gc
+import threading
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any, Iterator
+
+import av
+
+from ...core.neural_bridge import _DLPackPlane
+from ...core.jobs import Cancelled
+
+
+CUDA_SUCCESS = 0
+CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED = 704
+CU_STREAM_NON_BLOCKING = 1
+CU_MEMHOSTALLOC_PORTABLE = 1
+
+
+class CudaTransferError(RuntimeError):
+ pass
+
+
+class _CudaApi:
+ """The small CUDA Driver API subset needed for adapter-to-adapter copies."""
+
+ def __init__(self) -> None:
+ loader = getattr(ctypes, "WinDLL", ctypes.CDLL)
+ self.library = loader("nvcuda.dll")
+ self._bind("cuInit", [ctypes.c_uint])
+ self._bind("cuDeviceGet", [ctypes.POINTER(ctypes.c_int), ctypes.c_int])
+ self._bind("cuDeviceCanAccessPeer", [ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_int])
+ self._bind("cuDevicePrimaryCtxRetain", [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int])
+ self._bind_any("primary_release", ("cuDevicePrimaryCtxRelease_v2", "cuDevicePrimaryCtxRelease"),
+ [ctypes.c_int])
+ self._bind("cuCtxGetCurrent", [ctypes.POINTER(ctypes.c_void_p)])
+ self._bind("cuCtxSetCurrent", [ctypes.c_void_p])
+ self._bind("cuCtxEnablePeerAccess", [ctypes.c_void_p, ctypes.c_uint])
+ self._bind("cuStreamCreate", [ctypes.POINTER(ctypes.c_void_p), ctypes.c_uint])
+ self._bind_any("stream_destroy", ("cuStreamDestroy_v2", "cuStreamDestroy"),
+ [ctypes.c_void_p])
+ self._bind("cuStreamSynchronize", [ctypes.c_void_p])
+ self._bind_any("mem_alloc", ("cuMemAlloc_v2", "cuMemAlloc"),
+ [ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t])
+ self._bind_any("mem_free", ("cuMemFree_v2", "cuMemFree"), [ctypes.c_uint64])
+ self._bind("cuMemcpyPeerAsync", [ctypes.c_uint64, ctypes.c_void_p, ctypes.c_uint64,
+ ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p])
+ self._bind("cuMemHostAlloc", [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t, ctypes.c_uint])
+ self._bind("cuMemFreeHost", [ctypes.c_void_p])
+ self._bind_any("copy_dtoh_async", ("cuMemcpyDtoHAsync_v2", "cuMemcpyDtoHAsync"),
+ [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_size_t, ctypes.c_void_p])
+ self._bind_any("copy_htod_async", ("cuMemcpyHtoDAsync_v2", "cuMemcpyHtoDAsync"),
+ [ctypes.c_uint64, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p])
+ self._bind("cuGetErrorString", [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)])
+ self.check(self.cuInit(0), "cuInit")
+
+ def _bind(self, name: str, arguments: list[Any]) -> Any:
+ function = getattr(self.library, name)
+ function.argtypes = arguments
+ function.restype = ctypes.c_int
+ setattr(self, name, function)
+ return function
+
+ def _bind_any(self, attribute: str, names: tuple[str, ...], arguments: list[Any]) -> Any:
+ for name in names:
+ function = getattr(self.library, name, None)
+ if function is not None:
+ function.argtypes = arguments
+ function.restype = ctypes.c_int
+ setattr(self, attribute, function)
+ return function
+ raise CudaTransferError(f"The NVIDIA driver does not export {names[0]}.")
+
+ def check(self, result: int, operation: str, *, allowed: tuple[int, ...] = ()) -> int:
+ result = int(result)
+ if result == CUDA_SUCCESS or result in allowed:
+ return result
+ message = ctypes.c_char_p()
+ detail = ""
+ try:
+ if self.cuGetErrorString(result, ctypes.byref(message)) == CUDA_SUCCESS and message.value:
+ detail = message.value.decode("utf-8", "replace")
+ except Exception:
+ pass
+ raise CudaTransferError(
+ f"{operation} failed with CUDA result {result}" + (f": {detail}" if detail else "") + ".")
+
+ def device(self, ordinal: int) -> int:
+ value = ctypes.c_int()
+ self.check(self.cuDeviceGet(ctypes.byref(value), int(ordinal)), "cuDeviceGet")
+ return int(value.value)
+
+ def primary_context(self, device: int) -> ctypes.c_void_p:
+ value = ctypes.c_void_p()
+ self.check(self.cuDevicePrimaryCtxRetain(ctypes.byref(value), int(device)),
+ "cuDevicePrimaryCtxRetain")
+ return value
+
+ @contextmanager
+ def current(self, context: ctypes.c_void_p) -> Iterator[None]:
+ previous = ctypes.c_void_p()
+ self.check(self.cuCtxGetCurrent(ctypes.byref(previous)), "cuCtxGetCurrent")
+ self.check(self.cuCtxSetCurrent(context), "cuCtxSetCurrent")
+ try:
+ yield
+ finally:
+ self.check(self.cuCtxSetCurrent(previous), "cuCtxSetCurrent(restore)")
+
+
+@dataclass(slots=True)
+class _TransferSurface:
+ owner: "CudaTransferPool"
+ pointer: int
+ host_pointer: int
+ width: int
+ height: int
+ stride: int
+ bits: int
+ references: int = 0
+
+ @property
+ def byte_count(self) -> int:
+ return self.stride * (self.height + (self.height + 1) // 2)
+
+ def retain(self) -> None:
+ with self.owner.condition:
+ if self.references <= 0:
+ raise CudaTransferError("Cannot retain a released CUDA transfer surface.")
+ self.references += 1
+
+ def release(self) -> None:
+ with self.owner.condition:
+ if self.references <= 0:
+ return
+ self.references -= 1
+ if self.references == 0:
+ self.owner.condition.notify_all()
+
+ def to_av_frame(self) -> av.VideoFrame:
+ item_size = self.bits // 8
+ y = _DLPackPlane(
+ pointer=self.pointer,
+ shape=(self.height, self.width),
+ strides=(self.stride // item_size, 1),
+ bits=self.bits,
+ device_id=self.owner.destination_ordinal,
+ retain=self.retain,
+ release=self.release,
+ )
+ uv = _DLPackPlane(
+ pointer=self.pointer + self.stride * self.height,
+ shape=((self.height + 1) // 2, (self.width + 1) // 2, 2),
+ strides=(self.stride // item_size, 2, 1),
+ bits=self.bits,
+ device_id=self.owner.destination_ordinal,
+ retain=self.retain,
+ release=self.release,
+ )
+ try:
+ frame = av.VideoFrame.from_dlpack(
+ [y, uv], format="p010le" if self.bits == 16 else "nv12",
+ width=self.width, height=self.height,
+ device_id=self.owner.destination_ordinal, primary_ctx=True,
+ )
+ except BaseException:
+ self.release()
+ raise
+ self.release() # the two DLPack tensors now own the creator reference
+ return frame
+
+
+class CudaTransferPool:
+ """Eight reusable destination surfaces for explicit cross-GPU NVENC."""
+
+ def __init__(self, source_ordinal: int, destination_ordinal: int, controller: Any,
+ *, capacity: int = 8, timeout: float = 180.0) -> None:
+ if int(source_ordinal) == int(destination_ordinal):
+ raise ValueError("A cross-GPU transfer pool requires two different CUDA adapters.")
+ self.api = _CudaApi()
+ self.source_ordinal = int(source_ordinal)
+ self.destination_ordinal = int(destination_ordinal)
+ self.controller = controller
+ self.capacity = int(capacity)
+ self.timeout = float(timeout)
+ self.condition = threading.Condition()
+ self.surfaces: list[_TransferSurface] = []
+ self.pool_waits = 0
+ self.peer_copy_bytes = 0
+ self.pinned_download_bytes = 0
+ self.pinned_upload_bytes = 0
+ self.copy_seconds = 0.0
+ self.closed = False
+
+ self.source_device = self.api.device(self.source_ordinal)
+ self.destination_device = self.api.device(self.destination_ordinal)
+ self.source_context = self.api.primary_context(self.source_device)
+ try:
+ self.destination_context = self.api.primary_context(self.destination_device)
+ except BaseException:
+ self.api.check(self.api.primary_release(self.source_device),
+ "cuDevicePrimaryCtxRelease(source)")
+ raise
+ self.source_stream = ctypes.c_void_p()
+ self.destination_stream = ctypes.c_void_p()
+ try:
+ with self.api.current(self.source_context):
+ self.api.check(self.api.cuStreamCreate(ctypes.byref(self.source_stream),
+ CU_STREAM_NON_BLOCKING),
+ "cuStreamCreate(source)")
+ with self.api.current(self.destination_context):
+ self.api.check(self.api.cuStreamCreate(ctypes.byref(self.destination_stream),
+ CU_STREAM_NON_BLOCKING),
+ "cuStreamCreate(destination)")
+ can_access = ctypes.c_int()
+ self.api.check(self.api.cuDeviceCanAccessPeer(
+ ctypes.byref(can_access), self.destination_device, self.source_device),
+ "cuDeviceCanAccessPeer")
+ self.peer_access = bool(can_access.value)
+ if self.peer_access:
+ try:
+ with self.api.current(self.destination_context):
+ self.api.check(
+ self.api.cuCtxEnablePeerAccess(self.source_context, 0),
+ "cuCtxEnablePeerAccess",
+ allowed=(CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED,),
+ )
+ except CudaTransferError:
+ self.peer_access = False
+ except BaseException:
+ self.close(abort=True)
+ raise
+
+ @property
+ def memory_path(self) -> str:
+ return "cuda_cross_gpu_peer" if self.peer_access else "cuda_cross_gpu_pinned_bounce"
+
+ def _allocate(self, width: int, height: int, stride: int, bits: int) -> _TransferSurface:
+ pointer = ctypes.c_uint64()
+ byte_count = int(stride) * (int(height) + (int(height) + 1) // 2)
+ with self.api.current(self.destination_context):
+ self.api.check(self.api.mem_alloc(ctypes.byref(pointer), byte_count), "cuMemAlloc(cross-GPU surface)")
+ host_pointer = ctypes.c_void_p()
+ if not self.peer_access:
+ self.api.check(self.api.cuMemHostAlloc(
+ ctypes.byref(host_pointer), byte_count, CU_MEMHOSTALLOC_PORTABLE),
+ "cuMemHostAlloc(cross-GPU bounce)")
+ surface = _TransferSurface(
+ self, int(pointer.value), int(host_pointer.value or 0),
+ int(width), int(height), int(stride), int(bits), references=1,
+ )
+ self.surfaces.append(surface)
+ return surface
+
+ def _acquire(self, width: int, height: int, stride: int, bits: int) -> _TransferSurface:
+ deadline = time.monotonic() + self.timeout
+ while True:
+ if self.controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ with self.condition:
+ for surface in self.surfaces:
+ if (surface.references == 0 and surface.width == width and
+ surface.height == height and surface.stride == stride and surface.bits == bits):
+ surface.references = 1
+ return surface
+ if len(self.surfaces) < self.capacity:
+ return self._allocate(width, height, stride, bits)
+ self.pool_waits += 1
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise CudaTransferError("Cross-GPU CUDA surface pool remained exhausted.")
+ self.condition.wait(timeout=min(0.01, remaining))
+ gc.collect()
+
+ def transfer(self, frame: av.VideoFrame) -> tuple[av.VideoFrame, dict[str, Any]]:
+ if self.closed:
+ raise CudaTransferError("Cross-GPU CUDA transfer pool is closed.")
+ if frame.format.name != "cuda" or len(frame.planes) < 2:
+ raise CudaTransferError("Cross-GPU transfer requires a CUDA NV12/P010 frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ bits = 16 if sw_format in {"p010", "p010le"} else 8 if sw_format == "nv12" else 0
+ if not bits:
+ raise CudaTransferError(f"Unsupported cross-GPU CUDA format: {sw_format or 'unknown'}.")
+ width, height = int(frame.width), int(frame.height)
+ source_stride = int(frame.planes[0].line_size)
+ if int(frame.planes[1].line_size) != source_stride:
+ raise CudaTransferError("CUDA luma and chroma pitches differ; cross-GPU copy cannot be pooled safely.")
+ surface = self._acquire(width, height, source_stride, bits)
+ y_bytes = source_stride * height
+ uv_bytes = source_stride * ((height + 1) // 2)
+ source_y = int(frame.planes[0].buffer_ptr)
+ source_uv = int(frame.planes[1].buffer_ptr)
+ tick = time.perf_counter()
+ try:
+ if self.peer_access:
+ with self.api.current(self.destination_context):
+ for destination, source, count in (
+ (surface.pointer, source_y, y_bytes),
+ (surface.pointer + y_bytes, source_uv, uv_bytes),
+ ):
+ self.api.check(self.api.cuMemcpyPeerAsync(
+ destination, self.destination_context, source, self.source_context,
+ count, self.destination_stream), "cuMemcpyPeerAsync")
+ self.api.check(self.api.cuStreamSynchronize(self.destination_stream),
+ "cuStreamSynchronize(peer copy)")
+ self.peer_copy_bytes += y_bytes + uv_bytes
+ else:
+ with self.api.current(self.source_context):
+ self.api.check(self.api.copy_dtoh_async(
+ ctypes.c_void_p(surface.host_pointer), source_y, y_bytes, self.source_stream),
+ "cuMemcpyDtoHAsync(luma)")
+ self.api.check(self.api.copy_dtoh_async(
+ ctypes.c_void_p(surface.host_pointer + y_bytes), source_uv, uv_bytes,
+ self.source_stream), "cuMemcpyDtoHAsync(chroma)")
+ self.api.check(self.api.cuStreamSynchronize(self.source_stream),
+ "cuStreamSynchronize(pinned download)")
+ with self.api.current(self.destination_context):
+ self.api.check(self.api.copy_htod_async(
+ surface.pointer, ctypes.c_void_p(surface.host_pointer), y_bytes,
+ self.destination_stream), "cuMemcpyHtoDAsync(luma)")
+ self.api.check(self.api.copy_htod_async(
+ surface.pointer + y_bytes, ctypes.c_void_p(surface.host_pointer + y_bytes),
+ uv_bytes, self.destination_stream), "cuMemcpyHtoDAsync(chroma)")
+ self.api.check(self.api.cuStreamSynchronize(self.destination_stream),
+ "cuStreamSynchronize(pinned upload)")
+ self.pinned_download_bytes += y_bytes + uv_bytes
+ self.pinned_upload_bytes += y_bytes + uv_bytes
+ elapsed = time.perf_counter() - tick
+ self.copy_seconds += elapsed
+ output = surface.to_av_frame()
+ except BaseException:
+ surface.release()
+ raise
+ output.pts, output.time_base = frame.pts, frame.time_base
+ if frame.duration is not None:
+ output.duration = frame.duration
+ return output, {
+ "memory_path": self.memory_path,
+ "copy_ms": elapsed * 1000.0,
+ "peer_copy_bytes": y_bytes + uv_bytes if self.peer_access else 0,
+ "pinned_download_bytes": 0 if self.peer_access else y_bytes + uv_bytes,
+ "pinned_upload_bytes": 0 if self.peer_access else y_bytes + uv_bytes,
+ }
+
+ def diagnostics(self) -> dict[str, Any]:
+ return {
+ "memory_path": self.memory_path,
+ "peer_access": self.peer_access,
+ "peer_copy_bytes": self.peer_copy_bytes,
+ "pinned_download_bytes": self.pinned_download_bytes,
+ "pinned_upload_bytes": self.pinned_upload_bytes,
+ "copy_seconds": self.copy_seconds,
+ "surface_pool_capacity": self.capacity,
+ "surface_pool_allocated": len(self.surfaces),
+ "surface_pool_waits": self.pool_waits,
+ }
+
+ def close(self, *, abort: bool = False) -> None:
+ if self.closed:
+ return
+ deadline = time.monotonic() + (2.0 if abort else self.timeout)
+ with self.condition:
+ while any(surface.references for surface in self.surfaces):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ if abort:
+ # The encoder may still own a DLPack pointer after a failed
+ # encode. Leaking those few allocations is safer than a
+ # use-after-free in the NVIDIA driver during unwinding.
+ self.closed = True
+ return
+ raise CudaTransferError("Encoder retained cross-GPU surfaces past shutdown.")
+ self.condition.wait(timeout=min(0.05, remaining))
+ try:
+ with self.api.current(self.destination_context):
+ if self.destination_stream.value:
+ self.api.check(self.api.cuStreamSynchronize(self.destination_stream),
+ "cuStreamSynchronize(destination close)")
+ for surface in self.surfaces:
+ if surface.pointer:
+ self.api.check(self.api.mem_free(surface.pointer), "cuMemFree(cross-GPU surface)")
+ surface.pointer = 0
+ if surface.host_pointer:
+ self.api.check(self.api.cuMemFreeHost(ctypes.c_void_p(surface.host_pointer)),
+ "cuMemFreeHost(cross-GPU bounce)")
+ surface.host_pointer = 0
+ if self.destination_stream.value:
+ self.api.check(self.api.stream_destroy(self.destination_stream),
+ "cuStreamDestroy(destination)")
+ self.destination_stream = ctypes.c_void_p()
+ with self.api.current(self.source_context):
+ if self.source_stream.value:
+ self.api.check(self.api.cuStreamSynchronize(self.source_stream),
+ "cuStreamSynchronize(source close)")
+ self.api.check(self.api.stream_destroy(self.source_stream),
+ "cuStreamDestroy(source)")
+ self.source_stream = ctypes.c_void_p()
+ finally:
+ self.api.check(self.api.primary_release(self.destination_device),
+ "cuDevicePrimaryCtxRelease(destination)")
+ self.api.check(self.api.primary_release(self.source_device),
+ "cuDevicePrimaryCtxRelease(source)")
+ self.closed = True
+
+ def __enter__(self) -> "CudaTransferPool":
+ return self
+
+ def __exit__(self, exc_type, exc, traceback) -> None:
+ self.close(abort=exc_type is not None)
diff --git a/src/upscale/video/host_pipeline.py b/src/upscale/video/host_pipeline.py
new file mode 100644
index 0000000..2fe71fa
--- /dev/null
+++ b/src/upscale/video/host_pipeline.py
@@ -0,0 +1,654 @@
+from __future__ import annotations
+
+"""In-process NVDEC/software -> RTX Video -> pinned host -> CPU encoder pipeline."""
+
+import ctypes
+import gc
+import math
+import os
+import queue
+import tempfile
+import threading
+import time
+import uuid
+from contextlib import suppress
+from dataclasses import dataclass
+from fractions import Fraction
+from pathlib import Path
+from typing import Any, Callable
+
+import av
+from av.codec.hwaccel import HWAccel
+
+from ...core import app_log, ffmpeg
+from ...core.disk_paths import OutputFile, prepare_output_dir
+from ...core.jobs import Cancelled
+from ...core.naming import output_filename
+from ...core.paths import JOBS
+from ...core.ffmpeg.codecs import _x265_hdr_params
+from .cuda_pipeline import (
+ _END,
+ _SoftwareNormalizer,
+ _chroma_location_code,
+ _get_bounded,
+ _matrix_code,
+ _primaries_code,
+ _put_bounded,
+ _range_code,
+ _set_color_properties,
+ _transfer_code,
+ normalize_frame_timing,
+)
+from .cuda_transfer import CU_MEMHOSTALLOC_PORTABLE, _CudaApi
+from .media import inspect_video, packed_bytes
+from .models import UpscaleCapabilities, UpscaleOptions, UpscaleResult, output_size
+from .native import (
+ FORMAT_NV12,
+ FORMAT_P010,
+ FORMAT_YUV422P10,
+ RTXVideoSession,
+ probe_capabilities,
+)
+
+
+@dataclass(slots=True)
+class _EncoderSettings:
+ codec: str
+ display_name: str
+ pixel_format: str
+ native_format: int
+ options: dict[str, str]
+ bit_rate: int
+ quality: dict[str, Any]
+
+
+def _codec_available(name: str) -> bool:
+ try:
+ av.codec.Codec(name, "w")
+ return True
+ except Exception:
+ return False
+
+
+def _encoder_settings(options: UpscaleOptions, width: int, height: int,
+ rate: Fraction, ten_bit: bool) -> _EncoderSettings:
+ codec = ffmpeg._normalize_codec(options.codec)
+ quality = ffmpeg.resolve_encoding_quality(
+ options.quality, options.codec, width, height, float(rate), hdr_mode=ten_bit)
+ bit_rate = int(quality.get("target_bitrate_kbps") or 0) * 1000
+ common: dict[str, str] = {}
+ if quality["mode"] == "constant-quality":
+ common["crf"] = "0"
+ if codec == "H.264":
+ return _EncoderSettings(
+ "libx264", "libx264", "yuv420p", FORMAT_NV12,
+ {"preset": "slow", **common}, bit_rate, quality)
+ if codec in {"H.265", "HEVC"}:
+ values = {"preset": "slow", **common}
+ if options.hdr_enabled:
+ colors = {"color_space": "bt2020nc", "color_transfer": "smpte2084",
+ "color_primaries": "bt2020", "hdr": True}
+ hdr_parameters = _x265_hdr_params(colors)
+ if hdr_parameters:
+ values["x265-params"] = hdr_parameters
+ return _EncoderSettings(
+ "libx265", "libx265", "yuv420p10le" if ten_bit else "yuv420p",
+ FORMAT_P010 if ten_bit else FORMAT_NV12, values, bit_rate, quality)
+ if codec == "AV1":
+ encoder = "libsvtav1" if _codec_available("libsvtav1") else "libaom-av1"
+ if not _codec_available(encoder):
+ raise RuntimeError(
+ "AV1 CPU encoding is unavailable: neither libsvtav1 nor libaom-av1 can initialize.")
+ values = ({"preset": "6"} if encoder == "libsvtav1" else {"cpu-used": "4"})
+ values.update(common)
+ if encoder == "libaom-av1" and quality["mode"] == "constant-quality":
+ values["b"] = "0"
+ return _EncoderSettings(
+ encoder, encoder, "yuv420p10le" if ten_bit else "yuv420p",
+ FORMAT_P010 if ten_bit else FORMAT_NV12, values, bit_rate, quality)
+ if codec == "ProRes Proxy":
+ values = {"profile": "0"}
+ if quality.get("bits_per_mb") is not None:
+ values["bits_per_mb"] = str(int(quality["bits_per_mb"]))
+ return _EncoderSettings(
+ "prores_ks", "prores_ks (Proxy)", "yuv422p10le",
+ FORMAT_YUV422P10, values, 0, quality)
+ raise ValueError(f"The in-process host pipeline does not support codec {options.codec!r}.")
+
+
+@dataclass(slots=True)
+class _PinnedSlot:
+ owner: "_PinnedFramePool"
+ pointer: int
+ plane_pointers: tuple[int, ...]
+ strides: tuple[int, ...]
+ rows: tuple[int, ...]
+ row_bytes: tuple[int, ...]
+ held: bool = False
+ pts: int | None = None
+ duration: int | None = None
+ time_base: Fraction | None = None
+
+ def release(self) -> None:
+ with self.owner.condition:
+ if self.held:
+ self.held = False
+ self.owner.condition.notify_all()
+
+ def to_av_frame(self) -> tuple[av.VideoFrame, float]:
+ tick = time.perf_counter()
+ try:
+ frame = av.VideoFrame(self.owner.width, self.owner.height, self.owner.pixel_format)
+ for index, plane in enumerate(frame.planes):
+ source = self.plane_pointers[index]
+ source_stride = self.strides[index]
+ count = self.row_bytes[index]
+ rows = self.rows[index]
+ destination = int(plane.buffer_ptr)
+ destination_stride = int(plane.line_size)
+ if count == source_stride == destination_stride:
+ ctypes.memmove(destination, source, count * rows)
+ else:
+ for row in range(rows):
+ ctypes.memmove(destination + row * destination_stride,
+ source + row * source_stride, count)
+ frame.pts, frame.time_base = self.pts, self.time_base
+ if self.duration is not None:
+ frame.duration = self.duration
+ return frame, time.perf_counter() - tick
+ finally:
+ self.release()
+
+
+class _PinnedFramePool:
+ """Four preallocated page-locked frames shared by RTX and encoder stages."""
+
+ def __init__(self, ordinal: int, width: int, height: int, native_format: int,
+ pixel_format: str, controller: Any, *, capacity: int = 4,
+ timeout: float = 180.0) -> None:
+ self.api = _CudaApi()
+ self.ordinal, self.width, self.height = int(ordinal), int(width), int(height)
+ self.native_format, self.pixel_format = int(native_format), str(pixel_format)
+ self.controller, self.capacity, self.timeout = controller, int(capacity), float(timeout)
+ self.condition = threading.Condition()
+ self.pool_waits = 0
+ self.closed = False
+ self.device = self.api.device(self.ordinal)
+ self.context = self.api.primary_context(self.device)
+ if native_format == FORMAT_NV12:
+ strides = (width, width)
+ rows = (height, (height + 1) // 2)
+ row_bytes = strides
+ elif native_format == FORMAT_P010:
+ strides = (width * 2, width * 2)
+ rows = (height, (height + 1) // 2)
+ row_bytes = strides
+ elif native_format == FORMAT_YUV422P10:
+ strides = (width * 2, ((width + 1) // 2) * 2, ((width + 1) // 2) * 2)
+ rows = (height, height, height)
+ row_bytes = strides
+ else:
+ self.api.check(self.api.primary_release(self.device), "cuDevicePrimaryCtxRelease")
+ raise ValueError("Unsupported pinned-frame format.")
+ self.frame_bytes = sum(stride * count for stride, count in zip(strides, rows))
+ self.slots: list[_PinnedSlot] = []
+ try:
+ with self.api.current(self.context):
+ for _ in range(self.capacity):
+ allocation = ctypes.c_void_p()
+ self.api.check(self.api.cuMemHostAlloc(
+ ctypes.byref(allocation), self.frame_bytes, CU_MEMHOSTALLOC_PORTABLE),
+ "cuMemHostAlloc(encoder frame)")
+ base = int(allocation.value or 0)
+ pointers, offset = [], 0
+ for stride, count in zip(strides, rows):
+ pointers.append(base + offset)
+ offset += stride * count
+ self.slots.append(_PinnedSlot(
+ self, base, tuple(pointers), tuple(strides), tuple(rows),
+ tuple(row_bytes)))
+ except BaseException:
+ self.close(abort=True)
+ raise
+
+ def acquire(self) -> _PinnedSlot:
+ deadline = time.monotonic() + self.timeout
+ with self.condition:
+ while True:
+ if self.controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ for slot in self.slots:
+ if not slot.held:
+ slot.held = True
+ return slot
+ self.pool_waits += 1
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise RuntimeError("Pinned encoder-frame pool remained exhausted.")
+ self.condition.wait(timeout=min(0.05, remaining))
+
+ def diagnostics(self) -> dict[str, Any]:
+ return {
+ "surface_pool_capacity": self.capacity,
+ "surface_pool_allocated": len(self.slots),
+ "surface_pool_waits": self.pool_waits,
+ "bytes_per_surface": self.frame_bytes,
+ "allocation": "cuda_page_locked_host",
+ }
+
+ def close(self, *, abort: bool = False) -> None:
+ if self.closed:
+ return
+ deadline = time.monotonic() + (2.0 if abort else self.timeout)
+ with self.condition:
+ while any(slot.held for slot in self.slots):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ if abort:
+ self.closed = True
+ return
+ raise RuntimeError("Encoder retained pinned frames past shutdown.")
+ self.condition.wait(timeout=min(0.05, remaining))
+ try:
+ with self.api.current(self.context):
+ for slot in self.slots:
+ if slot.pointer:
+ self.api.check(self.api.cuMemFreeHost(ctypes.c_void_p(slot.pointer)),
+ "cuMemFreeHost(encoder frame)")
+ slot.pointer = 0
+ finally:
+ self.api.check(self.api.primary_release(self.device), "cuDevicePrimaryCtxRelease")
+ self.closed = True
+
+
+def convert_video_inprocess_host(
+ source: Path,
+ options: UpscaleOptions,
+ *,
+ controller: Any,
+ progress: Callable[[float, str], None] | None,
+ output_dir: str | os.PathLike[str] | None,
+ metadata: dict[str, Any] | None = None,
+ capabilities: UpscaleCapabilities | None = None,
+) -> UpscaleResult:
+ """Run CPU/ProRes encoding without raw frame or NUT subprocess pipes."""
+ started = time.perf_counter()
+ report_path = app_log.session_path()
+ metadata = metadata or inspect_video(source, controller, reject_hdr=True)
+ width, height = int(metadata["width"]), int(metadata["height"])
+ output_width, output_height, _ = output_size(width, height, options)
+ capabilities = capabilities or probe_capabilities(options.ai_gpu_uuid, controller=controller)
+ ai_gpu = capabilities.gpu
+ ordinal = int(ai_gpu["cuda_ordinal"])
+ input_format = 2 if int(metadata["depth"]) > 8 else 1
+ ten_bit = bool(
+ options.hdr_enabled or
+ (input_format == 2 and ffmpeg.hdr_mode_supported(options.codec)) or
+ ffmpeg._normalize_codec(options.codec) == "ProRes Proxy")
+ encoder = _encoder_settings(options, output_width, output_height, metadata["rate"], ten_bit)
+ preview = options.preview_frames is not None or options.preview_seconds is not None
+ stamp = time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:8]
+ extension = {"MP4": ".mp4", "MKV": ".mkv", "MOV": ".mov"}[options.container]
+ destination = prepare_output_dir(output_dir)
+ kind = "RTXVIDEO_PREVIEW" if preview else "RTXVIDEO"
+ output = destination / output_filename(
+ source, extension, "Auto" if preview else options.rename_mode,
+ options.custom_suffix, f"{source.stem}_{kind}_{stamp}")
+ destination_file = OutputFile(output)
+ decoded_container: Any | None = None
+ encoded_container: Any | None = None
+ session: RTXVideoSession | None = None
+ pinned_pool: _PinnedFramePool | None = None
+ decode_thread: threading.Thread | None = None
+ encode_thread: threading.Thread | None = None
+ failures: list[BaseException] = []
+ stop, decode_stop = threading.Event(), threading.Event()
+ delivered = 0
+ decode_backends: set[str] = set()
+ timings: dict[str, float] = {
+ "decode_seconds": 0.0, "software_prepare_seconds": 0.0,
+ "bridge_input_seconds": 0.0, "ngx_seconds": 0.0,
+ "bridge_output_seconds": 0.0, "host_copy_seconds": 0.0,
+ "encode_seconds": 0.0,
+ }
+
+ def update(value: float, message: str) -> None:
+ if controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ if progress:
+ progress(max(0.0, min(1.0, value)), message)
+
+ def drain_slots(target: queue.Queue | None) -> None:
+ if target is None:
+ return
+ while True:
+ try:
+ item = target.get_nowait()
+ except queue.Empty:
+ return
+ if isinstance(item, _PinnedSlot):
+ item.release()
+
+ encode_queue: queue.Queue | None = None
+ try:
+ update(0.01, f"Preparing in-process CUDA and {encoder.display_name}")
+ JOBS.mkdir(exist_ok=True)
+ job = tempfile.TemporaryDirectory(prefix="rtx-video-host-", dir=JOBS)
+ try:
+ job_dir = Path(job.name)
+ temp_video = job_dir / ("encoded.mov" if encoder.codec == "prores_ks" else "encoded.mp4")
+ decode_device = HWAccel(
+ "cuda", device=str(ordinal), allow_software_fallback=True,
+ options={"primary_ctx": "1"}, is_hw_owned=True)
+ decoded_container = av.open(str(source), hwaccel=decode_device)
+ input_stream = decoded_container.streams.video[0]
+ input_stream.thread_type = "AUTO"
+ decoder_iterator = iter(decoded_container.decode(input_stream))
+ tick = time.perf_counter()
+ try:
+ first_frame = next(decoder_iterator)
+ except StopIteration as exc:
+ raise ValueError("The input contains no decodable video frames.") from exc
+ timings["decode_seconds"] += time.perf_counter() - tick
+
+ encoded_container = av.open(str(temp_video), mode="w")
+ output_stream = encoded_container.add_stream(encoder.codec, rate=metadata["rate"])
+ output_stream.width, output_stream.height = output_width, output_height
+ output_stream.pix_fmt = encoder.pixel_format
+ stream_tb = input_stream.time_base or Fraction(1, max(1, round(float(metadata["rate"]))))
+ output_stream.time_base = stream_tb
+ output_stream.codec_context.time_base = stream_tb
+ output_stream.codec_context.options = encoder.options
+ output_stream.codec_context.bit_rate = encoder.bit_rate
+ _set_color_properties(output_stream.codec_context, bool(options.hdr_enabled))
+ output_stream.codec_context.open()
+
+ session = RTXVideoSession(
+ width, height, output_width, output_height, options, input_format,
+ capabilities, controller)
+ pinned_pool = _PinnedFramePool(
+ ordinal, output_width, output_height, encoder.native_format,
+ {FORMAT_NV12: "nv12", FORMAT_P010: "p010le",
+ FORMAT_YUV422P10: "yuv422p10le"}[encoder.native_format], controller)
+ decode_queue: queue.Queue = queue.Queue(maxsize=4)
+ encode_queue = queue.Queue(maxsize=4)
+
+ def decode_worker() -> None:
+ def put(item: Any) -> bool:
+ while not stop.is_set() and not decode_stop.is_set():
+ if controller.cancel.is_set() or failures:
+ stop.set()
+ return False
+ try:
+ decode_queue.put(item, timeout=0.05)
+ return True
+ except queue.Full:
+ continue
+ return False
+ try:
+ if not put(first_frame):
+ return
+ while not stop.is_set() and not decode_stop.is_set():
+ frame_tick = time.perf_counter()
+ try:
+ frame = next(decoder_iterator)
+ except StopIteration:
+ timings["decode_seconds"] += time.perf_counter() - frame_tick
+ break
+ timings["decode_seconds"] += time.perf_counter() - frame_tick
+ if frame.is_corrupt:
+ raise RuntimeError("The video decoder returned a corrupt frame.")
+ if not put(frame):
+ return
+ except BaseException as exc:
+ failures.append(exc)
+ stop.set()
+ finally:
+ if not decode_stop.is_set():
+ put(_END)
+
+ encoded_frames = [0]
+
+ def encode_worker() -> None:
+ try:
+ while not stop.is_set():
+ item = _get_bounded(encode_queue, stop, controller, failures)
+ if item is _END:
+ break
+ encode_tick = time.perf_counter()
+ frame, copy_seconds = item.to_av_frame()
+ timings["host_copy_seconds"] += copy_seconds
+ if frame.format.name != encoder.pixel_format:
+ copy_tick = time.perf_counter()
+ frame = frame.reformat(format=encoder.pixel_format)
+ timings["host_copy_seconds"] += time.perf_counter() - copy_tick
+ for packet in output_stream.encode(frame):
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - encode_tick
+ encoded_frames[0] += 1
+ del item, frame
+ if not failures and not controller.cancel.is_set():
+ encode_tick = time.perf_counter()
+ for packet in output_stream.encode():
+ encoded_container.mux(packet)
+ timings["encode_seconds"] += time.perf_counter() - encode_tick
+ except BaseException as exc:
+ failures.append(exc)
+ stop.set()
+
+ decode_thread = threading.Thread(target=decode_worker, name="rtx-host-decoder", daemon=True)
+ encode_thread = threading.Thread(target=encode_worker, name="rtx-host-encoder", daemon=True)
+ pipeline_started = time.perf_counter()
+ decode_thread.start()
+ encode_thread.start()
+
+ default_duration = max(1, round(Fraction(1, 1) / metadata["rate"] / stream_tb))
+ origin_pts = round(Fraction(str(metadata["origin"])) / stream_tb)
+ first_time: Fraction | None = None
+ preview_origin: int | None = None
+ last_pts: int | None = None
+ timestamp_fallbacks = 0
+ stopped_early = False
+ software_normalizer: _SoftwareNormalizer | None = None
+ source_dimensions = (int(first_frame.width), int(first_frame.height))
+ estimated = int(metadata["frames"] or max(
+ 1, math.ceil(metadata["duration"] * float(metadata["rate"]))))
+ if options.preview_frames is not None:
+ estimated = min(estimated, int(options.preview_frames))
+ elif options.preview_seconds is not None:
+ estimated = min(estimated, max(
+ 1, math.ceil(options.preview_seconds * float(metadata["rate"]))))
+ last_update = 0.0
+ while True:
+ frame = _get_bounded(decode_queue, stop, controller, failures)
+ if frame is _END:
+ break
+ if (int(frame.width), int(frame.height)) != source_dimensions:
+ raise ValueError("Source dimensions changed during decoding.")
+ pts, duration, used_fallback = normalize_frame_timing(
+ frame_pts=frame.pts, frame_duration=frame.duration,
+ frame_time_base=frame.time_base, stream_time_base=stream_tb,
+ origin_pts=origin_pts, delivered=delivered,
+ default_duration=default_duration, last_pts=last_pts)
+ timestamp_fallbacks += int(used_fallback)
+ timestamp = Fraction(pts) * stream_tb
+ if first_time is None:
+ first_time = timestamp
+ if ((options.preview_frames is not None and delivered >= int(options.preview_frames)) or
+ (options.preview_seconds is not None and delivered and
+ float(timestamp - first_time) >= float(options.preview_seconds))):
+ stopped_early = True
+ decode_stop.set()
+ del frame
+ break
+ slot = pinned_pool.acquire()
+ try:
+ if frame.format.name == "cuda":
+ decode_backends.add("nvdec")
+ detail = session.process_cuda_to_host_planar(
+ frame, output_format=encoder.native_format,
+ plane_pointers=slot.plane_pointers, strides=slot.strides,
+ color_matrix=_matrix_code(metadata), color_range=_range_code(metadata),
+ color_primaries=_primaries_code(metadata),
+ color_transfer=_transfer_code(metadata),
+ chroma_location=_chroma_location_code(metadata),
+ rotation=int(metadata["rotation"]))
+ else:
+ decode_backends.add("software")
+ prepare_tick = time.perf_counter()
+ if software_normalizer is None:
+ software_normalizer = _SoftwareNormalizer(
+ frame, metadata, width, height, input_format == 2)
+ packed = software_normalizer.convert(frame)
+ timings["software_prepare_seconds"] += time.perf_counter() - prepare_tick
+ detail = session.process_host_to_host_planar(
+ packed, output_format=encoder.native_format,
+ plane_pointers=slot.plane_pointers, strides=slot.strides)
+ timings["bridge_input_seconds"] += detail["input_ms"] / 1000.0
+ timings["ngx_seconds"] += detail["ngx_ms"] / 1000.0
+ timings["bridge_output_seconds"] += detail["output_ms"] / 1000.0
+ if preview:
+ if preview_origin is None:
+ preview_origin = pts
+ slot.pts = pts - preview_origin
+ else:
+ slot.pts = pts
+ slot.time_base, slot.duration = stream_tb, duration
+ if not _put_bounded(encode_queue, slot, stop, controller, failures):
+ raise failures[0] if failures else Cancelled("Upscale stopped by user.")
+ except BaseException:
+ slot.release()
+ raise
+ delivered += 1
+ last_pts = pts
+ del frame, slot
+ now = time.perf_counter()
+ if now - last_update > 0.2:
+ update(
+ min(0.87, 0.05 + 0.82 * delivered / max(1, estimated)),
+ f"RTX Video pinned pipeline: {width}×{height} → {output_width}×{output_height}; "
+ f"{delivered} frames; {delivered / max(0.01, now-started):.1f} fps")
+ last_update = now
+ if not delivered:
+ raise ValueError("The input contains no decodable video frames.")
+ _put_bounded(encode_queue, _END, threading.Event(), controller, failures)
+ decode_thread.join(timeout=30)
+ encode_thread.join(timeout=600)
+ if decode_thread.is_alive() or encode_thread.is_alive():
+ raise RuntimeError("The bounded host-encoding pipeline did not drain cleanly.")
+ timings["pipeline_seconds"] = time.perf_counter() - pipeline_started
+ if failures:
+ raise failures[0]
+ if encoded_frames[0] != delivered:
+ raise RuntimeError(f"The encoder accepted {encoded_frames[0]} frames instead of {delivered}.")
+ encoded_container.close()
+ encoded_container = None
+ decoded_container.close()
+ decoded_container = None
+ gc.collect()
+ pool_status = pinned_pool.diagnostics()
+ pinned_pool.close()
+ pinned_pool = None
+ session_status = session.structured_status(
+ decode_backend="+".join(sorted(decode_backends)) or "unknown",
+ encode_backend=encoder.display_name)
+ session.close()
+ if session.completed_frames != delivered:
+ raise RuntimeError("RTX Video bridge completion does not match frame accounting.")
+ if not preview and (not metadata["frames"] or delivered != metadata["frames"]):
+ exact = ffmpeg.probe_video(source, count_mode="exact", strict_decode=True,
+ controller=controller)
+ if int(exact["frames"]) != delivered:
+ raise RuntimeError(
+ f"Source has {exact['frames']} frames but only {delivered} were processed.")
+ update(0.90, "Muxing original audio, subtitles, chapters, and metadata")
+ mux_tick = time.perf_counter()
+ ffmpeg.final_mux(
+ temp_video, source, destination_file.temporary, options.container, controller,
+ preserve_supported_subtitles=True, source_time_origin=metadata["origin"])
+ timings["final_mux_seconds"] = time.perf_counter() - mux_tick
+ update(0.96, "Verifying output frames, resolution, chroma, and HDR signaling")
+ verified = ffmpeg.probe_video(destination_file.temporary, count_mode="packets",
+ controller=controller)
+ if int(verified["frames"]) != delivered:
+ verified = ffmpeg.probe_video(
+ destination_file.temporary, count_mode="exact", strict_decode=True,
+ controller=controller)
+ if (int(verified["frames"]) != delivered or
+ (int(verified["width"]), int(verified["height"])) !=
+ (output_width, output_height)):
+ raise RuntimeError("Saved output does not match processed frame count or dimensions.")
+ saved = inspect_video(destination_file.temporary, controller)
+ if encoder.codec == "prores_ks" and saved["stream"].get("pix_fmt") != "yuv422p10le":
+ raise RuntimeError("ProRes output did not preserve direct 10-bit 4:2:2 chroma.")
+ if options.hdr_enabled and (
+ saved["depth"] < 10 or not saved["hdr"] or
+ saved["stream"].get("color_primaries") != "bt2020" or
+ saved["stream"].get("color_space") != "bt2020nc"):
+ raise RuntimeError("Saved file failed HDR color and bit-depth verification.")
+ if not options.hdr_enabled and saved["hdr"]:
+ raise RuntimeError("SDR output unexpectedly contains HDR signaling.")
+ if controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ elapsed = time.perf_counter() - started
+ throughput = delivered / max(timings["pipeline_seconds"], 1e-9)
+ bridge_seconds = (timings["bridge_input_seconds"] + timings["ngx_seconds"] +
+ timings["bridge_output_seconds"])
+ stage_ms = {
+ "decode_ms": timings["decode_seconds"] * 1000 / delivered,
+ "bridge_ms": bridge_seconds * 1000 / delivered,
+ "encode_ms": timings["encode_seconds"] * 1000 / delivered,
+ }
+ slowest_ms = max(stage_ms.values())
+ ceiling_fps = 1000.0 / slowest_ms if slowest_ms else 0.0
+ timings.update(
+ steady_state_fps=throughput,
+ slowest_stage_ceiling_fps=ceiling_fps,
+ ceiling_utilization=throughput / ceiling_fps if ceiling_fps else 0.0,
+ timestamp_fallbacks=timestamp_fallbacks)
+ memory_path = (
+ "cuda_to_pinned_host"
+ if decode_backends == {"nvdec"}
+ else "host_upload_cuda_to_pinned_host"
+ )
+ session_status["timings"] = dict(timings)
+ session_status["memory_path"] = memory_path
+ session_status["pinned_pool"] = pool_status
+ session_status["adapters"] = {"ai": ai_gpu, "decode": ai_gpu, "encode": "CPU"}
+ destination_file.publish()
+ app_log.info(
+ "upscale-host", f"done src={source.name} out={output.name} frames={delivered} "
+ f"elapsed={elapsed:.2f}s fps={delivered / max(elapsed, 1e-9):.1f} "
+ f"encoder={encoder.display_name}")
+ update(1.0, "Complete — in-process pinned RTX Video path confirmed")
+ return UpscaleResult(
+ str(output), report_path, delivered, output_width, output_height,
+ options.hdr_enabled, elapsed,
+ bridge_version=str(capabilities.bridge_version),
+ memory_path=memory_path,
+ decode_backend="+".join(sorted(decode_backends)) or "unknown",
+ encode_backend=encoder.display_name, timings=timings,
+ bridge_status=session_status)
+ finally:
+ job.cleanup()
+ except BaseException:
+ stop.set()
+ decode_stop.set()
+ for thread in (decode_thread, encode_thread):
+ if thread is not None:
+ thread.join(timeout=2)
+ drain_slots(encode_queue)
+ if encoded_container is not None:
+ with suppress(Exception):
+ encoded_container.close()
+ if decoded_container is not None:
+ with suppress(Exception):
+ decoded_container.close()
+ if session is not None and not session.closed:
+ with suppress(Exception):
+ session.close(abort=True)
+ if pinned_pool is not None and not pinned_pool.closed:
+ with suppress(Exception):
+ pinned_pool.close(abort=True)
+ destination_file.cleanup(rollback=True)
+ raise
diff --git a/src/upscale/video/models.py b/src/upscale/video/models.py
index eb84e88..3ea1b2d 100644
--- a/src/upscale/video/models.py
+++ b/src/upscale/video/models.py
@@ -1,10 +1,10 @@
from __future__ import annotations
import math
-from dataclasses import dataclass, fields
+from dataclasses import dataclass, field, fields
from fractions import Fraction
-from ...core.ffmpeg import CODEC_CHOICES, ENCODING_QUALITIES, hdr_mode_supported, validate_codec_container
+from ...core.ffmpeg import CODEC_CHOICES, ENCODING_QUALITIES, container_for_codec, hdr_mode_supported, validate_codec_container
from ...core.naming import validate_rename
VSR_QUALITIES = [("1 - Low", 1), ("2 - Medium", 2), ("3 - High", 3), ("4 - Ultra", 4)]
@@ -31,7 +31,7 @@ class UpscaleOptions:
hdr_peak_luminance: int = 1000
hdr_precision: str = "Packed 10-bit"
codec: str = "H.265 (NVIDIA NVENC)"
- container: str = "MP4"
+ container: str = "MKV"
quality: str = "Auto (Default)"
rename_mode: str = "Auto"
custom_suffix: str = "_Upscale"
@@ -86,7 +86,9 @@ SETTING_FIELDS = tuple(f.name for f in fields(UpscaleOptions) if f.name not in {
def options_from_settings(settings) -> UpscaleOptions:
- return UpscaleOptions(**{name: getattr(settings, "upscale_" + name) for name in SETTING_FIELDS},
+ values = {name: getattr(settings, "upscale_" + name) for name in SETTING_FIELDS}
+ values["container"] = container_for_codec(values["codec"])
+ return UpscaleOptions(**values,
ai_gpu_uuid=settings.ai_gpu_uuid, video_gpu_uuid=settings.video_gpu_uuid)
@@ -119,6 +121,8 @@ class UpscaleCapabilities:
hdr: dict
sdk_version: str = "1.1.0"
worker_version: str = "1.0.0"
+ bridge_version: str = ""
+ bridge_status: dict = field(default_factory=dict)
@dataclass(slots=True)
@@ -130,6 +134,12 @@ class UpscaleResult:
output_height: int
hdr: bool
elapsed_seconds: float
+ bridge_version: str = ""
+ memory_path: str = "host_staging"
+ decode_backend: str = "software"
+ encode_backend: str = ""
+ timings: dict = field(default_factory=dict)
+ bridge_status: dict = field(default_factory=dict)
@dataclass(slots=True)
diff --git a/src/upscale/video/native.py b/src/upscale/video/native.py
index 69ce5d6..86457b6 100644
--- a/src/upscale/video/native.py
+++ b/src/upscale/video/native.py
@@ -1,229 +1,774 @@
-from __future__ import annotations
-
-import collections
-import ctypes
-import json
-import struct
-import subprocess
-import threading
-import time
-from contextlib import nullcontext, suppress
-from functools import lru_cache
-
-from ...core import app_log
-from ...core.gpu_detection import detect_gpus
-from ...core.gpu_selection import resolve_ai_gpu
-from ...core.jobs import Cancelled, JobController, current_job_controller, use_job_controller
-from ...core.paths import RUNTIME
-from .models import UpscaleCapabilities, UpscaleOptions
-
-RUNTIME_DIR = RUNTIME / "rtx_video"
-WORKER = RUNTIME_DIR / "rtx-video-worker.exe"
-MAGIC, FRAME, VERSION = 0x31585452, 0x314D5246, 1
-
-
-def gpu_luid(gpu: dict) -> str:
- driver = ctypes.WinDLL("nvcuda.dll")
- device, mask = ctypes.c_int(), ctypes.c_uint()
- luid = (ctypes.c_char * 8)()
- if driver.cuInit(0) or driver.cuDeviceGet(ctypes.byref(device), int(gpu["cuda_ordinal"])) or driver.cuDeviceGetLuid(luid, ctypes.byref(mask), device):
- raise RuntimeError("Cannot match the selected NVIDIA GPU to its DirectX adapter.")
- return bytes(luid).hex()
-
-
-def runtime_files() -> None:
- for path in (WORKER, RUNTIME_DIR / "nvngx_vsr.dll", RUNTIME_DIR / "nvngx_truehdr.dll"):
- if not path.is_file():
- raise RuntimeError(f"RTX Video runtime is missing: {path}. Rebuild or restore the RTX Video runtime.")
-
-
-@lru_cache(maxsize=8)
-def _probe(luid: str, worker_stamp: int, vsr_stamp: int, hdr_stamp: int) -> dict:
- controller = current_job_controller() or JobController()
- process = subprocess.Popen([str(WORKER), "--probe", "--gpu-luid", luid], cwd=RUNTIME_DIR,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, creationflags=subprocess.CREATE_NO_WINDOW)
- controller.register(process)
- deadline = time.monotonic() + 90
- try:
- while True:
- if controller.cancel.is_set():
- raise Cancelled("Stopped while checking RTX Video capabilities.")
- if time.monotonic() >= deadline:
- raise RuntimeError("RTX Video capability check timed out. Check the NVIDIA driver and retry.")
- try:
- stdout, stderr = process.communicate(timeout=.2)
- break
- except subprocess.TimeoutExpired:
- continue
- if controller.cancel.is_set():
- raise Cancelled("Stopped while checking RTX Video capabilities.")
- finally:
- if process.poll() is None:
- process.kill()
- process.wait(timeout=5)
- controller.unregister(process)
- process.stdout.close()
- process.stderr.close()
- try:
- data = json.loads(stdout)
- except ValueError as exc:
- raise RuntimeError("RTX Video capability probe returned invalid data: " + stderr.decode("utf-8", "replace")[-3000:]) from exc
- if process.returncode or "error" in data:
- raise RuntimeError(data.get("error") or stderr.decode("utf-8", "replace")[-3000:])
- if data.get("protocol_version") != VERSION:
- raise RuntimeError("RTX Video worker version does not match this application.")
- return data
-
-
-def probe_capabilities(gpu_uuid: str = "auto", *, controller=None) -> UpscaleCapabilities:
- runtime_files()
- gpu = resolve_ai_gpu(detect_gpus(), gpu_uuid)
- luid = gpu_luid(gpu)
- with use_job_controller(controller) if controller is not None else nullcontext():
- if controller is not None and controller.cancel.is_set():
- raise Cancelled("Stopped before checking RTX Video capabilities.")
- data = _probe(luid, *(p.stat().st_mtime_ns for p in (WORKER, RUNTIME_DIR / "nvngx_vsr.dll", RUNTIME_DIR / "nvngx_truehdr.dll")))
- return UpscaleCapabilities(gpu, luid, data["vsr"], data["hdr"], data["sdk_version"], data["worker_version"])
-
-
-class RTXVideoSession:
- """Bounded synchronous frame exchange with a watchdog and cancellable process."""
-
- def __init__(self, width: int, height: int, output_width: int, output_height: int,
- options: UpscaleOptions, input_format: int, capabilities: UpscaleCapabilities,
- controller: JobController, *, timeout: float = 180):
- options.validate()
- runtime_files()
- for enabled, cap, name in ((options.vsr_enabled, capabilities.vsr, "VSR"), (options.hdr_enabled, capabilities.hdr, "HDR")):
- if enabled and not cap.get("available"):
- raise RuntimeError(f"RTX Video {name} is unavailable on {capabilities.gpu['name']}. "
- f"Minimum driver {cap.get('min_driver_major', 0)}.{cap.get('min_driver_minor', 0)}; "
- f"initialization result 0x{cap.get('init_result', 0):08X}.")
- self.output_format = (3 if options.hdr_precision == "FP16" else 2) if options.hdr_enabled else input_format
- self.input_bytes = width * height * 4
- self.output_bytes = output_width * output_height * (8 if self.output_format == 3 else 4)
- self.controller, self.timeout = controller, timeout
- self.logs = collections.deque(maxlen=150)
- self.process = subprocess.Popen([str(WORKER), "--serve", "--gpu-luid", capabilities.luid], cwd=RUNTIME_DIR,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- bufsize=0, creationflags=subprocess.CREATE_NO_WINDOW)
- controller.register(self.process)
- self.closed = False
- self.completed_frames = 0
- self.last_results = (0, 0)
- self.deadline = time.monotonic() + timeout
- self.timed_out = False
- self._done = threading.Event()
- self._reader = threading.Thread(target=self._read_logs, daemon=True)
- self._guard = threading.Thread(target=self._watchdog, daemon=True)
- self._reader.start(); self._guard.start()
- try:
- self._write(struct.pack("<15I", MAGIC, VERSION, width, height, output_width, output_height,
- input_format, self.output_format, options.vsr_enabled, int(options.vsr_quality),
- options.hdr_enabled, int(options.hdr_contrast), int(options.hdr_saturation),
- int(options.hdr_middle_gray), int(options.hdr_peak_luminance)))
- magic, status, version, ibytes, obytes = struct.unpack("<5I", self._read(20))
- if (magic, status, version, ibytes, obytes) != (MAGIC, 0, VERSION, self.input_bytes, self.output_bytes):
- raise RuntimeError("RTX Video worker returned an invalid setup response.")
- self.deadline = float("inf")
- except BaseException:
- self.close(abort=True)
- raise
-
- def _read_logs(self):
- for line in iter(self.process.stderr.readline, b""):
- text = line.decode("utf-8", "replace").rstrip()
- if text:
- self.logs.append(text)
-
- def _watchdog(self):
- while not self._done.wait(.2):
- if time.monotonic() > self.deadline:
- self.timed_out = True
- self.process.kill()
- break
-
- def _error(self):
- if self.controller.cancel.is_set():
- raise Cancelled("Upscale stopped by user.")
- self._reader.join(timeout=.2)
- detail = "timed out" if self.timed_out else "closed unexpectedly"
- app_log.error("rtx-worker", f"RTX Video worker {detail}", "\n".join(self.logs)[-500:])
- raise RuntimeError(f"RTX Video worker {detail}.\n" + "\n".join(self.logs)[-4000:])
-
- def _write(self, data):
- view = memoryview(data).cast("B")
- try:
- while view:
- n = self.process.stdin.write(view)
- if not n:
- self._error()
- view = view[n:]
- except (OSError, ValueError):
- self._error()
-
- def _read(self, count):
- data = bytearray(count)
- view = memoryview(data)
- while view:
- n = self.process.stdout.readinto(view)
- if not n:
- self._error()
- view = view[n:]
- return data
-
- def process_frame(self, pixels) -> bytearray:
- if self.closed:
- raise RuntimeError("RTX Video session is closed.")
- if self.controller.cancel.is_set():
- raise Cancelled("Upscale stopped by user.")
- if memoryview(pixels).nbytes != self.input_bytes:
- raise ValueError("Incorrect RTX Video input frame byte count.")
- self.deadline = time.monotonic() + self.timeout
- try:
- self._write(struct.pack("<3I", FRAME, self.completed_frames, self.input_bytes))
- self._write(pixels)
- magic, index, status, size, vsr, hdr = struct.unpack("<6I", self._read(24))
- if (magic, index) != (FRAME, self.completed_frames):
- raise RuntimeError("RTX Video frame identity mismatch.")
- if status:
- app_log.error("rtx-worker", f"RTX Video evaluation failed: VSR=0x{vsr:08X}, HDR=0x{hdr:08X}")
- raise RuntimeError(f"RTX Video evaluation failed: VSR=0x{vsr:08X}, HDR=0x{hdr:08X}.")
- if size != self.output_bytes:
- raise RuntimeError("RTX Video output frame byte count mismatch.")
- result = self._read(size)
- self.completed_frames += 1
- self.last_results = (vsr, hdr)
- return result
- finally:
- self.deadline = float("inf")
-
- def close(self, *, abort=False):
- if self.closed:
- return
- self.closed = True
- self._done.set()
- try:
- if abort and self.process.poll() is None:
- self.process.terminate()
- with suppress(OSError):
- self.process.stdin.close()
- try:
- self.process.wait(timeout=10)
- except subprocess.TimeoutExpired:
- self.process.kill(); self.process.wait(timeout=5)
- finally:
- self.controller.unregister(self.process)
- self._reader.join(timeout=1)
- self._guard.join(timeout=1)
- for stream in (self.process.stdout, self.process.stderr):
- stream.close()
- if not abort and self.process.returncode:
- app_log.error("rtx-worker", "RTX Video worker failed during shutdown", "\n".join(self.logs)[-500:])
- raise RuntimeError("RTX Video worker failed during shutdown.\n" + "\n".join(self.logs)[-3000:])
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, *_):
- self.close(abort=exc_type is not None)
+from __future__ import annotations
+
+"""Persistent in-process CUDA bridge for RTX Video VSR and TrueHDR.
+
+The legacy D3D11 worker remains packaged as an explicit developer A/B probe,
+but production frames never fall back to it. Host callers cross PCIe once in
+each direction; CUDA video callers stay on the selected adapter through DLPack.
+"""
+
+import collections
+import ctypes
+import gc
+import json
+import subprocess
+import threading
+import time
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from ...core.gpu_detection import detect_gpus
+from ...core.gpu_selection import resolve_ai_gpu
+from ...core.jobs import Cancelled, JobController, current_job_controller
+from ...core.neural_bridge import _DLPackPlane
+from ...core.ngx_runtime import NGX_RUNTIME_LOCK
+from ...core.paths import ROOT, RUNTIME
+from .models import UpscaleCapabilities, UpscaleOptions
+
+
+RUNTIME_DIR = RUNTIME / "rtx_video"
+BRIDGE = RUNTIME_DIR / "neuroframe_engine.dll"
+WORKER = (ROOT / "native (dev)" / "Upscale" / "RTX Video" / "rtx_video" /
+ "build" / "legacy" / "rtx-video-worker.exe")
+BRIDGE_ABI_VERSION = 1
+MEMORY_HOST, MEMORY_CUDA = 1, 2
+FORMAT_RGBA8, FORMAT_R10, FORMAT_RGBA16F, FORMAT_NV12, FORMAT_P010, FORMAT_YUV422P10 = 1, 2, 3, 4, 5, 6
+
+
+class FrameDescriptorV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("memory_type", ctypes.c_uint32), ("pixel_format", ctypes.c_uint32),
+ ("width", ctypes.c_uint32), ("height", ctypes.c_uint32),
+ ("planes", ctypes.c_uint64 * 4), ("strides", ctypes.c_uint32 * 4),
+ ("color_matrix", ctypes.c_uint32), ("color_range", ctypes.c_uint32),
+ ("color_primaries", ctypes.c_uint32), ("color_transfer", ctypes.c_uint32),
+ ("rotation", ctypes.c_uint32), ("reserved", ctypes.c_uint32 * 3),
+ ]
+
+ @classmethod
+ def empty(cls) -> "FrameDescriptorV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
+
+
+class SessionDescriptorV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("input_width", ctypes.c_uint32), ("input_height", ctypes.c_uint32),
+ ("output_width", ctypes.c_uint32), ("output_height", ctypes.c_uint32),
+ ("input_format", ctypes.c_uint32), ("output_format", ctypes.c_uint32),
+ ("vsr_enabled", ctypes.c_uint32), ("vsr_quality", ctypes.c_uint32),
+ ("hdr_enabled", ctypes.c_uint32), ("hdr_contrast", ctypes.c_uint32),
+ ("hdr_saturation", ctypes.c_uint32), ("hdr_middle_gray", ctypes.c_uint32),
+ ("hdr_peak_luminance", ctypes.c_uint32), ("reserved", ctypes.c_uint32 * 5),
+ ]
+
+
+class FrameResultV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("vsr_result", ctypes.c_uint32), ("hdr_result", ctypes.c_uint32),
+ ("cuda_result", ctypes.c_uint32), ("reserved0", ctypes.c_uint32),
+ ("upload_bytes", ctypes.c_uint64), ("download_bytes", ctypes.c_uint64),
+ ("input_milliseconds", ctypes.c_double), ("ngx_milliseconds", ctypes.c_double),
+ ("output_milliseconds", ctypes.c_double), ("total_milliseconds", ctypes.c_double),
+ ]
+
+ @classmethod
+ def empty(cls) -> "FrameResultV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
+
+
+class BridgeStatusV1(ctypes.Structure):
+ _fields_ = [
+ ("struct_size", ctypes.c_uint32), ("abi_version", ctypes.c_uint32),
+ ("flags", ctypes.c_uint32), ("gpu_ordinal", ctypes.c_int32),
+ ("vsr_init_result", ctypes.c_uint32), ("hdr_init_result", ctypes.c_uint32),
+ ("vsr_min_driver_major", ctypes.c_uint32),
+ ("vsr_min_driver_minor", ctypes.c_uint32),
+ ("hdr_min_driver_major", ctypes.c_uint32),
+ ("hdr_min_driver_minor", ctypes.c_uint32),
+ ("bridge_version", ctypes.c_char * 32), ("gpu_name", ctypes.c_char * 128),
+ ("last_error", ctypes.c_char * 512),
+ ]
+
+ @classmethod
+ def empty(cls) -> "BridgeStatusV1":
+ value = cls()
+ value.struct_size = ctypes.sizeof(cls)
+ value.abi_version = BRIDGE_ABI_VERSION
+ return value
+
+
+class RTXVideoBridgeError(RuntimeError):
+ pass
+
+
+class RTXVideoBridgePoisonedError(RTXVideoBridgeError):
+ pass
+
+
+def gpu_luid(gpu: dict) -> str:
+ driver = ctypes.WinDLL("nvcuda.dll")
+ device, mask = ctypes.c_int(), ctypes.c_uint()
+ luid = (ctypes.c_char * 8)()
+ if (driver.cuInit(0) or driver.cuDeviceGet(ctypes.byref(device), int(gpu["cuda_ordinal"])) or
+ driver.cuDeviceGetLuid(luid, ctypes.byref(mask), device)):
+ raise RuntimeError("Cannot match the selected NVIDIA GPU to its DirectX adapter.")
+ return bytes(luid).hex()
+
+
+def runtime_files(*, legacy_probe: bool = False) -> None:
+ required = [BRIDGE, RUNTIME_DIR / "nvngx_vsr.dll", RUNTIME_DIR / "nvngx_truehdr.dll"]
+ if legacy_probe:
+ required.append(WORKER)
+ for path in required:
+ if not path.is_file():
+ raise RuntimeError(f"RTX Video runtime is missing: {path}. Rebuild or restore the RTX Video runtime.")
+
+
+class _BridgeManager:
+ def __init__(self) -> None:
+ self.library: Any | None = None
+ self.gpu_ordinal: int | None = None
+ self.poisoned_reason = ""
+ self.ffmpeg_devices: dict[int, tuple[Any, ctypes.c_void_p]] = {}
+
+ def _ensure_ffmpeg_device(self, ordinal: int) -> None:
+ """Create and retain FFmpeg's primary CUDA context before NGX loads.
+
+ Some driver releases reject the first av_hwdevice_ctx_create after NGX
+ has initialized CUDA. A retained generic device makes subsequent
+ PyAV decoder and encoder contexts cheap references to the same primary
+ context, even when capability probing happens before a video is open.
+ """
+ if ordinal in self.ffmpeg_devices:
+ return
+ import av
+
+ av_root = Path(av.__file__).resolve().parent.parent
+ candidates = sorted((av_root / "av.libs").glob("avutil-*.dll"))
+ if not candidates:
+ raise RTXVideoBridgeError("PyAV's avutil runtime is missing; CUDA video contexts cannot be prepared.")
+ avutil = ctypes.CDLL(str(candidates[0]))
+ avutil.av_hwdevice_find_type_by_name.argtypes = [ctypes.c_char_p]
+ avutil.av_hwdevice_find_type_by_name.restype = ctypes.c_int
+ avutil.av_dict_set.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_char_p,
+ ctypes.c_char_p, ctypes.c_int]
+ avutil.av_dict_set.restype = ctypes.c_int
+ avutil.av_dict_free.argtypes = [ctypes.POINTER(ctypes.c_void_p)]
+ avutil.av_hwdevice_ctx_create.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int,
+ ctypes.c_char_p, ctypes.c_void_p, ctypes.c_int]
+ avutil.av_hwdevice_ctx_create.restype = ctypes.c_int
+ kind = int(avutil.av_hwdevice_find_type_by_name(b"cuda"))
+ options, reference = ctypes.c_void_p(), ctypes.c_void_p()
+ if kind < 0 or avutil.av_dict_set(ctypes.byref(options), b"primary_ctx", b"1", 0) < 0:
+ raise RTXVideoBridgeError("FFmpeg CUDA primary-context options could not be prepared.")
+ try:
+ result = int(avutil.av_hwdevice_ctx_create(
+ ctypes.byref(reference), kind, str(int(ordinal)).encode("ascii"), options, 0))
+ finally:
+ avutil.av_dict_free(ctypes.byref(options))
+ if result < 0 or not reference.value:
+ raise RTXVideoBridgeError(
+ f"FFmpeg could not create CUDA primary context {ordinal} before NGX (error {result})."
+ )
+ # Deliberately process-lifetime: releasing this before NGX can recreate
+ # the driver-ordering hang that this guard prevents.
+ self.ffmpeg_devices[ordinal] = (avutil, reference)
+
+ def _load(self) -> Any:
+ if self.library is not None:
+ return self.library
+ runtime_files()
+ try:
+ library = ctypes.WinDLL(str(BRIDGE))
+ except OSError as exc:
+ raise RTXVideoBridgeError(f"Could not load the RTX Video CUDA bridge: {exc}") from exc
+ library.rtxv_abi_version.restype = ctypes.c_uint32
+ library.rtxv_version.restype = ctypes.c_char_p
+ library.rtxv_init.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_int]
+ library.rtxv_init.restype = ctypes.c_int
+ library.rtxv_status.argtypes = [ctypes.c_void_p, ctypes.c_int]
+ library.rtxv_status.restype = ctypes.c_int
+ library.rtxv_get_status_v1.argtypes = [ctypes.POINTER(BridgeStatusV1)]
+ library.rtxv_get_status_v1.restype = ctypes.c_int
+ library.rtxv_session_create.argtypes = [ctypes.POINTER(SessionDescriptorV1), ctypes.c_void_p, ctypes.c_int]
+ library.rtxv_session_create.restype = ctypes.c_void_p
+ library.rtxv_session_release.argtypes = [ctypes.c_void_p]
+ library.rtxv_surface_acquire.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_int]
+ library.rtxv_surface_acquire.restype = ctypes.c_void_p
+ library.rtxv_surface_frame_desc.argtypes = [ctypes.c_void_p, ctypes.POINTER(FrameDescriptorV1)]
+ library.rtxv_surface_frame_desc.restype = ctypes.c_int
+ library.rtxv_surface_retain.argtypes = [ctypes.c_void_p]
+ library.rtxv_surface_release.argtypes = [ctypes.c_void_p]
+ library.rtxv_process_frame_v1.argtypes = [
+ ctypes.c_void_p, ctypes.POINTER(FrameDescriptorV1), ctypes.POINTER(FrameDescriptorV1),
+ ctypes.POINTER(FrameResultV1), ctypes.c_void_p, ctypes.c_int,
+ ]
+ library.rtxv_process_frame_v1.restype = ctypes.c_int
+ library.rtxv_convert_input_v1.argtypes = [
+ ctypes.c_void_p, ctypes.POINTER(FrameDescriptorV1),
+ ctypes.POINTER(FrameDescriptorV1), ctypes.c_void_p, ctypes.c_int,
+ ]
+ library.rtxv_convert_input_v1.restype = ctypes.c_int
+ if int(library.rtxv_abi_version()) != BRIDGE_ABI_VERSION:
+ raise RTXVideoBridgeError("RTX Video CUDA bridge ABI does not match this application.")
+ self.library = library
+ return library
+
+ def guard(self) -> None:
+ if self.poisoned_reason:
+ raise RTXVideoBridgePoisonedError(
+ f"The RTX Video CUDA engine is poisoned: {self.poisoned_reason}. "
+ "Restart the application before using RTX Video again."
+ )
+
+ def initialize(self, ordinal: int) -> dict[str, Any]:
+ with NGX_RUNTIME_LOCK:
+ self.guard()
+ if self.gpu_ordinal is not None and self.gpu_ordinal != ordinal:
+ raise RTXVideoBridgeError(
+ "RTX Video is already bound to another AI GPU. Restart the application "
+ "before changing the RTX Video AI adapter."
+ )
+ self._ensure_ffmpeg_device(int(ordinal))
+ library = self._load()
+ error = ctypes.create_string_buffer(4096)
+ if not library.rtxv_init(int(ordinal), str(RUNTIME_DIR), error, len(error)):
+ detail = error.value.decode("utf-8", "replace") or "unknown initialization failure"
+ if "poison" in detail.casefold() or "access violation" in detail.casefold():
+ self.poisoned_reason = detail
+ self.guard()
+ raise RTXVideoBridgeError(f"RTX Video CUDA initialization failed: {detail}")
+ self.gpu_ordinal = int(ordinal)
+ return self.status()
+
+ def status(self) -> dict[str, Any]:
+ library = self._load()
+ record = BridgeStatusV1.empty()
+ if not library.rtxv_get_status_v1(ctypes.byref(record)):
+ raise RTXVideoBridgeError("RTX Video CUDA bridge returned an invalid status record.")
+ flags = int(record.flags)
+ return {
+ "abi_version": int(record.abi_version),
+ "bridge_version": bytes(record.bridge_version).split(b"\0", 1)[0].decode("utf-8", "replace"),
+ "initialized": bool(flags & 1), "poisoned": bool(flags & 2),
+ "gpu_ordinal": int(record.gpu_ordinal),
+ "gpu_name": bytes(record.gpu_name).split(b"\0", 1)[0].decode("utf-8", "replace"),
+ "vsr_available": bool(flags & 4), "hdr_available": bool(flags & 8),
+ "vsr_min_driver": f"{record.vsr_min_driver_major}.{record.vsr_min_driver_minor}",
+ "hdr_min_driver": f"{record.hdr_min_driver_major}.{record.hdr_min_driver_minor}",
+ "vsr_init_result": int(record.vsr_init_result),
+ "hdr_init_result": int(record.hdr_init_result),
+ "last_error": bytes(record.last_error).split(b"\0", 1)[0].decode("utf-8", "replace"),
+ }
+
+ def call(self, label: str, function, keepalive: tuple[Any, ...], timeout: float) -> Any:
+ """Run a potentially wedged native call without releasing its arguments."""
+ self.guard()
+ done = threading.Event()
+ result: list[Any] = []
+ failure: list[BaseException] = []
+
+ def invoke() -> None:
+ try:
+ with NGX_RUNTIME_LOCK:
+ result.append(function())
+ except BaseException as exc:
+ failure.append(exc)
+ finally:
+ done.set()
+
+ thread = threading.Thread(target=invoke, name=f"rtx-video-{label}", daemon=True)
+ thread.start()
+ if not done.wait(timeout):
+ _ = keepalive
+ self.poisoned_reason = f"{label} exceeded {timeout:g} seconds"
+ self.guard()
+ if failure:
+ self.poisoned_reason = f"native exception during {label}: {failure[0]}"
+ self.guard()
+ return result[0]
+
+
+_MANAGER = _BridgeManager()
+
+
+def _feature_record(status: dict[str, Any], prefix: str) -> dict[str, Any]:
+ version = str(status.get(f"{prefix}_min_driver") or "0.0").split(".", 1)
+ return {
+ "available": bool(status.get(f"{prefix}_available")),
+ "min_driver_major": int(version[0] or 0),
+ "min_driver_minor": int(version[1] or 0) if len(version) > 1 else 0,
+ "init_result": int(status.get(f"{prefix}_init_result") or 0),
+ }
+
+
+def probe_capabilities(gpu_uuid: str = "auto", *, controller=None) -> UpscaleCapabilities:
+ gpu = resolve_ai_gpu(detect_gpus(), gpu_uuid)
+ if controller is not None and controller.cancel.is_set():
+ raise Cancelled("Stopped before checking RTX Video capabilities.")
+ status = _MANAGER.initialize(int(gpu["cuda_ordinal"]))
+ return UpscaleCapabilities(
+ gpu=gpu, luid=gpu_luid(gpu), vsr=_feature_record(status, "vsr"),
+ hdr=_feature_record(status, "hdr"), sdk_version="1.1.0",
+ worker_version="developer-probe-only",
+ bridge_version=str(status.get("bridge_version") or "unknown"),
+ bridge_status=status,
+ )
+
+
+@lru_cache(maxsize=8)
+def probe_legacy_capabilities(luid: str) -> dict[str, Any]:
+ """Explicit developer-only D3D11 subprocess capability probe."""
+ runtime_files(legacy_probe=True)
+ controller = current_job_controller() or JobController()
+ process = subprocess.Popen(
+ [str(WORKER), "--probe", "--gpu-luid", luid], cwd=WORKER.parent,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ creationflags=subprocess.CREATE_NO_WINDOW,
+ )
+ controller.register(process)
+ try:
+ stdout, stderr = process.communicate(timeout=90)
+ finally:
+ if process.poll() is None:
+ process.kill()
+ controller.unregister(process)
+ if process.returncode:
+ raise RuntimeError(stderr.decode("utf-8", "replace")[-3000:])
+ return json.loads(stdout)
+
+
+class RTXVideoCudaSurface:
+ def __init__(self, library: Any, handle: int, descriptor: FrameDescriptorV1, ordinal: int) -> None:
+ self.library, self.handle = library, int(handle)
+ self.descriptor, self.ordinal = descriptor, int(ordinal)
+ self.closed = False
+
+ def retain(self) -> None:
+ self.library.rtxv_surface_retain(ctypes.c_void_p(self.handle))
+
+ def release(self) -> None:
+ self.library.rtxv_surface_release(ctypes.c_void_p(self.handle))
+
+ def close(self) -> None:
+ if not self.closed:
+ self.release()
+ self.closed = True
+
+ def to_av_frame(self):
+ import av
+
+ descriptor = self.descriptor
+ bits = 16 if descriptor.pixel_format == FORMAT_P010 else 8
+ item_size = bits // 8
+ y_plane = _DLPackPlane(
+ pointer=int(descriptor.planes[0]), shape=(int(descriptor.height), int(descriptor.width)),
+ strides=(int(descriptor.strides[0]) // item_size, 1), bits=bits,
+ device_id=self.ordinal, retain=self.retain, release=self.release,
+ )
+ uv_plane = _DLPackPlane(
+ pointer=int(descriptor.planes[1]),
+ shape=((int(descriptor.height) + 1) // 2, (int(descriptor.width) + 1) // 2, 2),
+ strides=(int(descriptor.strides[1]) // item_size, 2, 1), bits=bits,
+ device_id=self.ordinal, retain=self.retain, release=self.release,
+ )
+ try:
+ frame = av.VideoFrame.from_dlpack(
+ [y_plane, uv_plane], format="p010le" if bits == 16 else "nv12",
+ width=int(descriptor.width), height=int(descriptor.height),
+ device_id=self.ordinal, primary_ctx=True,
+ )
+ except BaseException:
+ self.close()
+ raise
+ self.close()
+ return frame
+
+
+class RTXVideoSession:
+ """Persistent RTX Video session supporting host and CUDA-frame callers."""
+
+ def __init__(self, width: int, height: int, output_width: int, output_height: int,
+ options: UpscaleOptions, input_format: int, capabilities: UpscaleCapabilities,
+ controller: JobController, *, timeout: float = 180,
+ image_srgb: bool = False):
+ options.validate()
+ for enabled, cap, name in ((options.vsr_enabled, capabilities.vsr, "VSR"),
+ (options.hdr_enabled, capabilities.hdr, "HDR")):
+ if enabled and not cap.get("available"):
+ raise RTXVideoBridgeError(
+ f"RTX Video {name} is unavailable on {capabilities.gpu['name']}. "
+ f"Minimum driver {cap.get('min_driver_major', 0)}.{cap.get('min_driver_minor', 0)}; "
+ f"initialization result 0x{cap.get('init_result', 0):08X}."
+ )
+ self.library = _MANAGER._load()
+ self.controller, self.timeout = controller, float(timeout)
+ self.ordinal = int(capabilities.gpu["cuda_ordinal"])
+ self.width, self.height = int(width), int(height)
+ self.output_width, self.output_height = int(output_width), int(output_height)
+ self.input_format = int(input_format)
+ self.output_format = (FORMAT_RGBA16F if options.hdr_precision == "FP16" else FORMAT_R10) \
+ if options.hdr_enabled else self.input_format
+ self.input_bytes = self.width * self.height * 4
+ self.output_bytes = self.output_width * self.output_height * (8 if self.output_format == FORMAT_RGBA16F else 4)
+ self.logs = collections.deque(maxlen=150)
+ self.completed_frames = 0
+ self.last_results = (0, 0)
+ self.last_timing: dict[str, float] = {}
+ self.upload_bytes = self.download_bytes = self.pool_waits = 0
+ self.closed = False
+ descriptor = SessionDescriptorV1()
+ descriptor.struct_size, descriptor.abi_version = ctypes.sizeof(descriptor), BRIDGE_ABI_VERSION
+ descriptor.input_width, descriptor.input_height = self.width, self.height
+ descriptor.output_width, descriptor.output_height = self.output_width, self.output_height
+ descriptor.input_format, descriptor.output_format = self.input_format, self.output_format
+ descriptor.vsr_enabled, descriptor.vsr_quality = int(options.vsr_enabled), int(options.vsr_quality)
+ descriptor.hdr_enabled = int(options.hdr_enabled)
+ descriptor.hdr_contrast, descriptor.hdr_saturation = int(options.hdr_contrast), int(options.hdr_saturation)
+ descriptor.hdr_middle_gray, descriptor.hdr_peak_luminance = int(options.hdr_middle_gray), int(options.hdr_peak_luminance)
+ descriptor.reserved[0] = int(bool(image_srgb))
+ error = ctypes.create_string_buffer(4096)
+ with NGX_RUNTIME_LOCK:
+ _MANAGER.guard()
+ self.handle = int(self.library.rtxv_session_create(ctypes.byref(descriptor), error, len(error)) or 0)
+ if not self.handle:
+ detail = error.value.decode("utf-8", "replace") or "unknown session creation failure"
+ raise RTXVideoBridgeError(f"Could not create RTX Video CUDA session: {detail}")
+
+ def _check(self) -> None:
+ if self.closed:
+ raise RTXVideoBridgeError("RTX Video session is closed.")
+ if self.controller.cancel.is_set():
+ raise Cancelled("Upscale stopped by user.")
+ _MANAGER.guard()
+
+ @staticmethod
+ def _host_pointer(buffer: Any, expected: int) -> tuple[Any, int]:
+ view = memoryview(buffer).cast("B")
+ if view.nbytes != expected:
+ raise ValueError("Incorrect RTX Video frame byte count.")
+ if not view.contiguous:
+ raise ValueError("RTX Video host frames must be contiguous.")
+ if view.readonly:
+ owner: Any = bytearray(view)
+ view = memoryview(owner)
+ else:
+ owner = buffer
+ return owner, ctypes.addressof(ctypes.c_ubyte.from_buffer(view))
+
+ def _record(self, result: FrameResultV1) -> dict[str, Any]:
+ self.completed_frames += 1
+ self.last_results = (int(result.vsr_result), int(result.hdr_result))
+ self.upload_bytes += int(result.upload_bytes)
+ self.download_bytes += int(result.download_bytes)
+ self.last_timing = {
+ "input_ms": float(result.input_milliseconds), "ngx_ms": float(result.ngx_milliseconds),
+ "output_ms": float(result.output_milliseconds), "total_ms": float(result.total_milliseconds),
+ }
+ return {
+ **self.last_timing, "vsr_result": f"0x{int(result.vsr_result):08X}",
+ "hdr_result": f"0x{int(result.hdr_result):08X}", "cuda_result": int(result.cuda_result),
+ "upload_bytes": int(result.upload_bytes), "download_bytes": int(result.download_bytes),
+ }
+
+ def _evaluate(self, source: FrameDescriptorV1, destination: FrameDescriptorV1,
+ keepalive: tuple[Any, ...]) -> dict[str, Any]:
+ result, error = FrameResultV1.empty(), ctypes.create_string_buffer(4096)
+ ok = _MANAGER.call(
+ "frame evaluation",
+ lambda: self.library.rtxv_process_frame_v1(
+ ctypes.c_void_p(self.handle), ctypes.byref(source), ctypes.byref(destination),
+ ctypes.byref(result), error, len(error)),
+ (*keepalive, source, destination, result, error), self.timeout,
+ )
+ if not ok:
+ detail = error.value.decode("utf-8", "replace") or "unknown frame evaluation failure"
+ if _MANAGER.status().get("poisoned"):
+ _MANAGER.poisoned_reason = detail
+ _MANAGER.guard()
+ raise RTXVideoBridgeError(f"RTX Video CUDA evaluation failed: {detail}")
+ return self._record(result)
+
+ def process_frame(self, pixels) -> bytearray:
+ """Compatibility route for images/software decode: one upload/download."""
+ self._check()
+ source_owner, source_pointer = self._host_pointer(pixels, self.input_bytes)
+ output = bytearray(self.output_bytes)
+ _, destination_pointer = self._host_pointer(output, self.output_bytes)
+ source, destination = FrameDescriptorV1.empty(), FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_HOST, self.input_format
+ source.width, source.height, source.planes[0], source.strides[0] = self.width, self.height, source_pointer, self.width * 4
+ destination.memory_type, destination.pixel_format = MEMORY_HOST, self.output_format
+ destination.width, destination.height = self.output_width, self.output_height
+ destination.planes[0] = destination_pointer
+ destination.strides[0] = self.output_width * (8 if self.output_format == FORMAT_RGBA16F else 4)
+ self._evaluate(source, destination, (source_owner, pixels, output))
+ return output
+
+ def _acquire_surface(self, pixel_format: int) -> RTXVideoCudaSurface:
+ deadline, error = time.monotonic() + self.timeout, ctypes.create_string_buffer(4096)
+ while True:
+ self._check()
+ with NGX_RUNTIME_LOCK:
+ handle = int(self.library.rtxv_surface_acquire(
+ ctypes.c_void_p(self.handle), pixel_format, error, len(error)) or 0)
+ if handle:
+ descriptor = FrameDescriptorV1.empty()
+ if not self.library.rtxv_surface_frame_desc(ctypes.c_void_p(handle), ctypes.byref(descriptor)):
+ self.library.rtxv_surface_release(ctypes.c_void_p(handle))
+ raise RTXVideoBridgeError("RTX Video returned an invalid CUDA surface descriptor.")
+ return RTXVideoCudaSurface(self.library, handle, descriptor, self.ordinal)
+ detail = error.value.decode("utf-8", "replace")
+ if "pool is exhausted" not in detail:
+ raise RTXVideoBridgeError(f"Could not acquire RTX Video CUDA output surface: {detail}")
+ self.pool_waits += 1
+ gc.collect()
+ if time.monotonic() >= deadline:
+ raise RTXVideoBridgeError("RTX Video CUDA output surface pool remained exhausted.")
+ time.sleep(0.001)
+
+ def process_cuda_frame(self, frame: Any, *, color_matrix: int = 1, color_range: int = 0,
+ color_primaries: int = 1, color_transfer: int = 0,
+ chroma_location: int = 0, rotation: int = 0,
+ output_p010: bool = False) -> tuple[Any, dict[str, Any]]:
+ self._check()
+ if str(getattr(getattr(frame, "format", None), "name", "")) != "cuda":
+ raise RTXVideoBridgeError("RTX Video CUDA input must be a PyAV CUDA frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ pixel_format = {"nv12": FORMAT_NV12, "p010": FORMAT_P010, "p010le": FORMAT_P010}.get(sw_format)
+ if pixel_format is None or len(frame.planes) < 2:
+ raise RTXVideoBridgeError(f"Unsupported NVDEC surface format: {sw_format or 'unknown'}.")
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_CUDA, pixel_format
+ source.width, source.height = int(frame.width), int(frame.height)
+ source.planes[0], source.planes[1] = int(frame.planes[0].buffer_ptr), int(frame.planes[1].buffer_ptr)
+ source.strides[0], source.strides[1] = int(frame.planes[0].line_size), int(frame.planes[1].line_size)
+ source.color_matrix, source.color_range = int(color_matrix), int(color_range)
+ source.color_primaries, source.color_transfer = int(color_primaries), int(color_transfer)
+ source.rotation = int(rotation) % 360
+ source.reserved[0] = int(chroma_location)
+ surface = self._acquire_surface(FORMAT_P010 if output_p010 else FORMAT_NV12)
+ try:
+ details = self._evaluate(source, surface.descriptor, (frame, surface))
+ output = surface.to_av_frame()
+ except BaseException:
+ surface.close()
+ raise
+ output.pts, output.time_base = getattr(frame, "pts", None), getattr(frame, "time_base", None)
+ if getattr(frame, "duration", None) is not None:
+ output.duration = frame.duration
+ details.update(input_format=sw_format, output_format="p010le" if output_p010 else "nv12")
+ return output, details
+
+ def process_cuda_to_host_for_test(
+ self, frame: Any, *, color_matrix: int = 1, color_range: int = 0,
+ color_primaries: int = 1, color_transfer: int = 0,
+ chroma_location: int = 0, rotation: int = 0,
+ ) -> tuple[bytearray, dict[str, Any]]:
+ """Evaluate a CUDA input into packed host output for lossless A/B tests.
+
+ Production video uses :meth:`process_cuda_frame`; this explicit test
+ route intentionally downloads the result so legacy and CUDA engines can
+ be compared before a lossy delivery encode.
+ """
+ self._check()
+ if str(getattr(getattr(frame, "format", None), "name", "")) != "cuda":
+ raise RTXVideoBridgeError("RTX Video CUDA input must be a PyAV CUDA frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ pixel_format = {"nv12": FORMAT_NV12, "p010": FORMAT_P010,
+ "p010le": FORMAT_P010}.get(sw_format)
+ if pixel_format is None or len(frame.planes) < 2:
+ raise RTXVideoBridgeError(f"Unsupported NVDEC surface format: {sw_format or 'unknown'}.")
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_CUDA, pixel_format
+ source.width, source.height = int(frame.width), int(frame.height)
+ source.planes[0], source.planes[1] = (
+ int(frame.planes[0].buffer_ptr), int(frame.planes[1].buffer_ptr))
+ source.strides[0], source.strides[1] = (
+ int(frame.planes[0].line_size), int(frame.planes[1].line_size))
+ source.color_matrix, source.color_range = int(color_matrix), int(color_range)
+ source.color_primaries, source.color_transfer = int(color_primaries), int(color_transfer)
+ source.rotation, source.reserved[0] = int(rotation) % 360, int(chroma_location)
+ output = bytearray(self.output_bytes)
+ _, destination_pointer = self._host_pointer(output, self.output_bytes)
+ destination = FrameDescriptorV1.empty()
+ destination.memory_type, destination.pixel_format = MEMORY_HOST, self.output_format
+ destination.width, destination.height = self.output_width, self.output_height
+ destination.planes[0] = destination_pointer
+ destination.strides[0] = self.output_width * (
+ 8 if self.output_format == FORMAT_RGBA16F else 4)
+ details = self._evaluate(source, destination, (frame, output))
+ details.update(input_format=sw_format, output_format={
+ FORMAT_RGBA8: "rgba8", FORMAT_R10: "r10", FORMAT_RGBA16F: "rgba16f",
+ }[self.output_format])
+ return output, details
+
+ def _planar_destination(self, pixel_format: int, plane_pointers: tuple[int, ...],
+ strides: tuple[int, ...]) -> FrameDescriptorV1:
+ required = 3 if pixel_format == FORMAT_YUV422P10 else 2
+ if pixel_format not in {FORMAT_NV12, FORMAT_P010, FORMAT_YUV422P10}:
+ raise ValueError("Pinned host output must be NV12, P010, or YUV422P10.")
+ if len(plane_pointers) != required or len(strides) != required:
+ raise ValueError(f"Pinned host output requires {required} planes.")
+ if any(int(pointer) <= 0 for pointer in plane_pointers) or any(int(stride) <= 0 for stride in strides):
+ raise ValueError("Pinned host output planes and strides must be positive.")
+ destination = FrameDescriptorV1.empty()
+ destination.memory_type, destination.pixel_format = MEMORY_HOST, int(pixel_format)
+ destination.width, destination.height = self.output_width, self.output_height
+ for index, (pointer, stride) in enumerate(zip(plane_pointers, strides)):
+ destination.planes[index], destination.strides[index] = int(pointer), int(stride)
+ return destination
+
+ def process_cuda_to_host_planar(
+ self, frame: Any, *, output_format: int,
+ plane_pointers: tuple[int, ...], strides: tuple[int, ...],
+ color_matrix: int = 1, color_range: int = 0,
+ color_primaries: int = 1, color_transfer: int = 0,
+ chroma_location: int = 0, rotation: int = 0,
+ ) -> dict[str, Any]:
+ """Evaluate a CUDA source into caller-owned pinned planar host memory."""
+ self._check()
+ if str(getattr(getattr(frame, "format", None), "name", "")) != "cuda":
+ raise RTXVideoBridgeError("RTX Video CUDA input must be a PyAV CUDA frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ pixel_format = {"nv12": FORMAT_NV12, "p010": FORMAT_P010,
+ "p010le": FORMAT_P010}.get(sw_format)
+ if pixel_format is None or len(frame.planes) < 2:
+ raise RTXVideoBridgeError(f"Unsupported NVDEC surface format: {sw_format or 'unknown'}.")
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_CUDA, pixel_format
+ source.width, source.height = int(frame.width), int(frame.height)
+ source.planes[0], source.planes[1] = (
+ int(frame.planes[0].buffer_ptr), int(frame.planes[1].buffer_ptr))
+ source.strides[0], source.strides[1] = (
+ int(frame.planes[0].line_size), int(frame.planes[1].line_size))
+ source.color_matrix, source.color_range = int(color_matrix), int(color_range)
+ source.color_primaries, source.color_transfer = int(color_primaries), int(color_transfer)
+ source.rotation, source.reserved[0] = int(rotation) % 360, int(chroma_location)
+ destination = self._planar_destination(output_format, plane_pointers, strides)
+ details = self._evaluate(source, destination, (frame,))
+ details.update(input_format=sw_format, output_format={
+ FORMAT_NV12: "nv12", FORMAT_P010: "p010le", FORMAT_YUV422P10: "yuv422p10le",
+ }[output_format])
+ return details
+
+ def process_host_to_host_planar(
+ self, pixels: Any, *, output_format: int,
+ plane_pointers: tuple[int, ...], strides: tuple[int, ...],
+ ) -> dict[str, Any]:
+ """One host upload and one pinned planar download for software decoders."""
+ self._check()
+ source_owner, source_pointer = self._host_pointer(pixels, self.input_bytes)
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_HOST, self.input_format
+ source.width, source.height = self.width, self.height
+ source.planes[0], source.strides[0] = source_pointer, self.width * 4
+ destination = self._planar_destination(output_format, plane_pointers, strides)
+ details = self._evaluate(source, destination, (source_owner, pixels))
+ details.update(input_format="host-packed", output_format={
+ FORMAT_NV12: "nv12", FORMAT_P010: "p010le", FORMAT_YUV422P10: "yuv422p10le",
+ }[output_format])
+ return details
+
+ def convert_cuda_input_for_test(
+ self, frame: Any, *, color_matrix: int = 1, color_range: int = 0,
+ color_primaries: int = 1, color_transfer: int = 0,
+ chroma_location: int = 0, rotation: int = 0,
+ ) -> bytearray:
+ """Return the production pre-NGX packed input for deterministic tests."""
+ self._check()
+ if str(getattr(getattr(frame, "format", None), "name", "")) != "cuda":
+ raise RTXVideoBridgeError("RTX Video CUDA input must be a PyAV CUDA frame.")
+ sw_format = str(getattr(getattr(frame, "sw_format", None), "name", ""))
+ pixel_format = {"nv12": FORMAT_NV12, "p010": FORMAT_P010, "p010le": FORMAT_P010}.get(sw_format)
+ if pixel_format is None or len(frame.planes) < 2:
+ raise RTXVideoBridgeError(f"Unsupported NVDEC surface format: {sw_format or 'unknown'}.")
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_CUDA, pixel_format
+ source.width, source.height = int(frame.width), int(frame.height)
+ source.planes[0], source.planes[1] = int(frame.planes[0].buffer_ptr), int(frame.planes[1].buffer_ptr)
+ source.strides[0], source.strides[1] = int(frame.planes[0].line_size), int(frame.planes[1].line_size)
+ source.color_matrix, source.color_range = int(color_matrix), int(color_range)
+ source.color_primaries, source.color_transfer = int(color_primaries), int(color_transfer)
+ source.rotation, source.reserved[0] = int(rotation) % 360, int(chroma_location)
+ output = bytearray(self.input_bytes)
+ _, pointer = self._host_pointer(output, self.input_bytes)
+ destination = FrameDescriptorV1.empty()
+ destination.memory_type, destination.pixel_format = MEMORY_HOST, self.input_format
+ destination.width, destination.height = self.width, self.height
+ destination.planes[0], destination.strides[0] = pointer, self.width * 4
+ error = ctypes.create_string_buffer(4096)
+ ok = _MANAGER.call(
+ "input-conversion validation",
+ lambda: self.library.rtxv_convert_input_v1(
+ ctypes.c_void_p(self.handle), ctypes.byref(source), ctypes.byref(destination),
+ error, len(error),
+ ),
+ (frame, output, source, destination, error), self.timeout,
+ )
+ if not ok:
+ detail = error.value.decode("utf-8", "replace") or "unknown input-conversion failure"
+ raise RTXVideoBridgeError(f"RTX Video CUDA input conversion failed: {detail}")
+ return output
+
+ def process_host_to_cuda_frame(self, pixels: Any, *, output_p010: bool = False,
+ pts: int | None = None, time_base: Any = None,
+ duration: int | None = None) -> tuple[Any, dict[str, Any]]:
+ """Upload a normalized software-decoded frame once and return CUDA YUV."""
+ self._check()
+ source_owner, source_pointer = self._host_pointer(pixels, self.input_bytes)
+ source = FrameDescriptorV1.empty()
+ source.memory_type, source.pixel_format = MEMORY_HOST, self.input_format
+ source.width, source.height = self.width, self.height
+ source.planes[0], source.strides[0] = source_pointer, self.width * 4
+ surface = self._acquire_surface(FORMAT_P010 if output_p010 else FORMAT_NV12)
+ try:
+ details = self._evaluate(source, surface.descriptor, (source_owner, pixels, surface))
+ output = surface.to_av_frame()
+ except BaseException:
+ surface.close()
+ raise
+ output.pts = pts
+ if time_base is not None:
+ output.time_base = time_base
+ if duration is not None:
+ output.duration = duration
+ details.update(input_format="host-r10" if self.input_format == FORMAT_R10 else "host-rgba8",
+ output_format="p010le" if output_p010 else "nv12")
+ return output, details
+
+ def structured_status(self, *, decode_backend: str = "software", encode_backend: str = "") -> dict[str, Any]:
+ return {
+ **_MANAGER.status(),
+ "memory_path": "cuda_zero_copy" if self.upload_bytes == 0 and self.download_bytes == 0 else "host_staging",
+ "decode_backend": decode_backend, "encode_backend": encode_backend,
+ "frames": self.completed_frames,
+ "transfers": {"upload_bytes": self.upload_bytes, "download_bytes": self.download_bytes},
+ "last_timing": dict(self.last_timing),
+ "surface_pool": {"capacity": 8, "backpressure_waits": self.pool_waits},
+ }
+
+ def close(self, *, abort: bool = False) -> None:
+ del abort
+ if self.closed:
+ return
+ self.closed = True
+ handle, self.handle = self.handle, 0
+ with NGX_RUNTIME_LOCK:
+ self.library.rtxv_session_release(ctypes.c_void_p(handle))
+
+ def __enter__(self) -> "RTXVideoSession":
+ return self
+
+ def __exit__(self, exc_type, *_args) -> None:
+ self.close(abort=exc_type is not None)
diff --git a/src/upscale/video/processor.py b/src/upscale/video/processor.py
index 920da62..861eecf 100644
--- a/src/upscale/video/processor.py
+++ b/src/upscale/video/processor.py
@@ -1,197 +1,70 @@
-from __future__ import annotations
-
-import math
-import tempfile
-import time
-import uuid
-from contextlib import ExitStack, nullcontext, suppress
-from dataclasses import replace
-from fractions import Fraction
-from pathlib import Path
-
-import av
-
-from ...core import app_log, ffmpeg
-from ...core.disk_paths import OutputFile, prepare_output_dir
-from ...core.gpu_detection import detect_gpus
-from ...core.jobs import Cancelled, active_job
-from ...core.naming import output_filename
-from ...core.paths import JOBS
-from .media import PipeReader, PipeWriter, close_process, finish_process, inspect_video, output_filter, packed_bytes, result_frame, start_decoder
-from .models import UpscaleOptions, UpscaleResult, output_size
-from .native import RTXVideoSession, probe_capabilities
-
-
-def upscale_video(input_path, options: UpscaleOptions | None = None, progress=None, *, output_dir=None,
- controller=None, _owns_slot=False, _capabilities=None) -> UpscaleResult:
- options = replace(options) if options else UpscaleOptions()
- options.validate()
- source = Path(input_path).resolve()
- if not source.is_file():
- raise FileNotFoundError(source)
- context = nullcontext(controller) if _owns_slot else active_job(controller)
- with context as controller:
- return _process(source, options, progress, output_dir, controller, _capabilities)
-
-
-def _process(source, options, progress, output_dir, controller, capabilities):
- started = time.monotonic()
- stamp = time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:8]
- JOBS.mkdir(exist_ok=True)
- report_path = app_log.session_path()
- app_log.info("upscale", f"start src={source.name}")
- processes = []
- destination_file = None
- session = None
- delivered = 0
- def check_cancel():
- if controller.cancel.is_set():
- raise Cancelled("Upscale stopped by user.")
- def update(value, message):
- check_cancel()
- if progress:
- progress(value, message)
- try:
- update(.01, "Checking SDR source and RTX Video capabilities")
- meta = inspect_video(source, controller, reject_hdr=True)
- width, height = meta["width"], meta["height"]
- ow, oh, rounding = output_size(width, height, options)
- caps = capabilities or probe_capabilities(options.ai_gpu_uuid, controller=controller)
- video_gpu = ffmpeg.resolve_video_gpu(detect_gpus(), options.video_gpu_uuid, options.codec, ow, oh)
- destination = prepare_output_dir(output_dir)
- extension = {"MP4": ".mp4", "MKV": ".mkv", "MOV": ".mov"}[options.container]
- preview = options.preview_frames is not None or options.preview_seconds is not None
- kind = "RTXVIDEO_PREVIEW" if preview else "RTXVIDEO"
- output = destination / output_filename(source, extension, "Auto" if preview else options.rename_mode,
- options.custom_suffix, f"{source.stem}_{kind}_{stamp}")
- destination_file = OutputFile(output)
- input_format = 2 if meta["depth"] > 8 else 1
- ten_bit = options.hdr_enabled or (input_format == 2 and ffmpeg.hdr_mode_supported(options.codec))
- colors = {"color_space": "bt2020nc" if options.hdr_enabled else "bt709",
- "color_transfer": "smpte2084" if options.hdr_enabled else "bt709",
- "color_primaries": "bt2020" if options.hdr_enabled else "bt709", "hdr": options.hdr_enabled}
- with tempfile.TemporaryDirectory(prefix="rtx-video-", dir=JOBS) as job_dir, ExitStack() as stack:
- session = stack.enter_context(RTXVideoSession(width, height, ow, oh, options, input_format, caps, controller))
- # Matroska's millisecond time base would quantize fractional/VFR PTS.
- temp_video = Path(job_dir) / ("encoded.mov" if options.codec == "ProRes Proxy" else "encoded.mp4")
- encoder, enc_thread, enc_logs, selected, quality = ffmpeg.start_encoder(
- temp_video, options.codec, options.quality, controller, ow, oh, float(meta["rate"]),
- None if video_gpu is None else int(video_gpu["cuda_ordinal"]), video_gpu is not None,
- hdr_mode=ten_bit, hdr_metadata=colors, preserve_timestamps=True,
- video_filter=output_filter(session.output_format, options.hdr_enabled), keep_start_time=True,
- bounded_logs=True,
- )
- processes.append(encoder)
- stack.callback(close_process, encoder, controller, enc_thread)
- decoder, dec_thread, dec_logs, assumptions = start_decoder(source, meta, controller)
- processes.append(decoder)
- stack.callback(close_process, decoder, controller, dec_thread)
- decoded = stack.enter_context(av.open(PipeReader(decoder.stdout), mode="r", format="nut"))
- source_stream = decoded.streams.video[0]
- tb = source_stream.time_base
- nut = stack.enter_context(av.open(PipeWriter(encoder.stdin), mode="w", format="nut"))
- raw = nut.add_stream("rawvideo", rate=meta["rate"])
- raw.width, raw.height = ow, oh
- raw.pix_fmt = {1: "rgba", 2: "gbrp10le", 3: "gbrpf32le"}[session.output_format]
- raw.time_base = tb; raw.codec_context.time_base = tb
- first_time, last_pts, stopped_early = None, None, False
- eval_seconds = 0.0
- timestamps_fallback = 0
- estimated = max(1, meta["frames"] or math.ceil(meta["duration"] * float(meta["rate"])))
- limit = options.preview_frames
- if options.preview_seconds is not None:
- estimated = min(estimated, max(1, math.ceil(options.preview_seconds * float(meta["rate"]))))
- if limit:
- estimated = min(estimated, limit)
- last_update = 0
- for frame in decoded.decode(source_stream):
- check_cancel()
- if frame.pts is None:
- pts = round(Fraction(delivered, 1) / meta["rate"] / tb)
- timestamps_fallback += 1
- else:
- pts = round(frame.pts * frame.time_base / tb)
- timestamp = pts * tb
- if first_time is None:
- first_time = timestamp
- if (limit and delivered >= limit) or (options.preview_seconds is not None and delivered and timestamp - first_time >= options.preview_seconds):
- stopped_early = True
- break
- if last_pts is not None and pts <= last_pts:
- raise ValueError("Source timestamps are not strictly increasing; refusing to drop or duplicate frames.")
- if (frame.width, frame.height) != (width, height):
- raise ValueError("Source dimensions changed during decoding.")
- t0 = time.monotonic()
- pixels = session.process_frame(packed_bytes(frame))
- eval_seconds += time.monotonic() - t0
- result = result_frame(pixels, ow, oh, session.output_format)
- result.pts, result.time_base = pts, tb
- for packet in raw.encode(result):
- nut.mux(packet)
- delivered += 1
- last_pts = pts
- now = time.monotonic()
- if now - last_update > .2:
- update(min(.87, .05 + .82 * delivered / estimated),
- f"RTX Video: {width}×{height} → {ow}×{oh}; {delivered} frames; {delivered / max(.01, now-started):.1f} fps")
- last_update = now
- if not delivered:
- raise ValueError("The input contains no decodable video frames.")
- for packet in raw.encode():
- nut.mux(packet)
- nut.close(); encoder.stdin.close()
- decoded.close()
- finish_process(decoder, dec_thread, dec_logs, controller, expected_stop=stopped_early)
- session.close()
- finish_process(encoder, enc_thread, enc_logs, controller)
- if not preview and (not meta["frames"] or delivered != meta["frames"]):
- exact = ffmpeg.probe_video(source, count_mode="exact", strict_decode=True, controller=controller)
- if exact["frames"] != delivered:
- raise RuntimeError(f"Source has {exact['frames']} frames but only {delivered} were processed.")
- if session.completed_frames != delivered:
- raise RuntimeError("RTX Video worker processed an unexpected number of frames.")
- update(.90, "Muxing original audio and subtitles")
- ffmpeg.final_mux(temp_video, source, destination_file.temporary, options.container, controller,
- preserve_supported_subtitles=True, source_time_origin=meta["origin"])
- update(.96, "Verifying output frames, resolution, and HDR signaling")
- verified = ffmpeg.probe_video(destination_file.temporary, count_mode="packets", controller=controller)
- if verified["frames"] != delivered:
- verified = ffmpeg.probe_video(destination_file.temporary, count_mode="exact", strict_decode=True, controller=controller)
- if verified["frames"] != delivered or (verified["width"], verified["height"]) != (ow, oh):
- raise RuntimeError("Saved output does not match processed frame count or dimensions.")
- saved = inspect_video(destination_file.temporary, controller)
- if options.hdr_enabled and (saved["depth"] < 10 or not saved["hdr"] or
- saved["stream"].get("color_primaries") != "bt2020" or saved["stream"].get("color_space") != "bt2020nc"):
- raise RuntimeError("Saved file failed HDR color and bit-depth verification.")
- if not options.hdr_enabled and saved["hdr"]:
- raise RuntimeError("SDR output unexpectedly contains HDR signaling.")
- check_cancel()
- elapsed = time.monotonic() - started
- app_log.info(
- "upscale",
- f"done src={source.name} out={output.name} frames={delivered} "
- f"elapsed={elapsed:.0f}s",
- )
- destination_file.publish()
- return UpscaleResult(str(output), report_path, delivered, ow, oh, options.hdr_enabled, time.monotonic()-started)
- except BaseException as exc:
- cancelled = controller.cancel.is_set() or isinstance(exc, Cancelled)
- if not cancelled:
- tails: dict[str, object] = {}
- if session is not None:
- tails["worker"] = list(session.logs)[-40:]
- with suppress(Exception):
- tails["decoder"] = dec_logs.snapshot()[-40:]
- with suppress(Exception):
- tails["encoder"] = enc_logs.snapshot()[-40:]
- report_path = str(app_log.fail("upscale", f"upscale-{source.stem}", exc, tails))
- else:
- app_log.info("upscale", f"cancelled src={source.name}")
- if controller.cancel.is_set() and not isinstance(exc, Cancelled):
- raise Cancelled("Upscale stopped by user.") from exc
- raise
- finally:
- for process in processes:
- close_process(process, controller)
- if destination_file:
- destination_file.cleanup()
+from __future__ import annotations
+
+from contextlib import nullcontext
+from dataclasses import replace
+from pathlib import Path
+
+from ...core import app_log, ffmpeg
+from ...core.gpu_detection import detect_gpus
+from ...core.jobs import Cancelled, active_job
+from .cuda_pipeline import convert_video_cuda_nvenc, is_nvenc
+from .host_pipeline import convert_video_inprocess_host
+from .media import inspect_video
+from .models import UpscaleOptions, UpscaleResult, output_size
+from .native import probe_capabilities
+
+
+def upscale_video(input_path, options: UpscaleOptions | None = None, progress=None, *,
+ output_dir=None, controller=None, _owns_slot=False,
+ _capabilities=None) -> UpscaleResult:
+ options = replace(options) if options else UpscaleOptions()
+ options.container = ffmpeg.container_for_codec(options.codec)
+ options.validate()
+ source = Path(input_path).resolve()
+ if not source.is_file():
+ raise FileNotFoundError(source)
+ context = nullcontext(controller) if _owns_slot else active_job(controller)
+ with context as controller:
+ return _process(source, options, progress, output_dir, controller, _capabilities)
+
+
+def _process(source, options, progress, output_dir, controller, capabilities):
+ app_log.info("upscale", f"start src={source.name}")
+ try:
+ if progress:
+ progress(0.01, "Checking SDR source and RTX Video capabilities")
+ metadata = inspect_video(source, controller, reject_hdr=True)
+ output_width, output_height, _ = output_size(
+ metadata["width"], metadata["height"], options)
+ capabilities = capabilities or probe_capabilities(
+ options.ai_gpu_uuid, controller=controller)
+ requested_video_gpu = (
+ str(capabilities.gpu["uuid"])
+ if options.video_gpu_uuid == "auto"
+ else options.video_gpu_uuid
+ )
+ video_gpu = ffmpeg.resolve_video_gpu(
+ detect_gpus(), requested_video_gpu, options.codec,
+ output_width, output_height)
+ if is_nvenc(options.codec):
+ if video_gpu is None:
+ raise RuntimeError(
+ "An NVIDIA encoder was selected but the requested adapter cannot encode "
+ f"{output_width}×{output_height} with {options.codec}.")
+ return convert_video_cuda_nvenc(
+ source, options, controller=controller, progress=progress,
+ output_dir=output_dir, metadata=metadata,
+ capabilities=capabilities, video_gpu=video_gpu)
+ return convert_video_inprocess_host(
+ source, options, controller=controller, progress=progress,
+ output_dir=output_dir, metadata=metadata,
+ capabilities=capabilities)
+ except BaseException as exc:
+ cancelled = controller.cancel.is_set() or isinstance(exc, Cancelled)
+ if cancelled:
+ app_log.info("upscale", f"cancelled src={source.name}")
+ if not isinstance(exc, Cancelled):
+ raise Cancelled("Upscale stopped by user.") from exc
+ else:
+ app_log.fail("upscale", f"upscale-{source.stem}", exc)
+ raise
diff --git a/src/upscale/video/ui.py b/src/upscale/video/ui.py
index 3c1192e..64c5aa2 100644
--- a/src/upscale/video/ui.py
+++ b/src/upscale/video/ui.py
@@ -10,7 +10,7 @@ from ...core.batch_ui import (
build_path_controls, build_save_controls,
)
from ...core.disk_paths import resolve_inputs
-from ...core.ffmpeg import CODEC_CHOICES, ENCODING_QUALITIES, hdr_mode_supported
+from ...core.ffmpeg import CODEC_CHOICES, ENCODING_QUALITIES, container_for_codec, hdr_mode_supported
from ...core.naming import RENAME_MODES
from ...settings.storage import processing_gpu_settings
from .batch import upscale_videos
@@ -23,7 +23,9 @@ from .preview import display_result, preview_mode, preview_upscale
def options_from_values(values):
ai, video = processing_gpu_settings()
- return UpscaleOptions(**dict(zip(SETTING_FIELDS, values)), ai_gpu_uuid=ai, video_gpu_uuid=video)
+ selected = dict(zip(SETTING_FIELDS, values))
+ selected["container"] = container_for_codec(selected["codec"])
+ return UpscaleOptions(**selected, ai_gpu_uuid=ai, video_gpu_uuid=video)
def render_upscale_batch(paths, *values, progress=None, output_dir=None, controller=None, on_item_update=None, direct_disk=False):
@@ -153,7 +155,10 @@ def build_upscale_tab(settings):
c["quality"] = gr.Radio(ENCODING_QUALITIES, value=opts.quality, label="Encoding quality")
with gr.Row():
c["codec"] = gr.Dropdown(CODEC_CHOICES, value=opts.codec, label="Video codec")
- c["container"] = gr.Dropdown(("MP4", "MKV", "MOV"), value=opts.container, label="Container")
+ c["container"] = gr.Dropdown(
+ ("MP4", "MKV", "MOV"), value=container_for_codec(opts.codec),
+ label="Container (automatic)", interactive=False,
+ )
with gr.Row():
c["rename_mode"] = gr.Radio(RENAME_MODES, value=opts.rename_mode, label="Rename")
c["custom_suffix"] = gr.Textbox(value=opts.custom_suffix, label="Custom suffix", interactive=opts.rename_mode == "Custom")
@@ -172,8 +177,14 @@ def build_upscale_tab(settings):
preview_actions=[(tab.preview_frame, preview_frame), (tab.preview, preview_clip)],
)
c["hdr_enabled"].change(lambda enabled: gr.update(visible=enabled), inputs=c["hdr_enabled"], outputs=hdr_controls, queue=False)
- c["codec"].change(lambda codec: gr.update(interactive=True) if hdr_mode_supported(codec) else gr.update(value=False, interactive=False),
- inputs=c["codec"], outputs=c["hdr_enabled"], queue=False)
+ c["codec"].change(
+ lambda codec: (
+ gr.update(value=container_for_codec(codec)),
+ gr.update(interactive=True) if hdr_mode_supported(codec)
+ else gr.update(value=False, interactive=False),
+ ),
+ inputs=c["codec"], outputs=[c["container"], c["hdr_enabled"]], queue=False,
+ )
c["rename_mode"].change(lambda mode: gr.update(interactive=mode == "Custom"), inputs=c["rename_mode"], outputs=c["custom_suffix"], queue=False)
def sizing_controls(enabled, mode, lock):
custom = mode == "Custom dimensions"