Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

Release: streamline RCP inference and target video outputs (#45)

Yudong Jin41fe7ef8fbca
diff --git a/README.md b/README.md
index d6ac647..a895a22 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@
- Peaks at 22 GB of CUDA memory, enabling inference on consumer GPUs.
- Averages 27 seconds per 121-frame video on a single RTX 4090.
-- Requires neither input camera parameters nor a static camera.
+- No input camera parameters or static camera required.
## News
@@ -112,9 +112,7 @@ python inference.py \
│ ├── motion.json # source timeline and motion metadata
│ └── motion.safetensors # motion tensors
├── skeletons/00.mp4 ... <N-1>.mp4 # pose conditioning for each target view
-└── videos/
- ├── sparse/{00,04,09,12,14,19}.mp4 # RCP videos
- └── dense/00.mp4 ... <N-1>.mp4 # target videos
+└── videos/00.mp4 ... <N-1>.mp4 # target videos
```
### Custom Data
diff --git a/fdanyone/config.py b/fdanyone/config.py
index 297543a..0012f00 100644
--- a/fdanyone/config.py
+++ b/fdanyone/config.py
@@ -23,7 +23,6 @@ class InferenceConfig:
(30000, 1001),
)
temporal_sampling_policy: str = "nearest_source_pts_on_zero_based_cfr_clock"
- rcp_jpeg_quality: int = 85
skeleton_h264_crf: int = 17
target_h264_crf: int = 18
h264_preset: str = "medium"
diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.py
index 6dfddc8..07c9398 100644
--- a/fdanyone/model/inference.py
+++ b/fdanyone/model/inference.py
@@ -13,6 +13,7 @@ import math
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
+from fractions import Fraction
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, TypedDict
@@ -22,7 +23,6 @@ from fdanyone.config import INFERENCE, DenoisingProfile
from fdanyone.errors import FourDAnyoneError
from fdanyone.model.conditioning import (
PoseFeatureBank,
- PoseFeatureCache,
build_pose_feature_cache,
load_prompt_context,
)
@@ -35,9 +35,8 @@ from fdanyone.model.distributed import (
from fdanyone.model.loader import Denoiser, load_denoiser
from fdanyone.model.metrics import GenerationMetrics
from fdanyone.model.routing import Routes, routing_steps, validate_routes
-from fdanyone.model.vae import VaeExecutor, load_reference_videos
+from fdanyone.model.vae import VaeExecutor
from fdanyone.skeleton.pipeline import Conditioning
-from fdanyone.video import CanonicalClip
from fdanyone.views import ViewPlan
if TYPE_CHECKING:
@@ -61,7 +60,6 @@ class ParallelismReport(TypedDict):
class GeneratedViews:
"""Paths and measurements produced by one resolved view plan."""
- rcp_videos: tuple[Path, ...]
target_videos: tuple[Path, ...]
view_plan: ViewPlan
denoising_profile: DenoisingProfile
@@ -261,13 +259,12 @@ def _denoise_targets_single(
def _resolve_generation_plan(
*,
- clip: CanonicalClip,
conditioning: Conditioning,
devices: tuple[str, ...],
) -> _GenerationPlan:
import torch
- if conditioning.num_frames != INFERENCE.num_frames or len(clip.frames) != INFERENCE.num_frames:
+ if conditioning.num_frames != INFERENCE.num_frames:
raise FourDAnyoneError("Generation requires the frozen 121-frame contract.")
if not devices:
raise FourDAnyoneError("Generation requires at least one CUDA device.")
@@ -301,62 +298,6 @@ def _resolve_generation_plan(
)
-def _generate_rcp_and_references(
- *,
- denoiser: Denoiser,
- vae: VaeExecutor,
- clip: CanonicalClip,
- plan: _GenerationPlan,
- seed: int,
- source_latents: Tensor,
- context: Tensor,
- pose_cache: PoseFeatureCache,
- root: Path,
- metrics: GenerationMetrics,
-) -> tuple[Tensor, tuple[Path, ...]]:
- import torch
-
- rcp_pose_features = pose_cache.rcp
- if rcp_pose_features is None:
- raise FourDAnyoneError("RCP was enabled without precomputed proposal pose features.")
- with (
- metrics.stage("rcp_denoise"),
- _denoiser_on_device(denoiser, plan.primary_device),
- ):
- rcp_latents = _denoise_rcp(
- denoiser=denoiser,
- vae=vae,
- src_latents=source_latents,
- context=context,
- camera_ids=plan.view_plan.rcp_camera_ids,
- pose_features=rcp_pose_features,
- seed=seed,
- device=plan.primary_device,
- )
-
- with metrics.stage("rcp_decode_and_publish"):
- rcp_root = root / "rcp"
- rcp_root.mkdir()
- published = vae.publish_rcp(
- rcp_latents,
- plan.view_plan.rcp_camera_ids,
- rcp_root,
- clip,
- )
- _merge_view_stage_peak(metrics, "rcp_decode_and_publish", vae)
-
- with metrics.stage("rcp_reference_load"):
- # The released model consumes four JPEG-decoded proposal views. Keep
- # that numerical boundary even though the decoded tensors are local.
- reference_videos = load_reference_videos(published.frame_directories[:4], INFERENCE.num_frames)
-
- with metrics.stage("reference_encode"):
- reference_latents = vae.encode(reference_videos)
- target_sources = torch.cat([source_latents, reference_latents], dim=0)
- _merge_view_stage_peak(metrics, "reference_encode", vae)
- return target_sources, published.videos
-
-
def _merge_view_stage_peak(metrics: GenerationMetrics, stage: str, vae: VaeExecutor) -> None:
if not vae.last_peak_vram_bytes:
return
@@ -416,7 +357,6 @@ def _denoise_targets_multi_gpu(
def generate_views(
*,
- clip: CanonicalClip,
conditioning: Conditioning,
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
@@ -429,10 +369,11 @@ def generate_views(
) -> GeneratedViews:
"""Generate proposal and target views with the pipeline's resolved backend."""
+ import torch
+
if seed < 0:
raise FourDAnyoneError(f"seed must be non-negative, got {seed}.")
plan = _resolve_generation_plan(
- clip=clip,
conditioning=conditioning,
devices=devices,
)
@@ -449,6 +390,9 @@ def generate_views(
checkpoint_path=checkpoint_path,
devices=plan.candidate_devices,
)
+ rcp_pose_features = pose_cache.rcp
+ target_pose_features = pose_cache.target
+ del pose_cache
with metrics.stage("model_load"):
denoiser = (
@@ -470,29 +414,31 @@ def generate_views(
del source_video
_merge_view_stage_peak(metrics, "source_encode", vae)
- rcp_videos: tuple[Path, ...] = ()
target_sources = source_latents
if plan.view_plan.enable_rcp:
- if denoiser is None:
- raise RuntimeError("RCP requires a primary-process denoiser.")
- target_sources, rcp_videos = _generate_rcp_and_references(
- denoiser=denoiser,
- vae=vae,
- clip=clip,
- plan=plan,
- seed=seed,
- source_latents=source_latents,
- context=context,
- pose_cache=pose_cache,
- root=root,
- metrics=metrics,
- )
+ if denoiser is None or rcp_pose_features is None:
+ raise RuntimeError("RCP requires a primary denoiser and proposal pose features.")
+ with metrics.stage("rcp_denoise"), _denoiser_on_device(denoiser, plan.primary_device):
+ rcp_latents = _denoise_rcp(
+ denoiser=denoiser,
+ vae=vae,
+ src_latents=source_latents,
+ context=context,
+ camera_ids=plan.view_plan.rcp_camera_ids,
+ pose_features=rcp_pose_features,
+ seed=seed,
+ device=plan.primary_device,
+ )
+ # Target references use null pose features, not the RCP bank.
+ # Drop its sole owner before the DiT returns to host memory.
+ del rcp_pose_features
+ with metrics.stage("rcp_latent_handoff"):
+ target_sources = torch.cat([source_latents, rcp_latents[:4]], dim=0)
+ del rcp_latents
del source_latents
# Parallel VAE replicas are stage-local. Retaining only the prototype
# bounds parent host memory while distributed DiT workers load.
vae.release_replicas()
- target_pose_features = pose_cache.target
- del pose_cache
parallelism = None
with metrics.stage("target_denoise"):
@@ -548,11 +494,12 @@ def generate_views(
with metrics.stage("target_decode_and_publish"):
target_root = root / "target"
target_root.mkdir()
- target_videos = vae.publish_targets(target_latents, target_root, clip)
+ target_videos = vae.publish_targets(
+ target_latents, target_root, Fraction(conditioning.fps_num, conditioning.fps_den)
+ )
_merge_view_stage_peak(metrics, "target_decode_and_publish", vae)
result = GeneratedViews(
- rcp_videos=rcp_videos,
target_videos=target_videos,
view_plan=plan.view_plan,
denoising_profile=denoising_profile,
diff --git a/fdanyone/model/vae.py b/fdanyone/model/vae.py
index 00c6aff..4ff0c36 100644
--- a/fdanyone/model/vae.py
+++ b/fdanyone/model/vae.py
@@ -4,20 +4,19 @@ from __future__ import annotations
import gc
import logging
-from collections.abc import Callable, Iterable
+from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
from copy import deepcopy
-from dataclasses import dataclass
+from fractions import Fraction
from pathlib import Path
from threading import BoundedSemaphore, Event
from typing import TYPE_CHECKING, TypeVar
import numpy as np
-from PIL import Image
from fdanyone.config import INFERENCE
from fdanyone.errors import FourDAnyoneError
-from fdanyone.video import CanonicalClip, write_video
+from fdanyone.video import write_video
if TYPE_CHECKING:
from torch import Tensor
@@ -29,22 +28,6 @@ LOGGER = logging.getLogger("fdanyone")
Output = TypeVar("Output")
-@dataclass(frozen=True)
-class PublishedRcp:
- """Canonical RCP frame directories and videos."""
-
- frame_directories: tuple[Path, ...]
- videos: tuple[Path, ...]
-
-
-@dataclass(frozen=True)
-class _DecodedView:
- """Host-owned values consumed by publication sinks."""
-
- video: Tensor | None
- rgb_frames: tuple[np.ndarray, ...]
-
-
def _bf16_autocast():
import torch
@@ -60,36 +43,6 @@ def _rgb_frames(video: Tensor) -> tuple[np.ndarray, ...]:
)
-def _save_jpegs(video: Tensor, camera_id: int, root: Path) -> Path:
- import torchvision.transforms.functional as transform
-
- frame_dir = root / f"{camera_id:06d}"
- frame_dir.mkdir(parents=True, exist_ok=False)
- for frame_index in range(video.shape[1]):
- normalized = video[:, frame_index].float().mul_(0.5).add_(0.5).clamp_(0.0, 1.0)
- image = transform.to_pil_image(normalized)
- image.save(frame_dir / f"{frame_index:06d}.jpg", quality=INFERENCE.rcp_jpeg_quality)
- return frame_dir
-
-
-def load_reference_videos(frame_dirs: Iterable[Path], num_frames: int) -> Tensor:
- """Load the frozen JPEG reference boundary as ``[V,C,F,H,W]``."""
-
- import torch
- import torchvision.transforms.functional as transform
-
- videos = []
- for frame_dir in frame_dirs:
- frames = []
- for frame_index in range(num_frames):
- path = frame_dir / f"{frame_index:06d}.jpg"
- with Image.open(path) as image:
- frames.append(transform.to_tensor(image.convert("RGB")))
- videos.append(torch.stack(frames, dim=0))
- frame_first = torch.stack(videos, dim=0).mul_(2.0).sub_(1.0)
- return frame_first.permute(0, 2, 1, 3, 4)
-
-
class VaeExecutor:
"""Run independent views on an ordered CUDA device pool.
@@ -239,9 +192,7 @@ class VaeExecutor:
def _decode_and_publish(
self,
latents: Tensor,
- sink: Callable[[int, _DecodedView], Output],
- *,
- retain_video: bool,
+ sink: Callable[[int, tuple[np.ndarray, ...]], Output],
) -> tuple[Output, ...]:
import torch
@@ -277,13 +228,13 @@ class VaeExecutor:
decoded = self._decode_view(model, latents[index], device)
device_video = decoded[0].detach()
rgb_frames = _rgb_frames(device_video)
- video = device_video.to(device="cpu").contiguous() if retain_video else None
del decoded, device_video
future = codec_pool.submit(
sink,
index,
- _DecodedView(video=video, rgb_frames=rgb_frames),
+ rgb_frames,
)
+ del rgb_frames
future.add_done_callback(release_slot)
sink_futures[index] = future
except BaseException:
@@ -299,56 +250,21 @@ class VaeExecutor:
raise RuntimeError("VAE decode completed without every canonical view.")
return tuple(future.result() for future in sink_futures if future is not None)
- def publish_rcp(
- self,
- latents: Tensor,
- camera_ids: tuple[int, ...],
- output_dir: Path,
- clip: CanonicalClip,
- ) -> PublishedRcp:
- if latents.shape[0] != len(camera_ids):
- raise FourDAnyoneError(f"RCP decode expected {len(camera_ids)} latent views, got {latents.shape[0]}.")
- frame_root = output_dir / "frames"
- video_root = output_dir / "videos"
- frame_root.mkdir(parents=True, exist_ok=False)
- video_root.mkdir(parents=True, exist_ok=False)
-
- def publish(index: int, decoded: _DecodedView) -> tuple[Path, Path]:
- camera_id = camera_ids[index]
- LOGGER.info("Publishing RCP camera %02d", camera_id)
- if decoded.video is None:
- raise RuntimeError("RCP publication requires the decoded BF16 video.")
- frame_dir = _save_jpegs(decoded.video, camera_id, frame_root)
- video_path = write_video(
- iter(decoded.rgb_frames),
- video_root / f"{camera_id:02d}.mp4",
- clip.fps,
- crf=INFERENCE.target_h264_crf,
- preset=INFERENCE.h264_preset,
- )
- return frame_dir, video_path
-
- published = self._decode_and_publish(latents, publish, retain_video=True)
- return PublishedRcp(
- frame_directories=tuple(item[0] for item in published),
- videos=tuple(item[1] for item in published),
- )
-
- def publish_targets(self, latents: Tensor, output_dir: Path, clip: CanonicalClip) -> tuple[Path, ...]:
+ def publish_targets(self, latents: Tensor, output_dir: Path, fps: Fraction) -> tuple[Path, ...]:
video_root = output_dir / "videos"
video_root.mkdir(parents=True, exist_ok=False)
- def publish(camera_id: int, decoded: _DecodedView) -> Path:
+ def publish(camera_id: int, rgb_frames: tuple[np.ndarray, ...]) -> Path:
LOGGER.info("Publishing target camera %02d", camera_id)
return write_video(
- iter(decoded.rgb_frames),
+ iter(rgb_frames),
video_root / f"{camera_id:02d}.mp4",
- clip.fps,
+ fps,
crf=INFERENCE.target_h264_crf,
preset=INFERENCE.h264_preset,
)
- return self._decode_and_publish(latents, publish, retain_video=False)
+ return self._decode_and_publish(latents, publish)
def release_replicas(self) -> None:
"""Keep one CPU model while releasing stage-local parallel replicas."""
diff --git a/fdanyone/nerfstudio/exporter.py b/fdanyone/nerfstudio/exporter.py
index d23d8f5..26b8563 100644
--- a/fdanyone/nerfstudio/exporter.py
+++ b/fdanyone/nerfstudio/exporter.py
@@ -24,6 +24,7 @@ from fdanyone.nerfstudio.visual_hull import (
write_sparse_point_cloud,
)
from fdanyone.output_directory import read_output_metadata
+from fdanyone.result_videos import read_target_videos
# Nerfstudio casts mask pixels directly to bool, so soft BiRefNet predictions
# must cross a real decision boundary before serialization.
@@ -73,17 +74,6 @@ def _extract_frame(video_path: Path, frame_index: int) -> np.ndarray:
return image
-def _dense_video_paths(result: Path, cameras: list[dict]) -> tuple[Path, ...]:
- paths = []
- for camera in cameras:
- camera_id = int(camera["camera_id"])
- relative = f"videos/dense/{camera_id:02d}.mp4"
- if camera.get("video") != relative:
- raise FourDAnyoneError(f"Camera {camera_id:02d} points to the wrong target video.")
- paths.append(result / relative)
- return tuple(paths)
-
-
def _validate_raster(image: np.ndarray, camera: dict) -> None:
camera_id = int(camera["camera_id"])
expected = (int(camera["image_height"]), int(camera["image_width"]), 3)
@@ -187,7 +177,7 @@ def export_nerfstudio(
read_output_metadata(result)
cameras = _camera_records(_read_cameras(result))
transforms = _transforms(cameras)
- videos = _dense_video_paths(result, cameras)
+ videos = read_target_videos(result, cameras)
if output_dir is None:
destination = Path("data/ns_data") / result.name / f"frame_{frame_index:03d}"
diff --git a/fdanyone/output_writer.py b/fdanyone/output_writer.py
index 7b39f68..0c2c18f 100644
--- a/fdanyone/output_writer.py
+++ b/fdanyone/output_writer.py
@@ -14,18 +14,13 @@ from typing import TYPE_CHECKING
from fdanyone.config import INFERENCE
from fdanyone.device import CUDA_ALLOCATOR_CONF
from fdanyone.errors import FourDAnyoneError
+from fdanyone.result_videos import target_video_path
if TYPE_CHECKING:
from fdanyone.model.inference import GeneratedViews
from fdanyone.motion.result import MotionResult
from fdanyone.skeleton.pipeline import Conditioning
- from fdanyone.video import CanonicalClip
-
-
-def _copy_file(source: Path, destination: Path) -> Path:
- destination.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(source, destination)
- return destination
+ from fdanyone.video import ClipInfo
def _runtime_metadata(device: str) -> dict:
@@ -72,7 +67,7 @@ def _camera_rig_payload(payload: dict, cameras: list[dict]) -> dict:
"camera_to_world": camera["camera_to_world"],
"image_width": int(camera["image_width"]),
"image_height": int(camera["image_height"]),
- "video": f"videos/dense/{camera_id:02d}.mp4",
+ "video": str(target_video_path(camera_id)),
"skeleton_video": f"skeletons/{camera_id:02d}.mp4",
}
)
@@ -101,7 +96,7 @@ def _target_cameras(payload: object, expected_count: int) -> list[dict]:
def write_output(
*,
- clip: CanonicalClip,
+ clip: ClipInfo,
conditioning: Conditioning,
generated: GeneratedViews,
destination: str | Path,
@@ -109,17 +104,13 @@ def write_output(
model_identity: dict,
pipeline_started: float,
) -> dict:
- """Write proposal, target, skeleton, camera, and metadata artifacts into staging."""
+ """Write target, skeleton, camera, and metadata artifacts into staging."""
root = Path(destination).expanduser().resolve()
attention_backend = generated.attention_backend
view_plan = generated.view_plan
if conditioning.view_plan != view_plan:
raise FourDAnyoneError("Conditioning and generation resolved different view plans.")
- if len(generated.rcp_videos) != len(view_plan.rcp_camera_ids):
- raise FourDAnyoneError(
- f"Generation returned {len(generated.rcp_videos)} RCP videos, expected {len(view_plan.rcp_camera_ids)}."
- )
if len(generated.target_videos) != view_plan.num_target_views:
raise FourDAnyoneError(
f"Generation returned {len(generated.target_videos)} target videos, expected {view_plan.num_target_views}."
@@ -130,24 +121,14 @@ def write_output(
f"expected {view_plan.num_target_views}."
)
- sparse_root = root / "videos" / "sparse"
- dense_root = root / "videos" / "dense"
+ videos_root = root / "videos"
skeletons_root = root / "skeletons"
- dense_root.mkdir(parents=True, exist_ok=False)
+ videos_root.mkdir(parents=True, exist_ok=False)
skeletons_root.mkdir(exist_ok=False)
- if generated.rcp_videos:
- sparse_root.mkdir(exist_ok=False)
-
- output_sparse = tuple(
- _copy_file(source, sparse_root / f"{camera_id:02d}.mp4")
- for camera_id, source in zip(view_plan.rcp_camera_ids, generated.rcp_videos, strict=True)
- )
- output_dense = tuple(
- _copy_file(source, dense_root / f"{camera_id:02d}.mp4")
- for camera_id, source in enumerate(generated.target_videos)
- )
+ for camera_id, source in enumerate(generated.target_videos):
+ shutil.copy2(source, root / target_video_path(camera_id))
for camera_id, skeleton in enumerate(conditioning.target_skeletons):
- _copy_file(skeleton.path, skeletons_root / f"{camera_id:02d}.mp4")
+ shutil.copy2(skeleton.path, skeletons_root / f"{camera_id:02d}.mp4")
camera_payload = json.loads((conditioning.root / "cameras.json").read_text())
conditioning_metadata = json.loads((conditioning.root / "metadata.json").read_text())
@@ -178,7 +159,7 @@ def write_output(
"filename": clip.source_path.name,
"fps": f"{clip.fps_num}/{clip.fps_den}",
"start_time_seconds": float(clip.start_time),
- "num_frames": len(clip.frames),
+ "num_frames": clip.num_frames,
"width": clip.width,
"height": clip.height,
},
@@ -195,8 +176,7 @@ def write_output(
"model": dict(model_identity),
"generation": generation_metadata,
"output": {
- "rcp_views": len(output_sparse),
- "target_views": len(output_dense),
+ "target_views": view_plan.num_target_views,
"frames_per_video": INFERENCE.num_frames,
"width": INFERENCE.width,
"height": INFERENCE.height,
@@ -213,8 +193,7 @@ def write_output(
(root / name).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
return {
"attention_backend": attention_backend,
- "num_rcp_videos": len(output_sparse),
- "num_target_videos": len(output_dense),
+ "num_target_videos": view_plan.num_target_views,
"fps": f"{clip.fps_num}/{clip.fps_den}",
"peak_vram_allocated_bytes": generated.peak_vram_allocated_bytes,
"peak_vram_reserved_bytes": generated.peak_vram_reserved_bytes,
diff --git a/fdanyone/pipeline.py b/fdanyone/pipeline.py
index c0f3e2b..124045f 100644
--- a/fdanyone/pipeline.py
+++ b/fdanyone/pipeline.py
@@ -333,9 +333,12 @@ def run_pipeline(
raise ConfigurationError("Skeleton conditioning does not match the canonical clip timeline.")
# Re-decode the worker-produced source before it becomes a model tensor.
verify_lossless_video(clip, conditioning.source_video)
+ # Generation reads the verified working video. Only lightweight
+ # provenance is needed from the original full-resolution clip now.
+ clip_info = clip.info
+ del clip
PROGRESS.info("Generating target-view videos", extra={"fraction": 0.45})
generated = generate_views(
- clip=clip,
conditioning=conditioning,
checkpoint_path=checkpoint,
turbo_lora_path=turbo_lora,
@@ -348,7 +351,7 @@ def run_pipeline(
)
PROGRESS.info("Saving and validating generated videos", extra={"fraction": 0.92})
summary = write_output(
- clip=clip,
+ clip=clip_info,
conditioning=conditioning,
generated=generated,
destination=work,
diff --git a/fdanyone/result_videos.py b/fdanyone/result_videos.py
new file mode 100644
index 0000000..38e3ec1
--- /dev/null
+++ b/fdanyone/result_videos.py
@@ -0,0 +1,30 @@
+"""Canonical target-video paths shared by publication and result consumers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fdanyone.errors import FourDAnyoneError
+
+
+def target_video_path(camera_id: int) -> Path:
+ """Return the relative publication path for a target camera."""
+
+ return Path("videos") / f"{camera_id:02d}.mp4"
+
+
+def read_target_videos(root: Path, cameras: list[dict]) -> tuple[Path, ...]:
+ """Read the canonical camera-indexed videos, contained within the result."""
+
+ if not cameras or [camera.get("camera_id") for camera in cameras] != list(range(len(cameras))):
+ raise FourDAnyoneError("Target cameras must be ordered by camera ID.")
+ layout = tuple(target_video_path(index) for index in range(len(cameras)))
+ if tuple(camera.get("video") for camera in cameras) != tuple(str(path) for path in layout):
+ raise FourDAnyoneError("Camera video paths must use the videos/<camera_id>.mp4 layout.")
+
+ root = root.resolve()
+ paths = tuple(root / relative for relative in layout)
+ for path in paths:
+ if not path.is_file() or not path.resolve().is_relative_to(root):
+ raise FourDAnyoneError(f"Missing or external result file: {path.relative_to(root)}")
+ return paths
diff --git a/fdanyone/space/viewer.py b/fdanyone/space/viewer.py
index 005ee99..b5946f5 100644
--- a/fdanyone/space/viewer.py
+++ b/fdanyone/space/viewer.py
@@ -20,6 +20,7 @@ from fdanyone.config import FRAMING
from fdanyone.errors import FourDAnyoneError
from fdanyone.geometry.cameras import camera_grid, reference_intrinsics
from fdanyone.output_directory import read_output_metadata
+from fdanyone.result_videos import read_target_videos
from fdanyone.space import scene
_EXPORT_LOCK = threading.Lock()
@@ -60,12 +61,8 @@ def read_result(directory: str | Path) -> Result:
frames = int(metadata["output"]["frames_per_video"])
if fps <= 0 or frames != 121 or metadata["output"]["target_views"] != len(cameras):
raise ValueError("Output does not describe synchronized 121-frame videos")
- videos = []
+ videos = read_target_videos(root, cameras)
for camera in cameras:
- camera_id = camera["camera_id"]
- relative = f"videos/dense/{camera_id:02d}.mp4"
- if camera["video"] != relative:
- raise ValueError(f"Unexpected video path for camera {camera_id}")
intrinsic = np.asarray(camera["K"], dtype=np.float64)
transform = np.asarray(camera["camera_to_world"], dtype=np.float64)
if (
@@ -75,13 +72,12 @@ def read_result(directory: str | Path) -> Result:
or not np.isfinite(transform).all()
or not np.allclose(transform[3], [0, 0, 0, 1])
):
- raise ValueError(f"Invalid calibration for camera {camera_id}")
+ raise ValueError(f"Invalid calibration for camera {camera['camera_id']}")
if min(camera["image_width"], camera["image_height"]) <= 0:
raise ValueError("Invalid camera image dimensions")
- videos.append(_contained_file(root, relative))
except (OSError, ValueError, TypeError, KeyError, ZeroDivisionError) as exc:
raise FourDAnyoneError(f"Cannot open this 4DAnyone output: {exc}") from exc
- return Result(root, metadata, tuple(cameras), tuple(videos), fps, frames)
+ return Result(root, metadata, tuple(cameras), videos, fps, frames)
def _target_video(result: Result, camera_id: int, destination: Path, check_cancelled: Callable) -> tuple[int, int]:
diff --git a/fdanyone/video.py b/fdanyone/video.py
index 99ae159..69acd5e 100644
--- a/fdanyone/video.py
+++ b/fdanyone/video.py
@@ -120,6 +120,26 @@ class CanonicalFrame:
canonical_timestamp: Fraction
+@dataclass(frozen=True)
+class ClipInfo:
+ """Input provenance and raster information without decoded pixel storage."""
+
+ source_path: Path
+ fps: Fraction
+ start_time: Fraction
+ num_frames: int
+ height: int
+ width: int
+
+ @property
+ def fps_num(self) -> int:
+ return self.fps.numerator
+
+ @property
+ def fps_den(self) -> int:
+ return self.fps.denominator
+
+
@dataclass(frozen=True)
class CanonicalClip:
source_path: Path
@@ -132,6 +152,17 @@ class CanonicalClip:
frames: tuple[CanonicalFrame, ...]
rotation_degrees: int
+ @property
+ def info(self) -> ClipInfo:
+ return ClipInfo(
+ source_path=self.source_path,
+ fps=self.fps,
+ start_time=self.start_time,
+ num_frames=len(self.frames),
+ height=self.height,
+ width=self.width,
+ )
+
@property
def height(self) -> int:
return int(self.frames[0].rgb.shape[0])