Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

fix(corpus-builder): sanitize clip_id and prevent path traversal in candidate processing (#717)

Huged7f3a93f63be
diff --git a/tests/tools/test_corpus_builder_path_traversal.py b/tests/tools/test_corpus_builder_path_traversal.py
new file mode 100644
index 00000000..4b0e43cc
--- /dev/null
+++ b/tests/tools/test_corpus_builder_path_traversal.py
@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+
+import tools.video.stock_sources as stock_sources
+from lib.corpus import Corpus
+from tools.video.corpus_builder import (
+ CorpusBuilder,
+ _is_safe_clip_id,
+ _safe_child_path,
+)
+
+
+@dataclass
+class DummyCandidate:
+ source: str
+ source_id: str
+ source_url: str = "https://example.com/item"
+ download_url: str = "https://example.com/file.mp4"
+ kind: str = "video"
+ width: int = 1920
+ height: int = 1080
+ duration: float = 10.0
+ creator: str = "author"
+ license: str = "cc0"
+ source_tags: str = "nature water"
+ thumbnail_url: str = ""
+ extra: dict[str, Any] = field(default_factory=dict)
+
+ @property
+ def clip_id(self) -> str:
+ return f"{self.source}_{self.source_id}"
+
+
+class DummyTraversingSource:
+ name = "traversing_source"
+
+ def __init__(self, candidates: list[DummyCandidate]) -> None:
+ self.candidates = candidates
+ self.download_called_with: list[Path] = []
+
+ def is_available(self) -> bool:
+ return True
+
+ def search(self, query, filters):
+ return self.candidates
+
+ def download(self, candidate, out_path: Path) -> Path:
+ self.download_called_with.append(out_path)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ out_path.write_bytes(b"dummy video content" * 100)
+ return out_path
+
+
+ "clip_id",
+ [
+ "../../traversal",
+ "../../../etc/passwd",
+ "..\\..\\windows_traversal",
+ "clip/nested",
+ "clip\\nested",
+ "nested/../../escape",
+ "C:drive_escape",
+ "/absolute/root",
+ "\\backslash_root",
+ "\x00nullbyte",
+ "bad\x1fcontrol",
+ "",
+ " ",
+ ".",
+ "..",
+ "...",
+ ],
+)
+def test_is_safe_clip_id_rejects_malicious_and_traversal_ids(clip_id: str) -> None:
+ assert _is_safe_clip_id(clip_id) is False
+
+
+ "clip_id",
+ [
+ "pexels_12345",
+ "archive_org_nasa_apollo",
+ "pixabay_video_9876",
+ "clip-0",
+ "source_item-alpha.1",
+ "wikimedia_Apollo_11_launch",
+ ],
+)
+def test_is_safe_clip_id_accepts_valid_ids(clip_id: str) -> None:
+ assert _is_safe_clip_id(clip_id) is True
+
+
+def test_safe_child_path_confines_paths(tmp_path: Path) -> None:
+ base = tmp_path / "base"
+ base.mkdir()
+
+ # Valid relative paths
+ valid = _safe_child_path(base, "clips/valid.mp4")
+ assert valid is not None
+ assert valid.is_relative_to(base)
+ assert valid.name == "valid.mp4"
+
+ # Escaping relative paths
+ assert _safe_child_path(base, "clips/../../escape.mp4") is None
+ assert _safe_child_path(base, "../escape.mp4") is None
+ assert _safe_child_path(base, "..") is None
+ assert _safe_child_path(base, ".") is None
+
+
+def test_process_candidate_rejects_traversing_clip_id(tmp_path: Path) -> None:
+ corpus_dir = tmp_path / "corpus"
+ corp = Corpus(corpus_dir)
+ corp.ensure_dirs()
+
+ builder = CorpusBuilder()
+ cand = DummyCandidate(
+ source="evil",
+ source_id="../../escaped_clip",
+ )
+ src = DummyTraversingSource([cand])
+
+ class DummyCache:
+ def try_link(self, *args, **kwargs):
+ return False
+
+ def ingest(self, *args, **kwargs):
+ return False
+
+ with pytest.raises(ValueError, match="[Uu]nsafe clip_id"):
+ builder._process_candidate(
+ cand=cand,
+ src=src,
+ corp=corp,
+ query="test",
+ thumbs_per_video=3,
+ cache=DummyCache(),
+ run_cache_stats={"hits": 0, "misses": 0, "bytes_saved": 0},
+ )
+
+ # Ensure no file was created outside the corpus directory
+ assert not (tmp_path / "escaped_clip.mp4").exists()
+ assert not (tmp_path / "thumbnails" / "evil_../../escaped_clip").exists()
+ assert len(src.download_called_with) == 0
+
+
+def test_corpus_builder_execute_skips_and_records_unsafe_candidates(
+ monkeypatch, tmp_path: Path
+) -> None:
+ corpus_dir = tmp_path / "corpus"
+ escape_file = tmp_path / "escaped_clip.mp4"
+
+ cand_bad = DummyCandidate(
+ source="evil",
+ source_id="../../escaped_clip",
+ )
+ cand_good = DummyCandidate(
+ source="good",
+ source_id="clip123",
+ )
+
+ source = DummyTraversingSource([cand_bad, cand_good])
+ monkeypatch.setattr(stock_sources, "available_sources", lambda: [source])
+ monkeypatch.setattr(stock_sources, "source_summary", lambda: {})
+
+ # Mock _process_candidate for cand_good so we don't need real CLIP/opencv models
+ orig_process = CorpusBuilder._process_candidate
+
+ def mock_process_candidate(self, cand, *args, **kwargs):
+ if not _is_safe_clip_id(cand.clip_id):
+ return orig_process(self, cand, *args, **kwargs)
+ from lib.corpus import ClipRecord
+ return ClipRecord(
+ clip_id=cand.clip_id,
+ source=cand.source,
+ source_id=cand.source_id,
+ source_url=cand.source_url,
+ local_path=f"clips/{cand.clip_id}.mp4",
+ kind=cand.kind,
+ )
+
+ monkeypatch.setattr(CorpusBuilder, "_process_candidate", mock_process_candidate)
+
+ result = CorpusBuilder().execute({
+ "corpus_dir": str(corpus_dir),
+ "queries": [{"query": "nature"}],
+ "max_new_clips": 10,
+ })
+
+ assert result.success is True
+ assert result.data["clips_added"] == 1
+ assert result.data["clips_failed"] == 1
+ assert result.data["added_ids"] == ["good_clip123"]
+ assert not escape_file.exists()
diff --git a/tools/video/corpus_builder.py b/tools/video/corpus_builder.py
index 4aca5123..82eefe58 100644
--- a/tools/video/corpus_builder.py
+++ b/tools/video/corpus_builder.py
@@ -352,6 +352,16 @@ class CorpusBuilder(BaseTool):
if len(added_ids) >= max_new:
break
+ clip_id = getattr(cand, "clip_id", "")
+ if not _is_safe_clip_id(clip_id):
+ failed += 1
+ errors.append({
+ "phase": "process",
+ "clip_id": clip_id or "unknown",
+ "error": f"ValueError: Unsafe clip_id: {clip_id!r}",
+ })
+ continue
+
if skip_existing and corp.has(cand.clip_id):
skipped += 1
continue
@@ -495,6 +505,9 @@ class CorpusBuilder(BaseTool):
next run benefits. Cache faults never block the pipeline —
they degrade gracefully to normal downloads.
"""
+ if not _is_safe_clip_id(cand.clip_id):
+ raise ValueError(f"Unsafe clip_id: {cand.clip_id!r}")
+
import cv2
from lib.clip_embedder import embed_images, embed_texts, pool_frames
@@ -502,9 +515,12 @@ class CorpusBuilder(BaseTool):
# Pick file extension from the URL path (sources give us
# stable .mp4/.jpg/.png URLs) with a kind-aware fallback.
+
ext = _guess_ext(cand)
local_rel = Path("clips") / f"{cand.clip_id}{ext}"
- local_abs = corp.corpus_dir / local_rel
+ local_abs = _safe_child_path(corp.corpus_dir, local_rel)
+ if local_abs is None:
+ raise ValueError(f"Unsafe clip path resolved for {cand.clip_id!r}")
# Try the shared cache first. A hit links the cached blob
# into local_abs (same filesystem → hard link, cross-drive
@@ -560,7 +576,9 @@ class CorpusBuilder(BaseTool):
pass
thumb_dir_rel = Path("thumbnails") / cand.clip_id
- thumb_dir_abs = corp.corpus_dir / thumb_dir_rel
+ thumb_dir_abs = _safe_child_path(corp.corpus_dir, thumb_dir_rel)
+ if thumb_dir_abs is None:
+ raise ValueError(f"Unsafe thumbnail path resolved for {cand.clip_id!r}")
thumb_dir_abs.mkdir(parents=True, exist_ok=True)
width = cand.width
@@ -627,6 +645,34 @@ class CorpusBuilder(BaseTool):
# ----------------------------------------------------------------------
+def _is_safe_clip_id(clip_id: str) -> bool:
+ """Validate that clip_id cannot cause path traversal or filesystem escapes."""
+ if not clip_id or not isinstance(clip_id, str):
+ return False
+ if "\x00" in clip_id:
+ return False
+ if any(sep in clip_id for sep in ("/", "\\", ":")) or ".." in clip_id:
+ return False
+ if any(ord(c) < 32 or ord(c) == 127 for c in clip_id):
+ return False
+ stripped = clip_id.strip()
+ if not stripped or stripped in {".", ".."}:
+ return False
+ return True
+
+
+def _safe_child_path(parent_dir: Path, rel_path: Path | str) -> Optional[Path]:
+ """Resolve a child path within parent_dir, guaranteeing no traversal."""
+ try:
+ resolved_parent = Path(parent_dir).resolve()
+ candidate = (Path(parent_dir) / rel_path).resolve()
+ if candidate.is_relative_to(resolved_parent) and candidate != resolved_parent:
+ return candidate
+ except (ValueError, RuntimeError, OSError):
+ return None
+ return None
+
+
def _guess_ext(cand) -> str:
"""Extract a sensible file extension from a candidate's URL."""
known = {".mp4", ".mov", ".mkv", ".webm", ".ogv", ".m4v",