Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

31.7 KB
Raw Blame
"""Validate latent WebDataset shards against the project's training contract.

The validator is read-only. It fully parses the annotation JSONL, then checks a
deterministic sample of tar shards and latent payloads by default. Use
``--all-shards --all-samples`` for full annotated coverage.

Exit code is 0 when no validation errors are found and 1 otherwise.
"""

from __future__ import annotations

import argparse
import io
import json
import sys
import tarfile
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

import numpy as np
import torch

SHARD_FILENAME_PREFIX = "latent_"
CANONICAL_LATENT_FIELD = "latent.pth"
MAX_JSON_MEMBER_BYTES = 16 * 1024 * 1024

DiskShape = tuple[int, int, int, int]
TrainingShape = tuple[int, int, int, int]


@dataclass(frozen=True)
class AnnotationRecord:
    """One validated annotation entry."""

    line_no: int
    shard: str
    clip_name: str
    caption: str


@dataclass(frozen=True)
class TrainingContract:
    """Effective shape and internal causal contract of one public recipe."""

    path: Path
    full_shape: tuple[int, int, int, int, int]
    block_size: int | None
    causal: bool
    independent_first_frame: bool
    model_name: str | None

    @property
    def sample_shape(self) -> TrainingShape:
        """Return the configured per-sample layout ``[T, C, H, W]``."""
        return self.full_shape[1:]


@dataclass(frozen=True)
class ValidationOptions:
    """Inputs and coverage controls for one validation run."""

    data_path: Path
    annotation_path: Path
    caption_field: str = "global_caption_en"
    expected_shape: DiskShape | None = None
    config_path: Path | None = None
    sample_shards: int = 3
    samples_per_shard: int = 3
    all_shards: bool = False
    all_samples: bool = False
    allow_unsafe_pickle: bool = False
    max_payload_bytes: int = 2 * 1024 * 1024 * 1024
    max_errors: int = 50


@dataclass
class ValidationReport:
    """Validation findings and compact coverage statistics."""

    max_errors: int
    error_count: int = 0
    warning_count: int = 0
    errors: list[str] = field(default_factory=list)
    warnings: list[str] = field(default_factory=list)
    notes: list[str] = field(default_factory=list)
    annotation_entries: int = 0
    annotation_shards: int = 0
    checked_shards: int = 0
    checked_samples: int = 0
    matched_annotation_entries: int = 0
    observed_shapes: Counter[DiskShape] = field(default_factory=Counter)
    observed_dtypes: Counter[str] = field(default_factory=Counter)
    training_contract: TrainingContract | None = None
    reference_shape: DiskShape | None = None

    @property
    def ok(self) -> bool:
        return self.error_count == 0

    def error(self, message: str) -> None:
        self.error_count += 1
        if len(self.errors) < self.max_errors:
            self.errors.append(message)

    def warning(self, message: str) -> None:
        self.warning_count += 1
        if len(self.warnings) < self.max_errors:
            self.warnings.append(message)

    def note(self, message: str) -> None:
        if message not in self.notes:
            self.notes.append(message)


def parse_disk_shape(value: str) -> DiskShape:
    """Parse ``C,T,H,W`` (or ``CxTxHxW``) into four positive integers."""
    normalized = value.lower().replace("x", ",")
    parts = [part.strip() for part in normalized.split(",") if part.strip()]
    if len(parts) != 4:
        raise argparse.ArgumentTypeError(
            f"expected four dimensions in C,T,H,W order, got {value!r}"
        )
    try:
        shape = tuple(int(part) for part in parts)
    except ValueError as exc:
        raise argparse.ArgumentTypeError(f"shape contains a non-integer: {value!r}") from exc
    if any(dim <= 0 for dim in shape):
        raise argparse.ArgumentTypeError(f"shape dimensions must be positive: {value!r}")
    return shape  # type: ignore[return-value]


def _positive_int(value: str) -> int:
    parsed = int(value)
    if parsed <= 0:
        raise argparse.ArgumentTypeError("value must be a positive integer")
    return parsed


def _format_shape(shape: tuple[int, ...]) -> str:
    return "[" + ", ".join(str(dim) for dim in shape) + "]"


def _validate_shape_values(raw: Any, *, dimensions: int, field_name: str) -> tuple[int, ...]:
    if not isinstance(raw, (list, tuple)) or len(raw) != dimensions:
        raise ValueError(f"{field_name} must contain exactly {dimensions} dimensions, got {raw!r}")
    values: list[int] = []
    for value in raw:
        if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
            raise ValueError(f"{field_name} dimensions must be positive integers, got {raw!r}")
        values.append(value)
    return tuple(values)


def _load_training_contract(config_path: Path) -> TrainingContract:
    # Resolve shared defaults and the public recipe through the same boundary as
    # training, without importing the trainer or touching data/GPU/runtime outputs.
    project_root = Path(__file__).resolve().parents[1]
    if str(project_root) not in sys.path:
        sys.path.insert(0, str(project_root))
    from src.utils.config import resolve_config

    resolved = resolve_config(["--config", str(config_path), "--dry_run"], training=True)
    config = resolved.public
    full_shape = _validate_shape_values(
        list(config.image_or_video_shape), dimensions=5, field_name="image_or_video_shape"
    )
    return TrainingContract(
        path=resolved.config_path,
        full_shape=full_shape,
        block_size=config.num_frame_per_block,
        causal=True,
        independent_first_frame=config.independent_first_frame,
        model_name=config.model_name,
    )


def _validate_training_contract(
    report: ValidationReport,
    contract: TrainingContract,
    expected_shape: DiskShape | None,
) -> None:
    training_t, training_c, training_h, training_w = contract.sample_shape
    block_size = contract.block_size
    if contract.causal and block_size is not None:
        if contract.independent_first_frame:
            if (training_t - 1) % block_size != 0:
                report.error(
                    f"config {contract.path}: independent_first_frame=true requires "
                    f"(T - 1) % num_frame_per_block == 0, got T={training_t}, "
                    f"block={block_size}"
                )
        elif training_t % block_size != 0:
            report.error(
                f"config {contract.path}: independent_first_frame=false requires "
                f"T % num_frame_per_block == 0, got T={training_t}, block={block_size}"
            )

    if expected_shape is not None:
        disk_c, disk_t, disk_h, disk_w = expected_shape
        if (disk_c, disk_h, disk_w) != (training_c, training_h, training_w):
            report.error(
                "--expected-shape is spatially incompatible with config: disk "
                f"{_format_shape(expected_shape)} [C,T,H,W] versus config "
                f"{_format_shape(contract.sample_shape)} [T,C,H,W]"
            )
        if disk_t < training_t:
            report.error(
                "--expected-shape has fewer latent frames than config consumes: "
                f"disk T={disk_t}, config T={training_t}"
            )


def _load_annotations(
    options: ValidationOptions,
    report: ValidationReport,
) -> dict[str, list[AnnotationRecord]]:
    path = options.annotation_path
    if not path.is_file():
        report.error(f"annotation JSONL does not exist or is not a file: {path}")
        return {}
    if path.suffix.lower() != ".jsonl":
        report.warning(f"annotation file does not use the recommended .jsonl suffix: {path}")

    by_shard: dict[str, list[AnnotationRecord]] = {}
    seen_pairs: dict[tuple[str, str], int] = {}
    try:
        annotation_file = path.open("r", encoding="utf-8")
    except OSError as exc:
        report.error(f"cannot open annotation JSONL {path}: {exc}")
        return {}

    with annotation_file:
        for line_no, raw_line in enumerate(annotation_file, 1):
            line = raw_line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
            except json.JSONDecodeError as exc:
                report.error(f"{path}:{line_no}: invalid JSON: {exc}")
                continue
            if not isinstance(entry, dict):
                report.error(f"{path}:{line_no}: each line must be a JSON object")
                continue

            shard = entry.get("shard")
            clip_name = entry.get("clip_name")
            caption = entry.get(options.caption_field)
            if not isinstance(shard, str) or not shard.strip():
                report.error(f"{path}:{line_no}: missing or empty string field 'shard'")
                continue
            if (
                shard != shard.strip()
                or shard in {".", ".."}
                or "/" in shard
                or "\\" in shard
                or "\x00" in shard
            ):
                report.error(
                    f"{path}:{line_no}: shard must be a filename-safe logical id, got {shard!r}"
                )
                continue
            if shard.startswith(SHARD_FILENAME_PREFIX) or shard.endswith(".tar"):
                report.error(
                    f"{path}:{line_no}: shard must omit the 'latent_' prefix and '.tar' suffix"
                )
                continue
            if not isinstance(clip_name, str) or not clip_name.strip():
                report.error(f"{path}:{line_no}: missing or empty string field 'clip_name'")
                continue
            if "\x00" in clip_name:
                report.error(f"{path}:{line_no}: clip_name contains a NUL byte")
                continue
            if not isinstance(caption, str) or not caption.strip():
                report.error(
                    f"{path}:{line_no}: missing or empty caption field {options.caption_field!r}"
                )
                continue

            pair = (shard, clip_name)
            if pair in seen_pairs:
                report.error(
                    f"{path}:{line_no}: duplicate (shard, clip_name) {pair!r}; "
                    f"first seen at line {seen_pairs[pair]}"
                )
                continue
            seen_pairs[pair] = line_no
            by_shard.setdefault(shard, []).append(
                AnnotationRecord(
                    line_no=line_no,
                    shard=shard,
                    clip_name=clip_name,
                    caption=caption,
                )
            )

    report.annotation_entries = sum(len(records) for records in by_shard.values())
    report.annotation_shards = len(by_shard)
    if report.annotation_entries == 0:
        report.error(
            f"annotation JSONL has no valid entries (caption field={options.caption_field!r}): {path}"
        )
    return by_shard


def _evenly_spaced[T](items: list[T], count: int) -> list[T]:
    if count >= len(items):
        return list(items)
    if count == 1:
        return [items[0]]
    indices = [round(index * (len(items) - 1) / (count - 1)) for index in range(count)]
    return [items[index] for index in indices]


def _split_webdataset_member(name: str) -> tuple[str, str] | None:
    """Mirror WebDataset's first-dot split into ``(__key__, field)``."""
    basename_start = name.rfind("/") + 1
    dot_index = name.find(".", basename_start)
    if dot_index <= basename_start or dot_index == len(name) - 1:
        return None
    return name[:dot_index], name[dot_index + 1 :]


def _coerce_latent(payload: Any) -> torch.Tensor:
    """Mirror the latent payload forms accepted by ``src/dataset/tensor.py``."""
    if isinstance(payload, torch.Tensor):
        return payload
    if isinstance(payload, np.ndarray):
        return torch.from_numpy(payload)
    if isinstance(payload, dict):
        for key in ("vae_latent", "latent", "latents", "data"):
            if key in payload:
                value = payload[key]
                if isinstance(value, torch.Tensor):
                    return value
                if isinstance(value, np.ndarray):
                    return torch.from_numpy(value)
                raise TypeError(f"payload[{key!r}] has unsupported type {type(value).__name__}")
        tensor_values = [value for value in payload.values() if isinstance(value, torch.Tensor)]
        if len(tensor_values) == 1:
            return tensor_values[0]
        raise KeyError(
            "dict payload has no recognized latent key "
            "(vae_latent/latent/latents/data) or unique tensor value"
        )
    raise TypeError(f"unsupported payload type {type(payload).__name__}")


def _extract_member_bytes(
    archive: tarfile.TarFile,
    member: tarfile.TarInfo,
    *,
    limit: int,
) -> bytes:
    if member.size > limit:
        raise ValueError(
            f"member is {member.size} bytes, exceeding the configured limit of {limit} bytes"
        )
    extracted = archive.extractfile(member)
    if extracted is None:
        raise OSError("tar member cannot be opened as a regular file")
    with extracted:
        return extracted.read()


def _load_json_member(
    archive: tarfile.TarFile,
    member: tarfile.TarInfo,
) -> Any:
    raw = _extract_member_bytes(archive, member, limit=MAX_JSON_MEMBER_BYTES)
    return json.loads(raw.decode("utf-8"))


def _validate_optional_json(
    archive: tarfile.TarFile,
    fields: dict[str, tarfile.TarInfo],
    *,
    shard: str,
    key: str,
    latent: torch.Tensor,
    report: ValidationReport,
) -> None:
    actions_member = fields.get("actions.json")
    if actions_member is not None:
        try:
            _load_json_member(archive, actions_member)
        except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
            report.error(f"shard={shard!r}, key={key!r}: invalid actions.json: {exc}")

    meta_member = fields.get("meta.json")
    if meta_member is None:
        return
    try:
        metadata = _load_json_member(archive, meta_member)
    except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
        report.error(f"shard={shard!r}, key={key!r}: invalid meta.json: {exc}")
        return
    if not isinstance(metadata, dict):
        report.error(f"shard={shard!r}, key={key!r}: meta.json must contain a JSON object")
        return

    metadata_key = metadata.get("clip_name")
    if metadata_key is not None and metadata_key != key:
        report.error(f"shard={shard!r}, key={key!r}: meta.json clip_name is {metadata_key!r}")
    metadata_shape = metadata.get("latent_shape")
    if metadata_shape is not None:
        try:
            parsed_shape = _validate_shape_values(
                metadata_shape,
                dimensions=4,
                field_name="meta.json latent_shape",
            )
        except ValueError as exc:
            report.error(f"shard={shard!r}, key={key!r}: {exc}")
        else:
            if parsed_shape != tuple(latent.shape):
                report.error(
                    f"shard={shard!r}, key={key!r}: meta.json latent_shape "
                    f"{_format_shape(parsed_shape)} does not match payload "
                    f"{_format_shape(tuple(latent.shape))}"
                )
    metadata_dtype = metadata.get("latent_dtype")
    if metadata_dtype is not None:
        expected_dtype = str(latent.dtype).removeprefix("torch.").lower()
        actual_dtype = str(metadata_dtype).removeprefix("torch.").lower()
        if actual_dtype != expected_dtype:
            report.error(
                f"shard={shard!r}, key={key!r}: meta.json latent_dtype "
                f"{metadata_dtype!r} does not match payload {expected_dtype!r}"
            )


def _validate_observed_shape(
    latent: torch.Tensor,
    *,
    shard: str,
    key: str,
    options: ValidationOptions,
    report: ValidationReport,
) -> None:
    if latent.ndim != 4:
        report.error(
            f"shard={shard!r}, key={key!r}: latent must be 4-D [C,T,H,W], "
            f"got {_format_shape(tuple(latent.shape))}"
        )
        return
    shape: DiskShape = tuple(latent.shape)  # type: ignore[assignment]
    report.observed_shapes[shape] += 1
    dtype_name = str(latent.dtype).removeprefix("torch.")
    report.observed_dtypes[dtype_name] += 1

    if report.reference_shape is None:
        report.reference_shape = shape
    elif shape != report.reference_shape:
        report.error(
            f"shard={shard!r}, key={key!r}: non-uniform latent shape "
            f"{_format_shape(shape)}; first checked shape was "
            f"{_format_shape(report.reference_shape)}"
        )

    if options.expected_shape is not None and shape != options.expected_shape:
        report.error(
            f"shard={shard!r}, key={key!r}: latent shape {_format_shape(shape)} "
            f"does not match --expected-shape {_format_shape(options.expected_shape)}"
        )

    contract = report.training_contract
    if contract is not None:
        training_t, training_c, training_h, training_w = contract.sample_shape
        disk_c, disk_t, disk_h, disk_w = shape
        if (disk_c, disk_h, disk_w) != (training_c, training_h, training_w):
            report.error(
                f"shard={shard!r}, key={key!r}: disk [C,H,W] "
                f"{_format_shape((disk_c, disk_h, disk_w))} does not match config "
                f"{_format_shape((training_c, training_h, training_w))}"
            )
        if disk_t < training_t:
            report.error(
                f"shard={shard!r}, key={key!r}: disk T={disk_t} is shorter than "
                f"config T={training_t}"
            )
        elif disk_t > training_t:
            report.note(
                f"stored T={disk_t} is compatible with config T={training_t}; "
                "the trainer consumes only the leading configured frames"
            )


def _validate_latent_member(
    archive: tarfile.TarFile,
    member: tarfile.TarInfo,
    fields: dict[str, tarfile.TarInfo],
    *,
    shard: str,
    key: str,
    options: ValidationOptions,
    report: ValidationReport,
) -> None:
    try:
        raw = _extract_member_bytes(
            archive,
            member,
            limit=options.max_payload_bytes,
        )
    except Exception as exc:
        report.error(
            f"shard={shard!r}, key={key!r}: cannot read latent member: {type(exc).__name__}: {exc}"
        )
        return

    try:
        payload = torch.load(
            io.BytesIO(raw),
            map_location="cpu",
            weights_only=not options.allow_unsafe_pickle,
        )
    except Exception as exc:
        safety_hint = ""
        if not options.allow_unsafe_pickle:
            safety_hint = (
                "; safe weights-only loading rejected this payload. Only for trusted data, "
                "retry with --allow-unsafe-pickle"
            )
        report.error(
            f"shard={shard!r}, key={key!r}: torch.load failed: "
            f"{type(exc).__name__}: {exc}{safety_hint}"
        )
        return

    try:
        latent = _coerce_latent(payload)
    except Exception as exc:
        report.error(
            f"shard={shard!r}, key={key!r}: unsupported latent payload: {type(exc).__name__}: {exc}"
        )
        return

    report.checked_samples += 1
    if not latent.is_floating_point():
        report.error(
            f"shard={shard!r}, key={key!r}: latent dtype must be floating point, got {latent.dtype}"
        )
    elif not bool(torch.isfinite(latent).all()):
        report.error(f"shard={shard!r}, key={key!r}: latent contains NaN or Inf values")

    _validate_observed_shape(
        latent,
        shard=shard,
        key=key,
        options=options,
        report=report,
    )
    if latent.ndim == 4:
        _validate_optional_json(
            archive,
            fields,
            shard=shard,
            key=key,
            latent=latent,
            report=report,
        )


def _scan_shard(
    shard: str,
    records: list[AnnotationRecord],
    *,
    options: ValidationOptions,
    report: ValidationReport,
) -> None:
    tar_path = options.data_path / f"{SHARD_FILENAME_PREFIX}{shard}.tar"
    if not tar_path.is_file():
        report.error(f"shard={shard!r}: derived tar file does not exist: {tar_path}")
        return

    annotated_keys = {record.clip_name for record in records}
    groups: dict[str, dict[str, tarfile.TarInfo]] = {}
    try:
        archive = tarfile.open(tar_path, mode="r:*")
    except (OSError, tarfile.TarError) as exc:
        report.error(f"shard={shard!r}: cannot open tar {tar_path}: {exc}")
        return

    with archive:
        try:
            for member in archive:
                if not member.isfile():
                    continue
                split = _split_webdataset_member(member.name)
                if split is None:
                    continue
                key, suffix = split
                if key not in annotated_keys:
                    continue
                normalized_suffix = suffix.lower()
                fields = groups.setdefault(key, {})
                if normalized_suffix in fields:
                    report.error(
                        f"shard={shard!r}, key={key!r}: duplicate tar field {normalized_suffix!r}"
                    )
                    continue
                fields[normalized_suffix] = member
        except (OSError, tarfile.TarError) as exc:
            report.error(f"shard={shard!r}: failed while scanning tar {tar_path}: {exc}")
            return

        loadable_members: dict[str, tarfile.TarInfo] = {}
        for record in records:
            fields = groups.get(record.clip_name)
            if fields is None:
                report.error(
                    f"shard={shard!r}, key={record.clip_name!r}: annotation key "
                    "has no matching tar sample"
                )
                continue
            latent_member = fields.get(CANONICAL_LATENT_FIELD)
            if latent_member is None:
                fallback = next(
                    (
                        member
                        for suffix, member in fields.items()
                        if suffix == "pth" or suffix.endswith(".pth")
                    ),
                    None,
                )
                if fallback is None:
                    report.error(
                        f"shard={shard!r}, key={record.clip_name!r}: missing required "
                        f"{CANONICAL_LATENT_FIELD!r} field"
                    )
                    continue
                report.error(
                    f"shard={shard!r}, key={record.clip_name!r}: latent member "
                    f"{fallback.name!r} is runtime-tolerated but non-canonical; use "
                    f"<key>.{CANONICAL_LATENT_FIELD}"
                )
                latent_member = fallback
            loadable_members[record.clip_name] = latent_member
            report.matched_annotation_entries += 1

        selected_records = (
            records
            if options.all_samples
            else _evenly_spaced(records, min(options.samples_per_shard, len(records)))
        )
        for record in selected_records:
            member = loadable_members.get(record.clip_name)
            fields = groups.get(record.clip_name)
            if member is None or fields is None:
                continue
            _validate_latent_member(
                archive,
                member,
                fields,
                shard=shard,
                key=record.clip_name,
                options=options,
                report=report,
            )

    report.checked_shards += 1


def validate_dataset(options: ValidationOptions) -> ValidationReport:
    """Validate one latent WebDataset and return all bounded findings."""
    report = ValidationReport(max_errors=options.max_errors)
    if options.caption_field != "global_caption_en":
        report.warning(
            "This tool-only caption selection does not establish training compatibility; "
            "public training fixes the field to 'global_caption_en'."
        )
    if options.allow_unsafe_pickle:
        report.warning("unsafe pickle loading is enabled; validate only data from a trusted source")
    data_path = options.data_path
    if not data_path.is_dir():
        report.error(f"data path does not exist or is not a directory: {data_path}")

    try:
        by_shard = _load_annotations(options, report)
    except (OSError, UnicodeError) as exc:
        report.error(f"cannot read annotation JSONL {options.annotation_path}: {exc}")
        by_shard = {}

    if options.config_path is not None:
        try:
            contract = _load_training_contract(options.config_path)
        except Exception as exc:
            report.error(
                f"cannot load training config {options.config_path}: {type(exc).__name__}: {exc}"
            )
        else:
            report.training_contract = contract
            _validate_training_contract(report, contract, options.expected_shape)
    elif options.expected_shape is None:
        report.note(
            "no --config or --expected-shape was supplied; payload rank, finiteness, "
            "and sampled shape uniformity are checked, but not a target model shape"
        )

    if not data_path.is_dir() or not by_shard:
        return report

    shard_names = sorted(by_shard)
    selected_shards = (
        shard_names
        if options.all_shards
        else _evenly_spaced(shard_names, min(options.sample_shards, len(shard_names)))
    )
    if len(selected_shards) < len(shard_names):
        report.note(
            f"archive coverage is sampled: {len(selected_shards)}/{len(shard_names)} shards; "
            "use --all-shards for every referenced tar"
        )
    if not options.all_samples:
        report.note(
            f"payload decoding is sampled at up to {options.samples_per_shard} annotated "
            "samples per checked shard; use --all-samples for every annotated payload"
        )

    for shard in selected_shards:
        _scan_shard(
            shard,
            by_shard[shard],
            options=options,
            report=report,
        )
    return report


def format_report(options: ValidationOptions, report: ValidationReport) -> str:
    """Render a stable human-readable validation summary."""
    result = "PASS" if report.ok else "FAIL"
    lines = [
        f"Latent WebDataset validation: {result}",
        f"Data path: {options.data_path}",
        f"Annotation: {options.annotation_path}",
        (
            "Index: "
            f"{report.annotation_entries} valid entries across "
            f"{report.annotation_shards} shards"
        ),
        (
            "Coverage: "
            f"{report.checked_shards}/{report.annotation_shards} shards opened, "
            f"{report.matched_annotation_entries} annotated keys matched, "
            f"{report.checked_samples} payloads decoded"
        ),
    ]
    if options.expected_shape is not None:
        lines.append(f"Expected disk shape [C,T,H,W]: {_format_shape(options.expected_shape)}")
    if report.training_contract is not None:
        contract = report.training_contract
        model_suffix = f", model={contract.model_name}" if contract.model_name else ""
        lines.append(
            "Training contract [B,T,C,H,W]: "
            f"{_format_shape(contract.full_shape)} from {contract.path}{model_suffix}"
        )
        lines.append(
            "Block config: "
            f"causal={str(contract.causal).lower()}, "
            f"num_frame_per_block={contract.block_size}, "
            f"independent_first_frame={str(contract.independent_first_frame).lower()}"
        )
    if report.observed_shapes:
        observed_shapes = ", ".join(
            f"{_format_shape(shape)} x{count}"
            for shape, count in sorted(report.observed_shapes.items())
        )
        lines.append(f"Observed disk shapes [C,T,H,W]: {observed_shapes}")
    if report.observed_dtypes:
        observed_dtypes = ", ".join(
            f"{dtype} x{count}" for dtype, count in sorted(report.observed_dtypes.items())
        )
        lines.append(f"Observed dtypes: {observed_dtypes}")

    if report.notes:
        lines.append("Notes:")
        lines.extend(f"  - {note}" for note in report.notes)
    if report.warning_count:
        lines.append(f"Warnings ({report.warning_count}):")
        lines.extend(f"  - {warning}" for warning in report.warnings)
        hidden = report.warning_count - len(report.warnings)
        if hidden > 0:
            lines.append(f"  - ... {hidden} more warnings suppressed")
    if report.error_count:
        lines.append(f"Errors ({report.error_count}):")
        lines.extend(f"  - {error}" for error in report.errors)
        hidden = report.error_count - len(report.errors)
        if hidden > 0:
            lines.append(f"  - ... {hidden} more errors suppressed")
    return "\n".join(lines)


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        "--data-path",
        type=Path,
        required=True,
        help="Directory containing latent_<shard>.tar files.",
    )
    parser.add_argument(
        "--annotation-path",
        type=Path,
        required=True,
        help="UTF-8 annotation JSONL containing shard, clip_name, and caption fields.",
    )
    parser.add_argument(
        "--caption-field",
        default="global_caption_en",
        help="Tool-only annotation field check; public training uses global_caption_en.",
    )
    parser.add_argument(
        "--config",
        dest="config_path",
        type=Path,
        help=(
            "Public recipe (stem or path), merged with shared defaults, whose shape must be "
            "compatible with sampled disk latents."
        ),
    )
    parser.add_argument(
        "--expected-shape",
        type=parse_disk_shape,
        help="Exact on-disk latent shape in C,T,H,W order, e.g. 48,31,44,80.",
    )
    parser.add_argument(
        "--sample-shards",
        type=_positive_int,
        default=3,
        help="Deterministically sample this many shards (default: 3).",
    )
    parser.add_argument(
        "--samples-per-shard",
        type=_positive_int,
        default=3,
        help="Decode this many annotated payloads per checked shard (default: 3).",
    )
    parser.add_argument(
        "--all-shards",
        action="store_true",
        help="Open every shard referenced by the annotation JSONL.",
    )
    parser.add_argument(
        "--all-samples",
        action="store_true",
        help="Decode every annotated payload in each checked shard.",
    )
    parser.add_argument(
        "--allow-unsafe-pickle",
        action="store_true",
        help=(
            "Use torch.load(weights_only=False). This may execute pickle code; enable only "
            "for trusted datasets."
        ),
    )
    parser.add_argument(
        "--max-payload-mib",
        type=_positive_int,
        default=2048,
        help="Refuse to load a sampled .pth member larger than this many MiB (default: 2048).",
    )
    parser.add_argument(
        "--max-errors",
        type=_positive_int,
        default=50,
        help="Maximum errors and warnings printed for each category (default: 50).",
    )
    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    options = ValidationOptions(
        data_path=args.data_path,
        annotation_path=args.annotation_path,
        caption_field=args.caption_field,
        expected_shape=args.expected_shape,
        config_path=args.config_path,
        sample_shards=args.sample_shards,
        samples_per_shard=args.samples_per_shard,
        all_shards=args.all_shards,
        all_samples=args.all_samples,
        allow_unsafe_pickle=args.allow_unsafe_pickle,
        max_payload_bytes=args.max_payload_mib * 1024 * 1024,
        max_errors=args.max_errors,
    )
    report = validate_dataset(options)
    print(format_report(options, report))
    return 0 if report.ok else 1


if __name__ == "__main__":
    sys.exit(main())