Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
5.7 KB
Raw Blame
"""Deterministic uncompressed tar and relative-index production."""

from __future__ import annotations

import base64
import hashlib
import io
import os
import tarfile
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from solarwm.config.loader import canonical_json
from solarwm.data.index import validate_relative_key
from solarwm.errors import DataContractError

from .contracts import EncodedPayload


@dataclass(frozen=True)
class ShardReceipt:
    relative_path: str
    samples: int
    size: int
    digest: str
    md5_b64: str
    rows: tuple[Mapping[str, Any], ...]


def _digest(path: Path) -> tuple[str, str]:
    digest = hashlib.blake2s()
    md5 = hashlib.md5(usedforsecurity=False)
    with path.open("rb") as handle:
        while chunk := handle.read(16 * 1024 * 1024):
            digest.update(chunk)
            md5.update(chunk)
    return digest.hexdigest(), base64.b64encode(md5.digest()).decode()


def _tar_member(archive: tarfile.TarFile, name: str, value: bytes) -> None:
    info = tarfile.TarInfo(name)
    info.size = len(value)
    info.mtime = 0
    info.mode = 0o644
    info.uid = 0
    info.gid = 0
    info.uname = ""
    info.gname = ""
    archive.addfile(info, io.BytesIO(value))


def write_shard(
    root: str | Path,
    relative_path: str,
    samples: Sequence[EncodedPayload],
) -> ShardReceipt:
    """Write one shard without overwriting any existing output."""

    relative = validate_relative_key(relative_path)
    if not relative.endswith(".tar"):
        raise DataContractError("preencoded shards must use the .tar suffix")
    if not samples:
        raise DataContractError("cannot write an empty preencoded shard")
    ids = [sample.sample_id for sample in samples]
    if len(ids) != len(set(ids)):
        raise DataContractError("preencoded shard contains duplicate sample IDs")

    target = Path(root).resolve() / relative
    if target.exists():
        raise DataContractError(f"preencoded shard already exists: {target}")
    target.parent.mkdir(parents=True, exist_ok=True)
    temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.partial")
    rows: list[dict[str, Any]] = []
    try:
        with tarfile.open(temporary, "w:", format=tarfile.USTAR_FORMAT) as archive:
            for sample in samples:
                prefix = f"samples/{hashlib.blake2s(sample.sample_id.encode()).hexdigest()[:24]}"
                members: dict[str, str] = {}
                for suffix, value in sorted(sample.members.items()):
                    member = f"{prefix}/{suffix}"
                    _tar_member(archive, member, value)
                    members[suffix] = member
                provenance = {
                    "schema": "solarwm.preencoded-sample.v1",
                    "sample_id": sample.sample_id,
                    "key": sample.key,
                    "source_sample_id": sample.source_sample_id,
                    "start_frame": sample.start_frame,
                    "source_frame_indices": list(sample.source_frame_indices),
                    "encoder_contract_digest": sample.encoder_contract_digest,
                    "members": members,
                    "metadata": dict(sample.metadata),
                }
                provenance_member = f"{prefix}/provenance.json"
                _tar_member(archive, provenance_member, canonical_json(provenance))
                rows.append(
                    {
                        "sample_id": sample.sample_id,
                        "key": sample.key,
                        "source_sample_id": sample.source_sample_id,
                        "shard": relative,
                        "epoch_repeats": 1,
                        "start_frame": sample.start_frame,
                        "source_frame_indices": list(sample.source_frame_indices),
                        "encoder_contract_digest": sample.encoder_contract_digest,
                        "members": members,
                        "provenance_member": provenance_member,
                        "metadata": dict(sample.metadata),
                    }
                )
        with temporary.open("rb") as handle:
            os.fsync(handle.fileno())
        os.replace(temporary, target)
    finally:
        temporary.unlink(missing_ok=True)
    digest, md5_b64 = _digest(target)
    size = target.stat().st_size
    for row in rows:
        row["shard_size"] = size
        row["shard_digest"] = digest
        row["shard_md5_b64"] = md5_b64
    return ShardReceipt(relative, len(samples), size, digest, md5_b64, tuple(rows))


def write_index(path: str | Path, rows: Sequence[Mapping[str, Any]]) -> str:
    """Atomically write ordered JSONL and return its content digest."""

    if not rows:
        raise DataContractError("cannot write an empty preencoded index")
    sample_ids: set[str] = set()
    payload = bytearray()
    for row in rows:
        sample_id = str(row.get("sample_id") or "")
        if not sample_id or sample_id in sample_ids:
            raise DataContractError(f"missing or duplicate sample_id {sample_id!r}")
        sample_ids.add(sample_id)
        validate_relative_key(str(row.get("shard") or ""))
        payload.extend(canonical_json(dict(row)))
    target = Path(path)
    if target.exists():
        raise DataContractError(f"preencoded index already exists: {target}")
    target.parent.mkdir(parents=True, exist_ok=True)
    temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.partial")
    try:
        with temporary.open("wb") as handle:
            handle.write(payload)
            handle.flush()
            os.fsync(handle.fileno())
        os.replace(temporary, target)
    finally:
        temporary.unlink(missing_ok=True)
    return hashlib.blake2s(payload).hexdigest()