SceneMosaic: Efficient and Diverse Simulation-Ready Scene Generation via Hybrid Agentic Layout Evolution
Python
#!/usr/bin/env python3
"""Compute CLIP image embeddings for locally downloaded ambientCG materials."""
from __future__ import annotations
import argparse
import os
from pathlib import Path
import numpy as np
import torch
import yaml
from PIL import Image
from tqdm import tqdm
PROJECT_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MATERIALS_DIR = PROJECT_ROOT / "data" / "materials"
DEFAULT_EMBEDDINGS_DIR = PROJECT_ROOT / "data" / "embeddings"
DEFAULT_CACHE_DIR = PROJECT_ROOT / "vlm_agent" / "evolution" / "cache" / "open_clip"
MODEL_NAME = "ViT-H-14-378-quickgelu"
PRETRAINED = "dfn5b"
EXPECTED_DIMENSION = 1024
DEFAULT_TEXTURE_RESOLUTION = "2K"
DEFAULT_TEXTURE_FORMAT = "PNG"
TEXTURE_RESOLUTIONS = ("1K", "2K", "4K", "8K", "12K", "16K")
TEXTURE_FORMATS = ("PNG", "JPG")
def env_texture_resolution() -> str:
"""Resolve the texture resolution from the environment (default 2K)."""
value = os.environ.get("SCENEMOSAIC_MATERIAL_RESOLUTION")
return value if value in TEXTURE_RESOLUTIONS else DEFAULT_TEXTURE_RESOLUTION
def env_texture_format() -> str:
"""Resolve the texture format from the environment (default PNG)."""
value = os.environ.get("SCENEMOSAIC_MATERIAL_FORMAT")
return value if value in TEXTURE_FORMATS else DEFAULT_TEXTURE_FORMAT
def texture_path(
material_dir: Path,
channel: str,
*,
resolution: str | None = None,
file_format: str | None = None,
) -> Path:
"""Return the path of one ambientCG texture of ``material_dir``.
ambientCG names textures ``<material>_<res>-<fmt>_<channel>.<ext>`` where
the filename label uses the format in uppercase (e.g. ``2K-PNG_Color.png``
for PNG, ``2K-JPG_Color.jpg`` for JPG). ``channel`` is the map suffix such
as ``Color``, ``NormalGL`` or ``Roughness``. Unset arguments fall back to
the environment overrides, then to 2K PNG.
"""
resolution = resolution or env_texture_resolution()
file_format = file_format or env_texture_format()
extension = file_format.lower()
return material_dir / (
f"{material_dir.name}_{resolution}-{file_format}_{channel}.{extension}"
)
def discover_materials(
materials_dir: Path,
*,
resolution: str | None = None,
file_format: str | None = None,
) -> list[tuple[str, Path]]:
"""Return deterministic material IDs and their base-color textures."""
resolution = resolution or env_texture_resolution()
file_format = file_format or env_texture_format()
discovered: list[tuple[str, Path]] = []
if not materials_dir.is_dir():
raise FileNotFoundError(f"Material directory not found: {materials_dir}")
for material_dir in sorted(materials_dir.iterdir(), key=lambda path: path.name):
if not material_dir.is_dir() or material_dir.is_symlink():
continue
color_path = texture_path(
material_dir, "Color", resolution=resolution, file_format=file_format
)
if color_path.is_file() and not color_path.is_symlink():
discovered.append((material_dir.name, color_path))
if not discovered:
raise FileNotFoundError(
f"No ambientCG {resolution}-{file_format} Color textures found below {materials_dir}"
)
return discovered
def encode_materials(
materials: list[tuple[str, Path]],
*,
device: str,
batch_size: int,
cache_dir: Path,
) -> np.ndarray:
"""Encode base-color textures with the retrieval model used at runtime."""
import open_clip
cache_dir.mkdir(parents=True, exist_ok=True)
model, _, preprocess = open_clip.create_model_and_transforms(
MODEL_NAME,
pretrained=PRETRAINED,
device=device,
cache_dir=str(cache_dir),
)
model.eval()
encoded_batches: list[np.ndarray] = []
with torch.inference_mode():
for start in tqdm(
range(0, len(materials), batch_size),
desc="Computing ambientCG embeddings",
):
batch_items = materials[start : start + batch_size]
tensors = []
for _, color_path in batch_items:
with Image.open(color_path) as image:
tensors.append(preprocess(image.convert("RGB")))
image_batch = torch.stack(tensors).to(device)
features = model.encode_image(image_batch)
features = features / features.norm(dim=-1, keepdim=True)
encoded_batches.append(
features.detach().cpu().numpy().astype(np.float32, copy=False)
)
embeddings = np.concatenate(encoded_batches, axis=0)
if embeddings.shape != (len(materials), EXPECTED_DIMENSION):
raise ValueError(
"Unexpected embedding shape: "
f"received {embeddings.shape}, expected "
f"({len(materials)}, {EXPECTED_DIMENSION})"
)
if not np.isfinite(embeddings).all():
raise ValueError("Computed embeddings contain non-finite values")
return embeddings
def write_outputs(
output_dir: Path,
material_ids: list[str],
embeddings: np.ndarray,
*,
overwrite: bool,
) -> None:
"""Write the matrix and row index without exposing partial final files."""
output_dir.mkdir(parents=True, exist_ok=True)
embeddings_path = output_dir / "clip_embeddings.npy"
index_path = output_dir / "embedding_index.yaml"
existing = [path for path in (embeddings_path, index_path) if path.exists()]
if existing and not overwrite:
names = ", ".join(str(path) for path in existing)
raise FileExistsError(f"Output already exists: {names}; pass --overwrite")
embeddings_tmp = output_dir / ".clip_embeddings.npy.tmp"
index_tmp = output_dir / ".embedding_index.yaml.tmp"
with embeddings_tmp.open("wb") as handle:
np.save(handle, embeddings, allow_pickle=False)
index_tmp.write_text(
yaml.safe_dump(material_ids, sort_keys=False),
encoding="utf-8",
)
embeddings_tmp.replace(embeddings_path)
index_tmp.replace(index_path)
def main() -> int:
parser = argparse.ArgumentParser(
description="Compute CLIP embeddings for downloaded ambientCG materials"
)
parser.add_argument(
"--materials-dir",
type=Path,
default=DEFAULT_MATERIALS_DIR,
help="ambientCG material directory (default: <project>/data/materials)",
)
parser.add_argument(
"--output-dir",
type=Path,
default=DEFAULT_EMBEDDINGS_DIR,
help="embedding output directory (default: <project>/data/embeddings)",
)
parser.add_argument(
"--cache-dir",
type=Path,
default=DEFAULT_CACHE_DIR,
help=(
"OpenCLIP model cache "
"(default: <project>/vlm_agent/evolution/cache/open_clip)"
),
)
parser.add_argument(
"--device",
default=None,
help="Torch device; defaults to CUDA when available, otherwise CPU",
)
parser.add_argument(
"--resolution",
"-r",
type=str,
default=env_texture_resolution(),
choices=list(TEXTURE_RESOLUTIONS),
help=(
"Texture resolution (default: $SCENEMOSAIC_MATERIAL_RESOLUTION "
f"or {DEFAULT_TEXTURE_RESOLUTION})"
),
)
parser.add_argument(
"--format",
"-f",
type=str,
default=env_texture_format(),
choices=list(TEXTURE_FORMATS),
help=(
"Texture file format (default: $SCENEMOSAIC_MATERIAL_FORMAT "
f"or {DEFAULT_TEXTURE_FORMAT})"
),
)
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--overwrite", action="store_true")
args = parser.parse_args()
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
materials_dir = args.materials_dir.expanduser().resolve()
output_dir = args.output_dir.expanduser().resolve()
cache_dir = args.cache_dir.expanduser().resolve()
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
materials = discover_materials(
materials_dir,
resolution=args.resolution,
file_format=args.format,
)
embeddings = encode_materials(
materials,
device=device,
batch_size=args.batch_size,
cache_dir=cache_dir,
)
write_outputs(
output_dir,
[material_id for material_id, _ in materials],
embeddings,
overwrite=args.overwrite,
)
print(
f"Wrote {len(materials)} {EXPECTED_DIMENSION}D embeddings to {output_dir}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())