Any Time, Any Space: A World-State-Centric Framework for Agentic Video Generation
Python agentic-video-generationmutli-agentmutli-agent-systemspatial-consistencyvideo-generationvideo-generator
"""End-to-end execution loop with fake providers and real media files.
Story: s1 (room, new) -> s2 (room, continuous) -> s3 (yard, review always holds)
-> s4 (room, revisit, registration rejected) -> s5 (room, recapture of s1)
Checks the f/r/a decisions, memory binding, event retrieval and the final film.
"""
import asyncio
from dataclasses import replace
import json
import shutil
from pathlib import Path
from types import SimpleNamespace
import imageio.v2 as imageio
import numpy as np
import pytest
from PIL import Image
from atas.review.film_review import file_hash
from atas.runner.pipeline import Policy, StoryRunner
from atas.request import plan_from_payload
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg required")
def write_video(path: Path, seconds: float, color: int) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
with imageio.get_writer(str(path), fps=24, codec="libx264", macro_block_size=1) as writer:
for i in range(int(seconds * 24)):
writer.append_data(np.full((36, 64, 3), (color + i) % 255, dtype=np.uint8))
return path
def review_for(video, decision="pass"):
return {"schema_version": 1, "stage": "film", "decision": decision, "cache_key": "k" + file_hash(video)[:8],
"video_sha256": file_hash(video), "lineage": {}, "summary": "fake",
"checks": [{"criterion": c, "status": "pass" if decision != "hold" else "fail", "severity": "critical",
"confidence": 0.95, "evidence": "fake"} for c in
("subject", "framing", "motion", "action", "event", "world_consistency")],
"scene_observations": [{"label": "alice", "visibility": "visible", "confidence": 0.9,
"evidence": "fake", "location": "center"}],
"observed_dynamic_classes": [{"noun": "person", "evidence": "fake"}]}
class FakeGeometry:
enabled = True
rejected = {"s4"}
def __init__(self, *a, **k):
self.renders, self.recons, self.jobs = [], [], {}
def render(self, intent):
self.renders.append(intent)
job = SimpleNamespace(job_id=f"render-{intent['shot_id']}-{len(self.renders)}")
self.jobs[job.job_id] = {"status": "succeeded", "report": {"start_from_role": "previous_shot"}}
return job
crash = set()
def recon(self, **kw):
self.recons.append(kw)
shot = kw["shot_id"]
if shot in self.crash:
import atas.world.geometry_client as geo
raise geo.GeometryJobFailed("RuntimeError: Vista4D recon exited 1")
registered = shot not in self.rejected
job = SimpleNamespace(job_id=f"recon-{shot}")
self.jobs[job.job_id] = {"status": "succeeded", "job_id": job.job_id, "cameras_npz": "c", "l0_dir": "l0",
"l1_npz": "l1", "l1_point_count": 10,
"fuse": {"registered": registered, "publication_status": "pending_review",
"candidate_job_id": job.job_id, "reason": "low overlap"}}
return job
def wait(self, job_id, **kw):
return self.jobs[job_id]
def fetch_memory_input(self, job, dest):
return write_video(dest, 4, 100)
def _request(self, method, path, payload=None, **kw):
raise RuntimeError("entity projection unavailable in this fake")
@pytest.fixture
def story(tmp_path, monkeypatch):
import atas.shot.camera_planner as cp
import atas.review.editorial as ed
import atas.world.geometry_client as geo
import atas.shot.stills as im
import atas.review.film_review as ir
import atas.world.keywords as kw
import atas.shot.prompt_author as pa
import atas.runner.publication_recovery as pr
import atas.providers.wan.generation as wan
alice = tmp_path / "alice.png"
Image.new("RGB", (8, 8)).save(alice)
base = {"duration": 4, "entity_ids": ["alice"], "onstage_labels": ["alice"],
"character_reference_images": [str(alice)], "character_reference_map": {"alice": str(alice)},
"prompt": "Alice acts.", "camera_move_name": "push_in", "shot_size": "medium"}
rows = [
{"id": "s1", "location_id": "room", "camera_relation": "new_scene", "event_id": "ev1", "start": 0},
{"id": "s2", "location_id": "room", "camera_relation": "continuous", "event_id": "ev2", "start": 4},
{"id": "s3", "location_id": "yard", "camera_relation": "new_scene", "event_id": "ev3", "start": 8},
{"id": "s4", "location_id": "room", "camera_relation": "revisit", "event_id": "ev4", "start": 12},
{"id": "s5", "location_id": "room", "camera_relation": "recapture", "event_id": "ev1", "start": 16,
"render_mode": "recapture", "source_event_id": "ev1", "source_time_start": 0.0, "source_time_end": 4.0},
]
plan = plan_from_payload("demo", {"title": "demo", "character_reference_map": {"alice": str(alice)},
"shots": [{**base, **r} for r in rows]}, tmp_path / "plan.json")
fake_geometry = FakeGeometry()
monkeypatch.setattr(geo, "GeometryClient", lambda *a, **k: fake_geometry)
monkeypatch.setattr(cp, "plan_camera", lambda g, intent, *a, **k: {**intent, "camera_plan": {
"subjects": [{"label": "alice", "framing_role": "focal", "kind": "dynamic"}], "framing": {}, "motion": {}}})
monkeypatch.setattr(wan, "build_request", lambda shot, **k: (SimpleNamespace(
condition=SimpleNamespace(references=()), output_dir=k["output_dir"], shot=shot), [], []))
monkeypatch.setattr(pa, "author_prompts", lambda shot, *a, **k: shot)
async def generate(request):
return write_video(Path(request.output_dir) / "wan" / "result.mp4", request.shot.duration,
hash(request.shot.shot_id) % 200)
monkeypatch.setattr(wan, "generate_shot", generate)
for name in ("generate_cold_start_frame", "generate_anchor_still", "generate_geometry_aligned_still"):
def still(shot, *a, output=None, **k):
path = Path(output or a[0])
Image.new("RGB", (64, 36), (10, 20, 30)).save(path)
return path
monkeypatch.setattr(im, name, still)
monkeypatch.setattr(im, "write_composition_guide", lambda video, report, out: (Image.new("RGB", (8, 8)).save(out), out)[1])
def review(shot, video, folder, *, stage, **k):
return review_for(video, "hold" if shot.shot_id == "s3" else "pass")
monkeypatch.setattr(ir, "review_shot", review)
def best(shot, shot_dir):
video = Path(shot_dir) / "result.mp4"
return video, review_for(video, "hold"), {"same_scene": True, "actual_event": "partial"}, None
monkeypatch.setattr(ed, "select_best_available", best)
monkeypatch.setattr(kw, "sam_nouns_for_recon", lambda *a, **k: ("person",))
monkeypatch.setattr(pr, "publish_with_rebase", lambda g, d, current, payload: (
current, {"world_version": "v-" + payload["shot_id"], "publication_status": "committed"}))
return plan, fake_geometry, tmp_path
def test_full_loop_delivers_every_shot_and_admits_only_f_r_a(story):
plan, geometry, tmp_path = story
runner = StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs", policy=Policy(film_repairs=1))
manifest = asyncio.run(runner.run())
rows = {r["shot_id"]: r for r in manifest["shots"]}
assert manifest["shots_delivered"] == 5 and manifest["complete"] and Path(manifest["final_video"]).is_file()
# f / r / a decisions
assert rows["s1"]["admitted"] and rows["s2"]["admitted"] and rows["s5"]["admitted"]
assert rows["s3"]["film_status"] == "best_available" and not rows["s3"]["f"] and not rows["s3"]["admitted"]
assert rows["s4"]["f"] and not rows["s4"]["r"] and not rows["s4"]["admitted"]
assert "registration_rejected" in rows["s4"]["reason"]
# binding against admitted memory
assert [rows[s]["mode"] for s in ("s1", "s2", "s3", "s4", "s5")] == [
"cold_start", "continue", "cold_start", "revisit", "recapture"]
recapture = [r for r in geometry.renders if r["shot_id"] == "s5"][0]
assert recapture["source_shot_id"] == "s1" and recapture["geometry_layers"] == "l1+l2"
# f=0 never reaches reconstruction; world time reaches the GPU
assert {r["shot_id"] for r in geometry.recons} == {"s1", "s2", "s4", "s5"}
assert [r["world_time_start"] for r in geometry.recons if r["shot_id"] == "s2"] == [4.0]
ledger = json.loads((tmp_path / "runs/story_demo/world_ledger.json").read_text())
assert ledger["scenes"]["room"]["admitted"] == ["s1", "s2", "s5"]
assert "yard" not in ledger["scenes"] and list(ledger["events"]) == ["ev1", "ev2"]
# Resume: everything is final, nothing is regenerated or reconstructed again.
renders, recons = len(geometry.renders), len(geometry.recons)
report = json.loads((tmp_path / "runs/story_demo/report.json").read_text())
assert report["state_update_rate"] == 3 / 5 and report["modes"]["recapture"] == 1
again = asyncio.run(StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs").run())
assert again["shots_delivered"] == 5 and len(geometry.renders) == renders and len(geometry.recons) == recons
def test_reconstruction_crash_keeps_reviewed_film_and_never_refilms(story):
plan, geometry, tmp_path = story
geometry.crash = {"s2"}
runner = StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs", policy=Policy(film_repairs=1))
manifest = asyncio.run(runner.run())
rows = {r["shot_id"]: r for r in manifest["shots"]}
assert rows["s2"]["film_status"] == "reviewed" and rows["s2"]["f"] and not rows["s2"]["r"]
assert rows["s2"]["reason"].startswith("reconstruction_failed")
assert not (tmp_path / "runs/story_demo/s2__degraded").exists()
assert manifest["shots_delivered"] == 5
def test_unadmitted_event_source_is_retaken_so_the_recapture_has_its_event(story):
plan, geometry, tmp_path = story
shots = tuple(replace(s, raw={**s.raw, "recapture_source": True}) if s.shot_id == "s1" else s
for s in plan.shots)
plan = replace(plan, shots=shots)
first_take = geometry.recon
def recon(**kw):
job = first_take(**kw)
if kw["shot_id"] == "s1" and "__take2" not in str(kw["video"]):
geometry.jobs[job.job_id]["fuse"] = {"registered": False, "reason": "internal_reconstruction_inconsistent"}
return job
geometry.recon = recon
manifest = asyncio.run(StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs",
policy=Policy(film_repairs=1)).run())
rows = {r["shot_id"]: r for r in manifest["shots"]}
assert rows["s1"]["admitted"] and "__take2" in rows["s1"]["video"]
assert rows["s1"]["earlier_takes"][0]["reason"].startswith("registration_rejected")
assert rows["s5"]["mode"] == "recapture"
# an ordinary unadmitted shot is never retaken
assert not (tmp_path / "runs/story_demo/s4__take2").exists()
def test_resume_waits_for_inflight_reconstruction_instead_of_re_reviewing(story, monkeypatch):
import atas.review.film_review as ir
plan, geometry, tmp_path = story
asyncio.run(StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs", policy=Policy(film_repairs=1)).run())
shot_dir = tmp_path / "runs/story_demo/s4"
ledger = json.loads((tmp_path / "runs/story_demo/world_ledger.json").read_text())
ledger["shots"].pop("s4") # killed while s4 was reconstructing
(tmp_path / "runs/story_demo/world_ledger.json").write_text(json.dumps(ledger))
(shot_dir / "geometry_recon_state.json").write_text(json.dumps({"status": "waiting", "job_id": "recon-s4"}))
monkeypatch.setattr(ir, "review_shot", lambda *a, **k: pytest.fail("in-flight film must not be re-reviewed"))
again = asyncio.run(StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs", policy=Policy(film_repairs=1)).run())
row = {r["shot_id"]: r for r in again["shots"]}["s4"]
assert row["f"] and row["film_status"] == "reviewed"
def test_reviewed_film_is_kept_when_only_admission_keeps_failing(story):
import atas.world.geometry_client as geo
plan, geometry, tmp_path = story
def recon(**kw):
if kw["shot_id"] == "s2":
raise geo.GeometryWaitUncertain("geometry job polling unavailable")
return FakeGeometry.recon(geometry, **kw)
geometry.recon = recon
manifest = asyncio.run(StoryRunner(plan, run_id="run", run_dir=tmp_path / "runs",
policy=Policy(film_repairs=1, backoff_seconds=(0, 0, 0))).run())
row = {r["shot_id"]: r for r in manifest["shots"]}["s2"]
assert row["film_status"] == "reviewed" and row["f"] and not row["r"]
assert row["reason"].startswith("admission_failed") and "__degraded" not in str(row["video"])
assert not (tmp_path / "runs/story_demo/s2__degraded").exists()