Skip to content

armgit / atas

Public Imported from GitHub
0

Any Time, Any Space: A World-State-Centric Framework for Agentic Video Generation

10.1 KB
Raw Blame
import copy
import json
import os
from types import SimpleNamespace

import imageio.v2 as imageio
import numpy as np
import pytest

from atas_geometry.memory.observation import CameraCalibration
from atas_geometry.render.cinematic_camera import validate_plan, ground_subjects, aimed_camera, fit_reference, project, plan_query, coverage_risk
from atas_geometry.render.camera_context import camera_context
def plan(move="locked", side=35):
    return {"schema_version":1,"status":"ok", "subjects":[{"label":"Alice", "kind":"dynamic", "part":"face", "visible":True,
        "confidence":.95, "bbox":[.25,.2,.75,.8], "anchor_uv":[.5,.5]}],
        "framing":{"shot_size":"close_up", "screen_position":[.5,.5], "vertical_fraction":.65, "horizontal_fraction":.78,
            "side_degrees":side, "elevation_degrees":0}, "motion":{"name":move,"degrees":35 if move in {"arc_left","arc_right","orbit"} else 0,
            "travel_ratio":.25 if move != "locked" else 0}}


def scene():
    camera = CameraCalibration("source", np.eye(4), np.array([[90.,0,80],[0,90,45],[0,0,1]]), (160,90))
    points = np.array([[x,y,3.] for x in np.linspace(-.3,.3,20) for y in np.linspace(-.6,.6,30)])
    wall = np.array([[x,y,7.] for x in np.linspace(-8,8,160) for y in np.linspace(-5,5,90)])
    return camera, points, SimpleNamespace(xyz=wall)


def test_plan_rejects_unresolved_and_nonfinite():
    for mutate in (lambda p:p.update(status="unresolved"), lambda p:p["subjects"][0].update(confidence=.3),
        lambda p:p["subjects"][0].update(anchor_uv=[.99,.99]), lambda p:p["motion"].update(degrees=float("nan"))):
        p = plan()
        mutate(p)
        with pytest.raises(ValueError): validate_plan(p)


def test_screen_placement_preserves_intrinsics():
    camera, points, memory = scene()
    focus = np.array([0.,0,3.])
    target = aimed_camera(camera, np.array([1.,0,0]), focus, [.35,.42], np.array([0,-1.,0]))
    uv, _ = project(focus[None], target)
    np.testing.assert_allclose(uv[0], [.35,.42], atol=1e-6)
    np.testing.assert_array_equal(camera.intrinsic, target.intrinsic)


def test_face_region_changes_distance_not_whole_body_zoom():
    camera, body, _ = scene()
    focus = np.array([0.,0,3.])
    framing = plan()["framing"]
    face = body.copy()
    face[:, :2] *= .25
    full = fit_reference(camera, body, focus, framing, 0)
    close = fit_reference(camera, face, focus, framing, 0)
    assert np.linalg.norm(close.center_world-focus) < .4*np.linalg.norm(full.center_world-focus)
    uv, _ = project(face, close)
    height = np.diff(np.percentile(uv[:,1],[2,98]))[0]
    assert abs(height - .65) < .02


@pytest.mark.parametrize("move", ["locked","push_in","orbit","truck_right","crane_up"])
def test_continuous_first_pose_exact(move):
    camera, points, memory = scene()
    intent = plan(move)
    # Isolate endpoint inheritance: orbit/truck/crane cannot turn a medium
    # source into a close-up without adding an unrequested radial move.
    if move in {"orbit", "truck_right", "crane_up"}:
        intent["framing"]["vertical_fraction"] = .4
    query, report = plan_query(memory,camera,points,np.array([0.,0,3.]),points,intent,tuple(np.linspace(0,2,9)),"continuous")
    np.testing.assert_array_equal(query.cameras[0].world_to_camera, camera.world_to_camera)
    assert report["frame0_inherited"] is True
    if move == "locked":
        for c in query.cameras: np.testing.assert_array_equal(c.world_to_camera,camera.world_to_camera)


def test_cut_reselects_camera_and_arc_not_capped_at_15():
    camera, points, memory = scene()
    focus = np.array([0.,0,3.])
    query, report = plan_query(memory,camera,points,focus,points,plan("orbit"),tuple(np.linspace(0,2,9)),"cut_recompose")
    assert not np.allclose(query.cameras[0].center_world,camera.center_world)
    a,b = query.cameras[0].center_world-focus,query.cameras[-1].center_world-focus
    angle = np.degrees(np.arccos(np.clip(a@b/np.linalg.norm(a)/np.linalg.norm(b),-1,1)))
    assert angle > 15
    assert report["selected"]["motion_scale"] >= .7


def test_coverage_large_connected_hole_is_worse_than_scattered():
    camera, _, _ = scene()
    def unproject_grid(mask):
        y,x=np.nonzero(mask)
        u,v=(x+.5)/96*160,(y+.5)/54*90
        return np.column_stack(((u-80)/90*4,(v-45)/90*4,np.full(len(x),4)))
    scattered=np.indices((54,96)).sum(axis=0)%2==0
    large=np.ones((54,96),dtype=bool)
    large[:,:48]=False
    empty=np.empty((0,3))
    a=coverage_risk(unproject_grid(scattered),[camera],empty)
    b=coverage_risk(unproject_grid(large),[camera],empty)
    assert b["worst_largest_unknown_region"] > a["worst_largest_unknown_region"]
    assert a["policy"].startswith("soft_ranking")
    excluded=unproject_grid(~large)
    c=coverage_risk(unproject_grid(large),[camera],excluded)
    assert c["unknown_fraction"] < .01


def test_grounding_uses_named_region_without_asserting_global_identity():
    h,w=40,60
    y,x=np.indices((h,w))
    depth=np.full((h,w),3.)
    xyz=np.column_stack(((x.ravel()+.5-w/2)/40*3,(y.ravel()+.5-h/2)/40*3,depth.ravel()))
    observation=SimpleNamespace(xyz=xyz,foreground=np.ones(h*w))
    recon=SimpleNamespace(depth_at=lambda _:depth,unproject=lambda _:observation,exclude_sky=True,sky_files=[])
    points,focus,dynamic,audit=ground_subjects(recon,0,plan())
    assert len(points)>32 and len(dynamic)==h*w
    assert audit[0]["label"]=="Alice" and audit[0]["identity_verified"] is False
    observation.foreground[:]=0
    with pytest.raises(ValueError,match="seed depth"):
        ground_subjects(recon,0,plan())


def test_context_event_uses_actual_center_slice(tmp_path):
    source=tmp_path/"run/story/shots/source/l0"
    source.mkdir(parents=True)
    with imageio.get_writer(str(source/"video.mp4"),fps=6,macro_block_size=1) as writer:
        for i in range(6): writer.append_data(np.full((16,16,3),i*20,dtype=np.uint8))
    np.savez(source/"cameras.npz",cam_c2w=np.repeat(np.eye(4)[None],6,axis=0))
    scene_dir=tmp_path/"run/story/scenes/room"
    scene_dir.mkdir(parents=True)
    (scene_dir/"l1.npz").touch()
    meta={"run_id":"run","story_id":"story","scene_id":"room","shot_id":"source","event_id":"event",
        "duration":2.,"registered":True,"raw_to_scene":{"scale":1},"original_frame_count":12,"source_fps":6.}
    (source/"observation.json").write_text(json.dumps(meta))
    p={"run_id":"run","story_id":"story","scene_id":"room","shot_id":"new","source_shot_id":"source","source_event_id":"event",
        "event_id":"event","render_mode":"recapture","source_time_start":.5,"source_time_end":1.5,"duration":1.,"fps":6}
    context=camera_context(tmp_path,p)
    assert context["frame_index"]==0
    with pytest.raises(ValueError,match="center slice"):
        camera_context(tmp_path,{**p,"source_time_start":0.,"source_time_end":1.})


@pytest.mark.skipif("ATAS_TEST_GPU" not in os.environ,reason="explicit idle GPU required")
@pytest.mark.parametrize("event",[False,True])
def test_semantic_runtime_writes_real_mp4_and_camera(tmp_path,monkeypatch,event):
    from PIL import Image
    from atas_geometry.render import camera_runtime as runtime
    from atas_geometry.memory.camera_query import CameraQuery
    from atas_geometry.memory import vista4d_layers
    source=tmp_path/"run/story/shots/source/l0"
    source.mkdir(parents=True)
    w,h=64,36
    mask=np.zeros((h,w),dtype=bool)
    mask[8:28,22:42]=True
    depth=np.where(mask,3.,5.).astype(np.float32)
    color=np.full((h,w,3),180,dtype=np.uint8)
    color[mask]=[200,30,30]
    with imageio.get_writer(str(source/"video.mp4"),fps=2,macro_block_size=1) as writer:
        for _ in range(4):writer.append_data(color)
    np.savez(source/"cameras.npz",cam_c2w=np.repeat(np.eye(4)[None],4,axis=0),intrinsics=np.tile([40.,40.,32.,18.],(4,1)))
    for dirname in ("depths","dynamic_mask"):(source/dirname).mkdir()
    for i in range(4):
        (source/"depths"/f"{i:04d}.exr").touch()
        Image.fromarray(mask.astype(np.uint8)*255).save(source/"dynamic_mask"/f"{i:04d}.png")
    scene_dir=tmp_path/"run/story/scenes/room"
    scene_dir.mkdir(parents=True)
    (scene_dir/"home_shot_id.txt").write_text("source\n")
    (scene_dir/"l1.npz").touch()
    y,x=np.nonzero(~mask)
    xyz=np.column_stack(((x+.5-32)/40*5,(y+.5-18)/40*5,np.full(len(x),5)))
    memory=SimpleNamespace(xyz=xyz,rgb=np.full(xyz.shape,180,dtype=np.uint8),voxel_size=.125)
    monkeypatch.setattr(runtime.FusedSceneMemory,"load",lambda _:memory)
    monkeypatch.setattr(runtime,"_install_depth_reader",lambda:None)
    monkeypatch.setattr(vista4d_layers,"_read_exr_depth",lambda _:depth)
    p={"run_id":"run","story_id":"story","scene_id":"room","shot_id":"new","event_id":"event",
        "duration":2.,"fps":4,"camera_relation":"cut_recompose","geometry_layers":"l1"}
    if event:
        meta={"run_id":"run","story_id":"story","scene_id":"room","shot_id":"source","event_id":"event",
            "duration":2.,"registered":True,"original_frame_count":4,"source_fps":2.,
            "raw_to_scene":{"scale":1.,"rotation":np.eye(3).tolist(),"translation":[0.,0.,0.]}}
        (source/"observation.json").write_text(json.dumps(meta))
        p.update(render_mode="recapture",geometry_layers="l1+l2",camera_relation="recapture",source_shot_id="source",source_event_id="event",source_time_start=0.,source_time_end=2.)
    intent=plan("locked",side=25)
    intent["context"]=camera_context(tmp_path,p)
    p["camera_plan"]=intent
    out=tmp_path/"render/memory_input.mp4"
    report=runtime.render_semantic_camera(tmp_path,p,out,w,h,int(os.environ["ATAS_TEST_GPU"]))
    assert out.stat().st_size>0
    with imageio.get_reader(str(out)) as reader:assert reader.count_frames()==8
    query=CameraQuery.load(out.with_name("camera_query.json"))
    assert not np.allclose(query.cameras[0].center_world,[0,0,0])
    assert report["anchor_audit"][0]["identity_verified"] is False
    assert report["splat_mode"] == "pixel"
    assert report["max_point_radius"] == 0
    assert report["mean_coverage"] >= report["mean_point_center_coverage"] - 1e-6
    if event:
        assert report["dynamic_splat_mode"] == "source_pixel"
        assert report["max_dynamic_radius"] == 2
        assert len(report["source_frame_indices"])==8
        assert report["source_frame_indices"][-1]==3
        assert min(report["dynamic_points_per_frame"])>0
    else:assert report["dynamic_points_per_frame"]==[]