Skip to content

armgit / atas

Public Imported from GitHub
0

Any Time, Any Space: A World-State-Centric Framework for Agentic Video Generation

24.6 KB
Raw Blame
"""Ground and execute one authored camera path; report issues for explicit repair.

This is not a global identity tracker. VLM regions are frame-local evidence;
they never become identity_verified masks or permanent world-memory labels.
"""
from __future__ import annotations

import math
from dataclasses import replace

import numpy as np
from scipy.ndimage import binary_dilation, label
from scipy.spatial.transform import Rotation, Slerp

from atas_geometry.memory.camera_query import (
    _look_at_camera,
    _normalize,
    _rotate_axis,
    check_camera_query,
    generate_camera_query,
    sparse_geometry_sample,
)

MOVES = {"locked", "push_in", "pull_back", "orbit", "arc_left", "arc_right", "truck_left", "truck_right", "crane_up", "crane_down", "push_in_with_arc", "pan_left", "pan_right", "tilt_up", "tilt_down", "handheld", "track_left", "track_right", "track_forward", "track_back"}
ROTATIONAL_MOVES = {"pan_left", "pan_right", "tilt_up", "tilt_down", "handheld"}
TRACK_MODES = {"track_left":"truck_left", "track_right":"truck_right",
               "track_forward":"push_in", "track_back":"pull_back"}


def numbers(values, n, name, lo=None, hi=None):
    a = np.asarray(values, dtype=float)
    if a.shape != (n,) or not np.isfinite(a).all() or (lo is not None and np.any(a < lo)) or (hi is not None and np.any(a > hi)):
        raise ValueError(f"invalid {name}")
    return a


def validate_plan(plan):
    if plan.get("schema_version") != 1 or plan.get("status") != "ok":
        raise ValueError("camera planner could not resolve the requested shot")
    subjects = plan.get("subjects")
    if not isinstance(subjects, list) or not 1 <= len(subjects) <= 4:
        raise ValueError("camera intent needs one to four explicitly selected subjects")
    for item in subjects:
        if item.get("framing_role", "focal") not in {"focal", "context"}:
            raise ValueError("subject framing_role must be focal or context")
        if not str(item.get("label", "")).strip() or item.get("kind") not in {"dynamic", "static"}:
            raise ValueError("subject needs a label and dynamic/static kind")
        if item.get('part') not in {None, 'face', 'head', 'face/head', 'upper_body', 'full_body', 'body', 'object'}:
            raise ValueError('subject region has unsupported part; replan the visible focal part')
        box = numbers(item.get("bbox"), 4, "subject bbox", 0, 1)
        point = numbers(item.get("anchor_uv"), 2, "subject anchor", 0, 1)
        if np.any(box[2:] - box[:2] <= .005) or np.any(point < box[:2]) or np.any(point > box[2:]):
            raise ValueError("subject anchor must lie inside a nonempty bbox")
        confidence = float(item.get("confidence", 0))
        if not math.isfinite(confidence) or confidence < .75 or confidence > 1 or item.get("visible") is not True:
            raise ValueError("subject visually unresolved; do not substitute an anonymous cluster")
    framing = plan["framing"]
    numbers(framing.get("screen_position"), 2, "screen position", .1, .9)
    numbers([framing.get("vertical_fraction"), framing.get("horizontal_fraction")], 2, "frame fractions", .1, .9)
    numbers([framing.get("side_degrees"), framing.get("elevation_degrees")], 2, "view angles", -85, 85)
    motion = plan["motion"]
    if motion.get("name") not in MOVES:
        raise ValueError("unsupported camera motion")
    numbers([motion.get("degrees")], 1, "motion degrees", 0, 100)
    numbers([motion.get("travel_ratio")], 1, "travel ratio", 0, .7)
    if motion['name'] in ROTATIONAL_MOVES:
        if float(motion['degrees']) <= 0 or float(motion['travel_ratio']) != 0:
            raise ValueError('rotational motion requires positive degrees and zero travel')
        if motion['name']=='handheld' and float(motion['degrees'])>3:
            raise ValueError('handheld sway is limited to 3 degrees')
    if motion['name'] in TRACK_MODES and (float(motion['travel_ratio'])<=0 or float(motion['degrees'])!=0):
        raise ValueError('tracking requires positive travel and zero arc')
    framing["screen_position"] = [float(v) for v in framing["screen_position"]]
    for key in ("vertical_fraction", "horizontal_fraction", "side_degrees", "elevation_degrees"):
        framing[key] = float(framing[key])
    for key in ("degrees", "travel_ratio"):
        motion[key] = float(motion[key])
    for item in subjects:
        item["bbox"] = [float(v) for v in item["bbox"]]
        item["anchor_uv"] = [float(v) for v in item["anchor_uv"]]
        item["confidence"] = float(item["confidence"])
    return plan


def ground_subjects(recon, index, plan, *, subject_masks=None):
    """Use seed-local depth inside the VLM region; reject missing/ambiguous depth."""
    validate_plan(plan)
    depth = recon.depth_at(index)
    h, w = depth.shape
    observation = recon.unproject(index)
    # Reconstruct the same valid-pixel ordering as ReconStore, including sky.
    from atas_geometry.memory.vista4d_layers import _mask
    valid = np.isfinite(depth) & (depth > 1e-5)
    if recon.exclude_sky and index < len(recon.sky_files):
        valid &= ~_mask(recon.sky_files[index], depth.shape)
    ys, xs = np.nonzero(valid)
    uv = np.column_stack(((xs + .5) / w, (ys + .5) / h))
    if len(uv) != len(observation.xyz):
        raise ValueError("source pixel/depth indexing mismatch")
    sets, audits = [], []
    def unavailable_context(item, reason):
        if item.get("framing_role", "focal") != "context":
            return False
        # Empty groups preserve the subjects/groups/audits positional contract.
        # Unknown context is not geometry evidence and never affects focal fit.
        sets.append(np.empty((0, 3), dtype=np.float32))
        audits.append({"label": item["label"], "part": item.get("part"), "points": 0,
                       "framing_role": "context", "status": "unknown", "reason": reason,
                       "evidence": "unavailable", "identity_verified": False,
                       "part_semantics_verified": False})
        return True
    for item in plan["subjects"]:
        box, seed = np.asarray(item["bbox"]), np.asarray(item["anchor_uv"])
        inside = np.all((uv >= box[:2]) & (uv <= box[2:]), axis=1)
        desired = observation.foreground >= .5 if item["kind"] == "dynamic" else observation.foreground < .5
        identity_mask = (subject_masks or {}).get(item['label'])
        instance_fraction = None
        if identity_mask is not None:
            identity_mask = np.asarray(identity_mask, dtype=bool)
            if identity_mask.shape != depth.shape:
                raise ValueError('subject region identity mask/depth shape mismatch')
            # A seed proves one pixel's identity, not every foreground pixel in
            # the box. Constrain the entire selected region to that same track.
            desired &= identity_mask[ys, xs]
            py, px = np.nonzero(identity_mask)
            puv = np.column_stack(((px + .5) / w, (py + .5) / h))
            contained = np.all((puv >= box[:2]) & (puv <= box[2:]), axis=1)
            instance_fraction = float(contained.mean()) if len(puv) else 0.
            clipped = (identity_mask[0].any() or identity_mask[-1].any()
                       or identity_mask[:,0].any() or identity_mask[:,-1].any())
            # This is a contradiction check, NOT an anatomical face detector.
            # A cropped source may show only a face; do not reject that case.
            if item.get('part') in {'face','head','face/head','upper_body'} and not clipped and instance_fraction >= .9:
                raise ValueError(f"subject region for {item['label']} claims {item['part']} but contains "
                                 f"{instance_fraction:.1%} of the complete visible instance; replan the actual part bbox")
        inside &= desired
        near = inside & (np.linalg.norm((uv - seed) / np.maximum(box[2:] - box[:2], .01), axis=1) < .2)
        if np.count_nonzero(near) < 16:
            if unavailable_context(item, "no_reliable_seed_depth"):
                continue
            raise ValueError(f"no reliable seed depth for {item['label']}")
        z = depth[ys, xs]
        median = float(np.median(z[near]))
        mad = float(np.median(np.abs(z[near] - median)))
        if mad / median > .15:
            if unavailable_context(item, "ambiguous_seed_depth"):
                continue
            raise ValueError(f"ambiguous seed depth for {item['label']}")
        selected = inside & (np.abs(z - median) <= max(4 * mad, .15 * median))
        points = np.asarray(observation.xyz)[selected]
        if len(points) < 32:
            if unavailable_context(item, "insufficient_grounded_region"):
                continue
            raise ValueError(f"insufficient grounded region for {item['label']}")
        sets.append(points)
        audits.append({"label": item["label"], "part": item.get("part"), "points": len(points),
            "evidence": "verified_instance_intersected_vlm_region_seed_depth" if identity_mask is not None else "vlm_region_seed_depth",
            "identity_verified": identity_mask is not None, "confidence": item["confidence"],
            "instance_fraction_in_region": instance_fraction,
            "part_semantics_verified": False,
            "part_verification_policy": "anatomy_requires_visual_review; occupancy_is_not_semantic_acceptance"})
    # Keep every region for diagnostics, but never use scenery to size a close-up.
    if not any(len(p) for p in sets):
        raise ValueError("camera intent has no grounded focal subject")
    focus = np.mean([np.median(p, axis=0) for p in sets if len(p)], axis=0)
    all_dynamic = np.asarray(observation.xyz)[observation.foreground >= .5]
    return np.concatenate(sets), focus, all_dynamic, audits


def project(points, camera):
    p = np.asarray(points) @ camera.world_to_camera[:3, :3].T + camera.world_to_camera[:3, 3]
    uvh = p @ camera.intrinsic.T
    uv = uvh[:, :2] / np.maximum(p[:, 2:3], 1e-8)
    return uv / np.asarray(camera.image_size), p[:, 2]


def aimed_camera(reference, position, focus, screen, up):
    """Aim the desired subject at an image location without changing intrinsics."""
    c2w = _look_at_camera(position, focus, up)
    w, h = reference.image_size
    k = reference.intrinsic
    ray = _normalize(np.array([(screen[0] * w - k[0, 2]) / k[0, 0], (screen[1] * h - k[1, 2]) / k[1, 1], 1.]))
    target = np.array([0., 0, 1.])
    cross = np.cross(ray, target)
    if np.linalg.norm(cross) > 1e-9:
        rotation = Rotation.from_rotvec(_normalize(cross) * math.acos(float(np.clip(ray @ target, -1, 1)))).as_matrix()
        c2w[:3, :3] = c2w[:3, :3] @ rotation
    return replace(reference, camera_id="semantic_framing", world_to_camera=np.linalg.inv(c2w))


def fit_reference(reference, points, focus, framing, side):
    up = _normalize(-reference.camera_to_world[:3, 1])
    offset = reference.center_world - focus
    direction = _normalize(offset - up * (offset @ up))
    direction = _rotate_axis(direction, up, math.radians(side))
    elevation = math.radians(float(framing["elevation_degrees"]))
    direction = direction * math.cos(elevation) + up * math.sin(elevation)
    screen = np.asarray(framing["screen_position"])
    extent = max(float(np.linalg.norm(np.ptp(points, axis=0))), 1e-4)
    low, high = extent * .05, max(extent * 20, np.linalg.norm(offset) * 4)
    for _ in range(24):
        distance = (low + high) / 2
        candidate = aimed_camera(reference, focus + direction * distance, focus, screen, up)
        uv, z = project(points, candidate)
        bounds = np.percentile(uv, [2, 98], axis=0)
        size = bounds[1] - bounds[0]
        fits = (np.min(z) > 0 and size[0] <= framing["horizontal_fraction"]
            and size[1] <= framing["vertical_fraction"] and np.all(bounds[0] >= .03) and np.all(bounds[1] <= .97))
        if fits:
            high = distance
        else:
            low = distance
    return aimed_camera(reference, focus + direction * high, focus, screen, up)


def coverage_risk(points, cameras, excluded_points, subject_points=None):
    """Coarse projected support, NOT a calibrated final-pixel coverage gate."""
    values = []
    for camera in cameras:
        def mask_for(xyz):
            uv, z = project(xyz, camera)
            valid = (z > 0) & np.all((uv >= 0) & (uv < 1), axis=1)
            pixels = np.floor(uv[valid] * [96, 54]).astype(int)
            mask = np.zeros((54, 96), dtype=bool)
            mask[pixels[:, 1], pixels[:, 0]] = True
            return mask
        occupied = mask_for(points)
        excluded = binary_dilation(mask_for(excluded_points), iterations=1) if len(excluded_points) else np.zeros_like(occupied)
        evaluated = ~excluded
        unknown = evaluated & ~occupied
        components, _ = label(unknown)
        sizes = np.bincount(components.ravel())[1:]
        large = float(sizes.max(initial=0) / max(1, evaluated.sum()))
        center = np.zeros_like(occupied)
        center[13:41, 24:72] = True
        central = center & evaluated
        neighborhood = center.copy()
        if subject_points is not None and len(subject_points):
            uv, z = project(subject_points, camera)
            visible = uv[z > 0]
            if len(visible):
                bounds = np.percentile(visible, [2, 98], axis=0)
                bounds[0] -= .12
                bounds[1] += .12
                lo = np.floor(np.clip(bounds[0], 0, 1) * [96,54]).astype(int)
                hi = np.ceil(np.clip(bounds[1], 0, 1) * [96,54]).astype(int)
                neighborhood[:] = False
                neighborhood[lo[1]:hi[1],lo[0]:hi[0]] = True
        values.append({"unknown_fraction": float(unknown.sum() / max(1, evaluated.sum())),
            "largest_unknown_region": large,
            "central_unknown_fraction": float((unknown & center).sum() / max(1, central.sum())),
            "subject_neighborhood_unknown_fraction": float((unknown & neighborhood).sum() / max(1, (neighborhood & evaluated).sum())),
            "excluded_dynamic_fraction": float(excluded.mean())})
    summary = {key: float(np.mean([v[key] for v in values])) for key in values[0]}
    summary["worst_largest_unknown_region"] = max(v["largest_unknown_region"] for v in values)
    summary["risk_score"] = summary["unknown_fraction"] + summary["subject_neighborhood_unknown_fraction"] + 2 * summary["worst_largest_unknown_region"]
    summary["policy"] = "soft_ranking_within_same_intent_not_generation_veto"
    summary["estimator"] = "96x54_sparse_geometry_support_not_final_pixel_coverage"
    return summary


def focal_geometry(points, plan, groups):
    if groups is None:
        return points, None
    selected = [group for item, group in zip(plan["subjects"], groups)
                if item.get("framing_role", "focal") == "focal"]
    if not selected:
        raise ValueError("subject framing requires at least one focal region")
    return np.concatenate(selected), np.mean([np.median(p, axis=0) for p in selected], axis=0)


def framing_check(points, cameras, framing, *, continuous=False, locked=False):
    """Geometry framing validity, separate from collision safety and coverage ranking.

    Sample the path, not only its end. Continuous opening poses are immutable;
    their initial crop may recover, but may not get worse while approaching target.
    """
    samples = []
    # Deterministic spatially dense subset for the repeated projection check only.
    sample = points[::max(1, len(points)//16000)]
    for index in sorted(set(np.linspace(0, len(cameras)-1, min(9, len(cameras))).astype(int))):
        uv, z = project(sample, cameras[index])
        bounds = np.percentile(uv, [2, 98], axis=0)
        size = bounds[1]-bounds[0]
        clipping = float(np.mean((z <= 0) | np.any((uv < 0) | (uv > 1), axis=1)))
        samples.append({"frame_index": int(index), "clipped_fraction": clipping,
                        "bbox": bounds.reshape(-1).tolist(), "size": size.tolist()})
    end = samples[-1]
    size = np.asarray(end["size"])
    requested = np.array([framing["horizontal_fraction"], framing["vertical_fraction"]])
    fill = float(np.max(size / requested))
    allowed_crop = max(.05, samples[0]["clipped_fraction"]) if continuous else .05
    failures = []
    if any(s["clipped_fraction"] > allowed_crop + .005 for s in samples):
        failures.append("focal subject leaves frame during the camera path")
    if not (continuous and locked):
        if end["clipped_fraction"] > .05:
            failures.append("focal subject clipped at the framing target")
        if not .70 <= fill <= 1.25:
            failures.append("focal framing scale does not match requested part/shot size")
    return {"passed": not failures, "failures": failures, "samples": samples,
            "target_fill_ratio": fill, "policy": "focal_path_v2"}


def rotational_cameras(reference, times, move, degrees, *, continuous):
    """True camera-center rotation; no orbit, translation, zoom or 2D warp.

    A cut is fitted at the end of a pan/tilt. A continuous shot begins with the
    exact inherited pose. Handheld is deterministic low-frequency angular sway.
    """
    times=np.asarray(times,dtype=float)
    if len(times)<2 or not np.isfinite(times).all() or np.any(np.diff(times)<=0):
        raise ValueError('rotational camera requires increasing frame timestamps')
    phase=(times-times[0])/(times[-1]-times[0])
    angle=math.radians(degrees)
    cameras=[]
    for i,p in enumerate(phase):
        if continuous and i==0:
            cameras.append(reference);continue
        progress=.5-.5*math.cos(math.pi*p)
        if move=='handheld':
            t=times[i]-times[0]
            rotvec=angle*np.array([.45*math.sin(2*math.pi*.7*t),
                                  .65*math.sin(2*math.pi*.43*t),
                                  .25*math.sin(2*math.pi*.31*t)])
        else:
            amount=angle*(progress if continuous else progress-1)
            sign=-1 if move in {'pan_left','tilt_down'} else 1
            axis=np.array([1.,0,0]) if move.startswith('tilt_') else np.array([0.,1,0])
            rotvec=sign*amount*axis
        pose=reference.camera_to_world.copy()
        pose[:3,:3]=pose[:3,:3]@Rotation.from_rotvec(rotvec).as_matrix()
        cameras.append(replace(reference,camera_id=move,world_to_camera=np.linalg.inv(pose)))
    return tuple(cameras)


def plan_query(memory, reference, points, focus, dynamic_points, plan, times, relation, *, subject_groups=None,
               framing_policy="strict"):
    validate_plan(plan)
    if framing_policy not in {"strict", "advisory"}:
        raise ValueError("framing_policy must be strict or advisory")
    requested_framing_policy = framing_policy
    if relation == "recapture":
        framing_policy = "strict"
    points, focal_focus = focal_geometry(points, plan, subject_groups)
    if focal_focus is not None:
        focus = focal_focus
    if not len(points) or not np.isfinite(points).all() or not np.isfinite(focus).all():
        raise ValueError("camera requires finite nonempty grounded focal geometry")
    framing, motion = plan["framing"], plan["motion"]
    move = motion["name"]
    path_move = TRACK_MODES.get(move, move)
    continuous = relation == "continuous"
    side = float(framing["side_degrees"])
    geometry = sparse_geometry_sample(memory)
    # One authored trajectory. Corrections are explicit replans, never hidden search.
    start = reference if continuous else fit_reference(reference, points, focus, framing, side)
    base_travel = float(motion["travel_ratio"])
    if continuous and move in {"push_in", "pull_back"}:
        # Match the end framing by solving distance, not by a fixed tiny dolly.
        offset = reference.center_world - focus
        up = _normalize(-reference.camera_to_world[:3, 1])
        elevation = math.degrees(math.asin(float(np.clip(_normalize(offset) @ up, -1, 1))))
        fitted = fit_reference(reference, points, focus, {**framing, "elevation_degrees": elevation}, 0.)
        required = 1 - np.linalg.norm(fitted.center_world - focus) / np.linalg.norm(offset)
        if (move == "push_in" and required > 0) or (move == "pull_back" and required < 0):
            base_travel = min(abs(float(required)), .9)
    sign = -1 if path_move in {"arc_left", "truck_left", "crane_down"} else 1
    mode = ("orbit" if path_move in {"orbit", "arc_left", "arc_right"} else "dolly" if path_move in {"push_in", "pull_back"}
        else "truck" if path_move.startswith("truck") else "crane" if path_move.startswith("crane") else "cinematic_arc" if path_move == "push_in_with_arc" else "locked")
    travel = base_travel
    if path_move == "pull_back":
        travel = -travel
    elif mode in {"truck", "crane"}:
        travel *= sign
    trajectory_start = start
    if not continuous and path_move == "push_in":
        # Fit the close-up at the END, then move into it. Fitting the start
        # and advancing further was pushing already-fitted subjects out.
        position = focus + (start.center_world-focus) / (1-travel)
        trajectory_start = aimed_camera(start, position, focus, framing["screen_position"],
                                         _normalize(-reference.camera_to_world[:3, 1]))
    query = generate_camera_query(memory, times, trajectory_start, mode=mode,
        focus_world=focus, azimuth_degrees=0, orbit_degrees=sign * float(motion["degrees"]),
        travel_ratio=travel, lift_ratio=.04 if mode == "cinematic_arc" else 0,
        truck_ratio=0)
    up = _normalize(-reference.camera_to_world[:3, 1])
    cameras = []
    for i, camera in enumerate(query.cameras):
        desired = aimed_camera(start, camera.center_world, focus, framing["screen_position"], up)
        if continuous:
            if move == "locked" or i == 0:
                desired = reference
            else:
                progress = .5 - .5 * math.cos(math.pi * i / (len(times) - 1))
                rotations = Rotation.from_matrix(np.stack([reference.camera_to_world[:3, :3], desired.camera_to_world[:3, :3]]))
                pose = desired.camera_to_world.copy()
                pose[:3, :3] = Slerp([0, 1], rotations)(progress).as_matrix()
                desired = replace(desired, world_to_camera=np.linalg.inv(pose))
        cameras.append(desired)
    query = replace(query, cameras=tuple(cameras))
    if move in ROTATIONAL_MOVES:
        query=replace(query,cameras=rotational_cameras(start,times,move,
            float(motion['degrees']),continuous=continuous))
    if any(not np.isfinite(camera.world_to_camera).all() or not np.isfinite(camera.intrinsic).all()
           for camera in query.cameras):
        raise ValueError("authored camera path has nonfinite calibration")
    safety = check_camera_query(query, geometry, min_end_distance_ratio=.05, min_surface_clearance_ratio=.02)
    if not safety["passed"]:
        raise ValueError("no geometrically safe same-intent authored trajectory; locally revise it: " + str(safety["failures"]))
    framing_validity = framing_check(points, query.cameras, framing,
                                     continuous=continuous, locked=move == "locked")
    if not framing_validity["passed"] and framing_policy == "strict":
        raise ValueError("no geometrically safe same-intent authored trajectory with valid focal framing; "
                         "locally revise it: " + str(framing_validity))
    observed = [query.cameras[i] for i in sorted({0, len(times)//2, len(times)-1})]
    risk = coverage_risk(geometry, observed, dynamic_points, points)
    # Framing remains meaningful while a continuous shot approaches its end target.
    uv, z = project(points, query.cameras[-1])
    bounds = np.percentile(uv, [2, 98], axis=0)
    size = bounds[1] - bounds[0]
    framing_error = abs(float(size[1]) - framing["vertical_fraction"]) + float(np.linalg.norm(np.mean(bounds, axis=0) - framing["screen_position"]))
    clipping = float(np.mean((z <= 0) | np.any((uv < 0) | (uv > 1), axis=1)))
    selected = {"side_degrees": side if not continuous else None,
        "motion_scale": 1., "solved_travel_ratio": travel, "framing_error": framing_error,
        "subject_clipped_fraction": clipping,
        "framing_warnings": list(framing_validity["failures"]) if not framing_validity["passed"] else [],
        "coverage": {**risk, "policy": "diagnostic_only_not_trajectory_selection"},
        "geometry_safety": safety, "framing_validity": framing_validity}
    return query, {"contract": "semantic-camera-v1", "relation": relation, "move": move,
        "framing_policy": framing_policy, "requested_framing_policy": requested_framing_policy,
        "framing_warnings": selected["framing_warnings"],
        "subject_evidence": "frame_local_vlm_regions_not_persistent_identity",
        "selected": selected, "candidate_count": 1, "rejected_candidates": [],
        "ranking": "none_single_authored_path", "execution_policy": "single_authored_path_v1", "frame0_inherited": continuous,
        "requested_motion": motion, "requested_framing": framing,
        "candidates": [selected]}