Skip to content

armgit / atas

Public Imported from GitHub
0

Any Time, Any Space: A World-State-Centric Framework for Agentic Video Generation

12.1 KB
Raw Blame
"""Repair evidenced temporal drift before a new scene has a published world.

No existing L1 or accepted observation is edited. Reuses dense source RGB/masks,
and re-estimates depth/poses together against the first unchained VO window.
Image correspondences used for verification never fit a correction transform.
"""
from __future__ import annotations
import copy
import itertools
import json
from pathlib import Path
import time
import numpy as np

POLICY = 'initial-world-fixed-anchor-v3'


def temporal_evidence(source):
    import cv2
    from atas_geometry.register.matching import _lift_uv_robust
    from atas_geometry.register.anchored_fusion import static_depth
    from atas_geometry.register.matching import evidence_frames
    ids = evidence_frames(source.count)
    detector = cv2.SIFT_create(nfeatures=5000)
    matcher = cv2.BFMatcher()
    features = {}
    depths = []
    for i in ids:
        rgb = source.frames[i]
        depth = static_depth(source, i)
        depths.append(depth[depth > 0])
        mask = (depth > 0).astype(np.uint8)*255
        features[i] = detector.detectAndCompute(cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY),mask)
    positive = np.concatenate(depths)
    if not len(positive):
        return [], None
    tolerance = .02*float(np.median(positive))
    pairs = []
    for a,b in itertools.combinations(ids,2):
        ka,da=features[a]; kb,db=features[b]
        if da is None or db is None or min(len(da),len(db))<2: continue
        forward=[p[0] for p in matcher.knnMatch(da,db,k=2) if len(p)==2 and p[0].distance<.72*p[1].distance]
        reverse={p[0].queryIdx:p[0].trainIdx for p in matcher.knnMatch(db,da,k=2) if len(p)==2 and p[0].distance<.72*p[1].distance}
        matches=[m for m in forward if reverse.get(m.trainIdx)==m.queryIdx]
        if len(matches)<100: continue
        uv=np.array([ka[m.queryIdx].pt for m in matches]); vv=np.array([kb[m.trainIdx].pt for m in matches])
        cv2.setRNGSeed(0)
        matrix,inliers=cv2.findFundamentalMat(uv,vv,cv2.USAC_MAGSAC,1.5,.999,10000)
        if matrix is None or inliers is None: continue
        s,sv,_=_lift_uv_robust(source,a,uv); t,tv,_=_lift_uv_robust(source,b,vv)
        valid=sv&tv&inliers.ravel().astype(bool)
        if valid.sum()<100: continue
        error=np.linalg.norm(s[valid]-t[valid],axis=1)
        pairs.append(dict(frames=[a,b],uv=uv[valid],vv=vv[valid],error=error,
                          count=len(error),median=float(np.median(error))))
    return pairs,tolerance


GROSS = 4      # one pair beyond GROSS x (2 x tolerance) is drift on its own
MIN_PAIRS = 3
MIN_COVERAGE = .8


def coverage(pairs, count):
    """Fraction of the clip's time span bridged by supported frame pairs."""
    if count <= 1:
        return 1.
    covered = np.zeros(count - 1, bool)
    for p in pairs:
        if p['count'] >= 100:
            a, b = sorted(p['frames'])
            covered[a:b] = True
    return float(covered.mean())


def assess(pairs, tolerance, count):
    """Classify a first observation from fixed-coordinate pair disagreement.

    drift: two supported pairs beyond 2x tolerance, or one beyond GROSS times that.
    insufficient: too few supported pairs, or pairs that leave part of the clip
    unchecked -- absence of evidence is not evidence of consistency.
    """
    supported = [p for p in pairs if p['count'] >= 100]
    drifted = [p for p in supported if tolerance is not None and p['median'] > 2*tolerance]
    gross = [p for p in drifted if p['median'] > GROSS*2*tolerance]
    span = coverage(supported, count)
    return dict(supported_pairs=len(supported), drifted_pairs=len(drifted), gross_pairs=len(gross),
                coverage=span, drift=len(drifted) >= 2 or bool(gross),
                sufficient=tolerance is not None and len(supported) >= MIN_PAIRS and span >= MIN_COVERAGE)


def verify_candidate(candidate,pairs,tolerance):
    """Re-measure the original correspondences in the repaired world, no refit.

    Every pair keeps its evidence and ends within the drift line (2x tolerance):
    drifted pairs must be resolved, consistent pairs must stay consistent.
    A small change of an already-consistent pair is not a regression.
    """
    from atas_geometry.register.matching import _lift_uv_robust
    checks=[]
    for p in pairs:
        a,b=p['frames']
        s,sv,_=_lift_uv_robust(candidate,a,p['uv']); t,tv,_=_lift_uv_robust(candidate,b,p['vv'])
        valid=sv&tv
        error=np.linalg.norm(s[valid]-t[valid],axis=1)
        count=len(error)
        median=float(np.median(error)) if count else None
        fraction=float(np.mean(error<=2*tolerance)) if count else 0.
        checks.append(dict(frames=p['frames'],count=count,original_count=p['count'],
            before_median=p['median'],median=median,within_2t=fraction,
            passed=bool(count>=100 and count>=.8*p['count'] and median<=2*tolerance and fraction>=.65)))
    return checks, all(c['passed'] for c in checks)


def prepare_initial_world(source_path,folder,*,vista_repo,model_id,first_window=48,pixel_limit=255000):
    from atas_geometry.register.anchored_fusion import strict_store, reconstruct_conditioned, sha256, write_json
    tick=time.perf_counter()
    source_path=Path(source_path).resolve(); folder=Path(folder).resolve()
    source=strict_store(source_path)
    stamp=dict(policy=POLICY,source=str(source_path.resolve()),video=sha256(source_path/'video.mp4'),
        cameras=sha256(source_path/'cameras.npz'),frames=source.count,first_window=first_window,
        pixel_limit=pixel_limit,model_id=str(model_id),
        dense_manifest=sha256(source_path/'dense_complete.json') if (source_path/'dense_complete.json').is_file() else None)
    receipt=folder/'initial_world.json'
    if receipt.exists():
        old=json.loads(receipt.read_text())
        if old['input']!=stamp: raise ValueError('initial-world repair input changed')
        if old['status']=='repaired':
            result=strict_store(old['l0'])
            if result.count!=source.count: raise ValueError('initial-world candidate incomplete')
        return old
    folder.mkdir(parents=True,exist_ok=True)
    intent=folder/'input.json'
    if intent.exists() and json.loads(intent.read_text())!=stamp:
        raise ValueError('interrupted initial-world repair input changed')
    write_json(intent,stamp)
    pairs,tolerance=temporal_evidence(source)
    assessment=assess(pairs,tolerance,source.count)
    out=dict(input=stamp,tolerance=tolerance,source=str(source_path),l0=str(source_path),
        before=[{k:p[k] for k in ('frames','count','median')} for p in pairs],assessment=assessment,
        published=False)
    if assessment['sufficient'] and not assessment['drift']:
        out.update(status='unchanged',reason='consistent_with_sufficient_evidence')
    elif first_window<=0 or source.count<=first_window:
        # Too short for a fixed-anchor re-estimate; never publish unverified drift.
        out.update(status='rejected' if assessment['drift'] else 'unchanged',
                   reason='drift_in_single_window_clip' if assessment['drift'] else 'single_window_clip')
    else:
        # Drift, or too little evidence to call the clip consistent: re-estimate it
        # against its own first window as a fixed anchor. The anchored reconstruction
        # applies the same held-out/seam validation as any later registration, and
        # every available pair is re-measured on top.
        reference=copy.copy(source);reference.count=min(first_window,source.count)
        target=folder/'conditioned_l0'
        # Interrupted candidate is retained, never treated as a completed cache.
        report_path=target/'anchored_reconstruction.json'
        if report_path.exists():
            report=json.loads(report_path.read_text())
        else:
            if target.exists(): target.rename(folder/f'conditioned_interrupted_{time.time_ns()}')
            report=reconstruct_conditioned(reference,source,target,model_id=model_id,
                vista_repo=Path(vista_repo),pixel_limit=pixel_limit,local_repair=True)
        checks,passed=verify_candidate(strict_store(target),pairs,tolerance)
        good=bool(report['geometry_accepted'] and report['frame_count']==source.count and passed)
        out.update(status='repaired' if good else 'rejected',after=checks,
            reason=('drift_resolved' if assessment['drift'] else 'verified_by_anchored_reconstruction') if good
                   else ('drift_unresolved' if assessment['drift'] else 'consistency_unverifiable'),
            internal_geometry_accepted=report['geometry_accepted'],candidate_l0=str(target))
        if good:
            # Keep instance masks/time metadata, not just their dynamic union.
            for name in ('instance_tracks.json','dynamic_mask_by_instance','dynamic_mask_by_entity',
                         'dense_complete.json','reconstruction_timings.json'):
                if (source_path/name).exists() and not (target/name).exists():
                    (target/name).symlink_to((source_path/name).resolve())
            out['l0']=str(target)
    out['seconds']=time.perf_counter()-tick
    write_json(receipt,out)
    return out


def run_initial_world(source,folder,*,settings,gpu_index):
    """Service process delegates GPU work to the existing Vista runtime."""
    import os,subprocess
    from atas_geometry.reconstruct.vista4d import effective_pi3_settings
    pixel_limit, first_window = effective_pi3_settings(
        settings.pi3_pixel_limit, settings.pi3_vo_chunk_size, settings.low_vram)
    legacy_receipt = Path(source) / 'dense_complete.json'
    if legacy_receipt.is_file():
        contract = json.loads(legacy_receipt.read_text()).get('reconstruction_contract', {})
        if all(key in contract for key in ('pi3_pixel_limit', 'pi3_vo_chunk_size', 'low_vram')):
            pixel_limit, first_window = effective_pi3_settings(
                int(contract['pi3_pixel_limit']), int(contract['pi3_vo_chunk_size']),
                str(contract['low_vram']).lower() == 'true')
    # Prefer the actual source settings when recorded, not today's settings.
    actual_path = Path(source) / 'reconstruction_effective.json'
    if actual_path.is_file():
        actual = json.loads(actual_path.read_text())
        pixel_limit = int(actual['pi3_pixel_limit'])
        first_window = int(actual['pi3_vo_chunk_size'])
    from atas_geometry import GEOMETRY_ROOT as root
    folder=Path(folder);folder.mkdir(parents=True,exist_ok=True)
    command=[str(settings.vista4d_python),'-m','atas_geometry.reconstruct.initial_world',str(source),str(folder),
        '--vista-repo',str(settings.vista4d_repo),'--model-id',settings.pi3_model_id,
        '--first-window',str(first_window),'--pixel-limit',str(pixel_limit)]
    from atas_geometry.service.device_mapping import gpu_selector
    env=os.environ.copy();env.update(CUDA_VISIBLE_DEVICES=gpu_selector(gpu_index),CUDA_DEVICE_ORDER='PCI_BUS_ID',
        OPENBLAS_NUM_THREADS='4',OMP_NUM_THREADS='4',PYTHONUNBUFFERED='1',
        PYTHONPATH=str(root))
    with (folder/'initial_world.log').open('a') as log:
        offset=log.tell()
        try: subprocess.run(command,cwd=root,env=env,stdout=log,stderr=subprocess.STDOUT,check=True)
        except subprocess.CalledProcessError:
            log.flush()
            from atas_geometry.service.resource_errors import log_resource_error
            error=log_resource_error(folder/'initial_world.log',physical_gpu_index=gpu_index,
                                     stage='initial_world_alignment',start_offset=offset)
            if error: raise error
            raise
    result=json.loads((folder/'initial_world.json').read_text())
    if result['status']=='rejected':
        raise ValueError(f'initial world rejected ({result.get("reason")}); raw L0 and candidate retained at {folder}')
    return Path(result['l0']),result


if __name__=='__main__':
    import argparse
    p=argparse.ArgumentParser();p.add_argument('source');p.add_argument('folder')
    p.add_argument('--vista-repo',required=True);p.add_argument('--model-id',required=True)
    p.add_argument('--first-window',type=int,default=48);p.add_argument('--pixel-limit',type=int,default=255000)
    args=p.parse_args();print(json.dumps(prepare_initial_world(args.source,args.folder,
        vista_repo=args.vista_repo,model_id=args.model_id,first_window=args.first_window,pixel_limit=args.pixel_limit)))