Headless 3D and VFX engine for agents. Research preview.
"""Measure reprojection and hand clearance. Evidence stays in the media directory.
Held-out desk-prop tracks are independent of the camera fit. Glass reflections
are NOT stationary table features. Contact predictions are compared with a
second solve from held-out tracks: this is model disagreement, not ground truth.
"""
import argparse
import json
from pathlib import Path
import cv2
import numpy as np
from scipy.optimize import least_squares
from track import project, timestamps, FOCAL
from scipy.spatial.transform import Rotation
def stats(values):
values = np.asarray(values)
return dict(mean_px=float(values.mean()), max_px=float(values.max()))
def verify(media):
cv2.setNumThreads(4)
data = np.load(media/'lock-tracks.npz')
tracks, world, poses = (data[k] for k in ['tracks', 'world', 'poses'])
fit, held = data['fit_indices'], data['validation']
scene = json.loads((media/'clean.json').read_text())
authored = {tuple(t['path']): t['keys'] for layer in scene['animation'] for t in layer['tracks']}
for path in [('nodes','camera','translation'), ('nodes','camera','rotation'), ('materials','camera_optics','inputs','lens')]:
assert np.allclose([k[0] for k in authored[path]], np.arange(len(poses))/25, atol=1e-9), 'camera key times differ from plate clock'
translation = np.array([k[1] for k in authored[('nodes','camera','translation')]])
rotation = Rotation.from_quat([k[1] for k in authored[('nodes','camera','rotation')]]).as_matrix()
assert np.allclose(translation, poses[:,3:], atol=1e-6), 'scene camera differs from measured camera'
assert np.allclose(rotation, Rotation.from_rotvec(poses[:,:3]).as_matrix(), atol=1e-6)
assert all(abs(k[1]-FOCAL*36/1920)<1e-6 for k in authored[('materials','camera_optics','inputs','lens')])
assert scene['render']['resolution']==[1920,1080] and scene['render']['fps']==25
assert scene['render']['motion_blur']['shutter']==0
assert all(scene['images'][name].get('time') is None for name in ['plate','person','hand']), 'unexpected image.time override'
layout = json.loads((media/'proxy-layout.json').read_text())
contacts = np.array(layout['contact_points'])
projected = np.array([project(p, world) for p in poses])
residual = projected-tracks
contact_uv = np.array([project(p, contacts) for p in poses])
edge_world, edge_observed, edge_slope = (data[k] for k in ['edge_world','edge_observed','edge_slope'])
edge_projected = np.array([project(p, edge_world) for p in poses])
edge_residual = (edge_projected[:,:,1]-edge_observed[:,:,1]-edge_slope[:,None]*(edge_projected[:,:,0]-edge_observed[:,:,0])) / np.sqrt(1+edge_slope[:,None]**2)
independent = []
for i, (p, xy) in enumerate(zip(poses, tracks)):
def independent_residual(v):
edge = project(v, edge_world[1:2])[0]
normal = edge[1]-edge_observed[i,1,1]-edge_slope[i]*(edge[0]-edge_observed[i,1,0])
return np.r_[(project(v,world[held])-xy[held]).ravel(), 5*normal]
check = least_squares(independent_residual,
p, loss='soft_l1', f_scale=.5, max_nfev=100)
independent.append(project(check.x, contacts))
disagreement = np.linalg.norm(np.array(independent)-contact_uv, axis=2)
report = dict(camera_fit=stats(np.linalg.norm(residual[:, fit], axis=2)),
held_out_features=stats(np.linalg.norm(residual[:, held], axis=2)),
held_out_interframe=stats(np.linalg.norm(np.diff(residual[:, held], axis=0), axis=2)),
contact_model_disagreement=stats(disagreement),
held_out_table_edge_normal=stats(np.abs(edge_residual[:,[1,4]])),
held_out_left_edge_normal=stats(np.abs(edge_residual[:,1])),
held_out_central_edge_normal=stats(np.abs(edge_residual[:,4])),
held_out_table_edge_second_half=stats(np.abs(edge_residual[100:,[1,4]])),
table_lip_height_m=float(data['edge_height']),
fit_tracks=len(fit), held_out_tracks=len(held), frames=len(poses),
subpixel_tabletop_lock_proven=False,
limitations=['The table edge measures normal slip only; glass reflections cannot measure tangential contact slip.',
'Contact model disagreement is an uncertainty estimate, not measured ground-truth slip.',
'Focal length, table height and scale are estimated; lens distortion and rolling shutter are unsolved.'])
old = media/'original-camera-solve.json'
if old.exists():
old = json.loads(old.read_text())
prediction = np.array([tracks[0]@np.array(r['matrix'])[:, :2].T+np.array(r['matrix'])[:, 2] for r in old])
report['original_fit_on_same_held_out_features'] = stats(np.linalg.norm(prediction[:, held]-tracks[:, held], axis=2))
report['timing'] = {}
for name in ['plate.mp4', 'person.mkv', 'hand.mkv']:
ts = timestamps(media/name)
report['timing'][name] = dict(frames=len(ts), max_error_s=float(np.max(np.abs(ts-np.arange(len(ts))/25))))
# Conservative projected hull of all artifact vertices against the tracked hand.
seed = next(n for n in scene['nodes'] if n['name']=='seed')
assert np.allclose(seed['scale'], layout['seed_scale'])
assert np.allclose(seed['translation'], np.array(layout['artifact'])*(1-layout['seed_scale']))
vertices = []
for node in seed['children']:
name = scene['geometry'][node['geometry']]['attributes']['POSITION']
buffer = scene['buffers'][name]
points = np.fromfile(media/buffer['file'], dtype='<f4', count=buffer['count']*3,
offset=buffer['offset']).reshape(-1, 3)
vertices.extend(points*np.array(seed['scale'])+np.array(seed['translation']))
vertices = np.array(vertices)
clearance, overlaps, rows = [], [], []
out = media/'lock-overlays'
out.mkdir(exist_ok=True)
cap = cv2.VideoCapture(str(media/'plate.mp4'))
for i, pose in enumerate(poses):
ok, frame = cap.read()
if not ok: raise RuntimeError(f'plate missing frame {i}')
hull = cv2.convexHull(project(pose, vertices).astype('float32')).round().astype('int32')
seed_mask = np.zeros(frame.shape[:2], 'uint8')
cv2.fillConvexPoly(seed_mask, hull, 1)
hand = cv2.imread(str(media/'hand-mattes'/f'{i:06d}.png'), cv2.IMREAD_UNCHANGED)>32767
distance = cv2.distanceTransform((~hand).astype('uint8'), cv2.DIST_L2, cv2.DIST_MASK_PRECISE)
clearance.append(float(distance[seed_mask>0].min()))
overlaps.append(int(np.count_nonzero(hand & (seed_mask>0))))
cv2.polylines(frame, [hull], True, (0,255,255), 1)
for xy in contact_uv[i]: cv2.drawMarker(frame, tuple(np.rint(xy).astype(int)), (255,255,0), cv2.MARKER_CROSS, 9, 1)
for xy in edge_projected[i]: cv2.drawMarker(frame, tuple(np.rint(xy).astype(int)), (255,0,255), cv2.MARKER_CROSS, 9, 1)
for j in held:
a, b = tuple(np.rint(tracks[i,j]).astype(int)), tuple(np.rint(projected[i,j]).astype(int))
cv2.circle(frame, a, 3, (0,255,0), 1)
cv2.drawMarker(frame, b, (0,0,255), cv2.MARKER_CROSS, 7, 1)
cv2.putText(frame, f'{i:03d} hand clearance {clearance[-1]:.1f}px', (700,1045), 0,.7,(255,255,255),2)
cv2.imwrite(str(out/f'{i:06d}.jpg'), frame[650:1080,600:1500])
rows.append(dict(frame=i, time=i/25, contact_uv=contact_uv[i].tolist(),
held_out_observed=tracks[i,held].tolist(), held_out_projected=projected[i,held].tolist(),
table_edge_observed=edge_observed[i].tolist(), table_edge_projected=edge_projected[i].tolist(),
table_edge_normal_error_px=edge_residual[i].tolist(),
contact_model_disagreement_px=disagreement[i].tolist(), hand_clearance_px=clearance[-1],
hand_overlap_pixels=overlaps[-1],artifact_bounds_px=[hull[:,0,:].min(axis=0).tolist(),hull[:,0,:].max(axis=0).tolist()]))
cap.release()
report['artifact_size_px'] = dict(min_height=min(r['artifact_bounds_px'][1][1]-r['artifact_bounds_px'][0][1] for r in rows), max_height=max(r['artifact_bounds_px'][1][1]-r['artifact_bounds_px'][0][1] for r in rows))
report['hand'] = dict(min_clearance_px=min(clearance), max_overlap_pixels=max(overlaps),
closest_frame=int(np.argmin(clearance)), method='conservative projected convex hull vs SAM hand at 50% coverage')
(media/'lock-projections.json').write_text(json.dumps(rows, indent=2)+'\n')
(media/'lock-report.json').write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps(report, indent=2))
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('media',type=Path)
verify(p.parse_args().media.resolve())