Skip to content

Photoreal Blender scenes from a one-paragraph brief, built by AI agents: image→3D props on fal, scripted layout, lighting, camera and grade.

Python
20.3 KB
Raw Blame
// Where stage events come from: the live fal app (through the Next.js proxy) or a recorded run in public/replay/.
import { fal } from "@fal-ai/client";
import {
  STAGE_PREFIX,
  type BuildRequest,
  type BuildResponse,
  type Plan,
  type PlanRequest,
  type PlanResponse,
  type StageEvent,
} from "./types";
import { assetKey, parseLogs } from "./run";
import { type BuilderEvent, type BuilderRequest, type BuilderResponse, type ShotsRequest, type ShotsResponse } from "./builder";

import { BUILDER_APP, PIPELINE_APP, requireApp } from "./apps";

export const APP_ID = PIPELINE_APP;

export interface Sink {
  onEvent: (ev: StageEvent, t: number) => void;
}

export interface BuilderSink {
  onEvent: (ev: BuilderEvent) => void;
}

export interface Source {
  readonly mode: "live" | "replay";
  /** Seconds since the current request was submitted (replay: virtual seconds at the replay speed). */
  clock(): number;
  uploadReference(file: File): Promise<string>;
  plan(req: PlanRequest, sink: Sink): Promise<PlanResponse>;
  build(req: BuildRequest, sink: Sink): Promise<BuildResponse>;
  builder(req: BuilderRequest, sink: BuilderSink): Promise<BuilderResponse>;
}

// ---------------------------------------------------------------- live

// The proxy can require a shared password (ANYSCENE_ACCESS_KEY on the server). The viewer types it once; it is kept
// in this browser's localStorage and sent as x-anyscene-key to our own proxy only (the proxy never forwards it to fal).
export const ACCESS_KEY_STORAGE = "anyscene.access-key";
export function getAccessKey(): string {
  try {
    return localStorage.getItem(ACCESS_KEY_STORAGE) || "";
  } catch {
    return "";
  }
}
export function setAccessKey(k: string) {
  try {
    if (k) localStorage.setItem(ACCESS_KEY_STORAGE, k);
    else localStorage.removeItem(ACCESS_KEY_STORAGE);
  } catch {
    /* private window: the key lasts for this page only */
  }
  sessionKey = k;
}
let sessionKey = "";

// Who is building: an email or username the viewer gives once, kept in localStorage and sent to our proxy as
// x-anyscene-user / x-anyscene-name. The proxy tags each build with it (`owner`, listed in /builds) and counts it
// against the per-person daily limit. Self-declared, behind the shared password: a label, not authentication.
export interface Who {
  name: string;
  email: string;
}
export const WHO_STORAGE = "anyscene.who";
let sessionWho: Who | null = null;
export function getWho(): Who | null {
  if (sessionWho) return sessionWho;
  try {
    const w = JSON.parse(localStorage.getItem(WHO_STORAGE) || "null");
    return w?.email ? { name: String(w.name || ""), email: String(w.email) } : null;
  } catch {
    return null;
  }
}
export function setWho(w: Who | null) {
  sessionWho = w;
  try {
    if (w) localStorage.setItem(WHO_STORAGE, JSON.stringify(w));
    else localStorage.removeItem(WHO_STORAGE);
  } catch {
    /* private window: kept for this page only */
  }
}
/** The owner tag a build carries: "Name <email>". */
export const ownerEmail = (owner?: string | null) => (owner?.match(/<([^>]+)>/)?.[1] ?? owner ?? "").toLowerCase();
export const ownerName = (owner?: string | null) => owner?.replace(/\s*<[^>]*>\s*$/, "").trim() || ownerEmail(owner).split("@")[0];

/** Our own headers for /api/* calls: the access key and who is asking. */
export function anysceneHeaders(): Record<string, string> {
  const h: Record<string, string> = {};
  const key = sessionKey || getAccessKey();
  if (key) h["x-anyscene-key"] = key;
  const who = getWho();
  if (who) {
    h["x-anyscene-user"] = who.email;
    h["x-anyscene-name"] = encodeURIComponent(who.name);
  }
  return h;
}

let falConfigured = false;
export function configureFal() {
  if (falConfigured) return;
  fal.config({
    proxyUrl: "/api/fal/proxy",
    requestMiddleware: async (req) => ({ ...req, headers: { ...(req.headers || {}), ...anysceneHeaders() } }),
  });
  falConfigured = true;
}

export class LiveSource implements Source {
  readonly mode = "live" as const;
  private t0 = performance.now();

  clock() {
    return (performance.now() - this.t0) / 1000;
  }

  async uploadReference(file: File) {
    configureFal();
    return fal.storage.upload(file);
  }

  private async call<O>(endpoint: string, input: object, onEvent: (ev: StageEvent) => void, pollInterval = 1000): Promise<O> {
    configureFal();
    this.t0 = performance.now();
    const seen = new Set<string>();
    const take = (logs: unknown) => {
      for (const ev of parseLogs(logs as { message?: string }[] | undefined, seen, STAGE_PREFIX)) onEvent(ev);
    };
    const res = await fal.subscribe(endpoint, {
      input,
      logs: true,
      pollInterval,
      onQueueUpdate: (update) => take("logs" in update ? update.logs : undefined),
    });
    // The last poll that sees COMPLETED usually carries no logs, so the closing events ("blockout done" with the
    // comparison, "done" with package_url) would be lost. Ask for the logs once more.
    try {
      const st = await fal.queue.status(endpoint, { requestId: res.requestId, logs: true });
      take("logs" in st ? st.logs : undefined);
    } catch {
      /* the response itself still has everything the UI needs */
    }
    return res.data as O;
  }

  plan(req: PlanRequest, sink: Sink) {
    return this.call<PlanResponse>(`${requireApp(APP_ID, "pipeline")}/plan`, req, (ev) => sink.onEvent(ev, this.clock()));
  }

  build(req: BuildRequest, sink: Sink) {
    return this.call<BuildResponse>(`${requireApp(APP_ID, "pipeline")}/build`, req, (ev) => sink.onEvent(ev, this.clock()));
  }

  /** The agent builder: one request, up to fal's 2 h cap. Logs grow with every tool call, so poll every 2 s. */
  builder(req: BuilderRequest, sink: BuilderSink) {
    return this.call<BuilderResponse>(`${requireApp(BUILDER_APP, "builder")}/ref2scene`, req, (ev) => sink.onEvent(ev as unknown as BuilderEvent), 2000);
  }

  /** Scene to images / video on a finished build: same event stream as a build (the shots agent reports blocking,
   *  farm progress and finals). */
  shots(req: ShotsRequest, sink: BuilderSink) {
    return this.call<ShotsResponse>(`${requireApp(BUILDER_APP, "builder")}/shots`, req, (ev) => sink.onEvent(ev as unknown as BuilderEvent), 2000);
  }

  /** Attach to a build that is already running (started elsewhere: the CLI client, another tab) by its fal request id.
   *  The queue keeps the request's whole log, so the page catches up from the start and then follows it live. */
  async watchBuilder(requestId: string, sink: BuilderSink): Promise<BuilderResponse> {
    configureFal();
    this.t0 = performance.now();
    const endpoint = `${requireApp(BUILDER_APP, "builder")}/ref2scene`;
    const seen = new Set<string>();
    const take = (logs: unknown) => {
      for (const ev of parseLogs(logs as { message?: string }[] | undefined, seen, STAGE_PREFIX))
        sink.onEvent(ev as unknown as BuilderEvent);
    };
    for (;;) {
      const st = await fal.queue.status(endpoint, { requestId, logs: true });
      take("logs" in st ? st.logs : undefined);
      if (st.status === "COMPLETED") break;
      await new Promise((r) => setTimeout(r, 3000));
    }
    const res = await fal.queue.result(endpoint, { requestId });
    return res.data as BuilderResponse;
  }
}

// ---------------------------------------------------------------- replay

export interface ReplayManifest {
  name: string;
  plan_events: boolean;
  plan: boolean;
  reference: string | null;
  floor_plan: string | null;
  blockout_reference: string | null;
  blockout_overview: string | null;
  comparison: string | null;
  build_events: boolean;
  build: boolean;
  stills: string[];
  panoramas?: string[];
  scene_glb?: string | null;
  kind?: "pipeline" | "builder";
  events?: boolean; // builder: events.jsonl
}

type Recorded = StageEvent & { t: number };

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

export class ReplaySource implements Source {
  readonly mode = "replay" as const;
  speed: number;
  paused = false;
  private vt = 0;
  private last = performance.now();
  private base: string;
  private planCache?: Plan;

  constructor(
    readonly manifest: ReplayManifest,
    speed = 1,
  ) {
    this.speed = speed;
    this.base = `/replay/${manifest.name}/`;
  }

  static async load(name: string, speed: number): Promise<ReplaySource> {
    let res = await fetch(`/replay/${encodeURIComponent(name)}/manifest.json`);
    if (!res.ok) res = await fetch(`/replay/demo/manifest.json`);
    if (!res.ok) throw new Error(`No replay named "${name}" in web/fixtures (run npm run fixtures).`);
    return new ReplaySource(await res.json(), speed);
  }

  url(file: string | null | undefined) {
    if (!file) return "";
    return /^(https?:|data:|blob:|\/)/.test(file) ? file : this.base + file;
  }

  get referenceUrl() {
    return this.url(this.manifest.reference);
  }

  clock() {
    this.tick();
    return this.vt;
  }

  private tick() {
    const now = performance.now();
    if (!this.paused) this.vt += ((now - this.last) / 1000) * this.speed;
    this.last = now;
  }

  private reset() {
    this.vt = 0;
    this.last = performance.now();
  }

  private async lines(file: string): Promise<Recorded[]> {
    const txt = await (await fetch(this.base + file)).text();
    return txt
      .split(/\r?\n/)
      .filter((l) => l.trim())
      .map((l) => JSON.parse(l));
  }

  /** Play recorded events on the virtual clock, starting the first one at ~0.6 s. */
  private async play(events: Recorded[], sink: Sink, remap: (ev: Recorded) => Recorded) {
    const t0 = (events[0]?.t ?? 0) - 0.6;
    this.reset();
    for (const ev of events) {
      const at = ev.t - t0;
      while (this.clock() < at) await sleep(Math.min(100, Math.max(16, ((at - this.vt) * 1000) / this.speed)));
      const { t: _t, ...rest } = remap(ev);
      sink.onEvent(rest as StageEvent, at);
    }
  }

  async uploadReference(file: File) {
    return URL.createObjectURL(file);
  }

  /** Builder replay: events.jsonl on its own server clock (t), build.json as the response. Recorded still URLs are
   *  swapped for local still_*.png copies when the fixture has them. */
  async builder(_req: BuilderRequest, sink: BuilderSink): Promise<BuilderResponse> {
    const events = (await this.lines("events.jsonl")) as unknown as BuilderEvent[];
    const res: BuilderResponse = this.manifest.build
      ? await (await fetch(this.base + "build.json")).json()
      : { build_id: "", stills: [], scene_blend_url: null, scene_glb_url: null, panorama_urls: [], package_url: null, critiques: [], missing: [], agent_cost_usd: null, minutes: 0 };
    const local = new Map<string, string>();
    (res.stills ?? []).forEach((u, i) => this.manifest.stills[i] && local.set(u, this.url(this.manifest.stills[i])));
    const fix = (u?: string | null) => (u ? (local.get(u) ?? this.url(u)) : (u ?? undefined));
    this.reset();
    const t0 = (events[0]?.t ?? 0) - 0.6;
    for (const ev of events) {
      const at = ev.t - t0;
      while (this.clock() < at) await sleep(Math.min(100, Math.max(16, ((at - this.vt) * 1000) / this.speed)));
      sink.onEvent({ ...ev, preview_url: fix(ev.preview_url) ?? undefined, package_url: fix(ev.package_url) ?? undefined });
    }
    if (!this.manifest.build) return { ...res, _partial: true } as BuilderResponse;
    return {
      ...res,
      stills: (res.stills ?? []).map((u) => fix(u)!),
      panorama_urls: (res.panorama_urls ?? []).map((u) => fix(u)!),
      scene_glb_url: fix(res.scene_glb_url) ?? null,
      scene_blend_url: fix(res.scene_blend_url) ?? null,
      package_url: fix(res.package_url) ?? null,
    };
  }

  async loadPlan(): Promise<Plan> {
    if (!this.planCache) this.planCache = await (await fetch(this.base + "plan.json")).json();
    return this.planCache!;
  }

  async plan(req: PlanRequest, sink: Sink): Promise<PlanResponse> {
    const plan = await this.loadPlan();
    const m = this.manifest;
    let recordedId: string | undefined;
    if (req.plan_id && req.edit) {
      // A recorded edit isn't available: play a short revision and return the same plan.
      const n = plan.objects.length;
      const inv = plan.objects.filter((o) => o.source === "invented").length;
      const fake: Recorded[] = [
        { t: 0, stage: "received", status: "running", message: "plan request", elapsed_s: 0 },
        { t: 0.2, stage: "planning", status: "running", message: "revising the plan", elapsed_s: 0.2 },
        { t: 24, stage: "planning", status: "done", message: `${n} objects (${inv} invented), ${plan.lights.length} lights, ${plan.room.width} x ${plan.room.depth} m ${plan.setting}`, elapsed_s: 24 },
        { t: 24.3, stage: "drawing_floor_plan", status: "running", message: "", elapsed_s: 24.3 },
        { t: 25, stage: "drawing_floor_plan", status: "done", message: "", elapsed_s: 25, preview_url: m.floor_plan ?? undefined },
        { t: 25, stage: "blockout", status: "running", message: "grey-box renders from the reference camera and an overview", elapsed_s: 25 },
        { t: 58, stage: "blockout", status: "done", message: "", elapsed_s: 58, preview_url: m.comparison ?? undefined },
        { t: 58, stage: "done", status: "done", message: `plan ${req.plan_id}`, elapsed_s: 58 },
      ];
      await this.play(fake, sink, (ev) => ({ ...ev, preview_url: ev.preview_url ? this.url(ev.preview_url) : undefined }));
    } else {
      const events = m.plan_events ? await this.lines("plan_events.jsonl") : [];
      recordedId = events.map((e) => /plan (p_\w+)/.exec(e.message || "")?.[1]).find(Boolean);
      await this.play(events, sink, (ev) => {
        let p = ev.preview_url;
        if (ev.stage === "drawing_floor_plan" && p && m.floor_plan) p = m.floor_plan;
        return { ...ev, preview_url: p ? this.url(p) : undefined };
      });
    }
    const done = /plan (p_\w+)/.exec(JSON.stringify(m)) ?? null;
    return {
      plan_id: req.plan_id || (done ? done[1] : "p_4f46c795ea"),
      plan,
      floor_plan_url: this.url(m.floor_plan),
      blockout_reference_url: this.url(m.blockout_reference),
      blockout_overview_url: this.url(m.blockout_overview),
      comparison_url: this.url(m.comparison),
      timings: {},
    };
  }

  async build(req: BuildRequest, sink: Sink): Promise<BuildResponse> {
    const m = this.manifest;
    const plan = await this.loadPlan();
    const recorded = m.build_events ? await this.lines("build_events.jsonl") : synthesizeBuild(plan, req.plan_id);
    const result: Partial<BuildResponse> = m.build ? await (await fetch(this.base + "build.json")).json() : {};
    // Stills recorded with the run are served locally; map the remote still URLs onto them in order.
    const remoteStills = result.stills ?? [];
    const localStills = m.stills.map((s) => this.url(s));
    const stillMap = new Map(remoteStills.map((u, i) => [u, localStills[i] ?? u]));
    const local = (u?: string) => (u ? (stillMap.get(u) ?? this.url(u)) : undefined);
    await this.play(withPlaceholder360(recorded, m.panoramas ?? [], m.scene_glb ?? null), sink, (ev) => ({
      ...ev,
      preview_url: local(ev.preview_url),
      panorama_url: ev.panorama_url ? this.url(ev.panorama_url) : undefined,
      scene_glb_url: ev.scene_glb_url ? this.url(ev.scene_glb_url) : undefined,
    }));
    const last = recorded.findLast?.((e) => e.package_url) ?? [...recorded].reverse().find((e) => e.package_url);
    return {
      scene_id: result.scene_id ?? "s_replay",
      package_url: result.package_url ?? last?.package_url ?? "",
      scene_blend_url: result.scene_blend_url ?? "",
      stills: remoteStills.length ? remoteStills.map((u) => local(u)!) : localStills,
      props: result.props ?? [],
      standins_left: result.standins_left ?? [],
      timings: result.timings ?? {},
    };
  }
}

// ---------------------------------------------------------------- placeholder 360

/** The endpoint doesn't emit panoramas or a scene glTF yet: when a recording has none but fixtures/ has the
 *  placeholders, add them where the endpoint will (rendering: one panorama per viewpoint; packaging: the scene). */
function withPlaceholder360(events: Recorded[], panoramas: string[], sceneGlb: string | null): Recorded[] {
  const out = [...events];
  const hasPano = events.some((e) => e.panorama_url);
  const hasGlb = events.some((e) => e.scene_glb_url);
  const lastRender = [...events].reverse().find((e) => e.stage === "rendering" && e.status === "running");
  if (!hasPano && panoramas.length && lastRender) {
    panoramas.forEach((p, i) => {
      const vp = p.replace(/^panorama_|\.\w+$/g, "");
      const label = vp === "ref" ? "reference camera" : vp === "mid" ? "room centre" : vp;
      out.push({ t: lastRender.t + 0.2 + i * 0.2, stage: "rendering", status: "running", message: `360 plate: ${label} (placeholder)`, elapsed_s: lastRender.elapsed_s, panorama_url: p, viewpoint: `${label} (placeholder)` });
    });
  }
  const pack = events.find((e) => e.stage === "packaging");
  if (!hasGlb && sceneGlb && pack) {
    out.push({ t: pack.t + 0.5, stage: "packaging", status: "running", message: "scene glTF exported (placeholder)", elapsed_s: pack.elapsed_s, scene_glb_url: sceneGlb });
  }
  return out.sort((a, b) => a.t - b.t);
}

// ---------------------------------------------------------------- synthesized build

// Used only when no recorded /build run is in fixtures/: one tile per unique generated asset (as props.py groups
// them), references at 30-60 s, meshes at 3-9 min, a failure and two stand-ins, like a real draft build.
export const SAMPLE_GLB = "https://threejs.org/examples/models/gltf/DamagedHelmet/glTF-Binary/DamagedHelmet.glb";

function synthesizeBuild(plan: Plan, planId: string): Recorded[] {
  let seed = 7;
  const rnd = () => ((seed = (seed * 16807) % 2147483647) / 2147483647);
  const assets = new Map<string, { id: string; label: string; n: number; small: boolean }>();
  for (const o of plan.objects) {
    if (!o.generate) continue;
    const k = assetKey(o);
    const a = assets.get(k) ?? { id: k, label: o.label, n: 0, small: (o.category === "prop" || o.category === "clutter") && Math.max(o.w, o.d, o.h) < 0.3 };
    a.n++;
    assets.set(k, a);
  }
  const list = [...assets.values()];
  const N = list.length;
  const ev: Recorded[] = [
    { t: 0, stage: "received", status: "running", message: `build ${planId} (draft)`, elapsed_s: 0 },
    { t: 1.2, stage: "prop_references", status: "running", message: `${N} unique props (${list.reduce((s, a) => s + a.n, 0)} placements)`, elapsed_s: 1.2, progress: 0 },
  ];
  const meshes: { t: number; a: (typeof list)[0]; fail: boolean; model: string }[] = [];
  list.forEach((a, i) => {
    const tr = 30 + rnd() * 30;
    ev.push({ t: tr, stage: "prop_references", status: "running", message: `${a.id} reference ready`, elapsed_s: tr, prop: { id: a.id, label: a.label, ref_url: null, status: "meshing" } });
    const model = a.small ? "meshy" : "tripo-h31";
    const tm = i === 3 || i === 9 ? 1e9 : tr + (a.small ? 300 + rnd() * 240 : 150 + rnd() * 240);
    meshes.push({ t: tm, a, fail: i === 6, model });
  });
  meshes.sort((x, y) => x.t - y.t);
  let done = 0;
  for (const mm of meshes) {
    if (mm.t > 480) break;
    done++;
    ev.push({
      t: mm.t, stage: "generating_meshes", status: "running", message: `${done} of ${N} props have meshes`, elapsed_s: mm.t, progress: done / N,
      prop: { id: mm.a.id, label: mm.a.label, ref_url: null, model: mm.fail ? null : mm.model, mesh_url: mm.fail ? null : SAMPLE_GLB, status: mm.fail ? "failed" : "done" },
    });
  }
  const left = N - done;
  ev.push({ t: 481, stage: "generating_meshes", status: "done", message: `${done} of ${N} props ready, ${left} still running`, elapsed_s: 481 });
  ev.push({ t: 481, stage: "layout", status: "running", message: "building the scene from the plan", elapsed_s: 481 });
  ev.push({ t: 540, stage: "layout", status: "done", message: `${plan.objects.length} objects placed, ${left} stand-ins`, elapsed_s: 540 });
  ev.push({ t: 540, stage: "dressing", status: "running", message: `waiting up to 240 s for ${left} late meshes`, elapsed_s: 540 });
  ev.push({ t: 780, stage: "dressing", status: "done", message: `swapped ${done} props, ${left} stand-ins kept`, elapsed_s: 780 });
  ev.push({ t: 781, stage: "rendering", status: "running", message: "stills from the reference camera and two open views", elapsed_s: 781 });
  ev.push({ t: 900, stage: "rendering", status: "done", message: "3 stills", elapsed_s: 900 });
  ev.push({ t: 901, stage: "packaging", status: "running", message: "", elapsed_s: 901 });
  ev.push({ t: 930, stage: "done", status: "done", message: "scene s_synthetic in 930 s", elapsed_s: 930 });
  return ev.sort((a, b) => a.t - b.t);
}