The VS Code of video editing, built for agents.
TypeScript
Refactor CLI and DAPI integration
konstantin-paulus905d6491412f
diff --git a/apps/cli/package.json b/apps/cli/package.jsonindex 76346cd6..20c2302e 100644--- a/apps/cli/package.json+++ b/apps/cli/package.json@@ -17,8 +17,6 @@},"dependencies": {"@babel/core": "^7.29.7",- "@trpc/client": "^11.18.0",- "@trpc/server": "^11.18.0","@babel/preset-typescript": "^7.27.1","babel-preset-solid": "^1.9.12","commander": "^14.0.3",diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 6554862f..0b1a5726 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -6,17 +6,36 @@ import { connect } from "node:net";import { randomBytes } from "node:crypto";import type { AddressInfo } from "node:net";import { WebSocketServer } from "ws";-import { createTRPCClient, TRPCClientError } from "@trpc/client";-import type { TRPCLink } from "@trpc/client";-import { observable } from "@trpc/server/observable";import { SOCKET_PATH } from "@diffusionstudio/dapi/socket";+import { decodeReply } from "./protocol";++import type { ToolInput, ToolName, ToolResult } from "@diffusionstudio/dapi";import type { CliHandshake, CliHandshakeReply, CliReply, CliRequest } from "./protocol";-import type { AppRouter } from "../../web/src/context/dapi";const DEFAULT_TIMEOUT_MS = 60000;export const GENERATE_TIMEOUT_MS = 600000;export const EXPORT_TIMEOUT_MS = 3600000;+export type CallOptions = { timeoutMs?: number };++/**+ * Calls one tool in the running app. Typed by the catalog: the input is what+ * the tool's schema accepts, the result what its handler returns (bytes come+ * back as bytes; the wire's base64 is decoded here).+ */+export async function call<N extends ToolName>(+ name: N,+ input: ToolInput<N>,+ options: CallOptions = {},+): Promise<ToolResult<N>> {+ return (await transport({ path: name, input }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS)) as ToolResult<N>;+}++/** Liveness: answered by the renderer's transport before any handler registers. */+export async function ping(): Promise<void> {+ await transport({ path: "ping", input: undefined }, DEFAULT_TIMEOUT_MS);+}+// Asks the app (via the unix socket) to have the renderer dial our WebSocket// server. Main replies once the connect info has been delivered, so a// rejection here means the app is down or the renderer never became ready.@@ -55,10 +74,12 @@ function requestConnection(handshake: CliHandshake, timeoutMs: number): Promise<});}+// Each call runs over its own short-lived WebSocket server that the renderer+// dials in to. No cancellation: the CLI process exits when the command settles.async function transport(request: CliRequest, timeoutMs: number): Promise<unknown> {const token = randomBytes(16).toString("hex");- // Frame batches and other base64 replies can exceed ws's 100 MiB default,- // so disable the payload cap; the server only lives for one request.+ // Frame batches can exceed ws's 100 MiB default, so disable the payload+ // cap; the server only lives for one request.const wss = new WebSocketServer({ host: "127.0.0.1", port: 0, maxPayload: 0 });try {@@ -83,15 +104,13 @@ async function transport(request: CliRequest, timeoutMs: number): Promise<unknow}ws.on("message", (raw) => {try {- const parsed = JSON.parse(raw.toString()) as CliReply;+ const parsed = decodeReply(raw.toString());settle(() => resolve(parsed));} catch (e) {settle(() => reject(e instanceof Error ? e : new Error(String(e))));}});- ws.on("close", () =>- settle(() => reject(new Error("App disconnected before replying"))),- );+ ws.on("close", () => settle(() => reject(new Error("App disconnected before replying"))));ws.on("error", (err) => settle(() => reject(err)));ws.send(JSON.stringify(request));});@@ -114,45 +133,22 @@ async function transport(request: CliRequest, timeoutMs: number): Promise<unknow}}-// Terminating link: each operation runs over its own short-lived WebSocket-// server that the renderer dials in to. Long-running procedures pass-// { context: { timeoutMs } } at the call site.-const cliLink: TRPCLink<AppRouter> =- () =>- ({ op }) =>- observable((observer) => {- const timeoutMs =- typeof op.context.timeoutMs === "number" ? op.context.timeoutMs : DEFAULT_TIMEOUT_MS;- transport({ path: op.path, input: op.input }, timeoutMs)- .then((data) => {- observer.next({ result: { data } });- observer.complete();- })- .catch((err) => observer.error(TRPCClientError.from(err as Error)));- // No cancellation: the CLI process exits when the command settles.- return () => {};- });--export const editor = createTRPCClient<AppRouter>({ links: [cliLink] });--// Transport failures surface as TRPCClientError wrapping the socket error;-// unwrap to reach errno codes like ENOENT/ECONNREFUSED.+/** The errno of a transport failure (ENOENT/ECONNREFUSED when the app is down), if any. */export function errnoCode(e: unknown): string | undefined {- if (!(e instanceof TRPCClientError)) return undefined;- return (e.cause as NodeJS.ErrnoException | undefined)?.code;+ return (e as NodeJS.ErrnoException | undefined)?.code;}// Bridges the cold-start gap after launching the app. Main only delivers the-// handshake once the renderer has finished loading, and `ping` is answered-// by the always-mounted app router, so a single round-trip proves the app is-// fully up. The retry loop only handles the brief window before the handshake+// handshake once the renderer has finished loading, and `ping` is answered by+// the renderer's transport, so a single round-trip proves the app is fully+// up. The retry loop only handles the brief window before the handshake// socket itself binds (ENOENT/ECONNREFUSED).export async function waitForCliSocket(timeoutMs = 30000): Promise<void> {const start = Date.now();let lastError: unknown = null;while (Date.now() - start < timeoutMs) {try {- await editor.ping.query();+ await ping();return;} catch (e) {lastError = e;diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex e8e74f82..b5160bb3 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -11,16 +11,16 @@ import { dirname, isAbsolute, join, resolve } from "node:path";import { Command } from "commander";import { version } from "../../../package.json";import { parseTime, TIME_FPS } from "@diffusionstudio/jsx";-import { editor, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, waitForCliSocket } from "./cli-client";+import { call, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, ping, waitForCliSocket } from "./cli-client";import { listLocalFonts } from "./fonts";import { buildIssueBody, createIssue } from "./report";import { fetchVideo } from "./ytdlp";-import { MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi";+import { ISSUE_LOG_TAIL, MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi";import type { FrameQuality, LogEntry, LogLevel, TimecodedImage } from "@diffusionstudio/dapi";// Long-running commands (renders, AI generation) override the default 60s.-const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } };-const EXPORT = { context: { timeoutMs: EXPORT_TIMEOUT_MS } };+const GENERATE = { timeoutMs: GENERATE_TIMEOUT_MS };+const EXPORT = { timeoutMs: EXPORT_TIMEOUT_MS };const APP_NAME = "Diffusion Studio";@@ -107,7 +107,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`);mkdirSync(dir, { recursive: true });try {- const images = await editor.media.frame.query({+ const images = await call("media_grab", {...target,times,count,@@ -117,6 +117,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {auto: opts.auto,combine: !opts.separate,perSheet,+ uncapped: opts.uncapped,});writeImages(images, dir);} catch (e) {@@ -143,7 +144,7 @@ async function mediaProbe(ref: string): Promise<void> {const target = resolveAssetRef(ref);const stop = startSpinner("Probing asset");try {- const result = await editor.media.probe.query(target);+ const result = await call("media_probe", target);stop();console.log(JSON.stringify(result));} catch (e) {@@ -156,7 +157,7 @@ async function mediaTranscribe(ref: string): Promise<void> {const target = resolveAssetRef(ref);const stop = startSpinner("Transcribing asset");try {- const result = await editor.media.transcribe.query(target, GENERATE);+ const result = await call("media_transcribe", target, GENERATE);stop();console.log(JSON.stringify(result));} catch (e) {@@ -178,7 +179,8 @@ async function mediaListen(ref: string, opts: MediaListenOptions): Promise<void>const target = resolveAssetRef(ref);const stop = startSpinner("Analyzing asset");try {- const result = await editor.media.listen.query(+ const result = await call(+ "media_listen",{ ...target, prompt: opts.prompt, start, end, stripVideo: !opts.keepVideo },GENERATE,);@@ -207,9 +209,9 @@ function parseTimeArg(value: string, flag: string, allowNegative = false): numbe// image with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the// filename too.function writeImages(images: TimecodedImage[], dir: string): void {- for (const { timecode, base64 } of images) {+ for (const { timecode, png } of images) {const path = join(dir, `${timecode}.png`);- writeFileSync(path, Buffer.from(base64, "base64"));+ writeFileSync(path, png);console.log(JSON.stringify({ timecode, path }));}}@@ -256,9 +258,9 @@ async function mediaFilmstrip(ref: string, opts: MediaPreviewOptions): Promise<vmkdirSync(dirname(resolve(path)), { recursive: true });const stop = startSpinner("Rendering filmstrip");try {- const { base64, ...rest } = await editor.media.filmstrip.query({ ...target, start, end, scale });+ const { png, ...rest } = await call("media_filmstrip", { ...target, start, end, scale });stop();- writeFileSync(path, Buffer.from(base64, "base64"));+ writeFileSync(path, png);console.log(JSON.stringify({ path, ...rest }));} catch (e) {stop();@@ -273,9 +275,9 @@ async function mediaWaveform(ref: string, opts: MediaPreviewOptions): Promise<vomkdirSync(dirname(resolve(path)), { recursive: true });const stop = startSpinner("Rendering waveform");try {- const { base64, ...rest } = await editor.media.waveform.query({ ...target, start, end, scale });+ const { png, ...rest } = await call("media_waveform", { ...target, start, end, scale });stop();- writeFileSync(path, Buffer.from(base64, "base64"));+ writeFileSync(path, png);console.log(JSON.stringify({ path, ...rest }));} catch (e) {stop();@@ -293,10 +295,7 @@ async function captureNode(id: string, opts: CaptureOptions): Promise<void> {const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`);mkdirSync(dir, { recursive: true });try {- const images = await editor.capture.query(- { id, frames, combine: !opts.separate, perSheet },- GENERATE,- );+ const images = await call("capture", { id, frames, combine: !opts.separate, perSheet }, GENERATE);writeImages(images, dir);} catch (e) {handleSocketError(e);@@ -310,7 +309,7 @@ async function exportScene(id: string, output: string | undefined): Promise<voidconst path = output !== undefined ? resolve(process.cwd(), output) : undefined;const stop = startSpinner("Exporting scene");try {- const result = await editor.export.mutate({ id, path }, EXPORT);+ const result = await call("export", { id, path }, EXPORT);stop();console.log(JSON.stringify(result));} catch (e) {@@ -321,7 +320,7 @@ async function exportScene(id: string, output: string | undefined): Promise<voidasync function checkNode(id: string): Promise<void> {try {- const result = await editor.check.query({ id });+ const result = await call("check", { id });console.log(JSON.stringify(result));// Linter convention: issues found is a different failure than "could not run".if (result.issues.some((issue) => issue.severity === "error")) process.exitCode = 1;@@ -348,10 +347,10 @@ async function openProject(path: string | undefined, opts: OpenOptions): Promise// A cold launch needs the renderer up before the app can answer; when// nothing was launched there is nothing to wait for, so fail fast.if (launched) await waitForCliSocket();- else await editor.ping.query();+ else await ping();if (path !== undefined) {- const result = await editor.open.mutate({ dir: resolve(path) });+ const result = await call("open", { dir: resolve(path) });console.log(JSON.stringify(result));}} catch (e) {@@ -361,7 +360,7 @@ async function openProject(path: string | undefined, opts: OpenOptions): Promiseasync function context(): Promise<void> {try {- const result = await editor.context.query();+ const result = await call("context", {});console.log(JSON.stringify(result));} catch (e) {handleSocketError(e);@@ -370,7 +369,7 @@ async function context(): Promise<void> {async function whoami(): Promise<void> {try {- const result = await editor.whoami.query();+ const result = await call("whoami", {});console.log(JSON.stringify(result));} catch (e) {handleSocketError(e);@@ -397,7 +396,7 @@ async function showLogs(opts: LogsOptions): Promise<void> {}try {- const entries = await editor.logs.query({ tail, level: opts.level as LogLevel | undefined });+ const entries = await call("logs", { tail, level: opts.level as LogLevel | undefined });for (const entry of entries) console.log(formatLogEntry(entry));} catch (e) {handleSocketError(e);@@ -427,14 +426,14 @@ async function appScreenshot(opts: ScreenshotOptions): Promise<void> {const dir = opts.output ?? tmpdir();mkdirSync(dir, { recursive: true });try {- const { base64, width, height } = await editor.screenshot.query();+ const { png, width, height } = await call("screenshot", {});const taken = new Date();let attempt = 1;let path = join(dir, screenshotFilename(taken, attempt));while (existsSync(path)) {path = join(dir, screenshotFilename(taken, ++attempt));}- writeFileSync(path, Buffer.from(base64, "base64"));+ writeFileSync(path, png);console.log(JSON.stringify({ path, width, height }));} catch (e) {handleSocketError(e);@@ -443,8 +442,6 @@ async function appScreenshot(opts: ScreenshotOptions): Promise<void> {type IssueOptions = { body?: string; command?: string[]; logs?: string };-const ISSUE_LOG_TAIL = 50;-async function reportIssue(title: string, opts: IssueOptions): Promise<void> {const summary = title.trim();if (!summary) {@@ -468,7 +465,7 @@ async function reportIssue(title: string, opts: IssueOptions): Promise<void> {let appStatus = "not checked";if (tail > 0) {try {- logs = (await editor.logs.query({ tail })).map(formatLogEntry);+ logs = (await call("logs", { tail })).map(formatLogEntry);appStatus = "running";} catch (e) {const code = errnoCode(e);@@ -525,7 +522,7 @@ async function listModels(type: string | undefined): Promise<void> {process.exit(1);}try {- const models = await editor.models.query({ type: type as "image" | "video" | "audio" | undefined });+ const models = await call("models", { type: type as "image" | "video" | "audio" | undefined });for (const model of models) console.log(JSON.stringify(model));} catch (e) {handleSocketError(e);@@ -534,7 +531,7 @@ async function listModels(type: string | undefined): Promise<void> {async function listVoices(): Promise<void> {try {- const voices = await editor.voices.query();+ const voices = await call("voices", {});for (const voice of voices) console.log(JSON.stringify(voice));} catch (e) {handleSocketError(e);diff --git a/apps/cli/src/protocol.ts b/apps/cli/src/protocol.tsindex 9119d703..eda36e18 100644--- a/apps/cli/src/protocol.ts+++ b/apps/cli/src/protocol.ts@@ -33,3 +33,41 @@ export type CliRequest = {export type CliReply =| { ok: true; data: unknown }| { ok: false; error: string };++// Replies carry bytes (PNGs) inside JSON. A Uint8Array is written as+// `{ $bytes: <base64> }` and read back as a Uint8Array, so handlers and the+// CLI both see bytes and only this seam knows about base64. Chunked so a+// 100 MiB frame batch never builds a single giant argument list.+const BYTES_KEY = "$bytes";+const CHUNK = 0x8000;++export function encodeReply(reply: CliReply): string {+ return JSON.stringify(reply, (_key, value) =>+ value instanceof Uint8Array ? { [BYTES_KEY]: bytesToBase64(value) } : value,+ );+}++export function decodeReply(text: string): CliReply {+ return JSON.parse(text, (_key, value) =>+ isBytesEnvelope(value) ? base64ToBytes(value[BYTES_KEY]) : value,+ ) as CliReply;+}++function isBytesEnvelope(value: unknown): value is { [BYTES_KEY]: string } {+ return typeof value === "object" && value !== null && typeof (value as Record<string, unknown>)[BYTES_KEY] === "string";+}++function bytesToBase64(bytes: Uint8Array): string {+ let binary = "";+ for (let i = 0; i < bytes.length; i += CHUNK) {+ binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));+ }+ return btoa(binary);+}++function base64ToBytes(base64: string): Uint8Array {+ const binary = atob(base64);+ const bytes = new Uint8Array(binary.length);+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);+ return bytes;+}diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.jsonindex c3835eb7..ac304146 100644--- a/apps/cli/tsconfig.json+++ b/apps/cli/tsconfig.json@@ -2,40 +2,9 @@"extends": "../../tsconfig.base.json","compilerOptions": {"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",- // The AppRouter type import in cli-client.ts pulls renderer sources into- // this program, so the web app's compiler settings (DOM, solid JSX, vite- // types, aliases) apply here on top of the node ones.- "useDefineForClassFields": true,- "erasableSyntaxOnly": false,- "lib": [- "es2024",- "DOM",- "DOM.Iterable"- ],- "types": [- "node",- "vite-plugin-solid-svg/types-component-solid",- "vite/client",- "@types/audioworklet",- "@types/wicg-file-system-access",- "@webgpu/types"- ],- "resolveJsonModule": true,- "allowImportingTsExtensions": true,- "jsx": "preserve",- "jsxImportSource": "solid-js",- "baseUrl": "./",- "paths": {- "@/*": [- "../web/src/*"- ],- "@desktop/*": [- "../desktop/src/*"- ]- }+ "lib": ["es2024"],+ "types": ["node"],+ "resolveJsonModule": true},- "include": [- "src",- "../web/src/vite-env.d.ts"- ]+ "include": ["src"]}diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.tsindex ad16a026..bd79dd7c 100644--- a/apps/desktop/src/main.ts+++ b/apps/desktop/src/main.ts@@ -275,7 +275,9 @@ if (app.requestSingleInstanceLock()) {if (!mainWindow || mainWindow.isDestroyed()) throw new Error("No main window");const image = await mainWindow.webContents.capturePage(undefined, { stayHidden: true });const { width, height } = image.getSize();- return { base64: image.toPNG().toString("base64"), width, height };+ const png = image.toPNG();+ // A plain Uint8Array over the PNG, so the renderer sees bytes and not a Buffer.+ return { png: new Uint8Array(png.buffer, png.byteOffset, png.byteLength), width, height };});mainBridge.handle(MAIN_CHANNELS.HEADLESS_GET_MODE, () => isHeadless());mainBridge.handle(MAIN_CHANNELS.LOGS_GET, () => logBuffer);diff --git a/apps/web/src/app.tsx b/apps/web/src/app.tsxindex bb490a29..39361eb9 100644--- a/apps/web/src/app.tsx+++ b/apps/web/src/app.tsx@@ -10,7 +10,7 @@ import { AppContextMenu } from "@/components/app-context-menu";import { AuthProvider, useAuth } from '@/context/auth';import { PersistRoute } from '@/lib/persist-route';-import { EditorApi } from '@/context/dapi';+import { EditorApi } from '@/dapi';import { UpgradeDialog } from '@/components/upgrade-dialog';import { PurchaseSuccess } from '@/components/purchase-success';import { ScreenTooSmall } from '@/components/screen-too-small';diff --git a/apps/web/src/components/canvas/desktop-app-banner.tsx b/apps/web/src/components/canvas/desktop-app-banner.tsxindex c1637fc1..5db091c6 100644--- a/apps/web/src/components/canvas/desktop-app-banner.tsx+++ b/apps/web/src/components/canvas/desktop-app-banner.tsx@@ -5,7 +5,7 @@import { Show } from "solid-js";import { Button } from "@/components/ui/button";import { Icon } from "@/components/ui/icon";-import { useEditorApi } from "@/context/dapi";+import { useEditorApi } from "@/dapi";import { createStoredSignal } from "@/lib/store";import { downloadDesktopApp } from "@/lib/desktop-app";import { track } from "@/lib/analytics";diff --git a/apps/web/src/components/sidebar-left/project-menu/index.tsx b/apps/web/src/components/sidebar-left/project-menu/index.tsxindex 2d2735a5..d9838de6 100644--- a/apps/web/src/components/sidebar-left/project-menu/index.tsx+++ b/apps/web/src/components/sidebar-left/project-menu/index.tsx@@ -19,7 +19,7 @@ import {import { useNavigate } from "@solidjs/router";import { Show, onCleanup, onMount } from "solid-js";import { isInputTarget } from "@/utils";-import { useEditorApi } from "@/context/dapi";+import { useEditorApi } from "@/dapi";import { downloadDesktopApp } from "@/lib/desktop-app";import { FileMenu } from "./file-menu";import { EditMenu } from "./edit-menu";diff --git a/apps/web/src/components/sidebar-left/sidebar-left.tsx b/apps/web/src/components/sidebar-left/sidebar-left.tsxindex a179fce4..b71c9db2 100644--- a/apps/web/src/components/sidebar-left/sidebar-left.tsx+++ b/apps/web/src/components/sidebar-left/sidebar-left.tsx@@ -4,7 +4,7 @@import { Assets } from "./assets";import { useLayout } from "@/context/layout";-import { useEditorApi } from "@/context/dapi";+import { useEditorApi } from "@/dapi";import { createSignal, Show } from "solid-js";import { toast } from "somoto";import { Button } from "../ui/button";diff --git a/apps/web/src/context/dapi/api.tsx b/apps/web/src/context/dapi/api.tsxdeleted file mode 100644index 1619cf04..00000000--- a/apps/web/src/context/dapi/api.tsx+++ /dev/null@@ -1,145 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { createEffect, createContext, useContext, onCleanup } from "solid-js";-import { useNavigate } from "@solidjs/router";-import { useWorld } from '@diffusionstudio/koota-solid';-import { Project } from '@diffusionstudio/runtime';-import { useProject } from '@/context/project';-import { useAuth } from '@/context/auth';-import { useEngineContext } from '@/engine';-import { t, q, q0, m } from "@/lib/cli-rpc";-import { editorSession, requireEditorSession, setEditorSession } from "./session";-import { handleContextGet } from "./context";-import { createAssetResolver, handleMediaProbe, handleMediaFrame, handleMediaTranscribe, handleMediaFilmstrip, handleMediaWaveform, handleMediaListen } from "./media";-import { handleCapture } from "./capture";-import { handleCheck } from "./check";-import { handleExport } from "./export";-import { handleLogs } from "./logs";-import { handleModels } from "./models";-import { handleVoices } from "./voices";-import { cliBridge } from '@/lib/ipc';-import { createRouterCaller } from '@/lib/cli-rpc';-import { openProjectFolder } from '@/projects';-import { projectRoute } from '@/hooks/use-project-route';-import { assert } from "@/utils/common";-import { handleWindowScreenshot } from "./window";-import { useFullscreenState } from "@/hooks/use-fullscreen-state";--import type { JSX, Accessor } from 'solid-js';-import type { Navigator } from '@solidjs/router';-import type { User } from '@supabase/supabase-js';--type EditorApiProviderProps = {- children: JSX.Element;-};--type EditorApiContextValue = {- isFullscreen: Accessor<boolean>;- isDesktop: boolean;-};--const EditorApiContext = createContext<EditorApiContextValue>();--/**- * The one CLI router, registered for as long as the app runs. Every endpoint- * is reachable whether or not a project is open; the ones that need one read- * the session slot (see ./session) per request and fail with a clear error —- * or, for `context`, report that nothing is open. Renders nothing; must sit- * inside the router tree for `useNavigate` and inside the auth provider.- */-export function EditorApi() {- const navigate = useNavigate();- const auth = useAuth();-- const requireAuth = <I, O>(fn: (data: I) => Promise<O>) => (data: I) => {- assert(auth.isAuthenticated(), "Sign in required: AI generation needs a Diffusion Studio account.");- return fn(data);- };-- const getUser = () => {- const user = auth.user();- assert(user, "User not found");- return user;- };-- const router = createAppRouter({ navigate, getUser, requireAuth });- onCleanup(cliBridge.register(createRouterCaller(router)));- return null;-}--/**- * Publishes the editor session for the CLI router while the project is open,- * and provides the editor UI's own view of the app shell (fullscreen state,- * desktop-ness). Mounted per project page.- */-export function EditorApiProvider(props: EditorApiProviderProps) {- const project = useProject();- const isFullscreen = useFullscreenState();- const world = useWorld();- const engine = useEngineContext();-- createEffect(() => {- if (!window.desktop || project.id() !== world.get(Project)?.id) return;-- setEditorSession({ world, project, engine });- onCleanup(() => setEditorSession(null));- });-- return (- <EditorApiContext.Provider- value={{- isFullscreen,- isDesktop: !!window.desktop,- }}- >- {props.children}- </EditorApiContext.Provider>- );-}---type AppRouterDeps = {- navigate: Navigator;- getUser: () => User;- requireAuth: <I, O>(fn: (data: I) => Promise<O>) => (data: I) => Promise<O>;-};--function createAppRouter({ navigate, getUser, requireAuth }: AppRouterDeps) {- const resolveAsset = createAssetResolver(editorSession);-- return t.router({- ping: t.procedure.query(() => {}),- open: m(async ({ dir }: { dir: string }) => {- const project = await openProjectFolder(dir);- navigate(projectRoute(project.id || project.name));- return { id: project.id, name: project.displayName, dir: project.dir };- }),- whoami: t.procedure.query(() => getUser()),- context: q0(handleContextGet(editorSession)),- capture: q(handleCapture(requireEditorSession)),- check: q(handleCheck(requireEditorSession)),- export: m(handleExport(requireEditorSession)),- models: q(handleModels()),- logs: q(handleLogs()),- screenshot: q0(handleWindowScreenshot()),- voices: q0(handleVoices()),- media: t.router({- probe: q(handleMediaProbe(resolveAsset)),- frame: q(handleMediaFrame(resolveAsset)),- transcribe: q(handleMediaTranscribe(resolveAsset)),- filmstrip: q(handleMediaFilmstrip(resolveAsset)),- waveform: q(handleMediaWaveform(resolveAsset)),- listen: q(requireAuth(handleMediaListen(resolveAsset, editorSession))),- }),- });-}--export type AppRouter = ReturnType<typeof createAppRouter>;--export function useEditorApi() {- const ctx = useContext(EditorApiContext);- assert(ctx, "useEditorApi must be used within EditorApiProvider");- return ctx;-}diff --git a/apps/web/src/context/dapi/capture.ts b/apps/web/src/context/dapi/capture.tsdeleted file mode 100644index 375e7bae..00000000--- a/apps/web/src/context/dapi/capture.ts+++ /dev/null@@ -1,102 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import {- composeSheet,- createImageEncoder,- decodePngBase64,- planSheet,- planSheetSizes,- sheetTimecode,-} from "@diffusionstudio/encoder";--import { getParentNode, isScene, Source } from "@diffusionstudio/runtime";--import { createCapture } from "@/engine/capture";-import { resolveNode } from "./nodes";--import type { CaptureRequest, CaptureResult, TimecodedImage } from "@diffusionstudio/dapi";-import type { EditorSession } from "./session";--// Ceiling on the height a sheet cell renders a node at.-const SHEET_CAPTURE_HEIGHT = 1080;--export function handleCapture(session: () => EditorSession) {- return async ({ id, frames, combine = true, perSheet }: CaptureRequest): Promise<CaptureResult> => {- const { world, project } = session();- const node = resolveNode(world, id);-- // Scenes only: a capture's promise is that its frames are the frames an- // export encodes, and a scene is the unit an export renders — framing an- // arbitrary node would need its bounds measured across the requested- // positions first, and that pre-roll runs the project's code ahead of the- // frames being drawn, which is exactly what an export never does.- if (!isScene(node)) {- let scene = getParentNode(node);- while (scene !== null && !isScene(scene)) scene = getParentNode(scene);- const stamp = scene?.get(Source)?.value;- throw new Error(- stamp- ? `"${id}" is not a scene — capture renders what an export renders. Capture its scene "${stamp}" instead.`- : `"${id}" is not a scene — capture renders what an export renders, so it takes a scene id.`,- );- }-- // `undefined` means the export's first frame (the workarea's start).- let shots = frames;- if (shots === undefined || shots.length === 0) {- shots = [0];- }-- // The project re-rendered into a world of its own, reduced to this scene:- // the same arrangement an export runs against, and the encoder's to draw.- const capture = await createCapture(world, node, { dir: project.dir() });-- try {- const encoder = await createImageEncoder(capture.world, {- frames: shots,- resolution: 720,- });-- // Sheets render at their cell size instead of the flat 720p: with a few- // frames that is sharper than a standalone capture, never coarser. A- // scene is drawn, not decoded, so a small one is worth rendering past- // its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens.- const aspect = encoder.bounds.width / encoder.bounds.height;- const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT);- const sizes = combine ? planSheetSizes(shots.length, perSheet) : [];- const plans = sizes.map((n) => planSheet(n, { width: height * aspect, height }));- if (combine) {- encoder.resize(Math.max(...plans.map((plan) => plan.cellHeight)));- }-- const result = await encoder.render();-- if (result.type === "canceled") throw new Error("Capture canceled");- if (result.type === "error") throw result.error;-- if (!combine) return result.data;-- const sheets: TimecodedImage[] = [];- let offset = 0;- for (const [sheet, size] of sizes.entries()) {- const group = result.data.slice(offset, offset + size);- const cells = group.map(({ timecode }, k) => ({ at: shots[offset + k], timecode }));- offset += size;- const images = await Promise.all(group.map((image) => decodePngBase64(image.base64)));- sheets.push({- timecode: sheetTimecode(cells),- base64: await composeSheet(- images.map((image, k) => ({ image, label: group[k].timecode })),- plans[sheet],- ),- });- for (const image of images) image.close();- }- return sheets;- } finally {- capture.dispose();- }- };-}diff --git a/apps/web/src/context/dapi/context.ts b/apps/web/src/context/dapi/context.tsdeleted file mode 100644index 1f6a7bb8..00000000--- a/apps/web/src/context/dapi/context.ts+++ /dev/null@@ -1,99 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime";--import { getProjectsRoot } from "@/projects";--import type { Accessor } from "solid-js";-import type { Entity, World } from "koota";-import type { EditorSession } from "./session";--/**- * What `dapi context` reports: what the project's source cannot say. The JSX is- * the composition — its scenes, what is selected, which scene is active, the- * work area are all in the file, and a caller that wants them reads it. What is- * left over is which folder projects live under, which project folder the app- * has open, where its playhead sits, which font families are actually- * registered in the world drawing it. With no project open only the root is- * left to report, and the report says so.- */-export function handleContextGet(session: Accessor<EditorSession | null>) {- return async () => {- const rootDir = await getProjectsRoot();-- const open = session();- if (!open) return { rootDir, projectDir: null };-- const { world, project } = open;- const frameRate = world.get(FrameRate)?.value || 30;- const active = getActiveEntity(world);-- return {- rootDir,- projectDir: project.dir(),- // Seconds, the unit the source places clips in; null when no scene is- // active, which is when there is no playhead to report.- currentTime: active ? (active.get(Computed)?.localTime ?? 0) / frameRate : null,- // What text can be drawn with right now: registered in the world, not- // merely named in the source. The editor default is always among them.- fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])],- generations: collectGenerations(world),- };- }-}--type GenerationRow = {- /** The element's source stamp, `<file>:<key or position>`; null for an- * entity no element produced. */- element: string | null;- name: string | null;- state: "generating" | "failed" | "done";- /** What the generation failed with, on `failed` rows. */- error?: string;- /** The library path the generation landed as, on `done` rows — ready for- * `dapi media probe` and its siblings. */- asset?: string;-};--function collectGenerations(world: World): GenerationRow[] {- const library = world.get(Library);- const rows: GenerationRow[] = [];- const seen = new Set<string>();-- const push = (row: GenerationRow) => {- const key = JSON.stringify(row);- if (seen.has(key)) return;- seen.add(key);- rows.push(row);- };-- for (const entity of world.query(Generating)) {- push({ ...describeElement(entity), state: "generating" });- }-- for (const entity of world.query(SourceError)) {- const failure = entity.get(SourceError)!;- if (!failure.generated) continue;- push({ ...describeElement(entity), state: "failed", error: failure.value });- }-- for (const entity of world.query(AssetId)) {- // A stale binding — the element is off generating a new answer or failed- // getting one — is not a generation that is done.- if (entity.has(Generating) || entity.has(PendingSource) || entity.has(SourceError)) continue;- const asset = library?.get(entity.get(AssetId)!.value);- if (!asset?.generation) continue;- push({ ...describeElement(entity), state: "done", asset: asset.path });- }-- return rows;-}--function describeElement(entity: Entity): Pick<GenerationRow, "element" | "name"> {- return {- element: entity.get(Source)?.value || null,- name: entity.get(Name)?.value || null,- };-}diff --git a/apps/web/src/context/dapi/export.ts b/apps/web/src/context/dapi/export.tsdeleted file mode 100644index c4e96e85..00000000--- a/apps/web/src/context/dapi/export.ts+++ /dev/null@@ -1,141 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { canEncodeVideo } from "mediabunny";-import { computeOutputSize } from "@diffusionstudio/encoder";-import { Computed, FrameRate, getParentNode, isScene, Source, Workarea } from "@diffusionstudio/runtime";--import { renderOverlay, renderScene } from "@/context/render";-import { ElectronWritableFileHandle } from "@/lib/electron-file-writable";-import { ProjectConfig as ProjectConfigTrait } from "@/engine/traits";-import { sceneConfigKey } from "@/engine/project-config";-import {- getDefaultExportTemplate,- VIDEO_FORMAT_OPTIONS,-} from "@/components/sidebar-right/inspector/export-templates";-import { mainBridge } from "@/lib/ipc";-import { MAIN_CHANNELS } from "@desktop/main-channels";-import { resolveNode } from "./nodes";--import type { ExportRequest, ExportResult, ExportSettings } from "@diffusionstudio/dapi";-import type { ContainerFormat, ExportConfig } from "@/engine/project-config";-import type { EditorSession } from "./session";--/**- * The container the export writes: the output extension when a path was- * given (ffmpeg's rule — the file must be what its name says), the config's- * format otherwise, mp4 when neither says. A given path settles it entirely,- * so a bad config format only fails an export that would actually use it.- */-function resolveFormat(path: string | undefined, config: ExportConfig): ContainerFormat {- const supported = VIDEO_FORMAT_OPTIONS.map((format) => `.${format}`).join(", ");- if (path !== undefined) {- const extension = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase();- if (!extension || !VIDEO_FORMAT_OPTIONS.includes(extension as ContainerFormat)) {- throw new Error(`The output path must end in a container extension: ${supported}.`);- }- return extension as ContainerFormat;- }- const format = config.format ?? "mp4";- if (!VIDEO_FORMAT_OPTIONS.includes(format)) {- throw new Error(`The export entry names an unknown format "${format}" — use one of ${supported.replaceAll(".", "")}.`);- }- return format;-}--export function handleExport(session: () => EditorSession) {- return async ({ id, path }: ExportRequest): Promise<ExportResult> => {- const { world, project, engine } = session();-- const scene = resolveNode(world, id);- if (!isScene(scene)) {- let parent = getParentNode(scene);- while (parent !== null && !isScene(parent)) parent = getParentNode(parent);- const stamp = parent?.get(Source)?.value;- throw new Error(- stamp- ? `"${id}" is not a scene — a scene is the unit an export renders. Export its scene "${stamp}" instead.`- : `"${id}" is not a scene — a scene is the unit an export renders, so export takes a scene id.`,- );- }-- // The scene's entry in the project's package.json (`diffusion.export.<id>`)- // — the same one the app's export panel writes — so a CLI export- // reproduces the in-app one; a scene without an entry uses the default- // template, the way ⌘E does. `template` is only the preset's label.- const base = world.get(ProjectConfigTrait)?.exportOf(scene) ?? getDefaultExportTemplate();- const settings: ExportConfig = { format: base.format, video: base.video, audio: base.audio };-- const format = resolveFormat(path, settings);- const key = sceneConfigKey(scene) ?? id;- const target = path ?? `${project.dir()}/exports/${key.replace(/[^\w.-]+/g, "-")}.${format}`;-- // Fail before the render machinery spins up: an unencodable configuration- // is known right away (same precheck the UI export runs).- const videoEnabled = format !== "ogg" && settings.video?.enabled !== false;- const computed = scene.get(Computed);- const { width, height } = computeOutputSize(- computed?.width || 1920,- computed?.height || 1080,- settings.video?.resolution ?? 1080,- );- if (videoEnabled) {- const codec = settings.video?.codec ?? "avc";- const encodable = await canEncodeVideo(codec, {- width,- height,- bitrate: settings.video?.bitrate ?? 10e6,- });- if (!encodable) {- throw new Error(- `Cannot encode ${codec.toUpperCase()} at ${width}×${height}. ` +- "Set a lower resolution, a lower bitrate, or another codec in the scene's export entry.",- );- }- }-- const workarea = scene.get(Workarea);- const frames = workarea ? workarea.end - workarea.start : computed?.duration ?? 0;- const duration = frames / (world.get(FrameRate)?.value || 30);-- // renderScene owns the one render slot (it stops the live engine and- // raises the progress overlay), so refuse a second export rather than- // interleave. No await sits between this check and renderScene claiming- // the overlay, so two racing requests cannot both pass.- if (renderOverlay()) {- throw new Error("An export is already running — wait for it to finish, or cancel it in the app.");- }-- const handle = new ElectronWritableFileHandle(target);- try {- const result = await renderScene(engine, {- scene,- target: handle,- config: { ...settings, format },- dir: project.dir(),- });- if (result.type === "canceled") throw new Error("Export canceled in the app");- if (result.type === "error") throw result.error;- } catch (error) {- // Close the fd and drop the partial file; a failed export leaves nothing.- await handle.dispose().catch(() => {});- throw error;- }-- const stat = await mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_STAT, {- dir: project.dir(),- source: target,- });-- const config: ExportSettings = { format, video: settings.video, audio: settings.audio };- return {- path: target,- width: videoEnabled ? width : 0,- height: videoEnabled ? height : 0,- duration,- size: stat?.size ?? 0,- config,- };- };-}diff --git a/apps/web/src/context/dapi/logs.ts b/apps/web/src/context/dapi/logs.tsdeleted file mode 100644index e45d2389..00000000--- a/apps/web/src/context/dapi/logs.ts+++ /dev/null@@ -1,21 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { mainBridge } from "@/lib/ipc";-import { MAIN_CHANNELS } from "@desktop/main-channels";-import type { LogsRequest, LogLevel } from "@diffusionstudio/dapi";--const LEVEL_RANK: Record<LogLevel, number> = { debug: 0, info: 1, warning: 2, error: 3 };--export function handleLogs() {- return async (req: LogsRequest) => {- let entries = await mainBridge.call(MAIN_CHANNELS.LOGS_GET, undefined);- if (req.level !== undefined) {- const min = LEVEL_RANK[req.level];- entries = entries.filter((e) => LEVEL_RANK[e.level] >= min);- }- if (req.tail !== undefined) entries = entries.slice(-req.tail);- return entries;- };-}diff --git a/apps/web/src/context/dapi/media.ts b/apps/web/src/context/dapi/media.tsdeleted file mode 100644index 09c643f9..00000000--- a/apps/web/src/context/dapi/media.ts+++ /dev/null@@ -1,361 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';-import { pickInformativeTimes } from './frame-triage';-import { trpc } from '@/lib/trpc';-import { uploadBlob } from '@/lib/uploads';-import { composeSheet, planSheet, planSheetSizes, sheetTimecode } from '@diffusionstudio/encoder';-import { assert } from '@/utils';-import { startResumableSession, uploadResumableStream } from '@/lib/uploads';-import { filmstripAsset, formatTimecode, getAssetFile, getLibrary, Project, transcodeForAnalysis, transcodeForTranscription, waveformAsset } from '@diffusionstudio/runtime';-import { AssetLibrary, assetName, isAbsoluteSource, isUrlSource } from '@diffusionstudio/assets';-import { createProjectFS } from '@/projects/fs';--import type { Accessor } from 'solid-js';-import type { Asset } from '@diffusionstudio/assets';-import type { EditorSession } from './session';-import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/dapi";--type ResolveAsset = (path: string) => Promise<Asset>;--/**- * Resolves a command target by path. With a project open, its library answers:- * library paths look assets up, absolute paths and URLs are described in place- * without being added (transient assets). With none open, a throwaway library- * over a project-less FS describes absolute paths and URLs the same way — a- * fresh one per request, so nothing is remembered between calls.- */-export function createAssetResolver(session: Accessor<EditorSession | null>): ResolveAsset {- return (path) => {- const world = session()?.world;- if (world) return getLibrary(world).resolve(path);- assert(- isAbsoluteSource(path) || isUrlSource(path),- `Could not resolve "${path}": with no project open only absolute paths and URLs resolve — run \`dapi open <dir>\` to use library paths.`,- );- return new AssetLibrary(createProjectFS("")).resolve(path);- };-}--const PROBE_SAMPLE_PACKETS = 200;--export function handleMediaProbe(resolve: ResolveAsset) {- return async (req: MediaProbeRequest): Promise<unknown> => {- const asset = await resolve(req.path);-- const blob = await getAssetFile(asset);- const base = {- id: asset.id,- name: assetName(asset),- path: asset.path,- type: asset.type,- mimeType: asset.mimeType,- size: blob.size,- ...("width" in asset && { width: asset.width, height: asset.height }),- };-- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });- try {- const format = await input.getFormat();- const mimeType = await input.getMimeType();- const duration = await input.computeDuration();- const { images, ...tags } = await input.getMetadataTags();- delete tags.raw;-- const tracks: Array<Record<string, unknown>> = [];- for (const track of await input.getTracks()) {- const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);- tracks.push({- id: track.id,- type: track.type,- codec: track.codec,- language: track.languageCode,- firstTimestamp: await track.getFirstTimestamp(),- duration: await track.computeDuration(),- ...stats,- ...(track.isVideoTrack() && {- codedWidth: track.codedWidth,- codedHeight: track.codedHeight,- displayWidth: track.displayWidth,- displayHeight: track.displayHeight,- rotation: track.rotation,- }),- ...(track.isAudioTrack() && {- sampleRate: track.sampleRate,- channels: track.numberOfChannels,- }),- });- }-- return {- ...base,- format: format.name,- mimeType,- duration,- tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },- tracks,- };- } catch {- return { ...base, format: null, tracks: [] };- } finally {- input.dispose();- }- };-}--// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio-// preserved). A budget of 0 means native resolution. `small` keeps frames small-// enough for vision models and is the default.-const FRAME_QUALITY_BUDGETS = {- small: 384 * 384, // 147,456- medium: 768 * 768, // 589,824- large: 1536 * 1536, // 2,359,296- fullres: 0, // native-} as const;--// Default cap on frames returned by auto selection when `count` is not given.-const AUTO_MAX_FRAMES = 30;--export function handleMediaFrame(resolve: ResolveAsset) {- return async (req: MediaFrameRequest): Promise<MediaFrameResult> => {- const { times, count, start, end, quality, auto } = req;- const combine = req.combine ?? true;- const asset = await resolve(req.path);- const id = asset.id;- assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);-- // `count` samples evenly across a window (default the whole clip); `auto`- // scans the window and keeps frames where the footage settles into a new- // visual state, capped at `count` (resolved once the track is open).- // Otherwise grab the explicit `times` (falling back to a single frame at 0).- const from = Math.min(Math.max(start ?? 0, 0), asset.duration);- const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);- let requested: number[] = [];- if (auto || count !== undefined) {- assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`);- if (!auto && count !== undefined) {- const interval = (to - from) / count;- requested = Array.from({ length: count }, (_, i) => from + i * interval);- }- } else {- const raw = times && times.length ? times : [0];- // A negative time is an offset back from the end of the clip: -1 is one- // second before the end, -1f one frame before it.- requested = raw.map((t) => {- if (t >= 0) {- assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);- return t;- }- const resolved = asset.duration + t;- assert(resolved >= 0, `--time ${t} counts past the start of the clip (duration ${asset.duration.toFixed(2)}s).`);- return resolved;- });- }-- const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")];-- const blob = await getAssetFile(asset);- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });- try {- const track = await input.getPrimaryVideoTrack();- assert(track, `Asset ${id} has no video track.`);-- // Track timestamps may not start at 0; offset content time by the first.- const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;-- if (auto) {- const picked = await pickInformativeTimes(track, {- from: firstTimestamp + from,- to: firstTimestamp + to,- max: count ?? AUTO_MAX_FRAMES,- });- requested = picked.map((t) => Math.max(0, t - firstTimestamp));- }-- // Downscale to fit the pixel budget while preserving aspect ratio; setting- // only the width lets the sink derive a matching height.- const displayWidth = await track.getDisplayWidth();- const displayHeight = await track.getDisplayHeight();- let sourceWidth = displayWidth;- let sourceHeight = displayHeight;- if (budget > 0 && displayWidth * displayHeight > budget) {- const scale = Math.sqrt(budget / (displayWidth * displayHeight));- sourceWidth = Math.max(1, Math.round(displayWidth * scale));- sourceHeight = Math.max(1, Math.round(displayHeight * scale));- }-- // Lay the sheets out up front: the largest cell across them sets the- // decode size, so no frame is decoded bigger than it will be drawn.- const sizes = combine ? planSheetSizes(requested.length, req.perSheet) : [];- const plans = sizes.map((n) => planSheet(n, { width: sourceWidth, height: sourceHeight }));- const width = combine- ? Math.min(sourceWidth, Math.max(...plans.map((plan) => plan.cellWidth)))- : sourceWidth;-- // Decode in ascending order (the sink's fast path), remember each- // entry's original slot so output mirrors the requested order.- const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);-- // No pool: each yielded canvas is fresh, so converting to PNG can't race- // the generator's read-ahead reusing a pooled canvas.- const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined);- const timestamps = ordered.map(({ time }) => firstTimestamp + time);-- // Which sheet each frame belongs to, and where that sheet starts.- const sheetOf: number[] = [];- const sheetStart: number[] = [];- for (const [sheet, size] of sizes.entries()) {- sheetStart.push(sheetOf.length);- for (let k = 0; k < size; k++) sheetOf.push(sheet);- }- const missing = [...sizes];-- const cells: Array<{ at: number; timecode: string }> = new Array(requested.length);- const canvases: Array<CanvasImageSource | undefined> = new Array(requested.length);- const result: TimecodedImage[] = new Array(combine ? sizes.length : requested.length);-- let i = 0;- for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {- const { time, index } = ordered[i++];- assert(wrapped, `No frame found at ${time}s.`);- const timecode = formatTimecode(time, asset.frameRate);- cells[index] = { at: time, timecode };-- if (!combine) {- result[index] = { timecode, base64: await canvasToPngBase64(wrapped.canvas) };- continue;- }-- // Compose a sheet as soon as its last frame lands and drop the- // canvases, so a long run never holds every decoded frame at once.- canvases[index] = wrapped.canvas;- const sheet = sheetOf[index];- if (--missing[sheet] > 0) continue;-- const from = sheetStart[sheet];- const to = from + sizes[sheet];- result[sheet] = {- timecode: sheetTimecode(cells.slice(from, to)),- base64: await composeSheet(- canvases.slice(from, to).map((canvas, k) => ({ image: canvas!, label: cells[from + k].timecode })),- plans[sheet],- ),- };- for (let k = from; k < to; k++) canvases[k] = undefined;- }-- return result;- } finally {- input.dispose();- }- };-}--const transcripts = new Map<string, TranscriptSegment[]>();--export function handleMediaTranscribe(resolve: ResolveAsset) {- return async (req: MediaTranscribeRequest): Promise<MediaTranscribeResult> => {- const asset = await resolve(req.path);- const id = asset.id;- assert(- asset.type === "AUDIO" || asset.type === "VIDEO",- `Asset ${id} is not a video or audio asset.`,- );-- let transcript = transcripts.get(asset.id);- if (!transcript) {- const uploadId = crypto.randomUUID();- const audioFile = await transcodeForTranscription(asset);- const fileRef = await uploadBlob(audioFile, uploadId);- if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`);-- ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");- }-- transcripts.set(asset.id, transcript);- }-- return { segments: transcript };- };-}--export function handleMediaFilmstrip(resolve: ResolveAsset) {- return async (req: MediaFilmstripRequest): Promise<MediaFilmstripResult> => {- const asset = await resolve(req.path);- const { dataUrl, ...rest } = await filmstripAsset(asset, { start: req.start, end: req.end, scale: req.scale });- const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);- return { base64, ...rest };- };-}--export function handleMediaWaveform(resolve: ResolveAsset) {- return async (req: MediaWaveformRequest): Promise<MediaWaveformResult> => {- const asset = await resolve(req.path);- const { dataUrl, ...rest } = await waveformAsset(asset, { start: req.start, end: req.end, scale: req.scale });- const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);- return { base64, ...rest };- };-}--export function handleMediaListen(resolve: ResolveAsset, session: Accessor<EditorSession | null>) {- return async (req: MediaListenRequest): Promise<MediaListenResult> => {- const { prompt, start, end } = req;- let { stripVideo } = req;- const asset = await resolve(req.path);- const id = asset.id;- assert(- asset.type === "AUDIO" || asset.type === "VIDEO",- `Asset ${id} is not a video or audio asset.`,- );-- const hasWindow = start !== undefined || end !== undefined;- stripVideo = stripVideo !== false && asset.type === "VIDEO";-- const contentType =- asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";-- const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";- const uploadId = `${session()?.world.get(Project)?.id ?? "project"}-${id}-analyze${stripVideo ? "-audio" : ""}${window}`- .replace(/[^A-Za-z0-9._-]/g, "_");- const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({- action: "resumable",- id: uploadId,- contentType,- });-- // Transcoding pending- if (uploadUrl) {- const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo });- const sessionUrl = await startResumableSession(uploadUrl, contentType);- const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);- await transcoder.run?.();- await uploadPromise;- }-- const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });-- return { result: analysis, start, end };- };-}--async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<string> {- if (canvas instanceof OffscreenCanvas) {- const blob = await canvas.convertToBlob({ type: "image/png" });- return base64FromArrayBuffer(await blob.arrayBuffer());- }- return canvas.toDataURL("image/png").split(",")[1] ?? "";-}--function base64FromArrayBuffer(buf: ArrayBuffer): string {- const bytes = new Uint8Array(buf);- let binary = "";- const chunk = 0x8000;- for (let i = 0; i < bytes.length; i += chunk) {- binary += String.fromCharCode(...bytes.subarray(i, i + chunk));- }- return btoa(binary);-}diff --git a/apps/web/src/context/dapi/models.ts b/apps/web/src/context/dapi/models.tsdeleted file mode 100644index 2b5b07f5..00000000--- a/apps/web/src/context/dapi/models.ts+++ /dev/null@@ -1,47 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import {- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,-} from "@/components/genai/config";-import type { ModelsRequest, ModelInfo } from "@diffusionstudio/dapi";--export function handleModels() {- return async (req: ModelsRequest) => {- const out: ModelInfo[] = [];- if (!req.type || req.type === "image") {- for (const option of PROMPT_INPUT_IMAGE_MODEL_OPTIONS) {- out.push({- type: "image",- id: option.id,- name: option.name,- });- }- }- if (!req.type || req.type === "video") {- for (const option of PROMPT_INPUT_VIDEO_MODEL_OPTIONS) {- out.push({- type: "video",- id: option.id,- name: option.name,- durations: option.durations,- aspectRatios: option.aspectRatios,- features: option.features,- });- }- }- if (!req.type || req.type === "audio") {- for (const option of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) {- out.push({- type: "audio",- id: option.id,- name: option.name,- });- }- }- return out;- };-}diff --git a/apps/web/src/context/dapi/window.ts b/apps/web/src/context/dapi/window.tsdeleted file mode 100644index 5ae14da0..00000000--- a/apps/web/src/context/dapi/window.ts+++ /dev/null@@ -1,19 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { mainBridge } from "@/lib/ipc";-import { MAIN_CHANNELS } from "@desktop/main-channels";--export function handleGetFullscreenState() {- if (!window.desktop) return Promise.resolve(false);- return mainBridge.call(MAIN_CHANNELS.WINDOW_IS_FULLSCREEN, undefined);-}--export function handleWindowScreenshot() {- return () => mainBridge.call(MAIN_CHANNELS.WINDOW_CAPTURE, undefined);-}--export function handleWindowFullscreenChange(mutate: (fullscreen: boolean) => void) {- return ({ fullscreen }: { fullscreen: boolean }) => mutate(fullscreen);-}diff --git a/apps/web/src/dapi/api.tsx b/apps/web/src/dapi/api.tsxnew file mode 100644index 00000000..611f2246--- /dev/null+++ b/apps/web/src/dapi/api.tsx@@ -0,0 +1,94 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { createEffect, createContext, useContext, onCleanup } from "solid-js";+import { useNavigate } from "@solidjs/router";+import { useWorld } from "@diffusionstudio/koota-solid";+import { Project } from "@diffusionstudio/runtime";+import { DapiError } from "@diffusionstudio/dapi";+import { useProject } from "@/context/project";+import { useAuth } from "@/context/auth";+import { useEngineContext } from "@/engine";+import { openProjectFolder } from "@/projects";+import { projectRoute } from "@/hooks/use-project-route";+import { useFullscreenState } from "@/hooks/use-fullscreen-state";+import { assert } from "@/utils/common";+import { cliBridge } from "./bridge";+import { handlers } from "./handlers";+import { editorSession, requireEditorSession, setEditorSession } from "./session";++import type { JSX, Accessor } from "solid-js";+import type { ToolContext } from "./handler";++type EditorApiContextValue = {+ isFullscreen: Accessor<boolean>;+ isDesktop: boolean;+};++const EditorApiContext = createContext<EditorApiContextValue>();++/**+ * Registers the tool handlers for as long as the app runs. Every tool is+ * reachable whether or not a project is open; the ones that need one read+ * the session slot (see ./session) per call and fail with a clear error —+ * or, for `context`, report that nothing is open. Renders nothing; must sit+ * inside the router tree for `useNavigate` and inside the auth provider.+ */+export function EditorApi() {+ const navigate = useNavigate();+ const auth = useAuth();++ const context = (signal: AbortSignal): ToolContext => ({+ session: editorSession,+ requireSession: requireEditorSession,+ signal,+ app: {+ async openProject(dir) {+ const project = await openProjectFolder(dir);+ navigate(projectRoute(project.id || project.name));+ return { id: project.id, name: project.displayName, dir: project.dir };+ },+ user: () => auth.user() ?? null,+ requireUser() {+ const user = auth.user();+ if (!user) throw new DapiError("sign-in-required", "Sign in required: AI generation needs a Diffusion Studio account.");+ return user;+ },+ },+ });++ onCleanup(cliBridge.register(handlers, context));+ return null;+}++/**+ * Publishes the editor session for the tool handlers while the project is+ * open, and provides the editor UI's own view of the app shell (fullscreen+ * state, desktop-ness). Mounted per project page.+ */+export function EditorApiProvider(props: { children: JSX.Element }) {+ const project = useProject();+ const isFullscreen = useFullscreenState();+ const world = useWorld();+ const engine = useEngineContext();++ createEffect(() => {+ if (!window.desktop || project.id() !== world.get(Project)?.id) return;++ setEditorSession({ world, project, engine });+ onCleanup(() => setEditorSession(null));+ });++ return (+ <EditorApiContext.Provider value={{ isFullscreen, isDesktop: !!window.desktop }}>+ {props.children}+ </EditorApiContext.Provider>+ );+}++export function useEditorApi() {+ const ctx = useContext(EditorApiContext);+ assert(ctx, "useEditorApi must be used within EditorApiProvider");+ return ctx;+}diff --git a/apps/web/src/dapi/bridge.ts b/apps/web/src/dapi/bridge.tsnew file mode 100644index 00000000..2db5a591--- /dev/null+++ b/apps/web/src/dapi/bridge.ts@@ -0,0 +1,100 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { parseToolArgs } from "@diffusionstudio/dapi";+import { CLI_WIRE, encodeReply } from "@diffusionstudio/cli/protocol";++import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol";+import type { Handlers, ServedToolName, ToolContext } from "./handler";++type Pending = { req: CliRequest; ws: WebSocket };++/** Builds the per-call context; the bridge supplies the abort signal. */+export type ContextFactory = (signal: AbortSignal) => ToolContext;++/**+ * Answers tool calls from the CLI. Each CLI command hosts a short-lived+ * WebSocket server; main relays only the connect info (CLI_WIRE.CONNECT) and+ * we dial the CLI directly, so payloads never pass through main. One request+ * and one reply per connection.+ *+ * The bridge is transport: it validates arguments against the catalog, runs+ * the handler, and encodes the reply. Requests arriving before the handlers+ * register (page bootstrap) are held and answered on registration.+ */+class CliBridge {+ private handlers: Handlers | null = null;+ private context: ContextFactory | null = null;+ private held: Pending[] = [];++ constructor() {+ // Bind eagerly so CONNECT arrivals during page bootstrap are caught+ // rather than silently dropped before the handlers register.+ window.desktop?.on(CLI_WIRE.CONNECT, (payload) => {+ const { port, token } = payload as CliHandshake;+ const ws = new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`);+ ws.onmessage = (event) => {+ try {+ const req = JSON.parse(event.data as string) as CliRequest;+ void this.dispatch({ req, ws });+ } catch (err) {+ console.error("[cli-bridge] malformed CLI request", err);+ ws.close();+ }+ };+ });+ }++ register(handlers: Handlers, context: ContextFactory): () => void {+ this.handlers = handlers;+ this.context = context;+ const held = this.held;+ this.held = [];+ for (const pending of held) void this.dispatch(pending);+ return () => {+ if (this.handlers === handlers) {+ this.handlers = null;+ this.context = null;+ }+ };+ }++ private async dispatch(pending: Pending): Promise<void> {+ const { req, ws } = pending;+ // Liveness, answered by the transport itself: `open` waits on it after+ // launching the app, before any handler exists to answer anything else.+ if (req.path === "ping") {+ ws.send(encodeReply({ ok: true, data: undefined }));+ return;+ }+ if (!this.handlers || !this.context) {+ this.held.push(pending);+ return;+ }++ const controller = new AbortController();+ ws.addEventListener("close", () => controller.abort(), { once: true });++ let reply: CliReply;+ try {+ const name = req.path as ServedToolName;+ const handler = this.handlers[name];+ if (!handler) throw new Error(`Unknown tool "${req.path}"`);+ const args = parseToolArgs(name, req.input);+ // Every handler takes its own parsed args; the map's union type cannot+ // express that pairing, so the call site widens.+ const data = await (handler as (args: unknown, ctx: ToolContext) => Promise<unknown>)(args, this.context(controller.signal));+ reply = { ok: true, data };+ } catch (err) {+ reply = { ok: false, error: (err as Error).message };+ }+ try {+ ws.send(encodeReply(reply));+ } catch (err) {+ ws.send(encodeReply({ ok: false, error: `Failed to serialize reply for ${req.path}: ${(err as Error).message}` }));+ }+ }+}++export const cliBridge = new CliBridge();diff --git a/apps/web/src/dapi/handler.ts b/apps/web/src/dapi/handler.tsnew file mode 100644index 00000000..1a7f67ab--- /dev/null+++ b/apps/web/src/dapi/handler.ts@@ -0,0 +1,40 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { Accessor } from "solid-js";+import type { User } from "@supabase/supabase-js";+import type { ToolArgs, ToolName, ToolResult } from "@diffusionstudio/dapi";+import type { EditorSession } from "./session";++/**+ * What every handler gets besides its arguments. Built per call by the+ * bridge from what the app shell provides (see ./api), so handlers are plain+ * functions with no Solid in them.+ */+export type ToolContext = {+ /** The open project's session, or null at the dashboard. */+ session: Accessor<EditorSession | null>;+ /** The session, or a `no-project` error the caller can act on. */+ requireSession(): EditorSession;+ /** Fires when the caller goes away before the reply. */+ signal: AbortSignal;+ /** What only the app shell can do: navigate, and know who is signed in. */+ app: {+ openProject(dir: string): Promise<ToolResult<"open">>;+ user(): User | null;+ /** The signed-in user, or a `sign-in-required` error. */+ requireUser(): User;+ };+};++export type ToolHandler<N extends ToolName> = (args: ToolArgs<N>, ctx: ToolContext) => Promise<ToolResult<N>>;++/**+ * The tools the renderer answers. `logs` is a main-process tool in the+ * catalog and is forwarded from here until main hosts the server itself;+ * `fonts`, `fetch` and `report` have no renderer side at all.+ */+export type ServedToolName = Exclude<ToolName, "fonts" | "fetch" | "report">;++export type Handlers = { readonly [N in ServedToolName]: ToolHandler<N> };diff --git a/apps/web/src/dapi/handlers/capture.ts b/apps/web/src/dapi/handlers/capture.tsnew file mode 100644index 00000000..13b1ee41--- /dev/null+++ b/apps/web/src/dapi/handlers/capture.ts@@ -0,0 +1,53 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { createImageEncoder, decodePng } from "@diffusionstudio/encoder";+import { DapiError } from "@diffusionstudio/dapi";+import { createCapture } from "@/engine/capture";+import { requireScene } from "../lib/scene";+import { SheetCollector } from "../lib/sheets";++import type { ToolHandler } from "../handler";++// Ceiling on the height a sheet cell renders a node at.+const SHEET_CAPTURE_HEIGHT = 1080;++export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, perSheet }, ctx) => {+ const { world, project } = ctx.requireSession();+ const scene = requireScene(world, id, "capture");++ // `undefined` means the export's first frame (the workarea's start).+ const shots = frames && frames.length > 0 ? frames : [0];++ // The project re-rendered into a world of its own, reduced to this scene:+ // the same arrangement an export runs against, and the encoder's to draw.+ const target = await createCapture(world, scene, { dir: project.dir() });+ try {+ const encoder = await createImageEncoder(target.world, { frames: shots, resolution: 720 });++ // Sheets render at their cell size instead of the flat 720p: with a few+ // frames that is sharper than a standalone capture, never coarser. A+ // scene is drawn, not decoded, so a small one is worth rendering past+ // its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens.+ let sheets: SheetCollector | undefined;+ if (combine) {+ const aspect = encoder.bounds.width / encoder.bounds.height;+ const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT);+ sheets = new SheetCollector(shots.length, { width: height * aspect, height }, perSheet);+ encoder.resize(sheets.cellHeight);+ }++ const result = await encoder.render();+ if (result.type === "canceled") throw new DapiError("canceled", "Capture canceled");+ if (result.type === "error") throw result.error;+ if (!sheets) return result.data;++ for (const [index, { timecode, png }] of result.data.entries()) {+ await sheets.add(index, { at: shots[index]!, timecode, image: await decodePng(png) });+ }+ return sheets.result();+ } finally {+ target.dispose();+ }+};diff --git a/apps/web/src/context/dapi/check.ts b/apps/web/src/dapi/handlers/check.tssimilarity index 95%rename from apps/web/src/context/dapi/check.tsrename to apps/web/src/dapi/handlers/check.tsindex 0aabb6df..03209cb7 100644--- a/apps/web/src/context/dapi/check.ts+++ b/apps/web/src/dapi/handlers/check.ts@@ -8,11 +8,11 @@ import {SourceError, Workarea, framesToSeconds, getIntrinsicPaint, isText,} from "@diffusionstudio/runtime";-import { resolveNode } from "./nodes";+import { resolveNode } from "../lib/nodes";-import type { CheckIssue, CheckRequest, CheckResult } from "@diffusionstudio/dapi";+import type { CheckIssue } from "@diffusionstudio/dapi";import type { Entity } from "koota";-import type { EditorSession } from "./session";+import type { ToolHandler } from "../handler";// Absolute frames, [start, end).type Interval = { start: number; end: number };@@ -147,9 +147,8 @@ function findGaps(window: Interval, coverage: Interval[]): Interval[] {return gaps.filter((gap) => gap.end - gap.start >= 1);}-export function handleCheck(session: () => EditorSession) {- return async ({ id }: CheckRequest): Promise<CheckResult> => {- const { world } = session();+export const check: ToolHandler<"check"> = async ({ id }, ctx) => {+ const { world } = ctx.requireSession();const target = resolveNode(world, id);const fps = world.get(FrameRate)?.value ?? 30;@@ -207,5 +206,4 @@ export function handleCheck(session: () => EditorSession) {},issues,};- };-}+};diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.tsnew file mode 100644index 00000000..d75b349c--- /dev/null+++ b/apps/web/src/dapi/handlers/context.ts@@ -0,0 +1,82 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime";+import { getProjectsRoot } from "@/projects";++import type { Entity, World } from "koota";+import type { GenerationRow } from "@diffusionstudio/dapi";+import type { ToolHandler } from "../handler";++/**+ * What the project's source cannot say. The JSX is the composition — its+ * scenes, what is selected, which scene is active, the work area are all in+ * the file, and a caller that wants them reads it. What is left over is which+ * folder projects live under, which project folder the app has open, where its+ * playhead sits, which font families are actually registered in the world+ * drawing it. With no project open only the root is left to report.+ */+export const context: ToolHandler<"context"> = async (_, ctx) => {+ const rootDir = await getProjectsRoot();++ const open = ctx.session();+ if (!open) return { rootDir, projectDir: null };++ const { world, project } = open;+ const frameRate = world.get(FrameRate)?.value || 30;+ const active = getActiveEntity(world);++ return {+ rootDir,+ projectDir: project.dir(),+ // Seconds, the unit the source places clips in; null when no scene is+ // active, which is when there is no playhead to report.+ currentTime: active ? (active.get(Computed)?.localTime ?? 0) / frameRate : null,+ // What text can be drawn with right now: registered in the world, not+ // merely named in the source. The editor default is always among them.+ fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])],+ generations: collectGenerations(world),+ };+};++function collectGenerations(world: World): GenerationRow[] {+ const library = world.get(Library);+ const rows: GenerationRow[] = [];+ const seen = new Set<string>();++ const push = (row: GenerationRow) => {+ const key = JSON.stringify(row);+ if (seen.has(key)) return;+ seen.add(key);+ rows.push(row);+ };++ for (const entity of world.query(Generating)) {+ push({ ...describeElement(entity), state: "generating" });+ }++ for (const entity of world.query(SourceError)) {+ const failure = entity.get(SourceError)!;+ if (!failure.generated) continue;+ push({ ...describeElement(entity), state: "failed", error: failure.value });+ }++ for (const entity of world.query(AssetId)) {+ // A stale binding — the element is off generating a new answer or failed+ // getting one — is not a generation that is done.+ if (entity.has(Generating) || entity.has(PendingSource) || entity.has(SourceError)) continue;+ const asset = library?.get(entity.get(AssetId)!.value);+ if (!asset?.generation) continue;+ push({ ...describeElement(entity), state: "done", asset: asset.path });+ }++ return rows;+}++function describeElement(entity: Entity): Pick<GenerationRow, "element" | "name"> {+ return {+ element: entity.get(Source)?.value || null,+ name: entity.get(Name)?.value || null,+ };+}diff --git a/apps/web/src/dapi/handlers/export.ts b/apps/web/src/dapi/handlers/export.tsnew file mode 100644index 00000000..fc688cc8--- /dev/null+++ b/apps/web/src/dapi/handlers/export.ts@@ -0,0 +1,118 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { canEncodeVideo } from "mediabunny";+import { computeOutputSize } from "@diffusionstudio/encoder";+import { Computed, FrameRate, Workarea } from "@diffusionstudio/runtime";+import { DapiError } from "@diffusionstudio/dapi";++import { renderOverlay, renderScene } from "@/context/render";+import { ElectronWritableFileHandle } from "@/lib/electron-file-writable";+import { ProjectConfig as ProjectConfigTrait } from "@/engine/traits";+import { sceneConfigKey } from "@/engine/project-config";+import { getDefaultExportTemplate, VIDEO_FORMAT_OPTIONS } from "@/components/sidebar-right/inspector/export-templates";+import { mainBridge } from "@/lib/ipc";+import { MAIN_CHANNELS } from "@desktop/main-channels";+import { requireScene } from "../lib/scene";++import type { ExportSettings } from "@diffusionstudio/dapi";+import type { ContainerFormat, ExportConfig } from "@/engine/project-config";+import type { ToolHandler } from "../handler";++/**+ * The container the export writes: the output extension when a path was+ * given (ffmpeg's rule — the file must be what its name says), the config's+ * format otherwise, mp4 when neither says. A given path settles it entirely,+ * so a bad config format only fails an export that would actually use it.+ */+function resolveFormat(path: string | undefined, config: ExportConfig): ContainerFormat {+ const supported = VIDEO_FORMAT_OPTIONS.map((format) => `.${format}`).join(", ");+ if (path !== undefined) {+ const extension = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase();+ if (!extension || !VIDEO_FORMAT_OPTIONS.includes(extension as ContainerFormat)) {+ throw new DapiError("invalid-input", `The output path must end in a container extension: ${supported}.`);+ }+ return extension as ContainerFormat;+ }+ const format = config.format ?? "mp4";+ if (!VIDEO_FORMAT_OPTIONS.includes(format)) {+ throw new DapiError(+ "invalid-input",+ `The export entry names an unknown format "${format}" — use one of ${supported.replaceAll(".", "")}.`,+ );+ }+ return format;+}++export const exportScene: ToolHandler<"export"> = async ({ id, path }, ctx) => {+ const { world, project, engine } = ctx.requireSession();+ const scene = requireScene(world, id, "export");++ // The scene's entry in the project's package.json (`diffusion.export.<id>`)+ // — the same one the app's export panel writes — so a tool export+ // reproduces the in-app one; a scene without an entry uses the default+ // template, the way ⌘E does. `template` is only the preset's label.+ const base = world.get(ProjectConfigTrait)?.exportOf(scene) ?? getDefaultExportTemplate();+ const settings: ExportConfig = { format: base.format, video: base.video, audio: base.audio };++ const format = resolveFormat(path, settings);+ const key = sceneConfigKey(scene) ?? id;+ const target = path ?? `${project.dir()}/exports/${key.replace(/[^\w.-]+/g, "-")}.${format}`;++ // Fail before the render machinery spins up: an unencodable configuration+ // is known right away (same precheck the UI export runs).+ const videoEnabled = format !== "ogg" && settings.video?.enabled !== false;+ const computed = scene.get(Computed);+ const { width, height } = computeOutputSize(+ computed?.width || 1920,+ computed?.height || 1080,+ settings.video?.resolution ?? 1080,+ );+ if (videoEnabled) {+ const codec = settings.video?.codec ?? "avc";+ const encodable = await canEncodeVideo(codec, { width, height, bitrate: settings.video?.bitrate ?? 10e6 });+ if (!encodable) {+ throw new DapiError(+ "unsupported",+ `Cannot encode ${codec.toUpperCase()} at ${width}×${height}. ` ++ "Set a lower resolution, a lower bitrate, or another codec in the scene's export entry.",+ );+ }+ }++ const workarea = scene.get(Workarea);+ const frames = workarea ? workarea.end - workarea.start : computed?.duration ?? 0;+ const duration = frames / (world.get(FrameRate)?.value || 30);++ // renderScene owns the one render slot (it stops the live engine and+ // raises the progress overlay), so refuse a second export rather than+ // interleave. No await sits between this check and renderScene claiming+ // the overlay, so two racing requests cannot both pass.+ if (renderOverlay()) {+ throw new DapiError("busy", "An export is already running — wait for it to finish, or cancel it in the app.");+ }++ const handle = new ElectronWritableFileHandle(target);+ try {+ const result = await renderScene(engine, { scene, target: handle, config: { ...settings, format }, dir: project.dir() });+ if (result.type === "canceled") throw new DapiError("canceled", "Export canceled in the app");+ if (result.type === "error") throw result.error;+ } catch (error) {+ // Close the fd and drop the partial file; a failed export leaves nothing.+ await handle.dispose().catch(() => {});+ throw error;+ }++ const stat = await mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_STAT, { dir: project.dir(), source: target });++ const config: ExportSettings = { format, video: settings.video, audio: settings.audio };+ return {+ path: target,+ width: videoEnabled ? width : 0,+ height: videoEnabled ? height : 0,+ duration,+ size: stat?.size ?? 0,+ config,+ };+};diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.tsnew file mode 100644index 00000000..2277306b--- /dev/null+++ b/apps/web/src/dapi/handlers/index.ts@@ -0,0 +1,42 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { open } from "./open";+import { context } from "./context";+import { capture } from "./capture";+import { check } from "./check";+import { exportScene } from "./export";+import { models } from "./models";+import { voices } from "./voices";+import { whoami } from "./whoami";+import { logs } from "./logs";+import { screenshot } from "./screenshot";+import { mediaProbe } from "./media-probe";+import { mediaGrab } from "./media-grab";+import { mediaTranscribe } from "./media-transcribe";+import { mediaFilmstrip } from "./media-filmstrip";+import { mediaWaveform } from "./media-waveform";+import { mediaListen } from "./media-listen";++import type { Handlers } from "../handler";++/** Every tool the renderer answers, keyed by its catalog name. */+export const handlers: Handlers = {+ open,+ context,+ capture,+ check,+ export: exportScene,+ models,+ voices,+ whoami,+ logs,+ screenshot,+ media_probe: mediaProbe,+ media_grab: mediaGrab,+ media_transcribe: mediaTranscribe,+ media_filmstrip: mediaFilmstrip,+ media_waveform: mediaWaveform,+ media_listen: mediaListen,+};diff --git a/apps/web/src/dapi/handlers/logs.ts b/apps/web/src/dapi/handlers/logs.tsnew file mode 100644index 00000000..cdb909f8--- /dev/null+++ b/apps/web/src/dapi/handlers/logs.ts@@ -0,0 +1,22 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { mainBridge } from "@/lib/ipc";+import { MAIN_CHANNELS } from "@desktop/main-channels";++import type { LogLevel } from "@diffusionstudio/dapi";+import type { ToolHandler } from "../handler";++const LEVEL_RANK: Record<LogLevel, number> = { debug: 0, info: 1, warning: 2, error: 3 };++// Main owns the buffer; this is a forward until main answers tools itself.+export const logs: ToolHandler<"logs"> = async ({ tail, level }) => {+ let entries = await mainBridge.call(MAIN_CHANNELS.LOGS_GET, undefined);+ if (level !== undefined) {+ const min = LEVEL_RANK[level];+ entries = entries.filter((e) => LEVEL_RANK[e.level] >= min);+ }+ if (tail !== undefined) entries = entries.slice(-tail);+ return entries;+};diff --git a/apps/web/src/dapi/handlers/media-filmstrip.ts b/apps/web/src/dapi/handlers/media-filmstrip.tsnew file mode 100644index 00000000..7902b328--- /dev/null+++ b/apps/web/src/dapi/handlers/media-filmstrip.ts@@ -0,0 +1,15 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { filmstripAsset } from "@diffusionstudio/runtime";+import { resolveAsset } from "../lib/assets";+import { dataUrlToBytes } from "../lib/png";++import type { ToolHandler } from "../handler";++export const mediaFilmstrip: ToolHandler<"media_filmstrip"> = async ({ path, start, end, scale }, ctx) => {+ const asset = await resolveAsset(ctx, path);+ const { dataUrl, ...rest } = await filmstripAsset(asset, { start, end, scale });+ return { png: dataUrlToBytes(dataUrl), ...rest };+};diff --git a/apps/web/src/dapi/handlers/media-grab.ts b/apps/web/src/dapi/handlers/media-grab.tsnew file mode 100644index 00000000..ed12e9fb--- /dev/null+++ b/apps/web/src/dapi/handlers/media-grab.ts@@ -0,0 +1,132 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { ALL_FORMATS, BlobSource, CanvasSink, Input } from "mediabunny";+import { encodePng } from "@diffusionstudio/encoder";+import { formatTimecode, getAssetFile } from "@diffusionstudio/runtime";+import { DapiError } from "@diffusionstudio/dapi";+import { pickInformativeTimes } from "../lib/frame-triage";+import { requireAssetType, resolveAsset } from "../lib/assets";+import { SheetCollector } from "../lib/sheets";++import type { FrameQuality, TimecodedImage } from "@diffusionstudio/dapi";+import type { ToolHandler } from "../handler";++// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio+// preserved). A budget of 0 means native resolution. `small` keeps frames small+// enough for vision models.+const FRAME_QUALITY_BUDGETS: Record<FrameQuality, number> = {+ small: 384 * 384,+ medium: 768 * 768,+ large: 1536 * 1536,+ fullres: 0,+};++// Default cap on frames returned by auto selection when `count` is not given.+const AUTO_MAX_FRAMES = 30;++export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {+ const { times, count, start, end, quality, auto, combine, perSheet } = args;+ const asset = await resolveAsset(ctx, args.path);+ requireAssetType(asset, ["VIDEO"], "a video");++ // `count` samples evenly across a window (default the whole clip); `auto`+ // scans the window and keeps frames where the footage settles into a new+ // visual state, capped at `count` (resolved once the track is open).+ // Otherwise grab the explicit `times` (falling back to a single frame at 0).+ const from = Math.min(Math.max(start ?? 0, 0), asset.duration);+ const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);+ let requested: number[] = [];+ if (auto || count !== undefined) {+ if (to <= from) {+ throw new DapiError(+ "invalid-input",+ `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`,+ );+ }+ if (!auto && count !== undefined) {+ const interval = (to - from) / count;+ requested = Array.from({ length: count }, (_, i) => from + i * interval);+ }+ } else {+ requested = (times && times.length ? times : [0]).map((t) => resolveTime(t, asset.duration));+ }++ const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")];++ const blob = await getAssetFile(asset);+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const track = await input.getPrimaryVideoTrack();+ if (!track) throw new DapiError("wrong-kind", `Asset ${asset.id} has no video track.`);++ // Track timestamps may not start at 0; offset content time by the first.+ const firstTimestamp = (await track.getFirstTimestamp()) ?? 0;++ if (auto) {+ const picked = await pickInformativeTimes(track, {+ from: firstTimestamp + from,+ to: firstTimestamp + to,+ max: count ?? AUTO_MAX_FRAMES,+ });+ requested = picked.map((t) => Math.max(0, t - firstTimestamp));+ }++ // Downscale to fit the pixel budget while preserving aspect ratio.+ const displayWidth = await track.getDisplayWidth();+ const displayHeight = await track.getDisplayHeight();+ let sourceWidth = displayWidth;+ let sourceHeight = displayHeight;+ if (budget > 0 && displayWidth * displayHeight > budget) {+ const scale = Math.sqrt(budget / (displayWidth * displayHeight));+ sourceWidth = Math.max(1, Math.round(displayWidth * scale));+ sourceHeight = Math.max(1, Math.round(displayHeight * scale));+ }++ // Lay the sheets out up front: the largest cell across them sets the+ // decode size, so no frame is decoded bigger than it will be drawn.+ const sheets = combine ? new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet) : undefined;+ const width = sheets ? Math.min(sourceWidth, sheets.cellWidth) : sourceWidth;++ // Decode in ascending order (the sink's fast path), remember each+ // entry's original slot so output mirrors the requested order.+ const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);++ // No pool: each yielded canvas is fresh, so encoding it can't race the+ // generator's read-ahead reusing a pooled canvas.+ const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined);+ const separate: TimecodedImage[] = new Array(requested.length);++ let i = 0;+ for await (const wrapped of sink.canvasesAtTimestamps(ordered.map(({ time }) => firstTimestamp + time))) {+ const { time, index } = ordered[i++]!;+ if (!wrapped) throw new DapiError("not-found", `No frame found at ${time}s.`);+ const timecode = formatTimecode(time, asset.frameRate);+ if (sheets) await sheets.add(index, { at: time, timecode, image: wrapped.canvas });+ else separate[index] = { timecode, png: await encodePng(wrapped.canvas) };+ }++ return sheets ? sheets.result() : separate;+ } finally {+ input.dispose();+ }+};++/**+ * A time within the clip. A negative time is an offset back from the end:+ * -1 is one second before the end, -1f one frame before it.+ */+function resolveTime(t: number, duration: number): number {+ if (t >= 0) {+ if (t > duration) {+ throw new DapiError("invalid-input", `time ${t}s is past the asset's duration (${duration.toFixed(2)}s).`);+ }+ return t;+ }+ const resolved = duration + t;+ if (resolved < 0) {+ throw new DapiError("invalid-input", `time ${t} counts past the start of the clip (duration ${duration.toFixed(2)}s).`);+ }+ return resolved;+}diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.tsnew file mode 100644index 00000000..4a5cdd9d--- /dev/null+++ b/apps/web/src/dapi/handlers/media-listen.ts@@ -0,0 +1,39 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Project, transcodeForAnalysis } from "@diffusionstudio/runtime";+import { trpc } from "@/lib/trpc";+import { startResumableSession, uploadResumableStream } from "@/lib/uploads";+import { requireAssetType, resolveAsset } from "../lib/assets";++import type { ToolHandler } from "../handler";++export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end, stripVideo }, ctx) => {+ ctx.app.requireUser();+ const asset = await resolveAsset(ctx, path);+ requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");++ const hasWindow = start !== undefined || end !== undefined;+ const audioOnly = stripVideo !== false && asset.type === "VIDEO";+ const contentType = asset.type === "VIDEO" && !audioOnly ? "video/mp4" : "audio/ogg";++ // One upload per distinct analysis input, so a repeated question about the+ // same span reuses the transcode.+ const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";+ const uploadId = `${ctx.session()?.world.get(Project)?.id ?? "project"}-${asset.id}-analyze${audioOnly ? "-audio" : ""}${window}`+ .replace(/[^A-Za-z0-9._-]/g, "_");+ const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({ action: "resumable", id: uploadId, contentType });++ // An upload URL means the server does not have this input yet.+ if (uploadUrl) {+ const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo: audioOnly });+ const sessionUrl = await startResumableSession(uploadUrl, contentType);+ const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);+ await transcoder.run?.();+ await uploadPromise;+ }++ const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });+ return { result: analysis, start, end };+};diff --git a/apps/web/src/dapi/handlers/media-probe.ts b/apps/web/src/dapi/handlers/media-probe.tsnew file mode 100644index 00000000..525ddff5--- /dev/null+++ b/apps/web/src/dapi/handlers/media-probe.ts@@ -0,0 +1,73 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { ALL_FORMATS, BlobSource, Input } from "mediabunny";+import { getAssetFile } from "@diffusionstudio/runtime";+import { assetName } from "@diffusionstudio/assets";+import { resolveAsset } from "../lib/assets";++import type { ToolHandler } from "../handler";++const PROBE_SAMPLE_PACKETS = 200;++export const mediaProbe: ToolHandler<"media_probe"> = async ({ path }, ctx) => {+ const asset = await resolveAsset(ctx, path);+ const blob = await getAssetFile(asset);+ const base = {+ id: asset.id,+ name: assetName(asset),+ path: asset.path,+ type: asset.type,+ mimeType: asset.mimeType,+ size: blob.size,+ ...("width" in asset && { width: asset.width, height: asset.height }),+ };++ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });+ try {+ const format = await input.getFormat();+ const mimeType = await input.getMimeType();+ const duration = await input.computeDuration();+ const { images, ...tags } = await input.getMetadataTags();+ delete tags.raw;++ const tracks = [];+ for (const track of await input.getTracks()) {+ const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS);+ tracks.push({+ id: track.id,+ type: track.type,+ codec: track.codec,+ language: track.languageCode,+ firstTimestamp: await track.getFirstTimestamp(),+ duration: await track.computeDuration(),+ ...stats,+ ...(track.isVideoTrack() && {+ codedWidth: track.codedWidth,+ codedHeight: track.codedHeight,+ displayWidth: track.displayWidth,+ displayHeight: track.displayHeight,+ rotation: track.rotation,+ }),+ ...(track.isAudioTrack() && {+ sampleRate: track.sampleRate,+ channels: track.numberOfChannels,+ }),+ });+ }++ return {+ ...base,+ format: format.name,+ mimeType,+ duration,+ tags: { ...tags, ...(images?.length && { attachedImages: images.length }) },+ tracks,+ };+ } catch {+ return { ...base, format: null, tracks: [] };+ } finally {+ input.dispose();+ }+};diff --git a/apps/web/src/dapi/handlers/media-transcribe.ts b/apps/web/src/dapi/handlers/media-transcribe.tsnew file mode 100644index 00000000..234c2632--- /dev/null+++ b/apps/web/src/dapi/handlers/media-transcribe.ts@@ -0,0 +1,37 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { transcodeForTranscription } from "@diffusionstudio/runtime";+import { trpc } from "@/lib/trpc";+import { uploadBlob } from "@/lib/uploads";+import { requireAssetType, resolveAsset } from "../lib/assets";++import type { TranscriptSegment } from "@diffusionstudio/dapi";+import type { ToolHandler } from "../handler";++// Transcripts are remembered per asset for the app's lifetime: the audio+// does not change, and the transcription is the expensive part.+const transcripts = new Map<string, TranscriptSegment[]>();++export const mediaTranscribe: ToolHandler<"media_transcribe"> = async ({ path }, ctx) => {+ const asset = await resolveAsset(ctx, path);+ requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");++ let transcript = transcripts.get(asset.id);+ if (!transcript) {+ const uploadId = crypto.randomUUID();+ const audioFile = await transcodeForTranscription(asset);+ const fileRef = await uploadBlob(audioFile, uploadId);+ if (!fileRef) throw new Error(`Failed to upload asset ${asset.id} for transcription.`);++ ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));+ if (!transcript.length || transcript.every((s) => s.words.length === 0)) {+ throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");+ }++ transcripts.set(asset.id, transcript);+ }++ return { segments: transcript };+};diff --git a/apps/web/src/dapi/handlers/media-waveform.ts b/apps/web/src/dapi/handlers/media-waveform.tsnew file mode 100644index 00000000..e15591c8--- /dev/null+++ b/apps/web/src/dapi/handlers/media-waveform.ts@@ -0,0 +1,15 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { waveformAsset } from "@diffusionstudio/runtime";+import { resolveAsset } from "../lib/assets";+import { dataUrlToBytes } from "../lib/png";++import type { ToolHandler } from "../handler";++export const mediaWaveform: ToolHandler<"media_waveform"> = async ({ path, start, end, scale }, ctx) => {+ const asset = await resolveAsset(ctx, path);+ const { dataUrl, ...rest } = await waveformAsset(asset, { start, end, scale });+ return { png: dataUrlToBytes(dataUrl), ...rest };+};diff --git a/apps/web/src/dapi/handlers/models.ts b/apps/web/src/dapi/handlers/models.tsnew file mode 100644index 00000000..b9d575b9--- /dev/null+++ b/apps/web/src/dapi/handlers/models.ts@@ -0,0 +1,35 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import {+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS,+ PROMPT_INPUT_VIDEO_MODEL_OPTIONS,+ PROMPT_INPUT_AUDIO_MODEL_OPTIONS,+} from "@/components/genai/config";++import type { ModelInfo } from "@diffusionstudio/dapi";+import type { ToolHandler } from "../handler";++export const models: ToolHandler<"models"> = async ({ type }) => {+ const out: ModelInfo[] = [];+ if (!type || type === "image") {+ for (const { id, name } of PROMPT_INPUT_IMAGE_MODEL_OPTIONS) out.push({ type: "image", id, name });+ }+ if (!type || type === "video") {+ for (const option of PROMPT_INPUT_VIDEO_MODEL_OPTIONS) {+ out.push({+ type: "video",+ id: option.id,+ name: option.name,+ durations: option.durations,+ aspectRatios: option.aspectRatios,+ features: option.features,+ });+ }+ }+ if (!type || type === "audio") {+ for (const { id, name } of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) out.push({ type: "audio", id, name });+ }+ return out;+};diff --git a/apps/web/src/dapi/handlers/open.ts b/apps/web/src/dapi/handlers/open.tsnew file mode 100644index 00000000..f4c0dcfb--- /dev/null+++ b/apps/web/src/dapi/handlers/open.ts@@ -0,0 +1,7 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { ToolHandler } from "../handler";++export const open: ToolHandler<"open"> = ({ dir }, ctx) => ctx.app.openProject(dir);diff --git a/apps/web/src/dapi/handlers/screenshot.ts b/apps/web/src/dapi/handlers/screenshot.tsnew file mode 100644index 00000000..8c9d2c67--- /dev/null+++ b/apps/web/src/dapi/handlers/screenshot.ts@@ -0,0 +1,10 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { mainBridge } from "@/lib/ipc";+import { MAIN_CHANNELS } from "@desktop/main-channels";++import type { ToolHandler } from "../handler";++export const screenshot: ToolHandler<"screenshot"> = () => mainBridge.call(MAIN_CHANNELS.WINDOW_CAPTURE, undefined);diff --git a/apps/web/src/context/dapi/voices.ts b/apps/web/src/dapi/handlers/voices.tssimilarity index 57%rename from apps/web/src/context/dapi/voices.tsrename to apps/web/src/dapi/handlers/voices.tsindex 9109a8d0..e589b0c0 100644--- a/apps/web/src/context/dapi/voices.ts+++ b/apps/web/src/dapi/handlers/voices.ts@@ -4,12 +4,7 @@import { PROMPT_INPUT_VOICE_OPTIONS } from "@/components/genai/config";-export function handleVoices() {- return async () => {- return PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({- id: v.value,- label: v.label,- description: v.description,- }));- }-}+import type { ToolHandler } from "../handler";++export const voices: ToolHandler<"voices"> = async () =>+ PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ id: v.value, label: v.label, description: v.description }));diff --git a/apps/web/src/dapi/handlers/whoami.ts b/apps/web/src/dapi/handlers/whoami.tsnew file mode 100644index 00000000..23022db5--- /dev/null+++ b/apps/web/src/dapi/handlers/whoami.ts@@ -0,0 +1,7 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { ToolHandler } from "../handler";++export const whoami: ToolHandler<"whoami"> = async (_, ctx) => ctx.app.user();diff --git a/apps/web/src/context/dapi/index.ts b/apps/web/src/dapi/index.tssimilarity index 75%rename from apps/web/src/context/dapi/index.tsrename to apps/web/src/dapi/index.tsindex b86dc190..3d31fac2 100644--- a/apps/web/src/context/dapi/index.ts+++ b/apps/web/src/dapi/index.ts@@ -2,4 +2,4 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-export * from './api';+export { EditorApi, EditorApiProvider, useEditorApi } from "./api";diff --git a/apps/web/src/dapi/lib/assets.ts b/apps/web/src/dapi/lib/assets.tsnew file mode 100644index 00000000..08e54f3c--- /dev/null+++ b/apps/web/src/dapi/lib/assets.ts@@ -0,0 +1,41 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { getLibrary } from "@diffusionstudio/runtime";+import { AssetLibrary, isAbsoluteSource, isUrlSource } from "@diffusionstudio/assets";+import { DapiError } from "@diffusionstudio/dapi";+import { createProjectFS } from "@/projects/fs";++import type { Asset } from "@diffusionstudio/assets";+import type { ToolContext } from "../handler";++/**+ * Resolves a tool target by path. With a project open, its library answers:+ * library paths look assets up, absolute paths and URLs are described in place+ * without being added (transient assets). With none open, a throwaway library+ * over a project-less FS describes absolute paths and URLs the same way — a+ * fresh one per request, so nothing is remembered between calls.+ */+export function resolveAsset(ctx: ToolContext, path: string): Promise<Asset> {+ const world = ctx.session()?.world;+ if (world) return getLibrary(world).resolve(path);+ if (!isAbsoluteSource(path) && !isUrlSource(path)) {+ throw new DapiError(+ "no-project",+ `Could not resolve "${path}": with no project open only absolute paths and URLs resolve — open a project to use library paths.`,+ );+ }+ return new AssetLibrary(createProjectFS("")).resolve(path);+}++/** Narrows the asset to one of `types`, or throws a `wrong-kind` error naming what was expected. */+export function requireAssetType<T extends Asset["type"]>(+ asset: Asset,+ types: readonly T[],+ expected: string,+): asserts asset is Extract<Asset, { type: T }> {+ if (!(types as readonly string[]).includes(asset.type)) {+ throw new DapiError("wrong-kind", `Asset ${asset.id} is not ${expected}.`);+ }+}diff --git a/apps/web/src/context/dapi/frame-triage.ts b/apps/web/src/dapi/lib/frame-triage.tssimilarity index 100%rename from apps/web/src/context/dapi/frame-triage.tsrename to apps/web/src/dapi/lib/frame-triage.tsdiff --git a/apps/web/src/context/dapi/nodes.ts b/apps/web/src/dapi/lib/nodes.tssimilarity index 58%rename from apps/web/src/context/dapi/nodes.tsrename to apps/web/src/dapi/lib/nodes.tsindex 690fb67d..e93772df 100644--- a/apps/web/src/context/dapi/nodes.ts+++ b/apps/web/src/dapi/lib/nodes.ts@@ -4,10 +4,11 @@import { AdjustmentLayer, Geometry, Group, Scene, Source } from "@diffusionstudio/runtime";import { parseSource } from "@diffusionstudio/jsx";+import { DapiError } from "@diffusionstudio/dapi";import type { Entity, World } from "koota";-// Scenes are nodes too, so node-targeting endpoints accept them alongside+// Scenes are nodes too, so node-targeting tools accept them alongside// geometry, groups, and adjustment layers.export function isNode(entity: Entity): boolean {return entity.has(Geometry)@@ -16,13 +17,7 @@ export function isNode(entity: Entity): boolean {|| entity.has(Scene);}-/**- * The entity `id` names. An id is a source id — the durable name an element- * carries in the project's JSX — not a koota entity number, which is minted- * anew on every recompile and so names nothing across calls. The bare id is- * enough while it names one element; `file:id` settles a tie between files,- * and a positional stamp (`index.tsx:12`) addresses elements that have no id.- */+/** The one node an id names, or the error that says why there is not exactly one. */export function resolveNode(world: World, id: string): Entity {const matches = world.query(Source).filter((entity) => {const stamp = entity.get(Source)!.value;@@ -31,20 +26,23 @@ export function resolveNode(world: World, id: string): Entity {return parsed !== undefined && String(parsed.locator) === id;});if (!matches.length) {- throw new Error(`No such node: "${id}" — node ids are the id attributes in the project's JSX`);+ throw new DapiError("not-found", `No such node: "${id}" — node ids are the id attributes in the project's JSX`);}const nodes = matches.filter(isNode);if (!nodes.length) {- throw new Error(`"${id}" is not a node; target a scene, group, clip, or adjustment layer`);+ throw new DapiError("wrong-kind", `"${id}" is not a node; target a scene, group, clip, or adjustment layer`);}if (nodes.length > 1) {const stamps = [...new Set(nodes.map((node) => node.get(Source)!.value))];if (stamps.length === 1) {// One element, many entities: a loop renders its body once per item.- throw new Error(`"${id}" renders ${nodes.length} times (it sits in a loop) — target its scene instead`);+ throw new DapiError("ambiguous", `"${id}" renders ${nodes.length} times (it sits in a loop) — target its scene instead`);}- throw new Error(`"${id}" is ambiguous between ${stamps.map((stamp) => `"${stamp}"`).join(", ")} — use the file:id form`);+ throw new DapiError(+ "ambiguous",+ `"${id}" is ambiguous between ${stamps.map((stamp) => `"${stamp}"`).join(", ")} — use the file:id form`,+ );}return nodes[0]!;}diff --git a/apps/web/src/dapi/lib/png.ts b/apps/web/src/dapi/lib/png.tsnew file mode 100644index 00000000..a0973f95--- /dev/null+++ b/apps/web/src/dapi/lib/png.ts@@ -0,0 +1,9 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/** The bytes of a `data:` URL, for previews the runtime renders that way. */+export function dataUrlToBytes(dataUrl: string): Uint8Array {+ const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);+ return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));+}diff --git a/apps/web/src/dapi/lib/scene.ts b/apps/web/src/dapi/lib/scene.tsnew file mode 100644index 00000000..d62c79fc--- /dev/null+++ b/apps/web/src/dapi/lib/scene.ts@@ -0,0 +1,37 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { getParentNode, isScene, Source } from "@diffusionstudio/runtime";+import { DapiError } from "@diffusionstudio/dapi";+import { resolveNode } from "./nodes";++import type { Entity, World } from "koota";++/**+ * The scene an id names. A scene is the unit an export renders, and capture's+ * promise is that its frames are an export's frames, so both take scenes only:+ * framing an arbitrary node would need its bounds measured across the+ * requested positions first, and that pre-roll runs the project's code ahead+ * of the frames being drawn, which is exactly what an export never does.+ *+ * `verb` names the caller in the error ("capture", "export").+ */+export function requireScene(world: World, id: string, verb: string): Entity {+ const node = resolveNode(world, id);+ if (isScene(node)) return node;++ let scene = getParentNode(node);+ while (scene !== null && !isScene(scene)) scene = getParentNode(scene);+ const stamp = scene?.get(Source)?.value;+ throw new DapiError(+ "wrong-kind",+ stamp+ ? `"${id}" is not a scene — ${verb} renders what an export renders. ${capitalize(verb)} its scene "${stamp}" instead.`+ : `"${id}" is not a scene — ${verb} renders what an export renders, so it takes a scene id.`,+ );+}++function capitalize(word: string): string {+ return word.charAt(0).toUpperCase() + word.slice(1);+}diff --git a/apps/web/src/dapi/lib/sheets.ts b/apps/web/src/dapi/lib/sheets.tsnew file mode 100644index 00000000..76647f3b--- /dev/null+++ b/apps/web/src/dapi/lib/sheets.ts@@ -0,0 +1,87 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { composeSheet, planSheet, planSheetSizes, sheetTimecode } from "@diffusionstudio/encoder";++import type { SheetPlan } from "@diffusionstudio/encoder";+import type { TimecodedImage } from "@diffusionstudio/dapi";++export type SheetFrame = {+ /** Where the frame sits, in the caller's clock; orders cells within a sheet's label. */+ at: number;+ timecode: string;+ image: CanvasImageSource;+};++/**+ * Lays `total` frames out over contact sheets and composes each sheet as+ * soon as its last frame arrives, so a long run never holds every frame at+ * once. Frames may arrive in any order; sheets come out in order.+ *+ * Plans are made up front from the frames' source size so the caller can+ * decode or render no larger than a cell will be drawn (see `cellWidth`,+ * `cellHeight`).+ */+export class SheetCollector {+ readonly plans: SheetPlan[];+ /** The largest cell across sheets: nothing needs to be bigger than this. */+ readonly cellWidth: number;+ readonly cellHeight: number;++ private readonly sizes: number[];+ private readonly sheetOf: number[] = [];+ private readonly firstIndex: number[] = [];+ private readonly missing: number[];+ private readonly frames: Array<SheetFrame | undefined>;+ private readonly sheets: Array<TimecodedImage | undefined>;++ constructor(total: number, source: { width: number; height: number }, perSheet?: number) {+ const sizes = planSheetSizes(total, perSheet);+ this.sizes = sizes;+ this.plans = sizes.map((size) => planSheet(size, source));+ this.cellWidth = Math.max(...this.plans.map((plan) => plan.cellWidth));+ this.cellHeight = Math.max(...this.plans.map((plan) => plan.cellHeight));+ for (const [sheet, size] of sizes.entries()) {+ this.firstIndex.push(this.sheetOf.length);+ for (let k = 0; k < size; k++) this.sheetOf.push(sheet);+ }+ this.missing = [...sizes];+ this.frames = new Array(total);+ this.sheets = new Array(sizes.length);+ }++ /**+ * Adds the frame at `index` (its position in the requested order). When it+ * completes a sheet, that sheet is composed and its images released:+ * ImageBitmaps are closed, and every frame image is dropped from memory.+ */+ async add(index: number, frame: SheetFrame): Promise<void> {+ this.frames[index] = frame;+ const sheet = this.sheetOf[index]!;+ if (--this.missing[sheet]! > 0) return;++ const from = this.firstIndex[sheet]!;+ const to = from + this.sizes[sheet]!;+ const cells = this.frames.slice(from, to).filter((f): f is SheetFrame => f !== undefined);+ this.sheets[sheet] = {+ timecode: sheetTimecode(cells),+ png: await composeSheet(+ cells.map((cell) => ({ image: cell.image, label: cell.timecode })),+ this.plans[sheet]!,+ ),+ };+ for (const cell of cells) {+ if (cell.image instanceof ImageBitmap) cell.image.close();+ }+ this.frames.fill(undefined, from, to);+ }++ /** Every sheet, in order. Only complete once every frame has been added. */+ result(): TimecodedImage[] {+ return this.sheets.map((sheet, i) => {+ if (!sheet) throw new Error(`Contact sheet ${i} is missing frames`);+ return sheet;+ });+ }+}diff --git a/apps/web/src/context/dapi/session.ts b/apps/web/src/dapi/session.tssimilarity index 70%rename from apps/web/src/context/dapi/session.tsrename to apps/web/src/dapi/session.tsindex 4b4d1429..8a7819ae 100644--- a/apps/web/src/context/dapi/session.ts+++ b/apps/web/src/dapi/session.ts@@ -2,13 +2,13 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// The CLI router mounts once for the app's lifetime (see ./api); what changes-// as projects open and close is this slot. The editor publishes its world and-// project here while it is mounted, and project-bound endpoints read the slot-// per request instead of closing over a world at registration time.+// The tool handlers register once for the app's lifetime (see ./api); what+// changes as projects open and close is this slot. The editor publishes its+// world and project here while it is mounted, and project-bound handlers read+// the slot per request instead of closing over a world at registration time.import { createSignal } from "solid-js";-import { assert } from "@/utils/common";+import { DapiError } from "@diffusionstudio/dapi";import type { Accessor } from "solid-js";import type { World } from "koota";@@ -34,9 +34,9 @@ export const editorSession: Accessor<EditorSession | null> = session;/** Set by the editor while a project is open; cleared on its way out. */export const setEditorSession = setSession;-/** The session, or the failure a CLI caller can act on. */+/** The session, or the failure a caller can act on. */export function requireEditorSession(): EditorSession {const current = session();- assert(current, "No project open — run `dapi open <dir>` first.");+ if (!current) throw new DapiError("no-project", "No project open — open one first (`dapi open <dir>`).");return current;}diff --git a/apps/web/src/hooks/use-fullscreen-state.ts b/apps/web/src/hooks/use-fullscreen-state.tsindex 8823fcbe..e407d1b1 100644--- a/apps/web/src/hooks/use-fullscreen-state.ts+++ b/apps/web/src/hooks/use-fullscreen-state.ts@@ -4,7 +4,6 @@import { createEffect, createResource, onCleanup, onMount } from "solid-js";-import { handleGetFullscreenState, handleWindowFullscreenChange } from "@/context/dapi/window";import { mainBridge } from "@/lib/ipc";import { MAIN_CHANNELS } from "@desktop/main-channels";@@ -18,9 +17,10 @@ import { MAIN_CHANNELS } from "@desktop/main-channels";* lives here rather than inside the editor-only API provider.*/export function useFullscreenState() {- const [isFullscreen, { mutate }] = createResource(handleGetFullscreenState, {- initialValue: false,- });+ const [isFullscreen, { mutate }] = createResource(+ () => (window.desktop ? mainBridge.call(MAIN_CHANNELS.WINDOW_IS_FULLSCREEN, undefined) : Promise.resolve(false)),+ { initialValue: false },+ );createEffect(() => {document.documentElement.dataset.fullscreen = String(isFullscreen());@@ -29,10 +29,7 @@ export function useFullscreenState() {onMount(() => {if (!window.desktop) return;onCleanup(- mainBridge.handle(- MAIN_CHANNELS.WINDOW_FULLSCREEN_CHANGE,- handleWindowFullscreenChange(mutate),- ),+ mainBridge.handle(MAIN_CHANNELS.WINDOW_FULLSCREEN_CHANGE, ({ fullscreen }) => mutate(fullscreen)),);});diff --git a/apps/web/src/lib/cli-rpc.ts b/apps/web/src/lib/cli-rpc.tsdeleted file mode 100644index dbaf1e12..00000000--- a/apps/web/src/lib/cli-rpc.ts+++ /dev/null@@ -1,45 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { initTRPC } from "@trpc/server";-import type { AnyTRPCRouter } from "@trpc/server";-import type { ProcedureCaller, RouterCaller } from "./ipc";--// tRPC instance for the CLI-facing routers. Procedures close over their-// dependencies (engine, session), so there is no request context. The-// renderer IS the server here, so opt out of tRPC's browser guard.-export const t = initTRPC.create({ allowOutsideOfServer: true });--// Inputs come pre-typed from the tRPC client, so parsers are identity casts-// rather than schemas; validation stays where it always was, in the handlers.-const input =- <I,>() =>- (value: unknown) =>- value as I;--// Lift an existing unary handler into a procedure, inferring its input and-// output types. The cast collapses tRPC's conditional input type, which-// stays unresolved for a generic I. Zero-argument handlers use q0/m0: they-// skip .input() so the client can call them without an argument.-export const q = <O, I>(fn: (data: I) => O | Promise<O>) =>- t.procedure.input(input<I>()).query(({ input: data }) => fn(data as I));--export const m = <O, I>(fn: (data: I) => O | Promise<O>) =>- t.procedure.input(input<I>()).mutation(({ input: data }) => fn(data as I));--export const q0 = <O,>(fn: () => O | Promise<O>) => t.procedure.query(() => fn());--export const m0 = <O,>(fn: () => O | Promise<O>) => t.procedure.mutation(() => fn());--// Adapts a router for the CLI bridge: resolve a dot-joined procedure path to-// an invocable, or undefined when this router doesn't own the path (the-// bridge then tries other routers or holds the request).-export function createRouterCaller(router: AnyTRPCRouter): RouterCaller {- const caller = t.createCallerFactory(router)({});- return (path: string): ProcedureCaller | undefined => {- if (!(path in router._def.procedures)) return undefined;- return (data: unknown) =>- path.split(".").reduce<any>((o, key) => o[key], caller)(data) as Promise<unknown>;- };-}diff --git a/apps/web/src/lib/ipc.ts b/apps/web/src/lib/ipc.tsindex 54447eaf..090b6359 100644--- a/apps/web/src/lib/ipc.ts+++ b/apps/web/src/lib/ipc.ts@@ -12,23 +12,15 @@ import type {MainRequestChannel,MainRequestMap,} from "@desktop/main-channels";-import { CLI_WIRE } from "@diffusionstudio/cli/protocol";-import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol";type EventHandler<C extends MainEventChannel> = (data: MainEventMap[C]) => void;-export type ProcedureCaller = (input: unknown) => Promise<unknown>;--// Resolves a dot-joined tRPC procedure path to an invocable, or undefined-// when the router doesn't own that path.-export type RouterCaller = (path: string) => ProcedureCaller | undefined;-type Pending = {resolve: (value: unknown) => void;reject: (error: Error) => void;};-// Renderer↔main bridge. Symmetric with cliBridge: `handle` registers a+// Renderer↔main bridge: `handle` registers a// single-subscriber receiver for inbound channels (events from main); `call`// sends a request to main and awaits the reply.class MainBridge {@@ -94,87 +86,3 @@ class MainBridge {}export const mainBridge = new MainBridge();--type PendingCliRequest = { req: CliRequest; ws: WebSocket };--// CLI bridge — answers CLI requests. Each CLI command hosts a short-lived-// WebSocket server; main relays only the connect info (CLI_WIRE.CONNECT) and-// we dial the CLI directly, so payloads never pass through main. One request-// and one reply per connection. The bridge is transport-only: the tRPC-// router registers as a path resolver, and requests arriving before it-// mounts (or between remounts on project switches) are held and retried on-// the next registration — no explicit ready signal needed.-class CliBridge {- // Several routers coexist: the shell router (always mounted, owns `ping`- // and `open`) and the editor router (mounted per open project). Paths are- // disjoint; a request is answered by whichever router owns its path.- private routers = new Set<RouterCaller>();- private pending: PendingCliRequest[] = [];-- constructor() {- // Bind eagerly so CONNECT arrivals during page bootstrap are caught- // rather than silently dropped before any router registers.- if (window.desktop) {- window.desktop.on(CLI_WIRE.CONNECT, (payload) => {- const { port, token } = payload as CliHandshake;- const ws = new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`);- ws.onmessage = (event) => {- try {- const req = JSON.parse(event.data as string) as CliRequest;- void this.dispatch({ req, ws });- } catch (err) {- console.error("[cli-bridge] malformed CLI request", err);- ws.close();- }- };- });- }- }-- private resolve(path: string): ProcedureCaller | undefined {- for (const router of this.routers) {- const proc = router(path);- if (proc) return proc;- }- return undefined;- }-- private async dispatch(pending: PendingCliRequest): Promise<void> {- const { req, ws } = pending;- const proc = this.resolve(req.path);- if (!proc) {- this.pending.push(pending);- return;- }- let reply: CliReply;- try {- const data = await proc(req.input);- reply = { ok: true, data };- } catch (err) {- reply = { ok: false, error: (err as Error).message };- }- try {- ws.send(JSON.stringify(reply));- } catch (err) {- ws.send(- JSON.stringify({- ok: false,- error: `Failed to serialize reply for ${req.path}: ${(err as Error).message}`,- }),- );- }- }-- register(router: RouterCaller): () => void {- if (!window.desktop) return () => {};- this.routers.add(router);- const held = this.pending;- this.pending = [];- for (const pending of held) void this.dispatch(pending);- return () => {- this.routers.delete(router);- };- }-}--export const cliBridge = new CliBridge();diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsxindex 81e054ef..1c3d9437 100644--- a/apps/web/src/pages/editor.tsx+++ b/apps/web/src/pages/editor.tsx@@ -8,7 +8,7 @@ import { Timeline, Layers } from "@/components/timeline";import { Soundboard, Inspector } from "@/components/sidebar-right";import { FloatingProjectHeader, SidebarLeft } from "@/components/sidebar-left";import { useLayout, MIN_TIMELINE_HEIGHT } from "@/context/layout";-import { useEditorApi } from "@/context/dapi";+import { useEditorApi } from "@/dapi";import { RULER_HEIGHT } from "@/engine/timeline";import { createEffect, onCleanup, untrack } from 'solid-js';import { toast } from 'somoto';diff --git a/apps/web/src/pages/project.tsx b/apps/web/src/pages/project.tsxindex f6cf3ada..f0ff44ca 100644--- a/apps/web/src/pages/project.tsx+++ b/apps/web/src/pages/project.tsx@@ -7,7 +7,7 @@ import { Navigate, useNavigate } from '@solidjs/router';import { EditorPage } from './editor';import { LayoutProvider } from "@/context/layout";import { PromptInputProvider } from "@/context/prompt-input";-import { EditorApiProvider } from '@/context/dapi';+import { EditorApiProvider } from '@/dapi';import { ExportProvider } from '@/context/export';import { ProjectProvider } from '@/context/project';import { projectRoute, useProjectRef } from '@/hooks/use-project-route';diff --git a/docs/mcp-server.md b/docs/mcp-server.mdindex 5b93ba17..d32644b3 100644--- a/docs/mcp-server.md+++ b/docs/mcp-server.md@@ -193,7 +193,7 @@ Removed from the CLI package: `cli-client.ts`, `cli-channels.ts`, the tRPC and `The CLI ships inside the app bundle, so there is no compatibility window to maintain between old CLIs and new apps. This can land as one feature branch in ordered steps, each of which builds and type-checks on its own.1. **Catalog.** Create `packages/dapi` with the tool catalog, `Time`, and `DapiError`, plus tests. Point the renderer handlers' request and result types at it. Delete `apps/cli/src/cli-channels.ts`; `apps/cli/src/protocol.ts` keeps only the handshake and envelope types of the current transport. No behaviour change. *Landed on `mcp-support`.*-2. **Renderer.** Replace the tRPC router in `api.tsx` with a handler map keyed by tool name, validated by the catalog schemas, with `AbortSignal` and progress plumbed in. Replace the WebSocket half of `CliBridge` with the `dapi:call` IPC handling. At this step the old CLI stops working; that is expected.+2. **Renderer.** Replace the tRPC router in `api.tsx` with one handler per tool behind a uniform `(args, ctx)` signature, validated by the catalog schemas, with an `AbortSignal` per call. Handlers return bytes; the WebSocket transport carries them as tagged base64 until step 3 replaces it. The CLI drops tRPC for a typed `call(name, input)` over the same catalog, so it keeps working at every step and its import of `apps/web` is gone. Progress reporting waits for step 3, with the transport that carries it. *Landed on `mcp-support`.*3. **Main.** Add `mcp-server.ts` with the socket transport, catalog registration, the in-flight map, result presentation, and the `logs`, `fonts`, `fetch` and `report` handlers. Delete `cli-server.ts`. Verify with the MCP Inspector against the socket.4. **CLI.** Rewrite `apps/cli` as `mcp`, `open`, `call` and the wrappers. Drop tRPC and `ws`. Update `cli-install.ts` and the settings UI to offer MCP registration.5. **Docs.** Update the skill and regenerate `reference/`. Update the installation reference the skill points to.diff --git a/package-lock.json b/package-lock.jsonindex f16d06a3..23f685b9 100644--- a/package-lock.json+++ b/package-lock.json@@ -25,8 +25,6 @@"@babel/core": "^7.29.7","@babel/preset-typescript": "^7.27.1","@diffusionstudio/dapi": "*",- "@trpc/client": "^11.18.0",- "@trpc/server": "^11.18.0","babel-preset-solid": "^1.9.12","commander": "^14.0.3","esbuild": "^0.28.1",diff --git a/packages/dapi/src/catalog.test.ts b/packages/dapi/src/catalog.test.tsindex cba7fd8e..c0641573 100644--- a/packages/dapi/src/catalog.test.ts+++ b/packages/dapi/src/catalog.test.ts@@ -26,10 +26,10 @@ describe("catalog", () => {}});- it("converts every schema to JSON Schema without throwing", () => {+ it("converts every schema to JSON Schema without throwing (bytes have no JSON form and pass as any)", () => {for (const tool of catalog) {expect(() => z.toJSONSchema(tool.input, { io: "input" }), `${tool.name} input`).not.toThrow();- expect(() => z.toJSONSchema(tool.output), `${tool.name} output`).not.toThrow();+ expect(() => z.toJSONSchema(tool.output, { unrepresentable: "any" }), `${tool.name} output`).not.toThrow();}});diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.tsindex d6ab997b..da23f6c8 100644--- a/packages/dapi/src/index.ts+++ b/packages/dapi/src/index.ts@@ -19,7 +19,8 @@ export type { TimeInput } from "./time";export { DapiError, isDapiError } from "./errors";export type { DapiErrorCode } from "./errors";-export { MAX_FRAMES_PER_SHEET } from "./schemas";+export { MAX_FRAMES_PER_SHEET, Bytes } from "./schemas";+export { parseToolArgs } from "./validate";export { FRAME_CAP } from "./tools/media-grab";export { ISSUE_LOG_TAIL } from "./tools/report";diff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.tsindex 22ffad93..7557a8c7 100644--- a/packages/dapi/src/schemas.ts+++ b/packages/dapi/src/schemas.ts@@ -29,12 +29,18 @@ export const AssetPath = zexport const MAX_FRAMES_PER_SHEET = 12;/**- * One written image: a single frame stamped with its timecode, or a contact- * sheet stamped with the span it covers (`0f-08s10f`).+ * Raw bytes as a handler produces them. How they travel is the transport's+ * business: base64 on a JSON wire, structured clone over IPC, a file on disk.+ */+export const Bytes = z.custom<Uint8Array>((value) => value instanceof Uint8Array, "expected bytes (a Uint8Array)");++/**+ * One image: a single frame stamped with its timecode, or a contact sheet+ * stamped with the span it covers (`0f-08s10f`).*/export const TimecodedImage = z.object({timecode: z.string(),- base64: z.string().describe("PNG bytes, base64"),+ png: Bytes,});/**diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.tsindex c6b40ba8..9c30c007 100644--- a/packages/dapi/src/tools/context.ts+++ b/packages/dapi/src/tools/context.ts@@ -32,11 +32,11 @@ export const context = defineTool({input: z.object({}),output: z.union([z.object({- rootDir: z.string().describe("folder projects live under"),+ rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"),projectDir: z.null(),}),z.object({- rootDir: z.string().describe("folder projects live under"),+ rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"),projectDir: z.string().describe("absolute path of the open project"),currentTime: z.number()diff --git a/packages/dapi/src/tools/media-filmstrip.ts b/packages/dapi/src/tools/media-filmstrip.tsindex 1de24cc2..dd9cdd1c 100644--- a/packages/dapi/src/tools/media-filmstrip.ts+++ b/packages/dapi/src/tools/media-filmstrip.ts@@ -4,7 +4,7 @@import { z } from "zod";import { defineTool } from "../tool";-import { AssetPath, checkWindow, windowFields } from "../schemas";+import { AssetPath, Bytes, checkWindow, windowFields } from "../schemas";/** The window and scale that filmstrip and waveform share. */export const previewFields = {@@ -22,6 +22,6 @@ export const mediaFilmstrip = defineTool({description:"Render a grid of thumbnails sampled across the timeline to a PNG (local render, no credits), each row stamped with an HH:MM:SS:FF ruler. A fast, token-efficient video track preview; narrow the window to zoom into a region of interest. Video only (use media_waveform for audio).",input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow),- output: z.looseObject({ base64: z.string().describe("PNG bytes, base64") }),+ output: z.looseObject({ png: Bytes }),runsIn: "renderer",});diff --git a/packages/dapi/src/tools/media-waveform.ts b/packages/dapi/src/tools/media-waveform.tsindex 61a17c45..d0691d5d 100644--- a/packages/dapi/src/tools/media-waveform.ts+++ b/packages/dapi/src/tools/media-waveform.ts@@ -4,7 +4,7 @@import { z } from "zod";import { defineTool } from "../tool";-import { AssetPath, checkWindow } from "../schemas";+import { AssetPath, Bytes, checkWindow } from "../schemas";import { previewFields } from "./media-filmstrip";export const mediaWaveform = defineTool({@@ -14,7 +14,7 @@ export const mediaWaveform = defineTool({"Render the audio track of a video or audio file as a waveform PNG (local render, no credits) with a timestamp ruler: loudness over time, with silent stretches highlighted in red. A fast, token-efficient audio track preview; the silent spans are also returned as second ranges.",input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow),output: z.looseObject({- base64: z.string().describe("PNG bytes, base64"),+ png: Bytes,silences: z.array(z.object({ start: z.number(), end: z.number() })).describe("seconds"),}),runsIn: "renderer",diff --git a/packages/dapi/src/tools/screenshot.ts b/packages/dapi/src/tools/screenshot.tsindex c5f7b5ed..9f54bd58 100644--- a/packages/dapi/src/tools/screenshot.ts+++ b/packages/dapi/src/tools/screenshot.ts@@ -4,6 +4,7 @@import { z } from "zod";import { defineTool } from "../tool";+import { Bytes } from "../schemas";export const screenshot = defineTool({name: "screenshot",@@ -12,7 +13,7 @@ export const screenshot = defineTool({"Capture the entire application window as a PNG — the full UI as the user sees it (panels, timeline, asset library, canvas viewport), at the window's current size. The tool for checking what the app itself looks like; to render a node or scene cleanly for composition checks use capture instead.",input: z.object({}),output: z.object({- base64: z.string().describe("PNG bytes, base64"),+ png: Bytes,width: z.number(),height: z.number(),}),diff --git a/packages/dapi/src/tools/whoami.ts b/packages/dapi/src/tools/whoami.tsindex d1ba3f9d..f2d5cf0b 100644--- a/packages/dapi/src/tools/whoami.ts+++ b/packages/dapi/src/tools/whoami.ts@@ -10,6 +10,6 @@ export const whoami = defineTool({title: "Signed-in account",description: "Report the authenticated account, or null if signed out.",input: z.object({}),- output: z.looseObject({ id: z.string(), email: z.string().optional() }).nullable(),+ output: z.object({ id: z.string(), email: z.string().optional() }).nullable(),runsIn: "renderer",});diff --git a/packages/dapi/src/validate.test.ts b/packages/dapi/src/validate.test.tsnew file mode 100644index 00000000..6a14b389--- /dev/null+++ b/packages/dapi/src/validate.test.ts@@ -0,0 +1,28 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { describe, expect, it } from "vitest";+import { isDapiError } from "./errors";+import { parseToolArgs } from "./validate";++describe("parseToolArgs", () => {+ it("returns parsed arguments with defaults applied", () => {+ expect(parseToolArgs("capture", { id: "intro" })).toEqual({ id: "intro", combine: true });+ });++ it("throws an invalid-input DapiError that names every bad field on one line", () => {+ try {+ parseToolArgs("media_grab", { path: "/c.mp4", times: ["abc"], count: 0 });+ } catch (e) {+ expect(isDapiError(e) && e.code).toBe("invalid-input");+ const message = (e as Error).message;+ expect(message).toMatch(/^Invalid arguments for media_grab: /);+ expect(message).toMatch(/times\.0: expected a time/);+ expect(message).toMatch(/count: /);+ expect(message).not.toContain("\n");+ return;+ }+ throw new Error("expected a throw");+ });+});diff --git a/packages/dapi/src/validate.ts b/packages/dapi/src/validate.tsnew file mode 100644index 00000000..89c5e776--- /dev/null+++ b/packages/dapi/src/validate.ts@@ -0,0 +1,25 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { z } from "zod";+import { toolByName } from "./catalog";+import type { ToolArgs, ToolName } from "./catalog";+import { DapiError } from "./errors";++/**+ * Validates a tool's raw arguments against its schema. Every issue is+ * reported on one line, keyed by the field it points at, so an agent can+ * correct the call from the message alone.+ */+export function parseToolArgs<N extends ToolName>(name: N, raw: unknown): ToolArgs<N> {+ const result = toolByName(name).input.safeParse(raw);+ if (result.success) return result.data as ToolArgs<N>;+ throw new DapiError("invalid-input", `Invalid arguments for ${name}: ${formatIssues(result.error.issues)}`, {+ cause: result.error,+ });+}++function formatIssues(issues: readonly z.core.$ZodIssue[]): string {+ return issues.map((issue) => `${issue.path.map(String).join(".") || "input"}: ${issue.message}`).join("; ");+}diff --git a/packages/encoder/src/contact-sheet.ts b/packages/encoder/src/contact-sheet.tsindex 37259a8b..ac14ed39 100644--- a/packages/encoder/src/contact-sheet.ts+++ b/packages/encoder/src/contact-sheet.ts@@ -3,6 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { stampTimestampLabel } from '@diffusionstudio/runtime';+import { encodePng } from './png';/*** Contact sheets: a handful of frames merged into one labelled image, so a@@ -112,7 +113,7 @@ export function planSheetSizes(total: number, perSheet?: number): number[] {export async function composeSheet(frames: Array<{ image: CanvasImageSource; label: string }>,plan: SheetPlan,-): Promise<string> {+): Promise<Uint8Array> {const canvas = new OffscreenCanvas(plan.width, plan.height);const ctx = canvas.getContext('2d')!;@@ -133,17 +134,5 @@ export async function composeSheet(ctx.restore();}- const blob = await canvas.convertToBlob({ type: 'image/png' });- const dataUrl = await new Promise<string>((resolve, reject) => {- const reader = new FileReader();- reader.onload = () => resolve(reader.result as string);- reader.onerror = () => reject(reader.error);- reader.readAsDataURL(blob);- });- return dataUrl.slice(dataUrl.indexOf(',') + 1);-}--export async function decodePngBase64(base64: string): Promise<ImageBitmap> {- const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));- return createImageBitmap(new Blob([bytes], { type: 'image/png' }));+ return encodePng(canvas);}diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.tsindex c0bf0f48..03919812 100644--- a/packages/encoder/src/image-encoder.ts+++ b/packages/encoder/src/image-encoder.ts@@ -11,12 +11,13 @@ import {import { captureScene, normalizeSceneTransform, resolverSystem, warmupAssets } from './encoder';import { scaleSize } from './utils';+import { encodePng } from './png';import type { World } from 'koota';import type { ImageEncoderConfig } from './interfaces';-/** One capture: the PNG plus the timecode of the frame rendered, e.g. `01s15f`. */-export type CapturedImage = { base64: string; timecode: string };+/** One capture: the PNG bytes plus the timecode of the frame rendered, e.g. `01s15f`. */+export type CapturedImage = { png: Uint8Array; timecode: string };export type ImageExportResult =| { type: 'success'; data: CapturedImage[] }@@ -25,7 +26,7 @@ export type ImageExportResult =/*** Renders single frames of the scene a capture world holds into standalone- * PNGs (base64, no data-url prefix).+ * PNGs.** `world` is the caller's, built for this capture and holding the scene as* the stage's only child — the same arrangement `createEncoder` takes, and@@ -125,7 +126,7 @@ export async function createImageEncoder(world: World, config: ImageEncoderConfirenderSystem(world);images.set(frame, {- base64: await toBase64Png(canvas),+ png: await encodePng(canvas),timecode: formatTimecode(playheadSeconds, frameRate),});}@@ -148,15 +149,3 @@ export async function createImageEncoder(world: World, config: ImageEncoderConfi};}-async function toBase64Png(canvas: HTMLCanvasElement): Promise<string> {- const blob = await new Promise<Blob>((resolve, reject) => {- canvas.toBlob((value) => value ? resolve(value) : reject(new Error('Could not encode PNG')), 'image/png');- });- const dataUrl = await new Promise<string>((resolve, reject) => {- const reader = new FileReader();- reader.onload = () => resolve(reader.result as string);- reader.onerror = () => reject(reader.error);- reader.readAsDataURL(blob);- });- return dataUrl.split(',')[1] ?? '';-}diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.tsindex ab8b77dd..433dd698 100644--- a/packages/encoder/src/index.ts+++ b/packages/encoder/src/index.ts@@ -8,5 +8,6 @@ export * from './encoder';export * from './format';export * from './image-encoder';export * from './interfaces';+export * from './png';export * from './types';export * from './utils';diff --git a/packages/encoder/src/png.ts b/packages/encoder/src/png.tsnew file mode 100644index 00000000..905b6190--- /dev/null+++ b/packages/encoder/src/png.ts@@ -0,0 +1,21 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// PNG in and out of a canvas, as bytes. Transports decide how bytes travel+// (base64 on a JSON wire, structured clone over IPC, a file on disk); nothing+// in the render path should have to know.++export async function encodePng(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Uint8Array> {+ const blob =+ canvas instanceof OffscreenCanvas+ ? await canvas.convertToBlob({ type: 'image/png' })+ : await new Promise<Blob>((resolve, reject) => {+ canvas.toBlob((value) => (value ? resolve(value) : reject(new Error('Could not encode PNG'))), 'image/png');+ });+ return new Uint8Array(await blob.arrayBuffer());+}++export function decodePng(png: Uint8Array): Promise<ImageBitmap> {+ return createImageBitmap(new Blob([png as BlobPart], { type: 'image/png' }));+}