Skip to content

Code-first video & image, rendered natively on WebGPU.

ai lib

okanasldcee493d26a7
diff --git a/.changeset/framefields-ai.md b/.changeset/framefields-ai.md
new file mode 100644
index 00000000..9038e4ed
--- /dev/null
+++ b/.changeset/framefields-ai.md
@@ -0,0 +1,6 @@
+---
+"framefields": minor
+"@framefields/ai": minor
+---
+
+Add `ai`: AI models (images, video, music, voice, …) through the framefields AI gateway, authenticated with `FRAMEFIELDS_API_KEY` and billed to your framefields account. `subscribe`, `run`, `stream`, `queue` and `storage`, with no dependencies: `import { ai } from "framefields"`.
diff --git a/packages/ai/README.md b/packages/ai/README.md
new file mode 100644
index 00000000..0818af03
--- /dev/null
+++ b/packages/ai/README.md
@@ -0,0 +1,29 @@
+# @framefields/ai
+
+AI models (images, video, music, sound effects, voice, upscaling, background removal) through the framefields AI gateway, billed to your framefields account. No dependencies. Use it from the main package:
+
+```ts
+import { ai } from "framefields";
+
+// FRAMEFIELDS_API_KEY from the environment, or:
+ai.config({ apiKey: process.env.FRAMEFIELDS_API_KEY });
+
+const { data } = await ai.subscribe<{ images: { url: string }[] }>("flux/schnell", {
+ input: { prompt: "Neon street at night" },
+});
+console.log(data.images[0].url);
+```
+
+| Method | Use |
+| --- | --- |
+| `ai.run(model, { input })` | One request; best for fast models. |
+| `ai.subscribe(model, { input, onQueueUpdate, logs })` | Queues the job, polls, returns the output. Use for video and music. |
+| `ai.stream(model, { input })` | Iterate partial output; `await stream.done()` for the last. |
+| `ai.queue.submit / status / result / cancel` | Run many jobs and collect them later by `requestId`. |
+| `ai.storage.upload(blob)` | Upload a local file and pass the URL as input. `Blob`s inside `input` upload automatically. |
+
+Failed calls throw `ApiError` (`status`, `type`, `body`, `requestId`), or `ValidationError` with `fieldErrors` when the input doesn't match the model's schema. `type` is the gateway's code, e.g. `insufficient_credits` (top up at https://framefields.dev/dashboard/billing) or `rate_limited`.
+
+Model ids and their input schemas: https://framefields.dev/models. Keys: https://framefields.dev/dashboard/keys.
+
+`FRAMEFIELDS_AI_PROXY_URL` overrides the gateway URL, for running the gateway locally.
diff --git a/packages/ai/package.json b/packages/ai/package.json
new file mode 100644
index 00000000..d13e6017
--- /dev/null
+++ b/packages/ai/package.json
@@ -0,0 +1,39 @@
+{
+ "name": "@framefields/ai",
+ "version": "2.3.2",
+ "private": false,
+ "description": "AI models (images, video, music, voice) through the framefields AI gateway, authenticated with a FRAMEFIELDS_API_KEY.",
+ "publishConfig": {
+ "access": "public"
+ },
+ "type": "module",
+ "sideEffects": false,
+ "license": "Apache-2.0",
+ "module": "./dist/index.mjs",
+ "types": "./dist/index.d.mts",
+ "exports": {
+ ".": {
+ "types": "./dist/index.d.mts",
+ "development": "./src/index.ts",
+ "import": "./dist/index.mjs",
+ "default": "./dist/index.mjs"
+ }
+ },
+ "files": [
+ "dist/**"
+ ],
+ "scripts": {
+ "build": "tsdown && node scripts/check-provider-neutral.mjs --dist",
+ "dev": "tsdown --watch",
+ "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist",
+ "test": "node scripts/check-provider-neutral.mjs && vitest run --passWithNoTests",
+ "typecheck": "tsc --noEmit -p ."
+ },
+ "devDependencies": {
+ "@framefields/tsconfig": "workspace:*",
+ "@types/node": "^24.1.0",
+ "tsdown": "^0.19.0",
+ "typescript": "^5.8.3",
+ "vitest": "4.1.5"
+ }
+}
diff --git a/packages/ai/scripts/check-provider-neutral.mjs b/packages/ai/scripts/check-provider-neutral.mjs
new file mode 100644
index 00000000..fa76f752
--- /dev/null
+++ b/packages/ai/scripts/check-provider-neutral.mjs
@@ -0,0 +1,53 @@
+// Fails when the package source, docs or build output depend on or disclose the upstream SDK / hosts.
+// Note: Model endpoint identifiers (e.g. "fal-ai/flux/schnell") are allowed.
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative } from "node:path";
+
+const root = new URL("..", import.meta.url).pathname;
+// `--dist` also scans the build output; the build script passes it right after building.
+const scan = ["src", "README.md", "package.json"];
+if (process.argv.includes("--dist")) scan.push("dist");
+// Encoded so this file does not match itself.
+const names = ["ZmFs", "ZmFsLWFp", "ZmFsX2Fp"].map((b) => atob(b));
+const pattern = new RegExp(`\\b(${names.join("|")})\\b|\\.${names[0]}\\.`, "i");
+
+function* files(path) {
+ let stat;
+ try {
+ stat = statSync(path);
+ } catch {
+ return;
+ }
+ if (!stat.isDirectory()) {
+ yield path;
+ return;
+ }
+ for (const entry of readdirSync(path)) yield* files(join(path, entry));
+}
+
+const hits = [];
+for (const target of scan) {
+ for (const file of files(join(root, target))) {
+ readFileSync(file, "utf8")
+ .split("\n")
+ .forEach((line, i) => {
+ // Strip permitted model names, prefixes and target paths before scanning
+ const stripped = line
+ .replace(
+ /(?:run|queue|storage)?\/?fal(?:-ai)?\/[a-zA-Z0-9_\-./]+/gi,
+ "",
+ )
+ .replace(/fal(?:-ai)?\//gi, "");
+ if (pattern.test(stripped))
+ hits.push(`${relative(root, file)}:${i + 1}`);
+ });
+ }
+}
+
+if (hits.length) {
+ console.error(
+ `Upstream provider leak in @framefields/ai:\n ${hits.join("\n ")}`,
+ );
+ process.exit(1);
+}
+process.exit(0);
diff --git a/packages/ai/src/errors.ts b/packages/ai/src/errors.ts
new file mode 100644
index 00000000..81bb18e3
--- /dev/null
+++ b/packages/ai/src/errors.ts
@@ -0,0 +1,96 @@
+/** One field a model rejected: where (`loc`), why (`msg`) and the rule (`type`). */
+export interface ValidationErrorInfo {
+ msg: string;
+ loc: Array<string | number>;
+ type: string;
+}
+
+export interface ApiErrorArgs {
+ message: string;
+ status: number;
+ body?: unknown;
+ requestId?: string;
+ type?: string;
+}
+
+/**
+ * A non-2xx answer from the gateway or the model. `type` is the gateway's error code
+ * (`insufficient_credits`, `rate_limited`, `key_limit_reached`, …) when the gateway refused
+ * the call; it is unset when the model itself failed.
+ */
+export class ApiError<Body = unknown> extends Error {
+ readonly status: number;
+ readonly body: Body;
+ readonly requestId: string;
+ readonly type?: string;
+
+ constructor({ message, status, body, requestId, type }: ApiErrorArgs) {
+ super(message);
+ this.name = "ApiError";
+ this.status = status;
+ this.body = body as Body;
+ this.requestId = requestId ?? "";
+ this.type = type;
+ }
+}
+
+export interface ValidationErrorBody {
+ detail: ValidationErrorInfo[];
+}
+
+/** A `422`: the input did not match the model's schema. `fieldErrors` says which fields. */
+export class ValidationError extends ApiError<ValidationErrorBody> {
+ constructor(args: ApiErrorArgs) {
+ super(args);
+ this.name = "ValidationError";
+ }
+
+ get fieldErrors(): ValidationErrorInfo[] {
+ return Array.isArray(this.body?.detail) ? this.body.detail : [];
+ }
+
+ getFieldErrors(field: string): ValidationErrorInfo[] {
+ return this.fieldErrors.filter((e) => e.loc[e.loc.length - 1] === field);
+ }
+}
+
+interface ErrorBody {
+ error?: { type?: string; message?: string };
+ detail?: unknown;
+ message?: unknown;
+}
+
+function messageOf(body: ErrorBody | undefined, fallback: string): string {
+ if (body?.error?.message) return body.error.message;
+ if (typeof body?.detail === "string") return body.detail;
+ if (Array.isArray(body?.detail)) {
+ const first = body.detail[0] as Partial<ValidationErrorInfo> | undefined;
+ if (first?.msg) return `${first.msg} (${(first.loc ?? []).join(".")})`;
+ }
+ if (typeof body?.message === "string") return body.message;
+ return fallback;
+}
+
+/** Builds the error for a failed response; the body is read here, once. */
+export async function errorFromResponse(response: Response): Promise<ApiError> {
+ const text = await response.text().catch(() => "");
+ let body: unknown = text || undefined;
+ try {
+ body = text ? JSON.parse(text) : undefined;
+ } catch {}
+ const parsed =
+ typeof body === "object" && body !== null ? (body as ErrorBody) : undefined;
+ const args: ApiErrorArgs = {
+ message: messageOf(
+ parsed,
+ `${response.status} ${response.statusText}`.trim(),
+ ),
+ status: response.status,
+ body,
+ requestId: response.headers.get("x-framefields-request-id") ?? undefined,
+ type: parsed?.error?.type,
+ };
+ return response.status === 422 && Array.isArray(parsed?.detail)
+ ? new ValidationError(args)
+ : new ApiError(args);
+}
diff --git a/packages/ai/src/index.test.ts b/packages/ai/src/index.test.ts
new file mode 100644
index 00000000..45650803
--- /dev/null
+++ b/packages/ai/src/index.test.ts
@@ -0,0 +1,281 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ ApiError,
+ ai,
+ FRAMEFIELDS_AI_PROXY_URL,
+ type QueueStatus,
+ ValidationError,
+} from "./index.js";
+
+interface Call {
+ url: string;
+ method: string;
+ target: string | null;
+ headers: Headers;
+ body: unknown;
+}
+
+type Handler = (call: Call) => Response | Promise<Response>;
+
+const calls: Call[] = [];
+let handler: Handler;
+
+const json = (body: unknown, init: ResponseInit = {}) =>
+ new Response(JSON.stringify(body), {
+ ...init,
+ headers: { "content-type": "application/json", ...init.headers },
+ });
+
+async function readBody(init: RequestInit): Promise<unknown> {
+ if (typeof init.body === "string") return JSON.parse(init.body);
+ return init.body;
+}
+
+beforeEach(() => {
+ ai.config({ apiKey: "ff_test_123" });
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async (url: string, init: RequestInit = {}) => {
+ const headers = new Headers(init.headers);
+ const call: Call = {
+ url,
+ method: init.method ?? "GET",
+ target: headers.get("x-framefields-target"),
+ headers,
+ body: await readBody(init),
+ };
+ calls.push(call);
+ return handler(call);
+ }),
+ );
+});
+
+afterEach(() => {
+ calls.length = 0;
+ ai.config({ apiKey: undefined });
+ delete process.env.FRAMEFIELDS_API_KEY;
+ vi.unstubAllGlobals();
+});
+
+describe("ai.run", () => {
+ it("sends one call to the gateway, naming the target in our own header", async () => {
+ handler = () =>
+ json(
+ { images: [{ url: "u" }] },
+ { headers: { "x-framefields-request-id": "req_1" } },
+ );
+ const result = await ai.run<{ images: { url: string }[] }>("flux/schnell", {
+ input: { prompt: "x" },
+ });
+ expect(result).toEqual({
+ data: { images: [{ url: "u" }] },
+ requestId: "req_1",
+ });
+ expect(calls).toHaveLength(1);
+ const [call] = calls;
+ expect(call.url).toBe(FRAMEFIELDS_AI_PROXY_URL);
+ expect(call.method).toBe("POST");
+ expect(call.target).toBe("run/flux/schnell");
+ expect(call.body).toEqual({ prompt: "x" });
+ expect(call.headers.get("authorization")).toBe("Key ff_test_123");
+ expect([...call.headers.keys()].sort()).toEqual([
+ "accept",
+ "authorization",
+ "content-type",
+ "x-framefields-target",
+ ]);
+ });
+
+ it("works with fal-ai/ prefixed model names as well as short model names", async () => {
+ handler = () => json({ ok: true });
+ await ai.run("fal-ai/flux/schnell", { input: { prompt: "hi" } });
+ expect(calls[0].target).toBe("run/fal-ai/flux/schnell");
+ });
+
+ it("reads FRAMEFIELDS_API_KEY from the environment", async () => {
+ ai.config({ apiKey: undefined });
+ process.env.FRAMEFIELDS_API_KEY = "ff_env";
+ handler = () => json({});
+ await ai.run("flux/schnell", { input: { prompt: "x" } });
+ expect(calls[0].headers.get("authorization")).toBe("Key ff_env");
+ });
+
+ it("fails before any request when no key is set", async () => {
+ ai.config({ apiKey: undefined });
+ await expect(ai.run("flux/schnell", { input: {} })).rejects.toThrow(
+ /FRAMEFIELDS_API_KEY/,
+ );
+ expect(calls).toHaveLength(0);
+ });
+});
+
+describe("errors", () => {
+ it("exposes the gateway's error type", async () => {
+ handler = () =>
+ json(
+ {
+ error: {
+ type: "insufficient_credits",
+ message: "Insufficient credits",
+ },
+ },
+ { status: 402 },
+ );
+ const err = await ai.run("flux/schnell", { input: {} }).catch((e) => e);
+ expect(err).toBeInstanceOf(ApiError);
+ expect(err.status).toBe(402);
+ expect(err.type).toBe("insufficient_credits");
+ expect(err.message).toBe("Insufficient credits");
+ });
+
+ it("turns a 422 with field details into a ValidationError", async () => {
+ handler = () =>
+ json(
+ {
+ detail: [
+ { loc: ["body", "prompt"], msg: "field required", type: "missing" },
+ ],
+ },
+ { status: 422 },
+ );
+ const err = await ai.run("flux/schnell", { input: {} }).catch((e) => e);
+ expect(err).toBeInstanceOf(ValidationError);
+ expect(err.getFieldErrors("prompt")).toHaveLength(1);
+ expect(err.message).toBe("field required (body.prompt)");
+ });
+});
+
+describe("ai.subscribe", () => {
+ it("submits, polls until completed, then fetches the result", async () => {
+ const statuses = ["IN_QUEUE", "IN_PROGRESS", "COMPLETED"];
+ handler = ({ target }) => {
+ if (target === "queue/flux/schnell") {
+ return json({
+ status: "IN_QUEUE",
+ request_id: "r1",
+ queue_position: 0,
+ });
+ }
+ if (target?.startsWith("queue/flux/schnell/requests/r1/status")) {
+ return json({ status: statuses.shift(), request_id: "r1", logs: [] });
+ }
+ if (target === "queue/flux/schnell/requests/r1")
+ return json({ ok: true });
+ return json({}, { status: 404 });
+ };
+ const updates: QueueStatus["status"][] = [];
+ const onEnqueue = vi.fn();
+ const result = await ai.subscribe("flux/schnell", {
+ input: { prompt: "x" },
+ pollInterval: 1,
+ logs: true,
+ onEnqueue,
+ onQueueUpdate: (s) => updates.push(s.status),
+ });
+ expect(result).toEqual({ data: { ok: true }, requestId: "r1" });
+ expect(onEnqueue).toHaveBeenCalledWith("r1");
+ expect(updates).toEqual(["IN_QUEUE", "IN_PROGRESS", "COMPLETED"]);
+ expect(calls[1].target).toBe(
+ "queue/flux/schnell/requests/r1/status?logs=1",
+ );
+ });
+
+ it("retries a status check that hit a transient error", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout"] });
+ let failures = 1;
+ handler = () =>
+ failures-- > 0
+ ? json(
+ { error: { type: "upstream_error", message: "busy" } },
+ { status: 503 },
+ )
+ : json({ status: "COMPLETED", request_id: "r1", logs: [] });
+ const pending = ai.queue.status("flux/schnell", { requestId: "r1" });
+ await vi.runAllTimersAsync();
+ await expect(pending).resolves.toMatchObject({ status: "COMPLETED" });
+ expect(calls).toHaveLength(2);
+ vi.useRealTimers();
+ });
+
+ it("cancels with PUT", async () => {
+ handler = () => json({ status: "CANCELLATION_REQUESTED" });
+ await ai.queue.cancel("flux/schnell", { requestId: "r1" });
+ expect(calls[0]).toMatchObject({
+ method: "PUT",
+ target: "queue/flux/schnell/requests/r1/cancel",
+ });
+ });
+});
+
+describe("ai.storage", () => {
+ it("uploads Blobs found in the input and passes their URLs", async () => {
+ handler = ({ target, url }) => {
+ if (target === "storage/upload/initiate") {
+ return json({
+ upload_url: "https://upload.test/put",
+ file_url: "https://cdn.test/a.png",
+ });
+ }
+ if (url === "https://upload.test/put")
+ return new Response(null, { status: 200 });
+ return json({ done: true });
+ };
+ const image = new Blob([new Uint8Array([1, 2, 3])], { type: "image/png" });
+ await ai.run("upscale", { input: { images: [{ image }], scale: 2 } });
+
+ expect(calls.map((c) => c.target ?? c.url)).toEqual([
+ "storage/upload/initiate",
+ "https://upload.test/put",
+ "run/upscale",
+ ]);
+ expect(calls[0].body).toMatchObject({ content_type: "image/png" });
+ expect(calls[1].headers.get("authorization")).toBeNull();
+ expect(calls[2].body).toEqual({
+ images: [{ image: "https://cdn.test/a.png" }],
+ scale: 2,
+ });
+ });
+});
+
+describe("ai.stream", () => {
+ it("yields each event across chunk boundaries and resolves done() with the last", async () => {
+ const encoder = new TextEncoder();
+ const chunks = [
+ 'data: {"n":1}\n\nda',
+ 'ta: {"n":2}\r\n\r\n',
+ 'data: {"n":3}\n\n',
+ ];
+ handler = () =>
+ new Response(
+ new ReadableStream({
+ start(controller) {
+ for (const c of chunks) controller.enqueue(encoder.encode(c));
+ controller.close();
+ },
+ }),
+ {
+ headers: {
+ "content-type": "text/event-stream",
+ "x-framefields-request-id": "s1",
+ },
+ },
+ );
+ const stream = await ai.stream<{ n: number }>("chat", {
+ input: { prompt: "x" },
+ });
+ expect(calls[0]).toMatchObject({
+ target: "run/chat/stream",
+ method: "POST",
+ });
+ expect(calls[0].headers.get("accept")).toBe("text/event-stream");
+ expect(stream.requestId).toBe("s1");
+
+ const seen: number[] = [];
+ for await (const event of stream) {
+ seen.push(event.n);
+ if (event.n === 2) break;
+ }
+ expect(seen).toEqual([1, 2]);
+ await expect(stream.done()).resolves.toEqual({ n: 2 });
+ });
+});
diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts
new file mode 100644
index 00000000..5dce4c88
--- /dev/null
+++ b/packages/ai/src/index.ts
@@ -0,0 +1,91 @@
+import { type Queue, queue, subscribe } from "./queue.js";
+import { type Storage, storage } from "./storage.js";
+import { type ModelStream, stream } from "./stream.js";
+import { gatewayJson, setApiKey } from "./transport.js";
+import type {
+ ModelData,
+ Result,
+ RunOptions,
+ SubscribeOptions,
+} from "./types.js";
+
+export interface AiConfig {
+ /** A framefields API key (`ff_live_…`). Defaults to `process.env.FRAMEFIELDS_API_KEY`. */
+ apiKey?: string;
+}
+
+/**
+ * AI models (images, video, music, voice, …) through the framefields AI gateway, billed to
+ * the framefields account that owns the API key. Models are named by their catalog id,
+ * e.g. `ai.subscribe("flux/schnell", { input })`; https://framefields.dev/models lists them.
+ */
+export interface Ai {
+ /** Set the API key in code instead of `FRAMEFIELDS_API_KEY`. */
+ config(options: AiConfig): void;
+ /** Runs a model and waits for its output in one request. Best for fast models. */
+ run<Output = ModelData, Input = ModelData>(
+ model: string,
+ options: RunOptions<Input>,
+ ): Promise<Result<Output>>;
+ /** Queues a job, polls until it finishes, and returns its output. Use for slow models. */
+ subscribe<Output = ModelData, Input = ModelData>(
+ model: string,
+ options: SubscribeOptions<Input>,
+ ): Promise<Result<Output>>;
+ /** Runs a model that streams partial output. */
+ stream<Output = ModelData, Input = ModelData>(
+ model: string,
+ options: RunOptions<Input>,
+ ): Promise<ModelStream<Output>>;
+ /** Submit jobs and collect them later, by request id. */
+ readonly queue: Queue;
+ /** Upload local files for use as model input. */
+ readonly storage: Storage;
+}
+
+async function run<Output, Input>(
+ model: string,
+ { input, abortSignal }: RunOptions<Input>,
+): Promise<Result<Output>> {
+ return gatewayJson<Output>({
+ service: "run",
+ path: model,
+ body: (await storage.transformInput(input)) ?? {},
+ signal: abortSignal,
+ });
+}
+
+export const ai: Ai = {
+ config(options) {
+ setApiKey(options.apiKey);
+ },
+ run,
+ subscribe,
+ stream,
+ queue,
+ storage,
+};
+
+export {
+ ApiError,
+ ValidationError,
+ type ValidationErrorInfo,
+} from "./errors.js";
+export type { Queue, WaitOptions } from "./queue.js";
+export type { Storage } from "./storage.js";
+export type { ModelStream } from "./stream.js";
+export { FRAMEFIELDS_AI_PROXY_URL } from "./transport.js";
+export type {
+ CompletedQueueStatus,
+ InProgressQueueStatus,
+ InQueueQueueStatus,
+ ModelData,
+ QueueStatus,
+ RequestLog,
+ RequestOptions,
+ Result,
+ RunOptions,
+ StatusOptions,
+ SubmitOptions,
+ SubscribeOptions,
+} from "./types.js";
diff --git a/packages/ai/src/queue.ts b/packages/ai/src/queue.ts
new file mode 100644
index 00000000..2d595578
--- /dev/null
+++ b/packages/ai/src/queue.ts
@@ -0,0 +1,163 @@
+import { ApiError } from "./errors.js";
+import { storage } from "./storage.js";
+import { gatewayJson } from "./transport.js";
+import type {
+ CompletedQueueStatus,
+ InQueueQueueStatus,
+ ModelData,
+ QueueStatus,
+ RequestOptions,
+ Result,
+ StatusOptions,
+ SubmitOptions,
+ SubscribeOptions,
+} from "./types.js";
+
+/** Statuses worth asking again for: rate limits and a gateway or model service hiccup. */
+const TRANSIENT = new Set([429, 502, 503, 504]);
+const READ_TRIES = 4;
+
+export interface Queue {
+ /** Queues a job and returns at once with its `request_id`. Billed when it runs. */
+ submit<Input = ModelData>(
+ model: string,
+ options: SubmitOptions<Input>,
+ ): Promise<InQueueQueueStatus>;
+ status(model: string, options: StatusOptions): Promise<QueueStatus>;
+ /** The output of a finished job. Fetching it again does not run or bill the model again. */
+ result<Output = ModelData>(
+ model: string,
+ options: RequestOptions,
+ ): Promise<Result<Output>>;
+ cancel(model: string, options: RequestOptions): Promise<void>;
+ /** Resolves when the job is `COMPLETED`, calling `onQueueUpdate` after each check. */
+ subscribeToStatus(
+ model: string,
+ options: WaitOptions,
+ ): Promise<CompletedQueueStatus>;
+}
+
+export type WaitOptions = StatusOptions &
+ Pick<SubscribeOptions, "pollInterval" | "onQueueUpdate">;
+
+const sleep = (ms: number, signal?: AbortSignal) =>
+ new Promise<void>((resolve, reject) => {
+ signal?.throwIfAborted();
+ const timer = setTimeout(resolve, ms);
+ signal?.addEventListener(
+ "abort",
+ () => {
+ clearTimeout(timer);
+ reject(signal.reason);
+ },
+ { once: true },
+ );
+ });
+
+/** Reads (status, result) are free and idempotent, so transient failures are retried. */
+async function read<T>(
+ run: () => Promise<T>,
+ signal?: AbortSignal,
+): Promise<T> {
+ for (let attempt = 1; ; attempt++) {
+ try {
+ return await run();
+ } catch (err) {
+ const transient = err instanceof ApiError && TRANSIENT.has(err.status);
+ if (!transient || attempt >= READ_TRIES) throw err;
+ await sleep(500 * 2 ** attempt, signal);
+ }
+ }
+}
+
+const requestPath = (model: string, requestId: string, suffix = "") =>
+ `${model}/requests/${encodeURIComponent(requestId)}${suffix}`;
+
+async function submit<Input>(
+ model: string,
+ { input, abortSignal }: SubmitOptions<Input>,
+): Promise<InQueueQueueStatus> {
+ const { data } = await gatewayJson<InQueueQueueStatus>({
+ service: "queue",
+ path: model,
+ body: (await storage.transformInput(input)) ?? {},
+ signal: abortSignal,
+ });
+ return data;
+}
+
+function status(
+ model: string,
+ { requestId, logs, abortSignal }: StatusOptions,
+): Promise<QueueStatus> {
+ return read(async () => {
+ const { data } = await gatewayJson<QueueStatus>({
+ service: "queue",
+ path: requestPath(model, requestId, "/status"),
+ query: { logs: logs ? "1" : undefined },
+ signal: abortSignal,
+ });
+ return data;
+ }, abortSignal);
+}
+
+function result<Output>(
+ model: string,
+ { requestId, abortSignal }: RequestOptions,
+): Promise<Result<Output>> {
+ return read(async () => {
+ const { data } = await gatewayJson<Output>({
+ service: "queue",
+ path: requestPath(model, requestId),
+ signal: abortSignal,
+ });
+ return { data, requestId };
+ }, abortSignal);
+}
+
+async function cancel(
+ model: string,
+ { requestId, abortSignal }: RequestOptions,
+): Promise<void> {
+ await gatewayJson<unknown>({
+ service: "queue",
+ path: requestPath(model, requestId, "/cancel"),
+ method: "PUT",
+ signal: abortSignal,
+ });
+}
+
+async function subscribeToStatus(
+ model: string,
+ options: WaitOptions,
+): Promise<CompletedQueueStatus> {
+ const interval = options.pollInterval ?? 500;
+ for (;;) {
+ const current = await status(model, options);
+ options.onQueueUpdate?.(current);
+ if (current.status === "COMPLETED") return current;
+ await sleep(interval, options.abortSignal);
+ }
+}
+
+export const queue: Queue = {
+ submit,
+ status,
+ result,
+ cancel,
+ subscribeToStatus,
+};
+
+/** Queues the job, waits for it, and returns its output. */
+export async function subscribe<Output = ModelData, Input = ModelData>(
+ model: string,
+ options: SubscribeOptions<Input>,
+): Promise<Result<Output>> {
+ const { request_id: requestId } = await submit(model, options);
+ options.onEnqueue?.(requestId);
+ await subscribeToStatus(model, { ...options, requestId });
+ return result<Output>(model, {
+ requestId,
+ abortSignal: options.abortSignal,
+ });
+}
diff --git a/packages/ai/src/storage.ts b/packages/ai/src/storage.ts
new file mode 100644
index 00000000..fbfd04a2
--- /dev/null
+++ b/packages/ai/src/storage.ts
@@ -0,0 +1,120 @@
+import { errorFromResponse } from "./errors.js";
+import { gatewayJson } from "./transport.js";
+
+/** Files above this size upload in parts. */
+const MULTIPART_THRESHOLD = 90 * 1024 * 1024;
+const PART_SIZE = 10 * 1024 * 1024;
+const PART_TRIES = 3;
+
+interface UploadTicket {
+ upload_url: string;
+ file_url: string;
+}
+
+interface UploadedPart {
+ partNumber: number;
+ etag: string;
+}
+
+export interface Storage {
+ /** Uploads a file and returns a URL any model accepts as input. */
+ upload(file: Blob): Promise<string>;
+ /** Replaces every `Blob` in an input (nested objects and arrays included) with its uploaded URL. */
+ transformInput<T>(input: T): Promise<T>;
+}
+
+const EXTENSIONS: Record<string, string> = {
+ "image/jpeg": "jpg",
+ "image/png": "png",
+ "image/webp": "webp",
+ "video/mp4": "mp4",
+ "audio/mpeg": "mp3",
+ "audio/wav": "wav",
+};
+
+function fileName(file: Blob, contentType: string): string {
+ if ("name" in file && typeof file.name === "string" && file.name) {
+ return file.name;
+ }
+ return `${Date.now()}.${EXTENSIONS[contentType] ?? "bin"}`;
+}
+
+function initiate(file: Blob, multipart: boolean): Promise<UploadTicket> {
+ const contentType = file.type || "application/octet-stream";
+ return gatewayJson<UploadTicket>({
+ service: "storage",
+ path: multipart ? "upload/initiate-multipart" : "upload/initiate",
+ body: { content_type: contentType, file_name: fileName(file, contentType) },
+ }).then((r) => r.data);
+}
+
+async function put(url: string, init: RequestInit): Promise<Response> {
+ const response = await fetch(url, init);
+ if (!response.ok) throw await errorFromResponse(response);
+ return response;
+}
+
+async function uploadPart(url: string, chunk: Blob): Promise<UploadedPart> {
+ for (let attempt = 1; ; attempt++) {
+ try {
+ const response = await put(url, { method: "PUT", body: chunk });
+ return (await response.json()) as UploadedPart;
+ } catch (err) {
+ if (attempt >= PART_TRIES) throw err;
+ }
+ }
+}
+
+async function uploadMultipart(file: Blob): Promise<string> {
+ const ticket = await initiate(file, true);
+ const base = new URL(ticket.upload_url);
+ const partUrl = (suffix: string | number) =>
+ `${base.origin}${base.pathname}/${suffix}${base.search}`;
+ const parts: UploadedPart[] = [];
+ for (let start = 0; start < file.size; start += PART_SIZE) {
+ const chunk = file.slice(start, Math.min(start + PART_SIZE, file.size));
+ parts.push(await uploadPart(partUrl(parts.length + 1), chunk));
+ }
+ await put(partUrl("complete"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ parts: parts.map(({ partNumber, etag }) => ({ partNumber, etag })),
+ }),
+ });
+ return ticket.file_url;
+}
+
+async function upload(file: Blob): Promise<string> {
+ if (file.size > MULTIPART_THRESHOLD) return uploadMultipart(file);
+ const ticket = await initiate(file, false);
+ await put(ticket.upload_url, {
+ method: "PUT",
+ body: file,
+ headers: { "Content-Type": file.type || "application/octet-stream" },
+ });
+ return ticket.file_url;
+}
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> =>
+ typeof value === "object" &&
+ value !== null &&
+ Object.getPrototypeOf(value) === Object.prototype;
+
+async function transformInput<T>(input: T): Promise<T> {
+ if (input instanceof Blob) return (await upload(input)) as T;
+ if (Array.isArray(input)) {
+ return (await Promise.all(input.map((item) => transformInput(item)))) as T;
+ }
+ if (isPlainObject(input)) {
+ const entries = await Promise.all(
+ Object.entries(input).map(
+ async ([key, value]) => [key, await transformInput(value)] as const,
+ ),
+ );
+ return Object.fromEntries(entries) as T;
+ }
+ return input;
+}
+
+export const storage: Storage = { upload, transformInput };
diff --git a/packages/ai/src/stream.ts b/packages/ai/src/stream.ts
new file mode 100644
index 00000000..98d52912
--- /dev/null
+++ b/packages/ai/src/stream.ts
@@ -0,0 +1,118 @@
+import { ApiError } from "./errors.js";
+import { storage } from "./storage.js";
+import { gatewayFetch } from "./transport.js";
+import type { ModelData, RunOptions } from "./types.js";
+
+/**
+ * A model's streamed output. Iterate it for each partial result as it arrives; `done()`
+ * resolves with the last one (and drains the stream if nobody iterates it).
+ */
+export interface ModelStream<Output = ModelData> extends AsyncIterable<Output> {
+ readonly requestId: string;
+ done(): Promise<Output>;
+ abort(): void;
+}
+
+/** Yields each `data:` payload of a server-sent event stream, joined across lines. */
+async function* serverSentEvents(
+ body: ReadableStream<Uint8Array>,
+): AsyncGenerator<{ event: string; data: string }> {
+ const decoder = new TextDecoder();
+ let buffer = "";
+ for await (const chunk of body) {
+ buffer += decoder.decode(chunk, { stream: true }).replace(/\r\n?/g, "\n");
+ let end = buffer.indexOf("\n\n");
+ while (end !== -1) {
+ const block = buffer.slice(0, end);
+ buffer = buffer.slice(end + 2);
+ const parsed = parseBlock(block);
+ if (parsed) yield parsed;
+ end = buffer.indexOf("\n\n");
+ }
+ }
+ const last = parseBlock(buffer);
+ if (last) yield last;
+}
+
+function parseBlock(block: string): { event: string; data: string } | null {
+ let event = "message";
+ const data: string[] = [];
+ for (const line of block.split("\n")) {
+ if (line.startsWith("data:")) data.push(line.slice(5).replace(/^ /, ""));
+ else if (line.startsWith("event:")) event = line.slice(6).trim();
+ }
+ return data.length ? { event, data: data.join("\n") } : null;
+}
+
+async function* outputs<Output>(
+ body: ReadableStream<Uint8Array>,
+ requestId: string,
+): AsyncGenerator<Output> {
+ for await (const { event, data } of serverSentEvents(body)) {
+ const payload = JSON.parse(data) as Output;
+ if (event === "error") {
+ throw new ApiError({
+ message: "The model stream ended with an error",
+ status: 500,
+ body: payload,
+ requestId,
+ });
+ }
+ yield payload;
+ }
+}
+
+/** Runs a streaming model; resolves once the stream is open. */
+export async function stream<Output = ModelData, Input = ModelData>(
+ model: string,
+ { input, abortSignal }: RunOptions<Input>,
+): Promise<ModelStream<Output>> {
+ const controller = new AbortController();
+ const signal = abortSignal
+ ? AbortSignal.any([abortSignal, controller.signal])
+ : controller.signal;
+ const response = await gatewayFetch({
+ service: "run",
+ path: `${model}/stream`,
+ body: (await storage.transformInput(input)) ?? {},
+ accept: "text/event-stream",
+ signal,
+ });
+ const requestId = response.headers.get("x-framefields-request-id") ?? "";
+ if (!response.body) {
+ throw new ApiError({ message: "Empty stream", status: 502, requestId });
+ }
+ const source = outputs<Output>(response.body, requestId);
+
+ let last: Output | undefined;
+ let finished: Promise<Output> | undefined;
+ async function* tracked(): AsyncGenerator<Output> {
+ for await (const value of source) {
+ last = value;
+ yield value;
+ }
+ }
+ const iterator = tracked();
+ const drain = async (): Promise<Output> => {
+ let step = await iterator.next();
+ while (!step.done) step = await iterator.next();
+ if (last === undefined) {
+ throw new ApiError({
+ message: "Stream had no output",
+ status: 502,
+ requestId,
+ });
+ }
+ return last;
+ };
+
+ return {
+ requestId,
+ [Symbol.asyncIterator]: () => iterator,
+ done: () => {
+ finished ??= drain();
+ return finished;
+ },
+ abort: () => controller.abort(),
+ };
+}
diff --git a/packages/ai/src/transport.ts b/packages/ai/src/transport.ts
new file mode 100644
index 00000000..14cbb862
--- /dev/null
+++ b/packages/ai/src/transport.ts
@@ -0,0 +1,85 @@
+import { errorFromResponse } from "./errors.js";
+
+/** The framefields AI gateway. Every request goes here, authenticated with a framefields key. */
+export const FRAMEFIELDS_AI_PROXY_URL =
+ "https://api.framefields.dev/v1/ai/proxy";
+
+/** The gateway service a call goes to: model runs, the job queue, or file storage. */
+export type Service = "run" | "queue" | "storage";
+
+export interface GatewayRequest {
+ service: Service;
+ /** Path under the service, e.g. `flux/schnell` or `flux/schnell/requests/<id>/status`. */
+ path: string;
+ method?: "GET" | "POST" | "PUT";
+ body?: unknown;
+ accept?: string;
+ query?: Record<string, string | undefined>;
+ signal?: AbortSignal;
+}
+
+let configuredKey: string | undefined;
+
+const env = (name: string): string | undefined =>
+ typeof process !== "undefined" ? process.env?.[name] : undefined;
+
+export function setApiKey(key: string | undefined): void {
+ configuredKey = key;
+}
+
+function apiKey(): string {
+ const key = configuredKey ?? env("FRAMEFIELDS_API_KEY");
+ if (!key) {
+ throw new Error(
+ "framefields ai: no API key. Set FRAMEFIELDS_API_KEY or call ai.config({ apiKey }). " +
+ "Create a key at https://framefields.dev/dashboard/keys",
+ );
+ }
+ return key;
+}
+
+function targetOf({ service, path, query }: GatewayRequest): string {
+ const params = new URLSearchParams();
+ for (const [name, value] of Object.entries(query ?? {})) {
+ if (value !== undefined) params.set(name, value);
+ }
+ const search = params.size ? `?${params}` : "";
+ return `${service}/${path.replace(/^\/+/, "")}${search}`;
+}
+
+/**
+ * Sends one call through the gateway and returns the response, or throws `ApiError` /
+ * `ValidationError` for a non-2xx answer. The key is checked before anything is sent.
+ */
+export async function gatewayFetch(request: GatewayRequest): Promise<Response> {
+ const headers: Record<string, string> = {
+ Authorization: `Key ${apiKey()}`,
+ Accept: request.accept ?? "application/json",
+ "x-framefields-target": targetOf(request),
+ };
+ const hasBody = request.body !== undefined;
+ if (hasBody) headers["Content-Type"] = "application/json";
+ const response = await fetch(
+ env("FRAMEFIELDS_AI_PROXY_URL") ?? FRAMEFIELDS_AI_PROXY_URL,
+ {
+ method: request.method ?? (hasBody ? "POST" : "GET"),
+ headers,
+ body: hasBody ? JSON.stringify(request.body) : undefined,
+ signal: request.signal,
+ },
+ );
+ if (!response.ok) throw await errorFromResponse(response);
+ return response;
+}
+
+/** `gatewayFetch` for JSON answers, with the gateway's request id alongside. */
+export async function gatewayJson<T>(
+ request: GatewayRequest,
+): Promise<{ data: T; requestId: string }> {
+ const response = await gatewayFetch(request);
+ const data = (await response.json()) as T;
+ return {
+ data,
+ requestId: response.headers.get("x-framefields-request-id") ?? "",
+ };
+}
diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts
new file mode 100644
index 00000000..64f5919d
--- /dev/null
+++ b/packages/ai/src/types.ts
@@ -0,0 +1,77 @@
+/** Model input or output when its shape isn't declared. Pass a type argument to name it. */
+export type ModelData = Record<string, unknown>;
+
+/** What a finished call returns: the model's output and the gateway's id for the request. */
+export interface Result<Output = ModelData> {
+ data: Output;
+ requestId: string;
+}
+
+export interface RunOptions<Input = ModelData> {
+ /** The model input. `Blob`s anywhere in it are uploaded first and replaced by URLs. */
+ readonly input?: Input;
+ /** Stops waiting. A queued job keeps running; cancel it with `ai.queue.cancel`. */
+ readonly abortSignal?: AbortSignal;
+}
+
+export interface RequestLog {
+ message: string;
+ level?: string;
+ source?: string;
+ timestamp: string;
+}
+
+export interface Metrics {
+ inference_time: number | null;
+}
+
+interface BaseQueueStatus {
+ status: "IN_QUEUE" | "IN_PROGRESS" | "COMPLETED";
+ request_id: string;
+}
+
+export interface InQueueQueueStatus extends BaseQueueStatus {
+ status: "IN_QUEUE";
+ queue_position: number;
+}
+
+export interface InProgressQueueStatus extends BaseQueueStatus {
+ status: "IN_PROGRESS";
+ logs: RequestLog[];
+}
+
+export interface CompletedQueueStatus extends BaseQueueStatus {
+ status: "COMPLETED";
+ logs: RequestLog[];
+ metrics?: Metrics;
+}
+
+export type QueueStatus =
+ | InQueueQueueStatus
+ | InProgressQueueStatus
+ | CompletedQueueStatus;
+
+export type SubmitOptions<Input = ModelData> = RunOptions<Input>;
+
+export interface RequestOptions {
+ /** The `request_id` returned by `ai.queue.submit`. */
+ readonly requestId: string;
+ readonly abortSignal?: AbortSignal;
+}
+
+export interface StatusOptions extends RequestOptions {
+ /** Include the model's log lines in the status. */
+ readonly logs?: boolean;
+}
+
+export interface SubscribeOptions<Input = ModelData>
+ extends SubmitOptions<Input> {
+ /** Milliseconds between status checks. Default 500. */
+ readonly pollInterval?: number;
+ /** Include the model's log lines in each `onQueueUpdate`. */
+ readonly logs?: boolean;
+ /** Called once with the request id, as soon as the job is queued. */
+ readonly onEnqueue?: (requestId: string) => void;
+ /** Called after every status check. */
+ readonly onQueueUpdate?: (status: QueueStatus) => void;
+}
diff --git a/packages/ai/tsconfig.json b/packages/ai/tsconfig.json
new file mode 100644
index 00000000..f104f891
--- /dev/null
+++ b/packages/ai/tsconfig.json
@@ -0,0 +1,8 @@
+{
+ "extends": "@framefields/tsconfig/base.json",
+ "compilerOptions": {
+ "lib": ["ES2022", "ESNext", "DOM", "DOM.Iterable", "DOM.AsyncIterable"]
+ },
+ "include": ["."],
+ "exclude": ["dist", "build", "node_modules"]
+}
diff --git a/packages/ai/tsdown.config.ts b/packages/ai/tsdown.config.ts
new file mode 100644
index 00000000..94257a01
--- /dev/null
+++ b/packages/ai/tsdown.config.ts
@@ -0,0 +1,10 @@
+import { defineConfig } from "tsdown";
+
+export default defineConfig({
+ entry: ["src/index.ts"],
+ format: ["esm"],
+ dts: true,
+ clean: true,
+ sourcemap: true,
+ treeshake: true,
+});
diff --git a/packages/framefields/package.json b/packages/framefields/package.json
index 2ab05aa0..49e190b8 100644
--- a/packages/framefields/package.json
+++ b/packages/framefields/package.json
@@ -114,6 +114,7 @@
"onnxruntime-web": "^1.24.0"
},
"devDependencies": {
+ "@framefields/ai": "workspace:*",
"@framefields/client-utils": "workspace:*",
"@framefields/compositions": "workspace:*",
"@framefields/core": "workspace:*",
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index 4f557dbc..70aed9a2 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -1,3 +1,14 @@
+export {
+ type Ai,
+ type AiConfig,
+ ApiError,
+ ai,
+ FRAMEFIELDS_AI_PROXY_URL,
+ type QueueStatus,
+ type Result,
+ type RunOptions,
+ ValidationError,
+} from "@framefields/ai";
export {
AudioSignal,
type AudioSignalOptions,
diff --git a/plugins/framefields/README.md b/plugins/framefields/README.md
index e86994e7..29c849e8 100644
--- a/plugins/framefields/README.md
+++ b/plugins/framefields/README.md
@@ -4,7 +4,7 @@ Agent skills for [framefields](https://github.com/gatewai-dev/framefields), a co
| Skill | Use it for |
| --- | --- |
-| `framefields` | Starting a project: install from npm, scaffold a composition and render script, first verified render |
+| `framefields` | Starting a project: install from npm, scaffold a composition and render script, first verified render; generating images, video and audio with AI models (`ai`) |
| `framefields-compose` | Compositions, layer trees, layout, animation and easing, beat grids, film structure |
| `framefields-effects` | Post-processing effects, color grading, VFX, and on-device vision (tracking, segmentation, pose) |
| `framefields-render` | Headless rendering, frame-grid inspection, pixel probes, MP4 delivery checks |
@@ -17,7 +17,7 @@ This plugin contains only skills: Markdown instructions for your agent. It bundl
## Data and privacy
-The plugin sends no data anywhere. When code that uses framefields' on-device vision features runs, the SDK downloads pinned, Apache-2.0 model weights from Hugging Face the first time each model is used and verifies them by SHA-256. Nothing else leaves your machine.
+The plugin sends no data anywhere. When code that uses framefields' on-device vision features runs, the SDK downloads pinned, Apache-2.0 model weights from Hugging Face the first time each model is used and verifies them by SHA-256. When code calls `ai` (AI generation), the request, its inputs and any files uploaded with `ai.storage` go to the framefields gateway at `api.framefields.dev` and on to the model provider that runs the model; the gateway records usage for billing but does not store prompts or outputs. Nothing else leaves your machine.
## License
diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.md
index 5959c3c7..c7ad6170 100644
--- a/plugins/framefields/skills/framefields-compose/SKILL.md
+++ b/plugins/framefields/skills/framefields-compose/SKILL.md
@@ -510,3 +510,4 @@ Layer.vfx({ effectType: "Aurora", width: W, height: H, colorStart: "#2bff88", co
8. **Premultiplied Alpha:** Alpha shaders operate with premultiplied alpha. Never use `"src-alpha"` factor in custom blends.
9. **Lint before rendering:** `comp.lint()` lists colors/gradients that would draw transparent, low-contrast text, off-canvas layers, shadow settings that do nothing and `roughness`/`metallic` without an environment. `renderVideo({ qa: true })` includes these findings.
10. **Lottie vs. Image Layers:** Use `Layer.lottie("file.json" | "file.lottie", { width, height, fit })` for Lottie vector animations. Never pass `.json` or `.lottie` to `Layer.image()`, which only decodes raster formats and throws/warns if given a vector animation.
+11. **AI-generated media are assets, not build steps:** Images, clips, music or voice made with `ai` (from `framefields`) are generated once by a separate script into `assets/generated/` and loaded like any other file. Never call `ai` from `buildFilm()` or a scene function: it would spend money and change the film on every render, preview and test. The `framefields` skill (section 6) covers the key, the script and the model catalog.
diff --git a/plugins/framefields/skills/framefields/SKILL.md b/plugins/framefields/skills/framefields/SKILL.md
index 3b6d1d15..3a22c7bb 100644
--- a/plugins/framefields/skills/framefields/SKILL.md
+++ b/plugins/framefields/skills/framefields/SKILL.md
@@ -1,6 +1,6 @@
---
name: framefields
-description: Start a new framefields video project, or add framefields to an existing Node.js project. Use when the user wants to make a video with framefields and the project has no framefields setup yet, or asks how to install framefields, scaffold a composition and render script, or get from an empty folder to a first MP4. Hands off to framefields-compose, framefields-effects, and framefields-render for the details.
+description: Start a new framefields video project, or add framefields to an existing Node.js project. Use when the user wants to make a video with framefields and the project has no framefields setup yet, or asks how to install framefields, scaffold a composition and render script, get from an empty folder to a first MP4, or generate images, video, music or voice for a film with AI models (`ai` from framefields, the framefields AI gateway, FRAMEFIELDS_API_KEY). Hands off to framefields-compose, framefields-effects, and framefields-render for the details.
---
# framefields
@@ -15,6 +15,7 @@ This skill takes a project from nothing to a verified first render. For anything
| Scenes, layout, typography, animation, beat grids, film structure | `framefields-compose` |
| Post-processing, color grading, VFX, on-device vision (tracking, cutouts, pose) | `framefields-effects` |
| Frame inspection, frame grids, pixel tests, video export and QA | `framefields-render` |
+| Generating images, video, music, voice with AI models (`ai`) | this skill, section 6 |
---
@@ -278,6 +279,63 @@ The framefields render worker (Docker, RunPod pod or serverless) renders a pushe
A failed job returns `{ code, hint, retryable }`; resubmit unchanged only when `retryable` is true. Job format and error codes: https://framefields.dev/docs/guides/render-service/
-## 6. Grow the film
+## 6. Generate assets with AI
+
+`framefields` exports `ai`: hundreds of AI models (images, video, music, sound effects, voice, upscaling, background removal) through the framefields AI gateway, each named by its catalog id and billed to the user's framefields account. `run`, `subscribe`, `stream`, `queue` and `storage` cover every model.
+
+```typescript
+import { ai } from "framefields";
+
+const { data, requestId } = await ai.subscribe<{ images: { url: string }[] }>("flux/schnell", {
+ input: { prompt: "Neon street at night, rain, 35mm", image_size: "landscape_16_9" },
+});
+console.log(data.images[0].url);
+```
+
+**The key.** Requests authenticate with `FRAMEFIELDS_API_KEY` (an `ff_live_…` key from https://framefields.dev/dashboard/keys). Put it in the project's `.env` (gitignored) and load it with `tsx --env-file=.env`, or call `ai.config({ apiKey })`. Never write the key into source, and never ask the user to paste it into chat: ask them to add it to `.env` themselves. Without a key, every call throws before any request is sent. Failures throw `ApiError` with the gateway's `type`: on `insufficient_credits` (status `402`) tell the user to top up at https://framefields.dev/dashboard/billing rather than retrying; `key_limit_reached` means the key's monthly cap; `rate_limited` and `too_many_inflight_requests` mean wait and retry. A `ValidationError` (`422`) lists the bad input fields in `fieldErrors`.
+
+**Generate once, into `assets/`, in its own script.** Generation costs money and is not deterministic, so it never runs inside `buildFilm()`, the preview, tests or the render worker. Write `src/generate.ts`, run it on purpose, download every output into `assets/generated/` (tracked in Git LFS, as above), and reference the local file from the film. Generated output URLs expire, and the worker renders from the repository.
+
+```typescript
+// src/generate.ts — run with: tsx --env-file=.env src/generate.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import { ai } from "framefields";
+
+const OUT = path.resolve(import.meta.dirname, "../assets/generated");
+await fs.mkdir(OUT, { recursive: true });
+
+async function save(url: string, name: string): Promise<string> {
+ const file = path.join(OUT, name);
+ await fs.writeFile(file, Buffer.from(await (await fetch(url)).arrayBuffer()));
+ return file;
+}
+
+const still = await ai.subscribe<{ images: { url: string }[] }>("flux/schnell", {
+ input: { prompt: "Neon street at night, rain, 35mm", image_size: "landscape_16_9", seed: 7 },
+});
+await save(still.data.images[0].url, "street.jpg");
+
+// Feed a local file to a model: put a Blob in the input and it is uploaded for you
+// (or call ai.storage.upload(blob) yourself and pass the URL).
+const image_url = new Blob([await fs.readFile(path.join(OUT, "street.jpg"))], { type: "image/jpeg" });
+const clip = await ai.subscribe<{ video: { url: string } }>("kling-video/v2.1/standard/image-to-video", {
+ input: { prompt: "slow push in, rain falling", image_url, duration: "5" },
+ logs: true,
+ onQueueUpdate: (u) => u.status === "IN_PROGRESS" && u.logs.forEach((l) => console.log(l.message)),
+});
+await save(clip.data.video.url, "street.mp4");
+process.exit(0);
+```
+
+Then `Layer.image("assets/generated/street.jpg")` / `Layer.video("assets/generated/street.mp4")` in the film.
+
+- **Pick the model and its input from the catalog.** https://framefields.dev/models lists every model with its id, category, price per unit and input schema. Each model has its own input and output fields (`images[0].url`, `video.url`, `audio.url`, `audio_file.url` …): read the schema and pass the output shape as the type argument, or log `data` once, instead of guessing.
+- **Long jobs** (video, music): `subscribe` waits and polls. For several at once, `ai.queue.submit(model, { input })` each, keep every `request_id`, then `ai.queue.result(model, { requestId })`; an interrupted script can collect the results instead of paying again. Reading a status or result is free and retried on transient errors; submitting is never retried, so it can't bill twice.
+- **Ask before spending.** Before a batch, or before any video model, tell the user which models you will call, how many times, and the catalog price; generate a single draft first and show it.
+- Pass a fixed `seed` where the model accepts one, and keep the prompts in `generate.ts`, so a result can be regenerated.
+- Frame-rate and size: generated clips rarely match the composition's `fps` or aspect. Use `fit: "cover"` on the layer, and check the clip's real duration before cutting to it.
+
+## 7. Grow the film
Once the first render works, move to the specialist skills: `framefields-compose` for splitting the film into scene modules on a beat grid, `framefields-effects` for grading, grain, and vision-driven effects, and `framefields-render` for tests and delivery checks. Keep `theme.ts` as the single source of sizes, colors, fonts, and timing as the project grows.
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c2f3364b..62e9ed9e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2598,6 +2598,24 @@ importers:
specifier: ^0.4.0
version: 0.4.0
+ packages/ai:
+ devDependencies:
+ '@framefields/tsconfig':
+ specifier: workspace:*
+ version: link:../tsconfig
+ '@types/node':
+ specifier: ^24.1.0
+ version: 24.10.14
+ tsdown:
+ specifier: ^0.19.0
+ typescript:
+ specifier: ^5.8.3
+ version: 5.9.3
+ vitest:
+ specifier: ^4.1.11
+
packages/client-utils:
dependencies:
'@framefields/core':
@@ -2766,6 +2784,9 @@ importers:
specifier: ^4.3.6
version: 4.3.6
devDependencies:
+ '@framefields/ai':
+ specifier: workspace:*
+ version: link:../ai
'@framefields/client-utils':
specifier: workspace:*
version: link:../client-utils