Code-first video & image, rendered natively on WebGPU.
ci(renderer-image): publish to GHCR on main and release tags
okanasla0081627ab00
diff --git a/.github/workflows/renderer-image.yml b/.github/workflows/renderer-image.ymlindex c01eaacb..5217ddd8 100644--- a/.github/workflows/renderer-image.yml+++ b/.github/workflows/renderer-image.yml@@ -1,24 +1,38 @@-# Builds and publishes the renderer worker image (apps/renderer-service).-# ghcr.io/gatewai-dev/framefields-renderer:<version> (slim)-# ghcr.io/gatewai-dev/framefields-renderer:<version>-full (with a C/C++ toolchain)-# Runs on tags renderer-v*, or by hand (publish=false only builds).+# Builds and publishes the renderer worker image (apps/renderer-service) to GHCR.+# push to main → :main, :sha-<short> (+ -full variants)+# tag renderer-v1.2.3 → :1.2.3, :1.2, :latest (+ -full variants)+# pull request → build + smoke test only, nothing pushed+# manual run → :<branch>, :sha-<short> (publish input, default on)+# The full variant adds a C/C++ toolchain.name: renderer-imageon:push:+ branches: [main]tags: ["renderer-v*"]+ paths:+ - "apps/renderer-service/**"+ - "packages/**"+ - "nodes/**"+ - "pnpm-lock.yaml"+ - ".github/workflows/renderer-image.yml"pull_request:paths:- "apps/renderer-service/**"- "packages/**"- "nodes/**"- "pnpm-lock.yaml"+ - ".github/workflows/renderer-image.yml"workflow_dispatch:inputs:publish:description: Push to GHCRtype: boolean- default: false+ default: true++concurrency:+ group: renderer-image-${{ github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}permissions:contents: read@@ -51,33 +65,75 @@ jobs:strategy:matrix:variant: [slim, full]+ env:+ PUBLISH: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish) }}steps:- uses: actions/checkout@v4- id: meta- run: |- version=$(jq -r .version apps/renderer-service/package.json)- suffix=$([ "${{ matrix.variant }}" = full ] && echo "-full" || echo "")- echo "tags=${IMAGE}:${version}${suffix},${IMAGE}:latest${suffix}" >> "$GITHUB_OUTPUT"+ uses: docker/metadata-action@v5+ with:+ images: ${{ env.IMAGE }}+ flavor: |+ latest=false+ suffix=${{ matrix.variant == 'full' && '-full' || '' }}+ tags: |+ type=ref,event=branch+ type=sha,prefix=sha-+ type=match,pattern=renderer-v(\d+\.\d+\.\d+.*),group=1+ type=match,pattern=renderer-v(\d+\.\d+),group=1+ type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/renderer-v') }}+ labels: |+ org.opencontainers.image.title=framefields-renderer+ org.opencontainers.image.description=framefields render worker (git checkout, install, GPU render, upload)- uses: docker/setup-buildx-action@v3+ - name: Build+ uses: docker/build-push-action@v6+ with:+ context: .+ file: apps/renderer-service/Dockerfile+ platforms: linux/amd64+ build-args: VARIANT=${{ matrix.variant }}+ tags: framefields-renderer:ci+ labels: ${{ steps.meta.outputs.labels }}+ load: true+ cache-from: type=gha,scope=renderer-${{ matrix.variant }}+ cache-to: type=gha,mode=max,scope=renderer-${{ matrix.variant }}+ - name: Smoke test (no GPU)+ run: |+ # Tools, pinned host keys, and no engine baked in.+ docker run --rm --entrypoint bash framefields-renderer:ci -c \+ 'git --version && git lfs version && ffprobe -version | head -1 && test -s /app/deploy/known_hosts && test ! -e /app/node_modules/framefields'+ # Boots and reports healthy; renders on the CPU without a GPU.+ docker run -d --name smoke -p 8082:8082 -e FF_MODE=http \+ -e RENDERER_API_KEY=ci-smoke-key-0123456789 -e FF_REPO_ALLOWLIST='github.com/example/*' \+ framefields-renderer:ci+ for i in $(seq 1 30); do+ if curl -fsS localhost:8082/health; then echo; break; fi+ sleep 1+ done+ curl -fsS localhost:8082/health | jq -e '.status == "ok" and .components.gpu.renderer == "cpu"' \+ || { docker logs smoke; exit 1; }+ docker rm -f smoke- uses: docker/login-action@v3- if: github.event_name != 'pull_request'+ if: env.PUBLISH == 'true'with:registry: ghcr.iousername: ${{ github.actor }}password: ${{ secrets.GITHUB_TOKEN }}- - uses: docker/build-push-action@v6+ - name: Publish+ if: env.PUBLISH == 'true'+ uses: docker/build-push-action@v6with:context: .file: apps/renderer-service/Dockerfileplatforms: linux/amd64build-args: VARIANT=${{ matrix.variant }}tags: ${{ steps.meta.outputs.tags }}- push: ${{ startsWith(github.ref, 'refs/tags/renderer-v') || inputs.publish == true }}- load: ${{ matrix.variant == 'slim' }}+ labels: ${{ steps.meta.outputs.labels }}+ push: truecache-from: type=gha,scope=renderer-${{ matrix.variant }}- cache-to: type=gha,mode=max,scope=renderer-${{ matrix.variant }}- - name: Smoke test (no GPU; checks tools, pinned host keys, and that no engine is baked in)- if: matrix.variant == 'slim'+ - name: Summary+ if: env.PUBLISH == 'true'run: |- tag=$(echo "${{ steps.meta.outputs.tags }}" | cut -d, -f1)- docker run --rm --entrypoint bash "$tag" -c 'git --version && git lfs version && ffprobe -version | head -1 && test -s /app/deploy/known_hosts && test ! -e /app/node_modules/framefields'+ echo "### Published (${{ matrix.variant }})" >> "$GITHUB_STEP_SUMMARY"+ echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/; s/$/`/' >> "$GITHUB_STEP_SUMMARY"diff --git a/.gitignore b/.gitignoreindex ecd00483..5800962c 100644--- a/.gitignore+++ b/.gitignore@@ -78,6 +78,8 @@ debug/examples/output/examples/**/output/**/output/+# Source module, not render output.+!apps/renderer-service/src/output/# Lazy-downloaded vision model weights (never source)*.onnxdiff --git a/apps/renderer-service/PLAN.md b/apps/renderer-service/PLAN.mdindex 3044a327..d2adc1be 100644--- a/apps/renderer-service/PLAN.md+++ b/apps/renderer-service/PLAN.md@@ -514,7 +514,11 @@ The service no longer uses `S3StorageService` from `@framefields/server-utils`,- **Two variants:** `:<ver>` (slim, no compiler; projects rely on prebuilt binaries) and `:<ver>-full` (adds `build-essential` and `python3-dev`).- Runs as user `framefields`. The `HEALTHCHECK` hits `/health`.- **Entrypoint (`docker/entrypoint.sh`):** `http` | `runpod-serverless` | `cli`. `xvfb-run` is used only with `FF_XVFB=1`, and the GPU is logged at boot (`nvidia-smi`).-- **Published to** `ghcr.io/gatewai-dev/framefields-renderer` by `.github/workflows/renderer-image.yml`.+- **Published to** `ghcr.io/gatewai-dev/framefields-renderer` by `.github/workflows/renderer-image.yml`, for linux/amd64:+ - pushes to `main` publish `:main` and `:sha-<short>`;+ - `renderer-v1.2.3` tags publish `:1.2.3`, `:1.2` and `:latest`;+ - the `full` variant gets the same tags with a `-full` suffix;+ - pull requests build and smoke-test only. The smoke test checks the tools and that no engine is baked in, then boots the worker and requires `/health` to be ok with `renderer: cpu`.**RunPod template:** `apps/renderer-service/deploy/runpod/`.diff --git a/apps/renderer-service/README.md b/apps/renderer-service/README.mdindex 9f1f9403..ff8ebbd6 100644--- a/apps/renderer-service/README.md+++ b/apps/renderer-service/README.md@@ -65,6 +65,18 @@ curl -s localhost:8082/v2/jobs/<id> -H "authorization: Bearer $RENDERER_curl -s -X POST localhost:8082/v2/jobs/<id>/cancel -H "authorization: Bearer $RENDERER_API_KEY"```+### Images++CI ([`renderer-image.yml`](../../.github/workflows/renderer-image.yml)) publishes `ghcr.io/gatewai-dev/framefields-renderer` (linux/amd64):++| Tag | When |+|---|---|+| `latest`, `1.2.3`, `1.2` | A `renderer-v1.2.3` git tag. Use these in production. |+| `main`, `sha-<commit>` | Each push to `main` that touches the renderer or the packages. |+| `<tag>-full` | The same tags, `full` variant. |++Pull requests only build and smoke-test the image. A manual run (**Actions → renderer-image → Run workflow**) publishes the selected branch.+Building the image yourself, from the repo root:```bashdiff --git a/apps/renderer-service/src/output/deliver.ts b/apps/renderer-service/src/output/deliver.tsnew file mode 100644index 00000000..1c377856--- /dev/null+++ b/apps/renderer-service/src/output/deliver.ts@@ -0,0 +1,127 @@+import { RenderError } from "../errors.js";+import { renderKeys } from "../jobs/key-template.js";+import type { DeliveredFile, Pipeline } from "../jobs/pipeline.js";+import {+ deliverInline,+ deliverLocal,+ deliverPresigned,+ deliverS3,+ readManifest,+ writeManifest,+} from "./sinks.js";+import { qaFailures, type VerifiedFile, verifyFile } from "./verify.js";++/**+ * Pipeline step: verifies every rendered file, applies the QA gate, and+ * delivers the files to the job's output target (PLAN.md §8.5).+ */+export const deliver: Pipeline["deliver"] = async (files, job, ctx) => {+ const { input, sha, ref } = job;+ const config = ctx.config;+ if (files.length === 0) {+ throw new RenderError("OUTPUT_MISSING", "The render produced no files.", { phase: "verify" });+ }++ ctx.progress({ phase: "verifying" });+ const verified: VerifiedFile[] = [];+ for (const file of files) {+ verified.push(await verifyFile(file, { maxBytes: config.storage.outputMaxBytes, signal: ctx.signal }));+ }++ const failOn = typeof input.render.qa === "object" ? input.render.qa.failOn : undefined;+ const qaIssues = verified.flatMap((f) =>+ qaFailures(f.qa, failOn as string[] | undefined).map((i) => ({ output: f.name, ...i })),+ );+ if (qaIssues.length > 0 && config.qaBlockUpload) {+ throw qaError(qaIssues, []);+ }++ ctx.progress({ phase: "uploading" });+ const out = input.output;+ const deliveries = verified.map((file) => ({ file, signal: ctx.signal }));+ const vars = { jobId: ctx.jobId, sha, ref };+ const named = verified.map((f) => ({ name: f.name, ext: f.ext }));+ let delivered: DeliveredFile[] = [];++ switch (out.type) {+ case "s3": {+ const bucket = out.bucket ?? (config.storage.bucket as string);+ const keys = renderKeys(out.key, named, vars);+ const metadata = {+ "ff-sha": sha,+ "ff-job": ctx.jobId,+ ...(input.idempotencyKey && { "ff-idempotency": input.idempotencyKey }),+ };+ for (const [i, d] of deliveries.entries()) {+ delivered.push(await deliverS3(d, config, bucket, keys[i], metadata));+ }+ if (input.idempotencyKey) {+ await writeManifest(config, bucket, {+ idempotencyKey: input.idempotencyKey,+ sha,+ outputs: delivered,+ createdAt: new Date().toISOString(),+ });+ }+ break;+ }+ case "presigned": {+ if (out.urls.length !== deliveries.length) {+ throw new RenderError(+ "JOB_INVALID",+ `The render produced ${deliveries.length} file(s) but output.urls has ${out.urls.length}.`,+ { phase: "upload", details: { files: named.map((n) => n.name) } },+ );+ }+ for (const [i, d] of deliveries.entries()) {+ delivered.push(await deliverPresigned(d, out.urls[i], out.headers));+ }+ break;+ }+ case "local": {+ const paths = renderKeys(out.path, named, vars);+ for (const [i, d] of deliveries.entries()) {+ delivered.push(await deliverLocal(d, config.storage.outputDir as string, paths[i]));+ }+ break;+ }+ case "inline": {+ const total = verified.reduce((sum, f) => sum + f.bytes, 0);+ // base64 grows ~4/3; keep the encoded payload under the limit.+ if (Math.ceil(total / 3) * 4 > config.storage.inlineMaxBytes) {+ throw new RenderError(+ "OUTPUT_TOO_LARGE",+ `Outputs total ${total} bytes, too large to return inline (FF_INLINE_MAX_BYTES=${config.storage.inlineMaxBytes}).`,+ { phase: "upload", hint: 'Use an "s3", "presigned" or "local" output.' },+ );+ }+ delivered = await Promise.all(deliveries.map(deliverInline));+ break;+ }+ }++ if (qaIssues.length > 0) throw qaError(qaIssues, delivered);+ return delivered;+};++function qaError(issues: Record<string, unknown>[], outputs: DeliveredFile[]): RenderError {+ return new RenderError("QA_FAILED", `QA found ${issues.length} issue(s) listed in render.qa.failOn.`, {+ phase: "verify",+ details: {+ issues: issues.slice(0, 50),+ // Delivered anyway (unless FF_QA_BLOCK_UPLOAD), so callers can inspect them.+ outputs: outputs.map(({ base64: _, ...rest }) => rest),+ },+ });+}++/**+ * Pipeline step: finds outputs an earlier attempt of the same job already+ * delivered for this commit (s3 + idempotencyKey only).+ */+export const findExisting: NonNullable<Pipeline["findExisting"]> = async (input, sha, ctx) => {+ if (!input.idempotencyKey || input.output.type !== "s3") return undefined;+ const bucket = input.output.bucket ?? (ctx.config.storage.bucket as string);+ const manifest = await readManifest(ctx.config, bucket, input.idempotencyKey, sha);+ return manifest?.outputs;+};diff --git a/apps/renderer-service/src/output/output.test.ts b/apps/renderer-service/src/output/output.test.tsnew file mode 100644index 00000000..144da8f0--- /dev/null+++ b/apps/renderer-service/src/output/output.test.ts@@ -0,0 +1,222 @@+import { execFileSync } from "node:child_process";+import fs from "node:fs/promises";+import http from "node:http";+import type { AddressInfo } from "node:net";+import os from "node:os";+import path from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { RenderError } from "../errors.js";+import type { RenderedFile, StepContext } from "../jobs/pipeline.js";+import { jobSchema } from "../jobs/schema.js";+import { testConfig } from "../test-helpers.js";+import { deliver, findExisting } from "./deliver.js";+import { qaFailures, verifyFile } from "./verify.js";++let tmp: string;+let server: http.Server;+let base: string;+const objects = new Map<string, { body: Buffer; headers: http.IncomingHttpHeaders }>();+/** Scripted responses for presigned PUTs, consumed in order. */+const presignedPlan: number[] = [];++beforeAll(async () => {+ tmp = await fs.mkdtemp(path.join(os.tmpdir(), "ff-output-"));+ // A tiny path-style S3 (PUT/GET/HEAD object) plus a presigned PUT endpoint.+ server = http.createServer(async (req, res) => {+ const chunks: Buffer[] = [];+ for await (const c of req) chunks.push(c as Buffer);+ const url = new URL(req.url ?? "/", "http://x");+ const key = decodeURIComponent(url.pathname);+ if (key.startsWith("/presigned/")) {+ const status = presignedPlan.shift() ?? 200;+ if (status === 200) objects.set(key, { body: Buffer.concat(chunks), headers: req.headers });+ res.writeHead(status).end(status === 403 ? "<Error><Message>Request has expired</Message></Error>" : "");+ return;+ }+ if (req.method === "PUT") {+ objects.set(key, { body: Buffer.concat(chunks), headers: req.headers });+ res.writeHead(200, { etag: '"x"' }).end();+ } else if (objects.has(key)) {+ const o = objects.get(key)!;+ res.writeHead(200, { "content-length": o.body.length, "content-type": "application/octet-stream" });+ res.end(req.method === "HEAD" ? undefined : o.body);+ } else {+ res.writeHead(404, { "content-type": "application/xml" });+ res.end(req.method === "HEAD" ? undefined : "<Error><Code>NoSuchKey</Code><Message>nope</Message></Error>");+ }+ });+ await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));+ base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+});+afterAll(async () => {+ server.close();+ await fs.rm(tmp, { recursive: true, force: true });+});++const hasFfmpeg = (() => {+ try {+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });+ return true;+ } catch {+ return false;+ }+})();++async function file(name: string, content: string | Buffer, ext = "bin"): Promise<RenderedFile> {+ const p = path.join(tmp, `${name}.${ext}`);+ await fs.writeFile(p, content);+ return { name, path: p, ext, contentType: "application/octet-stream" };+}++function mp4(name: string, frames: number): RenderedFile {+ const p = path.join(tmp, `${name}.mp4`);+ execFileSync("ffmpeg", ["-y", "-v", "error", "-f", "lavfi", "-i", "color=c=red:s=64x64:r=24", "-frames:v", String(frames), "-pix_fmt", "yuv420p", p]);+ return { name, path: p, ext: "mp4", contentType: "video/mp4", expectedFrames: frames };+}++async function codeOf(promise: Promise<unknown>) {+ try {+ await promise;+ return "no error";+ } catch (err) {+ return err instanceof RenderError ? err.code : String(err);+ }+}++describe("verifyFile", () => {+ it("hashes and sizes a file", async () => {+ const f = await verifyFile(await file("a", "hello"), { maxBytes: 100 });+ expect(f).toMatchObject({ bytes: 5, sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" });+ });++ it.each([+ ["missing", async () => ({ name: "x", path: path.join(tmp, "nope"), ext: "bin", contentType: "" }), "OUTPUT_MISSING"],+ ["empty", () => file("empty", ""), "OUTPUT_INVALID"],+ ["too large", () => file("big", "0123456789"), "OUTPUT_TOO_LARGE"],+ ["not media", () => file("fake", "not a video", "mp4"), "OUTPUT_INVALID"],+ ])("rejects %s output", async (_, make, code) => {+ expect(await codeOf(verifyFile(await make(), { maxBytes: 5 + (code === "OUTPUT_TOO_LARGE" ? 0 : 100) }))).toBe(code);+ });++ it.skipIf(!hasFfmpeg)("probes media and checks the frame count", async () => {+ const ok = await verifyFile(mp4("clip", 24), { maxBytes: 0 });+ expect(ok).toMatchObject({ frames: 24, durationMs: 1000 });+ const wrong = { ...mp4("short", 10), expectedFrames: 24 };+ expect(await codeOf(verifyFile(wrong, { maxBytes: 0 }))).toBe("OUTPUT_INVALID");+ });++ it("matches QA issues by code or severity", () => {+ const report = {+ issues: [+ { severity: "warning", code: "black_frames" },+ { severity: "info", code: "loudness_off_target" },+ ],+ };+ expect(qaFailures(report, ["black_frames"])).toHaveLength(1);+ expect(qaFailures(report, ["warning"])).toHaveLength(1);+ expect(qaFailures(report, ["frozen_video"])).toHaveLength(0);+ expect(qaFailures(report, undefined)).toHaveLength(0);+ });+});++describe("deliver", () => {+ const storageEnv = () => ({+ STORAGE_ENDPOINT: base,+ STORAGE_ACCESS_KEY_ID: "key",+ STORAGE_SECRET_ACCESS_KEY: "secret",+ STORAGE_BUCKET: "renders",+ STORAGE_FORCE_PATH_STYLE: "true",+ STORAGE_PUBLIC_BASE_URL: "https://cdn.example.com",+ FF_OUTPUT_DIR: path.join(tmp, "out"),+ });++ function run(output: unknown, files: RenderedFile[], extra: Record<string, unknown> = {}, env: Record<string, string> = {}) {+ const ctx: StepContext = {+ jobId: "job1",+ config: testConfig({ ...storageEnv(), ...env }),+ signal: new AbortController().signal,+ jobDir: tmp,+ progress: () => {},+ warn: () => {},+ };+ const input = jobSchema.parse({+ version: 2,+ source: { repo: "https://github.com/acme/films" },+ target: { entry: "src/film.ts" },+ output,+ ...extra,+ });+ return { promise: deliver(files, { input, sha: "9f2c1e4abcdef", ref: "main" }, ctx), input, ctx };+ }++ it("uploads to S3 with templated keys, metadata and public URLs", async () => {+ const files = [await file("main", "AAA", "mp4b"), await file("teaser", "BB", "mp4b")];+ const out = await run({ type: "s3", key: "acme/{shortSha}/{name}.{ext}" }, files).promise;+ expect(out.map((o) => o.key)).toEqual(["acme/9f2c1e4/main.mp4b", "acme/9f2c1e4/teaser.mp4b"]);+ expect(out[0].url).toBe("https://cdn.example.com/acme/9f2c1e4/main.mp4b");+ const stored = objects.get("/renders/acme/9f2c1e4/main.mp4b");+ expect(stored?.body.toString()).toBe("AAA");+ expect(stored?.headers["x-amz-meta-ff-sha"]).toBe("9f2c1e4abcdef");+ });++ it("writes an idempotency manifest that findExisting returns for the same commit", async () => {+ const files = [await file("main", "CCC", "mp4b")];+ const { promise, input, ctx } = run({ type: "s3", key: "idem/{name}.{ext}" }, files, { idempotencyKey: "launch-v12" });+ const delivered = await promise;+ expect(await findExisting?.(input, "9f2c1e4abcdef", ctx)).toEqual(delivered);+ expect(await findExisting?.(input, "0000000different", ctx)).toBeUndefined();+ // An output deleted from the bucket invalidates the manifest.+ objects.delete("/renders/idem/main.mp4b");+ expect(await findExisting?.(input, "9f2c1e4abcdef", ctx)).toBeUndefined();+ });++ it("PUTs presigned URLs, retrying 5xx and failing fast on expiry", async () => {+ presignedPlan.push(503, 200);+ const ok = await run({ type: "presigned", urls: [`${base}/presigned/a?X-Amz-Signature=s`] }, [await file("p", "PPP")]).promise;+ expect(ok[0].url).toBe(`${base}/presigned/a`);+ expect(objects.get("/presigned/a")?.body.toString()).toBe("PPP");++ presignedPlan.push(403);+ expect(await codeOf(run({ type: "presigned", urls: [`${base}/presigned/b`] }, [await file("p2", "x")]).promise)).toBe(+ "UPLOAD_URL_EXPIRED",+ );+ expect(+ await codeOf(run({ type: "presigned", urls: [`${base}/presigned/c`] }, [await file("q1", "x"), await file("q2", "y")]).promise),+ ).toBe("JOB_INVALID");+ });++ it("copies to FF_OUTPUT_DIR and refuses to collide or escape", async () => {+ const out = await run({ type: "local", path: "{ref}/{name}.{ext}" }, [await file("loc", "LLL")]).promise;+ expect(await fs.readFile(path.join(tmp, "out/main/loc.bin"), "utf8")).toBe("LLL");+ expect(out[0].path).toBe("main/loc.bin");+ expect(+ await codeOf(run({ type: "local", path: "same.{ext}" }, [await file("l1", "1"), await file("l2", "2")]).promise),+ ).toBe("JOB_INVALID");+ });++ it("returns small outputs inline and refuses large ones", async () => {+ const out = await run({ type: "inline" }, [await file("i", "hi")]).promise;+ expect(Buffer.from(out[0].base64 ?? "", "base64").toString()).toBe("hi");+ expect(+ await codeOf(run({ type: "inline" }, [await file("big", "x".repeat(100))], {}, { FF_INLINE_MAX_BYTES: "50" }).promise),+ ).toBe("OUTPUT_TOO_LARGE");+ });++ it("fails with QA_FAILED after delivering, unless uploads are blocked", async () => {+ const qa = { issues: [{ severity: "warning", code: "frozen_video", message: "frozen" }] };+ const files = [{ ...(await file("qa", "QQQ")), qa }];+ const render = { render: { qa: { failOn: ["frozen_video"] } } };+ try {+ await run({ type: "inline" }, files, render).promise;+ throw new Error("expected QA_FAILED");+ } catch (err) {+ expect((err as RenderError).code).toBe("QA_FAILED");+ expect((err as RenderError).details?.outputs).toHaveLength(1);+ }+ try {+ await run({ type: "inline" }, files, render, { FF_QA_BLOCK_UPLOAD: "true" }).promise;+ } catch (err) {+ expect((err as RenderError).details?.outputs).toEqual([]);+ }+ });+});diff --git a/apps/renderer-service/src/output/sinks.ts b/apps/renderer-service/src/output/sinks.tsnew file mode 100644index 00000000..dfb9e288--- /dev/null+++ b/apps/renderer-service/src/output/sinks.ts@@ -0,0 +1,233 @@+import { createReadStream } from "node:fs";+import fs from "node:fs/promises";+import path from "node:path";+import {+ GetObjectCommand,+ HeadObjectCommand,+ PutObjectCommand,+ S3Client,+ S3ServiceException,+} from "@aws-sdk/client-s3";+import { Upload } from "@aws-sdk/lib-storage";+import type { RendererConfig } from "../config/index.js";+import { RenderError } from "../errors.js";+import type { DeliveredFile } from "../jobs/pipeline.js";+import { redact } from "../redact.js";+import type { VerifiedFile } from "./verify.js";++const ATTEMPTS = 3;++/** Retries transient storage failures with backoff. */+async function withRetries<T>(signal: AbortSignal, fn: () => Promise<T>): Promise<T> {+ for (let attempt = 1; ; attempt++) {+ try {+ return await fn();+ } catch (err) {+ const classified = classifyUploadError(err);+ if (!classified.retryable || attempt >= ATTEMPTS || signal.aborted) throw classified;+ await new Promise((r) => setTimeout(r, 500 * 2 ** attempt));+ }+ }+}++export function classifyUploadError(err: unknown): RenderError {+ if (err instanceof RenderError) return err;+ const name = err instanceof S3ServiceException ? err.name : "";+ const status = err instanceof S3ServiceException ? err.$metadata?.httpStatusCode : undefined;+ const message = redact(err instanceof Error ? err.message : String(err));+ if (/AccessDenied|InvalidAccessKeyId|SignatureDoesNotMatch|InvalidToken|ExpiredToken|AllAccessDisabled/.test(name)) {+ return new RenderError("UPLOAD_AUTH_FAILED", `${name}: ${message}`, {+ phase: "upload",+ hint: "Check STORAGE_ACCESS_KEY_ID / STORAGE_SECRET_ACCESS_KEY and that they may write to the bucket.",+ cause: err,+ });+ }+ if (/NoSuchBucket/.test(name)) {+ return new RenderError("UPLOAD_AUTH_FAILED", `Bucket does not exist: ${message}`, { phase: "upload", cause: err });+ }+ if (/ENOSPC/.test(message)) return new RenderError("DISK_FULL", message, { phase: "upload" });+ return new RenderError("UPLOAD_FAILED", status ? `HTTP ${status}: ${message}` : message, {+ phase: "upload",+ cause: err,+ });+}++let client: S3Client | undefined;+export function s3Client(config: RendererConfig): S3Client {+ client ??= new S3Client({+ region: config.storage.region,+ endpoint: config.storage.endpoint,+ forcePathStyle: config.storage.forcePathStyle,+ credentials: {+ accessKeyId: config.storage.accessKeyId ?? "",+ secretAccessKey: config.storage.secretAccessKey ?? "",+ },+ maxAttempts: 2,+ });+ return client;+}++export function publicUrl(config: RendererConfig, key: string): string | undefined {+ const base = config.storage.publicBaseUrl;+ if (!base) return undefined;+ return `${base.replace(/\/+$/, "")}/${key.split("/").map(encodeURIComponent).join("/")}`;+}++export interface Delivery {+ file: VerifiedFile;+ signal: AbortSignal;+}++export async function deliverS3(+ { file, signal }: Delivery,+ config: RendererConfig,+ bucket: string,+ key: string,+ metadata: Record<string, string>,+): Promise<DeliveredFile> {+ await withRetries(signal, async () => {+ const upload = new Upload({+ client: s3Client(config),+ params: {+ Bucket: bucket,+ Key: key,+ Body: createReadStream(file.path),+ ContentType: file.contentType,+ ContentLength: file.bytes,+ Metadata: metadata,+ },+ queueSize: 4,+ partSize: 16 * 1024 * 1024,+ leavePartsOnError: false,+ });+ const onAbort = () => void upload.abort();+ signal.addEventListener("abort", onAbort, { once: true });+ try {+ await upload.done();+ } finally {+ signal.removeEventListener("abort", onAbort);+ }+ });+ return summary(file, { key, url: publicUrl(config, key) });+}++export async function deliverPresigned(+ { file, signal }: Delivery,+ url: string,+ headers: Record<string, string> = {},+): Promise<DeliveredFile> {+ await withRetries(signal, async () => {+ const res = await fetch(url, {+ method: "PUT",+ body: createReadStream(file.path) as unknown as RequestInit["body"],+ // Node's fetch needs this for streamed request bodies.+ duplex: "half",+ headers: {+ "content-type": file.contentType,+ "content-length": String(file.bytes),+ ...headers,+ },+ signal,+ } as RequestInit);+ if (res.ok) return;+ const text = await res.text().catch(() => "");+ if ((res.status === 403 || res.status === 400) && /expired|Request has expired|AccessDenied/i.test(text)) {+ throw new RenderError("UPLOAD_URL_EXPIRED", `The presigned URL was rejected (${res.status}).`, {+ phase: "upload",+ hint: "Sign upload URLs with an expiry at least as long as the job timeout.",+ });+ }+ const error = new RenderError("UPLOAD_FAILED", `Presigned PUT failed with HTTP ${res.status}.`, {+ phase: "upload",+ details: { body: redact(text.slice(0, 500)) },+ });+ if (res.status < 500 && res.status !== 429) {+ throw new RenderError("UPLOAD_AUTH_FAILED", error.message, { phase: "upload", details: error.details });+ }+ throw error;+ });+ return summary(file, { url: url.split("?")[0] });+}++/** Copies into FF_OUTPUT_DIR atomically (temp file + rename). */+export async function deliverLocal({ file }: Delivery, outputDir: string, relPath: string): Promise<DeliveredFile> {+ const root = path.resolve(outputDir);+ const target = path.resolve(root, relPath);+ if (!target.startsWith(root + path.sep)) {+ throw new RenderError("PATH_ESCAPE", `Output path ${relPath} leaves FF_OUTPUT_DIR.`, { phase: "upload" });+ }+ await fs.mkdir(path.dirname(target), { recursive: true });+ const tmp = `${target}.partial-${process.pid}`;+ try {+ await fs.copyFile(file.path, tmp);+ await fs.rename(tmp, target);+ } catch (err) {+ await fs.rm(tmp, { force: true });+ throw classifyUploadError(err);+ }+ return summary(file, { path: relPath });+}++export async function deliverInline({ file }: Delivery): Promise<DeliveredFile> {+ return summary(file, { base64: (await fs.readFile(file.path)).toString("base64") });+}++function summary(file: VerifiedFile, extra: Partial<DeliveredFile>): DeliveredFile {+ return {+ name: file.name,+ bytes: file.bytes,+ sha256: file.sha256,+ contentType: file.contentType,+ ...(file.durationMs !== undefined && { durationMs: file.durationMs }),+ ...(file.frames !== undefined && { frames: file.frames }),+ ...(file.qa !== undefined && { qa: file.qa }),+ ...extra,+ };+}++// ─── Idempotency manifest (PLAN.md §8.7) ─────────────────────────────────────++export interface Manifest {+ idempotencyKey: string;+ sha: string;+ outputs: DeliveredFile[];+ createdAt: string;+}++export function manifestKey(idempotencyKey: string): string {+ return `.ff-idempotency/${Buffer.from(idempotencyKey).toString("base64url")}.json`;+}++/** Returns the earlier result for this key when it rendered the same commit and its objects still exist. */+export async function readManifest(+ config: RendererConfig,+ bucket: string,+ idempotencyKey: string,+ sha: string,+): Promise<Manifest | undefined> {+ const s3 = s3Client(config);+ try {+ const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: manifestKey(idempotencyKey) }));+ const manifest = JSON.parse((await res.Body?.transformToString()) ?? "{}") as Manifest;+ if (manifest.sha !== sha || manifest.idempotencyKey !== idempotencyKey) return undefined;+ for (const output of manifest.outputs) {+ if (!output.key) return undefined;+ await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: output.key }));+ }+ return manifest;+ } catch (err) {+ if (err instanceof S3ServiceException && (err.name === "NoSuchKey" || err.name === "NotFound")) return undefined;+ throw classifyUploadError(err);+ }+}++export async function writeManifest(config: RendererConfig, bucket: string, manifest: Manifest): Promise<void> {+ await s3Client(config).send(+ new PutObjectCommand({+ Bucket: bucket,+ Key: manifestKey(manifest.idempotencyKey),+ Body: JSON.stringify(manifest),+ ContentType: "application/json",+ }),+ );+}diff --git a/apps/renderer-service/src/output/verify.ts b/apps/renderer-service/src/output/verify.tsnew file mode 100644index 00000000..f400fd89--- /dev/null+++ b/apps/renderer-service/src/output/verify.ts@@ -0,0 +1,116 @@+import { createHash } from "node:crypto";+import { createReadStream } from "node:fs";+import fs from "node:fs/promises";+import { RenderError } from "../errors.js";+import { CommandError, exec } from "../git/exec.js";+import type { RenderedFile } from "../jobs/pipeline.js";++export interface VerifiedFile extends RenderedFile {+ bytes: number;+ sha256: string;+ durationMs?: number;+ frames?: number;+}++const PROBED = new Set(["mp4", "webm", "m4a", "mp3", "gif"]);++/**+ * Checks a rendered file before it leaves the worker (PLAN.md §8.5): it must+ * exist, be non-empty and within the size limit, and media files must probe+ * cleanly with the frame count the engine promised (±1).+ */+export async function verifyFile(+ file: RenderedFile,+ options: { maxBytes: number; signal?: AbortSignal; probe?: boolean },+): Promise<VerifiedFile> {+ const stat = await fs.stat(file.path).catch(() => undefined);+ if (!stat?.isFile()) {+ throw new RenderError("OUTPUT_MISSING", `Output ${file.name} was not written.`, { phase: "verify" });+ }+ if (stat.size === 0) {+ throw new RenderError("OUTPUT_INVALID", `Output ${file.name} is empty.`, { phase: "verify" });+ }+ if (options.maxBytes > 0 && stat.size > options.maxBytes) {+ throw new RenderError(+ "OUTPUT_TOO_LARGE",+ `Output ${file.name} is ${stat.size} bytes, over FF_OUTPUT_MAX_BYTES (${options.maxBytes}).`,+ { phase: "verify" },+ );+ }++ const verified: VerifiedFile = { ...file, bytes: stat.size, sha256: await sha256File(file.path) };+ if (options.probe !== false && PROBED.has(file.ext)) {+ Object.assign(verified, await probe(file, options.signal));+ }+ return verified;+}++export async function sha256File(file: string): Promise<string> {+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(file)) hash.update(chunk as Buffer);+ return hash.digest("hex");+}++interface ProbeOutput {+ format?: { duration?: string };+ streams?: { codec_type?: string; nb_frames?: string; nb_read_frames?: string }[];+}++async function probe(file: RenderedFile, signal?: AbortSignal): Promise<{ durationMs?: number; frames?: number }> {+ let out: ProbeOutput;+ try {+ const { stdout } = await exec(+ "ffprobe",+ ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", file.path],+ { timeoutMs: 120_000, signal },+ );+ out = JSON.parse(stdout) as ProbeOutput;+ } catch (err) {+ if (err instanceof CommandError) {+ throw new RenderError("OUTPUT_INVALID", `Output ${file.name} is not readable media: ${err.message}`, {+ phase: "verify",+ });+ }+ if (/ENOENT/.test(String(err))) {+ // No ffprobe on this host: size and hash checks still ran.+ return {};+ }+ throw err;+ }++ const duration = Number(out.format?.duration);+ const video = out.streams?.find((s) => s.codec_type === "video");+ const frames = Number(video?.nb_frames ?? video?.nb_read_frames);+ const result = {+ ...(Number.isFinite(duration) && { durationMs: Math.round(duration * 1000) }),+ ...(Number.isFinite(frames) && frames > 0 && { frames }),+ };+ if (!out.streams?.length) {+ throw new RenderError("OUTPUT_INVALID", `Output ${file.name} has no streams.`, { phase: "verify" });+ }+ if (file.expectedFrames && result.frames !== undefined && Math.abs(result.frames - file.expectedFrames) > 1) {+ throw new RenderError(+ "OUTPUT_INVALID",+ `Output ${file.name} has ${result.frames} frames; the engine rendered ${file.expectedFrames}.`,+ { phase: "verify" },+ );+ }+ return result;+}++interface QaIssueLike {+ severity?: string;+ code?: string;+ message?: string;+}++/**+ * QA issues that fail the job under `render.qa.failOn`. Entries match an+ * issue code (`black_frames`, `frozen_video`, `audio_clipping`, …) or a+ * severity (`error`, `warning`).+ */+export function qaFailures(report: unknown, failOn: readonly string[] | undefined): QaIssueLike[] {+ if (!failOn?.length || !report || typeof report !== "object") return [];+ const issues = (report as { issues?: QaIssueLike[] }).issues ?? [];+ return issues.filter((i) => failOn.includes(i.code ?? "") || failOn.includes(i.severity ?? ""));+}