Open source CapCut alternative.
TypeScript
fix(cut): stop an export on a sticker it cannot read
David Lebfa5e57e005a
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex 8598624d..d479ebdb 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -32,7 +32,7 @@ import { createRasterCanvas, rasterCanvasToPng } from "./raster";import { clipLen, clipSpeed, getClipSpans, MICRO_GAP_S, openedTimeline, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";import { drawStreak, elementLook, isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type FlashFade, type FlashRhythm, type FlashTone, type GlitchKind, type LeakCourse, type Mask, type SpeedNode } from "@donkeycut/effects-kit";-import { renderElementFrames, renderElementPng, renderStillPng } from "./textRender";+import { renderElementFrames, renderElementPng, renderStillPng, requireStickers } from "./textRender";import { CLIP_MIN_SECONDS, clipCovers, clipFrameAt, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";import { liveReader } from "./liveReader";import { soundSourceOf, spanSoundSpec, spanUsesSoundFeature, specSound } from "./soundSource";@@ -1401,6 +1401,10 @@ export async function buildExportPayload(end: Math.min(o.end, duration),}));+ // Every sticker the cut shows is read before any is drawn: one that cannot+ // be read stops the export by name.+ await requireStickers(doc.overlays.filter((o) => o.start < duration), doc.assets);+for (let i = 0; i < doc.overlays.length; i++) {const o = doc.overlays[i];if (o.hidden || o.start >= duration) continue;diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex 833838fe..680a1c84 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -53,7 +53,7 @@ import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";import { MatteAlpha, matteStamp } from "./matteAlpha";import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";-import { cutRenderEnv, renderElementCanvas, renderStillPng } from "./textRender";+import { cutRenderEnv, renderElementCanvas, renderStillPng, requireStickers } from "./textRender";import { cardMatteKey } from "./cameraCard";import { assetIsSilent, behindSubjectOverlay, clipCovers, frameOf, frontSubjectOverlay, isEffectOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, removalActive } from "./types";import type { ClipAnim, ClipSpan, EffectOverlay, MediaAsset, Overlay, StickerOverlay } from "./types";@@ -1512,6 +1512,9 @@ export class FramePainter {this.comp.sourceProvider = sourceLookup(this.doc.assets, renderFile);await settleRenderColor(this.doc.assets);await ensureClipLuts(this.doc.clips.map((c) => this.comp.recipeFor(c)));+ // Every sticker the cut shows is read before the first frame: one that+ // cannot be read stops the render by name.+ await requireStickers(this.doc.overlays.filter((o) => o.start < this.duration), this.doc.assets);this.stacked = [...(await stampText(this.doc))].sort((a, b) => b.stackLane - a.stackLane);// The subject pass is created only when something reads the person matte;// prepare() makes rasters and the segmenter resident before the firstdiff --git a/site/src/cut/lib/stickerExport.test.ts b/site/src/cut/lib/stickerExport.test.tsnew file mode 100644index 00000000..fe179be1--- /dev/null+++ b/site/src/cut/lib/stickerExport.test.ts@@ -0,0 +1,83 @@+import { afterAll, beforeAll, expect, test } from "bun:test";+import type { StickerOverlay } from "@donkeycut/effects-kit";+import { installSkiaRaster } from "./headless/skiaRaster";+import { buildExportPayload, EXPORT_PRESETS, presetSettings } from "./exportClient";+import { renderProjectFrame } from "./exportRender";+import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";+import type { ExportDoc } from "./renderSnapshot";+import type { MediaAsset } from "./types";++// A sticker reaches an export as its asset's URL, read when the element is+// drawn. The preview skips a sticker it cannot read and asks again next frame;+// an export draws each frame once, so a read that fails there has to stop it+// by name — the worker's renders came back with the title and without the+// doodles drawn around it.++const SERVED = "https://media.test/sparkles.png";+const UNSIGNED = "https://media.test/unsigned.png";+const realFetch = globalThis.fetch;++beforeAll(async () => {+ expect(await installSkiaRaster()).toBe(true);++ // A white block on transparent, the shape of a doodle sheet.+ const sheet = createRasterCanvas(64, 96);+ const ctx = sheet.getContext("2d") as CanvasRenderingContext2D;+ ctx.fillStyle = "#ffffff";+ ctx.fillRect(0, 0, 64, 96);+ const png = await (await rasterCanvasToPng(sheet)).arrayBuffer();++ // The served URL answers with the picture; anything else gets what a read+ // without its signature gets.+ globalThis.fetch = (async (input: RequestInfo | URL) =>+ String(input) === SERVED+ ? new Response(png, { headers: { "content-type": "image/png" } })+ : new Response("Unauthorized", { status: 401 })) as typeof fetch;+});++afterAll(() => {+ globalThis.fetch = realFetch;+});++const sticker = (assetId: string): StickerOverlay => ({+ id: `o-${assetId}`, kind: "sticker", assetId, name: "Sparkles left",+ start: 0, end: 2, x: 0.5, y: 0.5, w: 0.5, h: 0.5,+});++const docOf = (assetId: string, url: string): ExportDoc => ({+ aspect: "16:9",+ assets: [{+ id: assetId, name: "Sparkles left", type: "image", origin: "sticker", duration: 0,+ width: 64, height: 96, fileName: "sparkles-left.png", url,+ } as MediaAsset],+ clips: [], audioClips: [], overlays: [sticker(assetId)],+ subtitles: { cues: [], showOnVideo: false, showOnTimeline: false },+});++const size = { width: 320, height: 180 };+const settings = () => presetSettings(EXPORT_PRESETS.find((p) => p.id === "tiktok")!, "16:9");++test("the spec export stops on a sticker it cannot read", async () => {+ await expect(buildExportPayload("project", docOf("stx-spec-unread", UNSIGNED), settings(), "export")).rejects.toThrow(+ /The sticker "Sparkles left" could not be read \(HTTP 401\)/+ );+});++test("the tab export stops on a sticker it cannot read", async () => {+ await expect(renderProjectFrame(docOf("stx-tab-unread", UNSIGNED), 1, size, (a) => a.url)).rejects.toThrow(+ /The sticker "Sparkles left" could not be read \(HTTP 401\)/+ );+});++test("a sticker that reads draws in both exports", async () => {+ const payload = await buildExportPayload("project", docOf("stx-spec-read", SERVED), settings(), "export");+ const still = await decodeRasterImage(payload.pngs.find((p) => p.name === "overlay_0.png")!.blob);+ const sheet = createRasterCanvas(still!.width, still!.height);+ const sctx = sheet.getContext("2d") as CanvasRenderingContext2D;+ sctx.drawImage(still!.source, 0, 0);+ expect(sctx.getImageData(still!.width / 2, still!.height / 2, 1, 1).data[3]).toBeGreaterThan(200);++ const frame = await renderProjectFrame(docOf("stx-tab-read", SERVED), 1, size, (a) => a.url);+ const ink = (frame.getContext("2d") as CanvasRenderingContext2D).getImageData(160, 90, 1, 1).data;+ expect(ink[0]).toBeGreaterThan(200);+});diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.tsindex a2e8206e..5a60364a 100644--- a/site/src/cut/lib/textRender.ts+++ b/site/src/cut/lib/textRender.ts@@ -126,12 +126,26 @@ function sweepStickers(asked: string): void {}}-function decodeSticker(url: string): Promise<StickerImage | null> {- return fetch(url)- .then((res) => (res.ok ? res.blob() : Promise.reject(new Error("fetch failed"))))- .then((blob) => decodeRasterImage(blob)- )- .catch(() => null);+/** Why each sticker's last read came back empty, by asset id: what an export+ * names when it stops on one. */+const stickerMisses = new Map<string, string>();++async function decodeSticker(asset: MediaAsset): Promise<StickerImage | null> {+ try {+ const res = await fetch(asset.url);+ if (!res.ok) {+ throw new Error(`HTTP ${res.status}`);+ }+ const img = await decodeRasterImage(await res.blob());+ if (!img) {+ throw new Error("not a picture");+ }+ stickerMisses.delete(asset.id);+ return img;+ } catch (err) {+ stickerMisses.set(asset.id, err instanceof Error ? err.message : String(err));+ return null;+ }}/** The kit render env bound to this app: Cut's font stacks, and sticker@@ -152,7 +166,7 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {// Asked for again: move it to the young end of the cache.stickerCache.delete(asset.id);} else {- hit = decodeSticker(asset.url).then((img) => {+ hit = decodeSticker(asset).then((img) => {// A failed decode is not worth pinning; the next render retries.if (!img) stickerCache.delete(asset.id);else {@@ -168,6 +182,29 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {};}+/**+ * Read every sticker an export draws before its first frame. The preview skips+ * a sticker it cannot read and asks again on the next frame; an export draws+ * each frame once, so a sticker it cannot read stops it by name. The pictures+ * stay in the sticker cache for the frames that follow.+ */+export async function requireStickers(overlays: Overlay[], assets: MediaAsset[]): Promise<void> {+ const env = cutRenderEnv(assets);+ for (const o of overlays) {+ // Lottie stickers seek their own player, and an element with no asset+ // draws nothing in the preview either.+ if (o.kind !== "sticker" || o.hidden || o.lottie || !o.assetId) {+ continue;+ }+ const asset = assets.find((a) => a.id === o.assetId);+ if (!asset || (await env.resolveAsset!(asset.id))) {+ continue;+ }+ const why = stickerMisses.get(asset.id) ?? "no picture";+ throw new Error(`The sticker "${o.name ?? asset.name}" could not be read (${why}).`);+ }+}+/*** Render an overlay element (text, shape, or sticker) to a transparent* full-frame PNG at the export resolution, matching the DOM preview's