Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): stop an export on a sticker it cannot read

David Lebfa5e57e005a
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 8598624d..d479ebdb 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -32,7 +32,7 @@ import { createRasterCanvas, rasterCanvasToPng } from "./raster";
import { clipLen, clipSpeed, getClipSpans, MICRO_GAP_S, openedTimeline, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
import { drawStreak, elementLook, isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type FlashFade, type FlashRhythm, type FlashTone, type GlitchKind, type LeakCourse, type Mask, type SpeedNode } from "@donkeycut/effects-kit";
-import { renderElementFrames, renderElementPng, renderStillPng } from "./textRender";
+import { renderElementFrames, renderElementPng, renderStillPng, requireStickers } from "./textRender";
import { CLIP_MIN_SECONDS, clipCovers, clipFrameAt, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";
import { liveReader } from "./liveReader";
import { soundSourceOf, spanSoundSpec, spanUsesSoundFeature, specSound } from "./soundSource";
@@ -1401,6 +1401,10 @@ export async function buildExportPayload(
end: Math.min(o.end, duration),
}));
+ // Every sticker the cut shows is read before any is drawn: one that cannot
+ // be read stops the export by name.
+ await requireStickers(doc.overlays.filter((o) => o.start < duration), doc.assets);
+
for (let i = 0; i < doc.overlays.length; i++) {
const o = doc.overlays[i];
if (o.hidden || o.start >= duration) continue;
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index 833838fe..680a1c84 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -53,7 +53,7 @@ import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
import { MatteAlpha, matteStamp } from "./matteAlpha";
import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";
import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";
-import { cutRenderEnv, renderElementCanvas, renderStillPng } from "./textRender";
+import { cutRenderEnv, renderElementCanvas, renderStillPng, requireStickers } from "./textRender";
import { cardMatteKey } from "./cameraCard";
import { assetIsSilent, behindSubjectOverlay, clipCovers, frameOf, frontSubjectOverlay, isEffectOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, removalActive } from "./types";
import type { ClipAnim, ClipSpan, EffectOverlay, MediaAsset, Overlay, StickerOverlay } from "./types";
@@ -1512,6 +1512,9 @@ export class FramePainter {
this.comp.sourceProvider = sourceLookup(this.doc.assets, renderFile);
await settleRenderColor(this.doc.assets);
await ensureClipLuts(this.doc.clips.map((c) => this.comp.recipeFor(c)));
+ // Every sticker the cut shows is read before the first frame: one that
+ // cannot be read stops the render by name.
+ await requireStickers(this.doc.overlays.filter((o) => o.start < this.duration), this.doc.assets);
this.stacked = [...(await stampText(this.doc))].sort((a, b) => b.stackLane - a.stackLane);
// The subject pass is created only when something reads the person matte;
// prepare() makes rasters and the segmenter resident before the first
diff --git a/site/src/cut/lib/stickerExport.test.ts b/site/src/cut/lib/stickerExport.test.ts
new file mode 100644
index 00000000..fe179be1
--- /dev/null
+++ b/site/src/cut/lib/stickerExport.test.ts
@@ -0,0 +1,83 @@
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import type { StickerOverlay } from "@donkeycut/effects-kit";
+import { installSkiaRaster } from "./headless/skiaRaster";
+import { buildExportPayload, EXPORT_PRESETS, presetSettings } from "./exportClient";
+import { renderProjectFrame } from "./exportRender";
+import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";
+import type { ExportDoc } from "./renderSnapshot";
+import type { MediaAsset } from "./types";
+
+// A sticker reaches an export as its asset's URL, read when the element is
+// drawn. The preview skips a sticker it cannot read and asks again next frame;
+// an export draws each frame once, so a read that fails there has to stop it
+// by name — the worker's renders came back with the title and without the
+// doodles drawn around it.
+
+const SERVED = "https://media.test/sparkles.png";
+const UNSIGNED = "https://media.test/unsigned.png";
+const realFetch = globalThis.fetch;
+
+beforeAll(async () => {
+ expect(await installSkiaRaster()).toBe(true);
+
+ // A white block on transparent, the shape of a doodle sheet.
+ const sheet = createRasterCanvas(64, 96);
+ const ctx = sheet.getContext("2d") as CanvasRenderingContext2D;
+ ctx.fillStyle = "#ffffff";
+ ctx.fillRect(0, 0, 64, 96);
+ const png = await (await rasterCanvasToPng(sheet)).arrayBuffer();
+
+ // The served URL answers with the picture; anything else gets what a read
+ // without its signature gets.
+ globalThis.fetch = (async (input: RequestInfo | URL) =>
+ String(input) === SERVED
+ ? new Response(png, { headers: { "content-type": "image/png" } })
+ : new Response("Unauthorized", { status: 401 })) as typeof fetch;
+});
+
+afterAll(() => {
+ globalThis.fetch = realFetch;
+});
+
+const sticker = (assetId: string): StickerOverlay => ({
+ id: `o-${assetId}`, kind: "sticker", assetId, name: "Sparkles left",
+ start: 0, end: 2, x: 0.5, y: 0.5, w: 0.5, h: 0.5,
+});
+
+const docOf = (assetId: string, url: string): ExportDoc => ({
+ aspect: "16:9",
+ assets: [{
+ id: assetId, name: "Sparkles left", type: "image", origin: "sticker", duration: 0,
+ width: 64, height: 96, fileName: "sparkles-left.png", url,
+ } as MediaAsset],
+ clips: [], audioClips: [], overlays: [sticker(assetId)],
+ subtitles: { cues: [], showOnVideo: false, showOnTimeline: false },
+});
+
+const size = { width: 320, height: 180 };
+const settings = () => presetSettings(EXPORT_PRESETS.find((p) => p.id === "tiktok")!, "16:9");
+
+test("the spec export stops on a sticker it cannot read", async () => {
+ await expect(buildExportPayload("project", docOf("stx-spec-unread", UNSIGNED), settings(), "export")).rejects.toThrow(
+ /The sticker "Sparkles left" could not be read \(HTTP 401\)/
+ );
+});
+
+test("the tab export stops on a sticker it cannot read", async () => {
+ await expect(renderProjectFrame(docOf("stx-tab-unread", UNSIGNED), 1, size, (a) => a.url)).rejects.toThrow(
+ /The sticker "Sparkles left" could not be read \(HTTP 401\)/
+ );
+});
+
+test("a sticker that reads draws in both exports", async () => {
+ const payload = await buildExportPayload("project", docOf("stx-spec-read", SERVED), settings(), "export");
+ const still = await decodeRasterImage(payload.pngs.find((p) => p.name === "overlay_0.png")!.blob);
+ const sheet = createRasterCanvas(still!.width, still!.height);
+ const sctx = sheet.getContext("2d") as CanvasRenderingContext2D;
+ sctx.drawImage(still!.source, 0, 0);
+ expect(sctx.getImageData(still!.width / 2, still!.height / 2, 1, 1).data[3]).toBeGreaterThan(200);
+
+ const frame = await renderProjectFrame(docOf("stx-tab-read", SERVED), 1, size, (a) => a.url);
+ const ink = (frame.getContext("2d") as CanvasRenderingContext2D).getImageData(160, 90, 1, 1).data;
+ expect(ink[0]).toBeGreaterThan(200);
+});
diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.ts
index a2e8206e..5a60364a 100644
--- a/site/src/cut/lib/textRender.ts
+++ b/site/src/cut/lib/textRender.ts
@@ -126,12 +126,26 @@ function sweepStickers(asked: string): void {
}
}
-function decodeSticker(url: string): Promise<StickerImage | null> {
- return fetch(url)
- .then((res) => (res.ok ? res.blob() : Promise.reject(new Error("fetch failed"))))
- .then((blob) => decodeRasterImage(blob)
- )
- .catch(() => null);
+/** Why each sticker's last read came back empty, by asset id: what an export
+ * names when it stops on one. */
+const stickerMisses = new Map<string, string>();
+
+async function decodeSticker(asset: MediaAsset): Promise<StickerImage | null> {
+ try {
+ const res = await fetch(asset.url);
+ if (!res.ok) {
+ throw new Error(`HTTP ${res.status}`);
+ }
+ const img = await decodeRasterImage(await res.blob());
+ if (!img) {
+ throw new Error("not a picture");
+ }
+ stickerMisses.delete(asset.id);
+ return img;
+ } catch (err) {
+ stickerMisses.set(asset.id, err instanceof Error ? err.message : String(err));
+ return null;
+ }
}
/** The kit render env bound to this app: Cut's font stacks, and sticker
@@ -152,7 +166,7 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {
// Asked for again: move it to the young end of the cache.
stickerCache.delete(asset.id);
} else {
- hit = decodeSticker(asset.url).then((img) => {
+ hit = decodeSticker(asset).then((img) => {
// A failed decode is not worth pinning; the next render retries.
if (!img) stickerCache.delete(asset.id);
else {
@@ -168,6 +182,29 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {
};
}
+/**
+ * Read every sticker an export draws before its first frame. The preview skips
+ * a sticker it cannot read and asks again on the next frame; an export draws
+ * each frame once, so a sticker it cannot read stops it by name. The pictures
+ * stay in the sticker cache for the frames that follow.
+ */
+export async function requireStickers(overlays: Overlay[], assets: MediaAsset[]): Promise<void> {
+ const env = cutRenderEnv(assets);
+ for (const o of overlays) {
+ // Lottie stickers seek their own player, and an element with no asset
+ // draws nothing in the preview either.
+ if (o.kind !== "sticker" || o.hidden || o.lottie || !o.assetId) {
+ continue;
+ }
+ const asset = assets.find((a) => a.id === o.assetId);
+ if (!asset || (await env.resolveAsset!(asset.id))) {
+ continue;
+ }
+ const why = stickerMisses.get(asset.id) ?? "no picture";
+ throw new Error(`The sticker "${o.name ?? asset.name}" could not be read (${why}).`);
+ }
+}
+
/**
* Render an overlay element (text, shape, or sticker) to a transparent
* full-frame PNG at the export resolution, matching the DOM preview's