Open source CapCut alternative.
TypeScript
fix(site): soften a posed clip from its overhang
David Lee5e13cd915a8
diff --git a/site/src/cut/lib/composite.test.ts b/site/src/cut/lib/composite.test.tsindex dc2b7174..766be86d 100644--- a/site/src/cut/lib/composite.test.ts+++ b/site/src/cut/lib/composite.test.ts@@ -339,4 +339,24 @@ describe("a keyed clip's softening", () => {);expect(moving).toContain("lighter");});++ test("a zoomed clip softens from the picture past the frame edge", () => {+ // A full-frame clip zoomed 1.5x hangs past the frame. Its streak and+ // blur have to read that overhang: a frame-sized scratch cut the picture+ // at the edge, and the soft pass pulled transparent black into a rim.+ const full = { x: 0, y: 0, w: 1, h: 1 };+ const zoomed = (t: number, x: number) => ({ t, x, y: 0.5, scale: 1.5, rotation: 0, opacity: 1, blur: 270 });+ const clip = clipOf({ frame: full, kf: [zoomed(0, 0.4), zoomed(1, 0.6)], motionBlur: 1 });+ const frame = new PixelCanvas(W, H);+ const lands: { width: number; dx: number }[] = [];+ const draw = frame.ctx.drawImage as (src: PixelCanvas, ...args: number[]) => void;+ frame.ctx.drawImage = (src: PixelCanvas, ...args: number[]) => {+ lands.push({ width: src.width, dx: args[0] });+ draw(src, ...args);+ };+ new FrameCompositor(frame as unknown as RasterSurface).drawIntoRect(frameOf(picture()), full, true, 1, 0.5, 1, clip);+ const last = lands.at(-1)!;+ expect(last.width).toBeGreaterThan(W);+ expect(last.dx).toBeLessThan(0);+ });});diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.tsindex 21ec8786..88cf955e 100644--- a/site/src/cut/lib/composite.ts+++ b/site/src/cut/lib/composite.ts@@ -88,6 +88,11 @@ const REC709_SOURCE: RecipeSource = { profile: "rec709" };/** How many clips a compositor remembers the last drawn LUT for. */const LAST_LUT_MAX = 64;+/** The posed picture's overhang grows in steps of this many px, so a streak+ * that lengthens frame by frame keeps its padded scratches at one size. */+const PAD_STEP = 32;+const padStep = (px: number) => (px > 0 ? Math.ceil(px / PAD_STEP) * PAD_STEP : 0);+/** Whether the clip wears its effects on its framed picture. A keyed clip* wears them on its source picture instead (see `gradedSource`). */const framedEffects = (clip: VideoClip) => !!clip.effects?.length && !removalActive(clip.removal);@@ -107,10 +112,14 @@ export class FrameCompositor {private clipFxScratch: Surface | null = null;/** The masked/keyframed-layer pass: the layer draws into `layerScratch`,* its mask's coverage paints into `maskScratch`, a keyframed pose blits- * through `poseScratch`, and the result composites back onto the frame. */+ * through `poseScratch`, and the result composites back onto the frame.+ * A streak smears into `smearScratch` and a subject matte lands in+ * `matteScratch`, both at the posed picture's padded size. */private layerScratch: Surface | null = null;private maskScratch: Surface | null = null;private poseScratch: Surface | null = null;+ private smearScratch: Surface | null = null;+ private matteScratch: Surface | null = null;/** Where a shadow is cast before the picture goes down over it. */private shadowScratch: Surface | null = null;/** The removal pass: the keyed picture builds in `removalScratch`, its@@ -240,6 +249,8 @@ export class FrameCompositor {| "layerScratch"| "maskScratch"| "poseScratch"+ | "smearScratch"+ | "matteScratch"| "shadowScratch"| "removalScratch"| "removalMatte"@@ -807,15 +818,27 @@ export class FrameCompositor {}let out: Surface = layer;let octx = lctx;+ // The soft pass reads past the frame edge — half the streak and three+ // blur radii — so the posed picture keeps that much overhang around the+ // frame. Example: a clip zoomed 1.5x and streaking keeps its own picture+ // at the border, where a frame-sized scratch would smear in a black rim.+ let padX = 0;+ let padY = 0;+ if (pose && look) {+ padX = padStep(Math.abs(look.streakX) / 2 + 3 * look.blur);+ padY = padStep(Math.abs(look.streakY) / 2 + 3 * look.blur);+ }+ const PW = W + 2 * padX;+ const PH = H + 2 * padY;if (pose) {- const { surface: posed } = this.scratch("poseScratch", W, H);+ const { surface: posed } = this.scratch("poseScratch", PW, PH);const pctx = posed.getContext("2d") as Ctx | null;if (!pctx) return;pctx.setTransform(1, 0, 0, 1, 0, 0);- pctx.clearRect(0, 0, W, H);+ pctx.clearRect(0, 0, PW, PH);pctx.save();pctx.globalAlpha = Math.max(0, Math.min(1, pose.opacity));- pctx.translate(pose.x * W, pose.y * H);+ pctx.translate(padX + pose.x * W, padY + pose.y * H);pctx.rotate((pose.rotation * Math.PI) / 180);pctx.scale(pose.scale, pose.scale);pctx.translate(-(rect.x + rect.w / 2) * W, -(rect.y + rect.h / 2) * H);@@ -827,13 +850,16 @@ export class FrameCompositor {// Motion blur smears the posed picture along its streak; the layer// scratch is free again by now, so it takes the smear.if (look && look.taps >= 2) {- lctx.setTransform(1, 0, 0, 1, 0, 0);- lctx.globalAlpha = 1;- lctx.globalCompositeOperation = "source-over";- lctx.clearRect(0, 0, W, H);- drawStreak(lctx, posed as CanvasImageSource, 0, 0, look);- out = layer;- octx = lctx;+ const { surface: smear } = this.scratch("smearScratch", PW, PH);+ const sctx = smear.getContext("2d") as Ctx | null;+ if (!sctx) return;+ sctx.setTransform(1, 0, 0, 1, 0, 0);+ sctx.globalAlpha = 1;+ sctx.globalCompositeOperation = "source-over";+ sctx.clearRect(0, 0, PW, PH);+ drawStreak(sctx, posed as CanvasImageSource, 0, 0, look);+ out = smear;+ octx = sctx;}}if (mask?.kind === "subject") {@@ -847,14 +873,16 @@ export class FrameCompositor {if (res && !res.alpha) {if (!mask.invert) return;} else if (res?.alpha) {- const cctx = cover.getContext("2d") as Ctx;- cctx.clearRect(0, 0, W, H);+ // The matte covers the frame; the overhang around it has no person.+ const { surface: padded } = this.scratch("matteScratch", PW, PH);+ const cctx = padded.getContext("2d") as Ctx;+ cctx.clearRect(0, 0, PW, PH);const feather = (mask.feather ?? 0) * (Math.min(W, H) / 1080);if (feather > 0 && "filter" in cctx) cctx.filter = `blur(${feather / 2}px)`;cctx.imageSmoothingEnabled = true;- cctx.drawImage(res.alpha, 0, 0, W, H);+ cctx.drawImage(res.alpha, padX, padY, W, H);cctx.filter = "none";- maskComposite(octx, cover as CanvasImageSource, mask.invert);+ maskComposite(octx, padded as CanvasImageSource, mask.invert);}}// Transition motion lands on the finished result, so a push carries the@@ -864,7 +892,7 @@ export class FrameCompositor {ctx.save();ctx.translate(Math.round(fx.dx ?? 0), Math.round(fx.dy ?? 0));}- const shade = clip.card ? null : this.shadowOf(out, clip.boxStyle?.shadow);+ const shade = clip.card ? null : this.shadowOf(out, clip.boxStyle?.shadow, padX, padY);// The keyed blur softens the picture and its shadow as they land, on// top of any filter the caller set (a blur transition's own).@@ -875,7 +903,7 @@ export class FrameCompositor {ctx.filter = before && before !== "none" ? `${before} ${own}` : own;}if (shade) ctx.drawImage(shade, 0, 0);- ctx.drawImage(out, 0, 0);+ ctx.drawImage(out, -padX, -padY);if (soft) {ctx.filter = before;}@@ -889,7 +917,7 @@ export class FrameCompositor {* frame beside it (the export's painted PNG) and look the same either way.* Null when the clip casts none.*/- private shadowOf(layer: Surface, sh: ClipShadow | undefined): Surface | null {+ private shadowOf(layer: Surface, sh: ClipShadow | undefined, padX = 0, padY = 0): Surface | null {const ctx = this.ctx();if (!ctx || !sh) return null;const W = this.canvas.width;@@ -907,10 +935,10 @@ export class FrameCompositor {sctx.shadowBlur = Math.max(0, sh.blur) * ds;sctx.shadowOffsetX = (sh.x ?? 0) * ds;sctx.shadowOffsetY = (sh.y ?? 0) * ds;- sctx.drawImage(layer as CanvasImageSource, 0, 0);+ sctx.drawImage(layer as CanvasImageSource, -padX, -padY);sctx.restore();sctx.globalCompositeOperation = "destination-out";- sctx.drawImage(layer as CanvasImageSource, 0, 0);+ sctx.drawImage(layer as CanvasImageSource, -padX, -padY);sctx.globalCompositeOperation = "source-over";return surface;}