Open source CapCut alternative.
TypeScript
fix(cut): read suggest_font boxes from the centre
David Leb1dec543d80e
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex d41aabb3..06756f7e 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -303,15 +303,15 @@ export const AI_PANEL_TOOLS = [{name: "suggest_font",description:- "Name the face a reference's type is set in: reads the source's frame at t, lifts the ink inside region off its ground, and ranks every face Donkey can set a title in — each bundled family at each bundled weight, the system set and the account's own fonts — by how closely its glyphs for the same words match, tracking solved so each runs as wide as the reference. Call it for each distinct face while rebuilding a reference's titles, so the font, weight, size and tracking come from the frame. Pass the words exactly as they read, case and line breaks included, and a region close around them read off watch_video at detail original. Returns the best faces with score (0..1), size (px at the 1080 short side, for the same ink height) and tracking (em), plus the ink's box in frame fractions. verdict no_match means nothing bundled is close: the reference uses a face Donkey does not bundle, so say so and name the stand-in you use.",+ "Name the face a reference's type is set in: reads the source's frame at t, lifts the ink inside region off its ground, and ranks every face Donkey can set a title in — each bundled family at each bundled weight, the system set and the account's own fonts — by how closely its glyphs for the same words match, tracking solved so each runs as wide as the reference. Call it for each distinct face while rebuilding a reference's titles, so the font, weight, size and tracking come from the frame. Pass the words exactly as they read, case and line breaks included, and a region close around them read off watch_video at detail original. Returns the best faces with score (0..1), size (px at the 1080 short side, for the same ink height) and tracking (em), plus the ink's box as centre and size in frame fractions — its x/y are add_title's. verdict no_match means nothing bundled is close: the reference uses a face Donkey does not bundle, so say so and name the stand-in you use.",inputSchema: obj({asset_id: str("Project asset id of the reference (video or image)"),t: num("Source second the type shows at (default 0)"),region: obj({- x: num("Left edge, frame fraction"),- y: num("Top edge, frame fraction"),+ x: num("Centre x, frame fraction"),+ y: num("Centre y, frame fraction"),w: num("Width, frame fraction"),h: num("Height, frame fraction"),},diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 55cffccb..8bf6c956 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -2144,7 +2144,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (!text) throw new ToolError("Pass text: the words the region shows, as they read, with a line break where the type breaks.");const r = (input.region ?? {}) as Record<string, unknown>;if (![r.x, r.y, r.w, r.h].every(isNum) || !((r.w as number) > 0 && (r.h as number) > 0))- throw new ToolError("Pass region: {x, y, w, h} in frame fractions, x/y its top-left corner, around the type and little else.");+ throw new ToolError("Pass region: {x, y, w, h} in frame fractions, x/y its centre, around the type and little else.");const t = clamp(isNum(input.t) ? input.t : 0, 0, Math.max(0, asset.duration - 0.001));// The frame, then the region of it at a size the scoring can use.@@ -2168,10 +2168,13 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const area = pictureArea(asset, input);const left = area.x - area.w / 2;const top = area.y - area.h / 2;- const x0 = (left + clamp(r.x as number, 0, 1) * area.w) * fw;- const y0 = (top + clamp(r.y as number, 0, 1) * area.h) * fh;- const sw = Math.max(1, Math.min(fw - x0, (r.w as number) * area.w * fw));- const sh = Math.max(1, Math.min(fh - y0, (r.h as number) * area.h * fh));+ // A region centred near an edge is cut at the frame.+ const rx0 = (left + (clamp(r.x as number, 0, 1) - (r.w as number) / 2) * area.w) * fw;+ const ry0 = (top + (clamp(r.y as number, 0, 1) - (r.h as number) / 2) * area.h) * fh;+ const x0 = Math.max(0, rx0);+ const y0 = Math.max(0, ry0);+ const sw = Math.max(1, Math.min(fw, rx0 + (r.w as number) * area.w * fw) - x0);+ const sh = Math.max(1, Math.min(fh, ry0 + (r.h as number) * area.h * fh) - y0);const k = Math.min(1, FONT_CROP_MAX / Math.max(sw, sh));const cw = Math.max(1, Math.round(sw * k));const ch = Math.max(1, Math.round(sh * k));@@ -2206,8 +2209,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {t: round2(t),text,ink: {- x: round3(((x0 + ref.box.x / k) / fw - left) / area.w),- y: round3(((y0 + ref.box.y / k) / fh - top) / area.h),+ x: round3(((x0 + (ref.box.x + ref.box.w / 2) / k) / fw - left) / area.w),+ y: round3(((y0 + (ref.box.y + ref.box.h / 2) / k) / fh - top) / area.h),w: round3(ref.box.w / k / fw / area.w),h: round3(ref.box.h / k / fh / area.h),},diff --git a/site/src/cut/lib/suggestFontTool.test.ts b/site/src/cut/lib/suggestFontTool.test.tsindex 09d7ec03..1be98af4 100644--- a/site/src/cut/lib/suggestFontTool.test.ts+++ b/site/src/cut/lib/suggestFontTool.test.ts@@ -70,7 +70,8 @@ beforeEach(() => {} as never);});-const REGION = { x: 0.2, y: 0.4, w: 0.6, h: 0.2 };+// Centre and size, frame fractions, like every picture tool.+const REGION = { x: 0.5, y: 0.5, w: 0.6, h: 0.2 };type Out = {verdict: string;ink: { x: number; y: number; w: number; h: number };@@ -88,8 +89,9 @@ test("the face the reference is set in comes first, with the size that matches i// 72px type on a 1080-tall frame is 72px at the 1080 short side.expect(Math.abs(out.matches[0].size - 72)).toBeLessThanOrEqual(3);expect(Math.abs(out.matches[0].tracking)).toBeLessThan(0.03);- // The ink, in frame fractions, sits centred where it was drawn.- expect(Math.abs(out.ink.x + out.ink.w / 2 - 0.5)).toBeLessThan(0.01);+ // The ink's centre, in frame fractions, is where it was drawn: the x/y+ // add_title takes as they are.+ expect(Math.abs(out.ink.x - 0.5)).toBeLessThan(0.01);expect(out.matches.length).toBeLessThanOrEqual(5);});@@ -102,7 +104,7 @@ test("type in a face Donkey does not bundle is reported as that", async () => {});test("a region with no type, a sound, and missing words are refused", async () => {- expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: { x: 0, y: 0, w: 0.1, h: 0.1 }, text: "TRAVEL" }))).toContain("No type");+ expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: { x: 0.05, y: 0.05, w: 0.1, h: 0.1 }, text: "TRAVEL" }))).toContain("No type");expect(await message(runAiTool("suggest_font", { asset_id: "song", t: 3, region: REGION, text: "TRAVEL" }))).toContain("audio");expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: REGION, text: " " }))).toContain("text");expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: { x: 0.5, y: 0.5, w: 0, h: 0.1 }, text: "TRAVEL" }))).toContain("region");@@ -115,11 +117,11 @@ test("inside a picture box, the region, the ink and the size are the box's", asyconst out = (await runAiTool("suggest_font", {asset_id: "ref",t: 3,- region: { x: 0.2, y: 0.3, w: 0.6, h: 0.4 },+ region: { x: 0.5, y: 0.5, w: 0.6, h: 0.4 },text: "TRAVEL DESTINATION",})) as Out & { picture?: unknown };expect(out.matches[0]).toMatchObject({ font: "oswald", weight: 700 });expect(Math.abs(out.matches[0].size - 144)).toBeLessThanOrEqual(6);- expect(Math.abs(out.ink.y + out.ink.h / 2 - 0.5)).toBeLessThan(0.02);+ expect(Math.abs(out.ink.y - 0.5)).toBeLessThan(0.02);expect(out.picture).toBeDefined();});