Skip to content

Open source CapCut alternative.

TypeScript

feat(site): score a named stand-in in suggest_font

David Lecfc11969a49a
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index 93230a8d..7db6cb9e 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -340,7 +340,7 @@ export const AI_PANEL_TOOLS = [
{
name: "suggest_font",
description:
- "Name the face a reference's type is set in: reads the source's frame at t, lifts the ink inside region off its ground, and ranks every face Donkey can set a title in — each bundled family at each bundled weight, the system set and the account's own fonts — by how closely its glyphs for the same words match, tracking solved so each runs as wide as the reference. Call it for each distinct face while rebuilding a reference's titles, so the font, weight, size and tracking come from the frame. Pass the words exactly as they read, case and line breaks included, and a region close around them read off watch_video at detail original. Returns the best faces with score (0..1), size (px at the 1080 short side, for the same ink height) and tracking (em), and x/y, where that face's title sits (add_title's x/y) for its ink to land on the reference's — a title is set from the middle of its line, so its ink centre is off it. `ink` is the ink's own box, centre and size in frame fractions. Cards in one face share its weight, size and tracking, but each sits its own way in its space, so read each card for its own x/y rather than setting the rest from the first one's place. Type set at an angle — a column word reading bottom to top — is read upright: pass `rotation`, the angle add_title would set it at (-90 reads bottom to top, 90 top to bottom), with a region around the word as it stands; one line of words in a region taller than wide is tried both ways round without it, and `rotation` comes back with the result to set on the title. verdict no_match means nothing bundled is close: the reference uses a face Donkey does not bundle, so say so and name the stand-in you use, of its kind — `kinds` lists every face by kind (serif, sans, script, display, mono), each as its id and a few words on its look. `inkPx` is how tall a line of the ink runs in the source; under 24 the result is `small`: the glyphs have blurred into their proportions, the ranking cannot tell a serif from a condensed sans, so name the face by eye at detail original and take the closest of its kind from `kinds`.",
+ "Name the face a reference's type is set in: reads the source's frame at t, lifts the ink inside region off its ground, and ranks every face Donkey can set a title in — each bundled family at each bundled weight, the system set and the account's own fonts — by how closely its glyphs for the same words match, tracking solved so each runs as wide as the reference. Call it for each distinct face while rebuilding a reference's titles, so the font, weight, size and tracking come from the frame. Pass the words exactly as they read, case and line breaks included, and a region close around them read off watch_video at detail original. Returns the best faces with score (0..1), size (px at the 1080 short side, for the same ink height) and tracking (em), and x/y, where that face's title sits (add_title's x/y) for its ink to land on the reference's — a title is set from the middle of its line, so its ink centre is off it. `ink` is the ink's own box, centre and size in frame fractions. Cards in one face share its weight, size and tracking, but each sits its own way in its space, so read each card for its own x/y rather than setting the rest from the first one's place. Type set at an angle — a column word reading bottom to top — is read upright: pass `rotation`, the angle add_title would set it at (-90 reads bottom to top, 90 top to bottom), with a region around the word as it stands; one line of words in a region taller than wide is tried both ways round without it, and `rotation` comes back with the result to set on the title. verdict no_match means nothing bundled is close: the reference uses a face Donkey does not bundle, so say so and name the stand-in you use, of its kind — `kinds` lists every face by kind (serif, sans, script, display, mono), each as its id and a few words on its look. Whatever face you set — a stand-in, or one you picked over the top match by eye — call again with `fonts: [its id]` for its own size, tracking and x/y on this ink; never guess a size. `inkPx` is how tall a line of the ink runs in the source; under 24 the result is `small`: the glyphs have blurred into their proportions, the ranking cannot tell a serif from a condensed sans, so name the face by eye at detail original and take the closest of its kind from `kinds`.",
inputSchema: obj(
{
asset_id: str("Project asset id of the reference (video or image)"),
@@ -357,6 +357,7 @@ export const AI_PANEL_TOOLS = [
text: str("The words the region shows, as they read, with a line break where the type breaks"),
rotation: num("Degrees the type is turned, as add_title's rotation: -90 reads bottom to top, 90 top to bottom (default: upright, or both ways for one line in a tall region)"),
count: num("How many faces to return, 1..10 (default 5)"),
+ fonts: { type: "array", items: { type: "string" }, description: "Font ids to score instead of every face — the stand-in you picked, so its size, tracking and x/y come from the frame" },
picture: pictureArg,
},
["asset_id", "region", "text"]
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 922053fe..909eae6c 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2635,6 +2635,14 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (![r.x, r.y, r.w, r.h].every(isNum) || !((r.w as number) > 0 && (r.h as number) > 0))
throw new ToolError("Pass region: {x, y, w, h} in frame fractions, x/y its centre, around the type and little else.");
const t = clamp(isNum(input.t) ? input.t : 0, 0, Math.max(0, asset.duration - 0.001));
+ // A stand-in already picked is scored alone, so its size and place come
+ // from the frame too.
+ const named = Array.isArray(input.fonts) ? input.fonts.filter((f): f is string => typeof f === "string") : [];
+ const faces = named.length > 0 ? bundledFaces().filter((f) => named.includes(f.font)) : bundledFaces();
+ const unknown = named.filter((id) => !faces.some((f) => f.font === id));
+ if (unknown.length > 0) {
+ throw new ToolError(`No face ${unknown.join(", ")} to set a title in. fonts takes ids from the font list (a font's id, as add_title's font).`);
+ }
// The frame, then the region of it at a size the scoring can use.
let source: CanvasImageSource;
@@ -2700,7 +2708,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (upright !== crop) upright.width = 0;
if (!ref.box) continue;
// Every face ranked; the size each would need to fill the ink's height.
- const ranked = await rankFaces(ref, text, bundledFaces(), fontStack).then(
+ const ranked = await rankFaces(ref, text, faces, fontStack).then(
(r) => r,
(e) => {
// A guess at the angle that reads wrong is one of the guesses.
diff --git a/site/src/cut/lib/suggestFontTool.test.ts b/site/src/cut/lib/suggestFontTool.test.ts
index 7aa64c71..2574493a 100644
--- a/site/src/cut/lib/suggestFontTool.test.ts
+++ b/site/src/cut/lib/suggestFontTool.test.ts
@@ -111,6 +111,19 @@ test("type in a face Donkey does not bundle is reported as that", async () => {
expect(out.note).toContain("does not bundle");
});
+test("a stand-in named in fonts is scored alone, with the size and place that match the ink", async () => {
+ // The reference's face isn't bundled; the stand-in picked for it still
+ // needs its size from the frame, not a guess.
+ shows = { text: "Travel Destination", css: '400 110px "Great Vibes"' };
+ const out = (await runAiTool("suggest_font", { asset_id: "ref", t: 3, region: REGION, text: "Travel Destination", fonts: ["anton"] })) as Out & {
+ matches: { x: number; y: number }[];
+ };
+ expect(out.matches.map((m) => m.font)).toEqual(["anton"]);
+ expect(out.matches[0].size).toBeGreaterThan(0);
+ expect(Math.abs(out.matches[0].x - 0.5)).toBeLessThan(0.03);
+ expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: REGION, text: "Travel", fonts: ["no-such-face"] }))).toContain("no-such-face");
+});
+
test("a region with no type, a sound, and missing words are refused", async () => {
expect(await message(runAiTool("suggest_font", { asset_id: "ref", t: 3, region: { x: 0.05, y: 0.05, w: 0.1, h: 0.1 }, text: "TRAVEL" }))).toContain("No type");
expect(await message(runAiTool("suggest_font", { asset_id: "song", t: 3, region: REGION, text: "TRAVEL" }))).toContain("audio");