Open source CapCut alternative.
TypeScript
feat(site): the assistant reads and sets preview guides
David Lee9569263954c
diff --git a/site/src/cut/components/TopBar.tools.ts b/site/src/cut/components/TopBar.tools.tsindex e8d2f361..54fbd472 100644--- a/site/src/cut/components/TopBar.tools.ts+++ b/site/src/cut/components/TopBar.tools.ts@@ -6,6 +6,7 @@*/import { obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";+import { GUIDE_IDS, GUIDE_PRESETS } from "@/cut/lib/guides";export const TOP_BAR_TOOLS = [{@@ -14,6 +15,30 @@ export const TOP_BAR_TOOLS = ["Set the project's output frame ratio as \"W:H\". The frame is the user's own setting, so call this only when they asked for a shape or a format (\"make it vertical\", \"for TikTok\", \"square\") — never inferred from a reference image's shape, a platform named in passing, or what would suit a generation you are about to run. Presets: 16:9 (YouTube), 9:16 (TikTok/Reels/Shorts), 1:1, 4:3, 3:4, 2:1 — but any ratio up to an 8:1 shape works, with whole or decimal sides (\"9:5\", \"2.39:1\" — stored reduced, so 2.39:1 becomes 239:100). The frame renders with its short side at 1080px: 9:16 → 1080×1920, 16:9 → 1920×1080, 9:5 → 1944×1080.",inputSchema: obj({ aspect: { type: "string", description: "Output ratio as \"W:H\", e.g. \"9:16\", \"1:1\", \"9:5\", \"2.39:1\"" } }, ["aspect"]),},+ {+ name: "set_guides",+ description:+ `Choose the guides drawn over the preview — lines and shaded keep-out regions that show where titles, stickers and captions can sit. They draw in the editor only and never export. Presets: ${GUIDE_PRESETS.map((p) => `${p.id} (${p.name}${p.sublabel ? `, ${p.sublabel.toLowerCase()}` : ""})`).join(", ")}. The short-form preset shades the phone UI of TikTok, Instagram Reels and YouTube Shorts merged — top bar, right-hand action rail, bottom caption block — and fits portrait frames. The custom preset draws the user's own lines, which they drag on the preview; \`lines\` writes that list (frame fractions: v is x positions, h is y) and turns custom on. \`show\` replaces the whole set; [] turns every guide off. editor_state project.guides then reports the set with the safeArea it leaves and the custom lines.`,+ inputSchema: obj(+ {+ show: {+ type: "array",+ items: { type: "string", enum: GUIDE_IDS },+ description: "Guide ids to show; replaces the current set",+ },+ lines: {+ type: "object",+ description: "Custom lines to draw, replacing the current ones: v = vertical line x positions, h = horizontal line y positions, each 0..1 of the frame",+ properties: {+ v: { type: "array", items: { type: "number" } },+ h: { type: "array", items: { type: "number" } },+ },+ additionalProperties: false,+ },+ },+ ["show"]+ ),+ },{name: "set_project_name",description: "Rename the current project.",diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.tsindex 77060b83..1910355e 100644--- a/site/src/cut/lib/aiContext.ts+++ b/site/src/cut/lib/aiContext.ts@@ -1,5 +1,6 @@"use client";+import { guideGeometry, safeAreaOf } from "./guides";import { hasOverlayAnim, retimeOf, speedCurveOf, type ClipSound, type SpeedNode } from "@donkeycut/effects-kit";import { chatOwner } from "./chatAssets";import { useGenerate } from "./generate";@@ -232,6 +233,24 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu...(s.fadeIn > 0 ? { fadeIn: r(s.fadeIn) } : {}),...(s.fadeOut > 0 ? { fadeOut: r(s.fadeOut) } : {}),background: s.background,+ // The guides showing on the preview, with the room they leave for+ // graphics: safeArea and keepOut are frame fractions (x, y, w, h).+ ...(s.guides.length > 0+ ? {+ guides: {+ on: s.guides,+ safeArea: safeAreaOf(s.guides, s.aspect),+ ...(s.guides.includes("custom") ? { customLines: s.guideLines } : {}),+ keepOut: guideGeometry(s.guides, s.aspect).boxes.map((b) => ({+ label: b.label,+ x: r(b.x),+ y: r(b.y),+ w: r(b.w),+ h: r(b.h),+ })),+ },+ }+ : {}),},playhead: r(playheadAt()),skimmer: skimAt() === null ? null : r(skimAt()!),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 2aa71478..de4da6dc 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1,5 +1,6 @@"use client";+import { GUIDE_IDS, isGuideId, safeAreaOf, sanitizeGuideLines, type GuideId } from "./guides";import {ALL_EFFECT_IDS,autoGradeFromImageData,@@ -3785,6 +3786,26 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {return { aspect: a, frame: `${f.w}x${f.h}` };},+ set_guides: (s, input) => {+ if (!Array.isArray(input.show)) throw new ToolError("show must be a list of guide ids.");+ const bad = input.show.filter((id) => !isGuideId(id));+ if (bad.length)+ throw new ToolError(`Unknown guide ${bad.join(", ")} — pick from ${GUIDE_IDS.join(", ")}.`);+ const show = input.show as GuideId[];+ if (input.lines !== undefined) {+ s.setGuideLines(sanitizeGuideLines(input.lines));+ if (!show.includes("custom")) show.push("custom");+ }+ s.setGuides(show);+ s.setGuidesHidden(false);+ const after = useEditor.getState();+ return {+ guides: after.guides,+ safeArea: safeAreaOf(after.guides, after.aspect),+ ...(after.guides.includes("custom") ? { customLines: after.guideLines } : {}),+ };+ },+set_project_fade: (s, input) => {if (!isNum(input.fadeIn) && !isNum(input.fadeOut))throw new ToolError("Pass fadeIn and/or fadeOut seconds (0 clears).");diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 70d4c61c..baa189cd 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -82,7 +82,7 @@ export const AI_TOOLS: AiToolDef[] = [/** Deep documentation the model can pull in on demand. */export const AI_SKILLS: Record<string, string> = {"editor-overview": `# Cut editor overview-Cut is a local, project-based short-video editor. Each project has an output aspect ratio "W:H" — presets 16:9, 9:16, 1:1, 4:3, 3:4, 2:1, or any custom ratio like 9:5 — rendered with the frame's short side at 1080px and switchable from the pill in the top bar; the current one is in editor_state project.aspect. Layout:+Cut is a local, project-based short-video editor. Each project has an output aspect ratio "W:H" — presets 16:9, 9:16, 1:1, 4:3, 3:4, 2:1, or any custom ratio like 9:5 — rendered with the frame's short side at 1080px and switchable from the pill in the top bar; the current one is in editor_state project.aspect. The Guides button beside the timeline zoom turns on preview guides — thirds, center, safe margins, the short-form keep-out zone shared by TikTok, Reels and Shorts, and custom lines the user drags into place (set_guides) — which editor_state project.guides reports with the safeArea they leave. Layout:- Left icon rail tabs: Media (user-imported files + Exports list), Library (shared reusable assets), Video (stock clip browser — footage + talking characters — beside the AI video generator), Image (stock images beside the AI image generator), Audio (AI voiceover + audio files), Text (title presets), Subtitles (transcript editor), Publish (caption/tags/sound metadata). Camera/mic recording lives in the top bar next to the aspect picker.- Media shows only what the user imported. Everything Cut makes carries an origin tag (generated, voiceover, recording, stock, freeze, chat — see \`media\` in editor_state) and lives where it was created — media you make previews on a card in this chat, panel renders sit in their job lists; a "…" menu on each files it into Media or the Library, and every card drags onto the timeline. Deleting a chat thread deletes the chat media the user never placed or filed away.- Center: the video preview canvas (composited at the project's frame size) with draggable text overlays and subtitle captions.@@ -407,6 +407,7 @@ Rules:- Keep replies short and concrete — one or two sentences. When the work is done: state what changed, in terms your tool results confirm, and end there. You may close with at most one suggestion, phrased as a statement ("A 0.5s crossfade on the middle cut would smooth the jump."); a suggestion the user has already passed on stays retired. A question belongs in a reply only when the request is genuinely ambiguous or you are blocked. The same cadence holds for answers in chat: give the answer and stop — no trailing offer, no "would you like me to", no "let me know if". You have no name and never name the app; a message that asks for nothing (a greeting, thanks) gets a short "How can I help?" / "What would you like to do?" back — words alone, no tool calls. No headings, no fluff, and never paste the editor_state snapshot or raw JSON back — say what it means in plain words (a failed render: name the error and offer to retry).- Edits are undoable (unlimited undo), so prefer doing over asking; only ask when the request is genuinely ambiguous. Ambiguous means the target itself is unresolvable — "shorten that clip" with nothing selected and several clips gets "Which clip?" and no edit; guessing a target and editing it is worse than the question. The existing cut is the user's work: adding media adds — never delete, trim, or reorder a timeline clip unless the user said to. Read "change it / make it X / try again / closer to the original" as a fresh take that lands on its own card beside the old, never as license to delete the old clip first; only an explicit "replace / delete / remove this" clears what's already there. Items you placed yourself while working the current ask are your own scaffolding: revise, retime, or delete them freely until the result matches the ask — tearing down your own misstep and rebuilding is part of doing the job. The guardrail covers the user's media and the settled work of earlier turns. Generation is different: undo removes the clip but the credits stay spent, so be certain the user asked for the media before calling a generation tool. Removing media is different too — delete_asset and library_organize deletes don't come back with undo, so they take an explicit ask naming the media.- Times are seconds. The frame follows project.aspect in editor_state — any "W:H" ratio, short side 1080px; project.frame has the exact pixels. The frame is the user's setting: work within the shape they chose, and change it only when they ask for a different one. A generation that renders in another shape letterboxes inside their frame — mention that if it matters, and leave the choice to them.+- When project.guides carries a safeArea, keep every title, sticker and caption inside it (frame fractions, x/y/w/h) and out of the keepOut boxes — that is where the platform draws its own UI over the video. Turn the short-form guide on with set_guides when the user names a platform they are posting to or asks to see the safe zone; guides are a view, so showing one changes nothing in the cut.- Exercise judgment on details the user left open, and state the choice you made. Judgment fills in the parameters of the asked-for change (style, length, placement); the scope stays the ask, and extra tools, tracks, or treatments beyond it wait for the user to ask. Editing defaults: a hard cut at every joint; when transitions are asked for, one style family per video at 0.4–0.6s. The editing-taste skill carries the full doctrine.- Styling is part of the ask, never a bonus. Words the user asked to put on screen — a title, the lines from the audio, a caption run — go on in one plain readable face at one place, timed to what they are for. Per-line typefaces, tilts, color cards, moves, word effects and a repainted frame are a design, and a design is something the user asks for: a lyric video, kinetic type, a named style or vibe, a reference to match, "make it fun". Call add_text_sequence with no \`look\` for the plain job, and leave the caption styling and the background as you found them.- Read list_skills / read_skill before working in an area you're unsure about — they document every setting. A tool whose description already answers the ask goes straight through.