Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
perf: ask one question per cut, and stop making shots too short to observe
rpone964563dcc8a1
diff --git a/.agents/skills/hypit/references/local-author-package.md b/.agents/skills/hypit/references/local-author-package.mdindex 1f21b942..0badf3c5 100644--- a/.agents/skills/hypit/references/local-author-package.md+++ b/.agents/skills/hypit/references/local-author-package.md@@ -14,6 +14,14 @@ Read all of these completely before editing:4. `packages/component-kit/README.md`5. The closest existing package's README, `manifest.ts`, `surface.ts`, `component.ts`, and`activation.ts`+6. For a package that produces a visual output, that same package's `render.ts`, `schedule.ts` and+ `fragment.ts` as well. The implementation section below requires a renderer, a Fragment and the+ lowering between them, and none of those live in the five files above — reading only those five+ means discovering the shapes by failing.++Read item 5's files in full. Read item 6's for their shapes: how a Program becomes elements, where+timing is resolved, what the Fragment declares. They are long, and copying one package's specifics is+a worse outcome than understanding its structure.## Package boundary@@ -67,6 +75,17 @@ nothing.- A component that cannot render on its own cannot produce the preview image its Surface owes. Treata missing preview as evidence of this mistake rather than a step to skip.+## Freeze the Types before writing in parallel++The nominal Types and the Manifest are what every other file agrees with, so write them first and+stop changing them. Once they are frozen, `render.ts`, `style.ts` and `schedule.ts` depend on the+Types rather than on each other and can be written in any order or at the same time. `surface.ts`,+`component.ts` and `fragment.ts` follow, because they wire what those three produce.++Do not start that parallel stretch while a Type is still in question. Interface drift mid-flight+costs more than the ordering saves, and the symptom — a Producer rejecting a value that looks right —+is expensive to read.+## Install and validateReuse an existing workspace when it already includes `packages/*`; otherwise make only the minimaldiff --git a/packages/reference-video-tools/src/media.ts b/packages/reference-video-tools/src/media.tsindex 0ad60ca8..5322c087 100644--- a/packages/reference-video-tools/src/media.ts+++ b/packages/reference-video-tools/src/media.ts@@ -68,7 +68,18 @@ async function contentBoundaries(path: string, duration: number): Promise<readonanchor = frames[index]!;}}- return [...starts, duration].filter((value, index, all) => index === 0 || value - all[index - 1]! > 0.3);+ // A boundary closer than this to the one before it does not describe a shot: it describes a+ // flicker, a flash frame or a fast pan. Clips that short are also rejected outright by the model,+ // so admitting them costs a request and returns nothing.+ const shortest = 1;+ const kept = [starts[0]!];+ for (const value of starts.slice(1)) {+ if (value - kept.at(-1)! >= shortest) kept.push(value);+ }+ // The tail is a boundary too, and a final fragment belongs to the shot before it rather than+ // becoming a shot of its own.+ if (duration - kept.at(-1)! < shortest && kept.length > 1) kept.pop();+ return [...kept, duration];}type Bounds = { readonly start: number; readonly end: number; readonly group: number; readonly part: number; readonly parts: number };diff --git a/packages/reference-video-tools/src/tools.ts b/packages/reference-video-tools/src/tools.tsindex 0273b180..05cbc470 100644--- a/packages/reference-video-tools/src/tools.ts+++ b/packages/reference-video-tools/src/tools.ts@@ -61,6 +61,14 @@ type ObservationTask = { readonly key: string; readonly run: () => Promise<Obserfunction observation(status: "complete" | "failed", text: string): Observation { return { status, text }; }+function positiveEnv(name: string): number | undefined {+ const raw = process.env[name]?.trim();+ if (raw === undefined || raw.length === 0) return undefined;+ const value = Number(raw);+ assert(Number.isSafeInteger(value) && value > 0, `${name} must be a positive integer`);+ return value;+}+function stateRoot(workspaceRoot: string, reference: string): string {return join(workspaceRoot, ".hypit", "reference-video-tools", reference);}@@ -100,12 +108,24 @@ const MIME_TYPES: Readonly<Record<string, string>> = {mp4: "video/mp4",};-async function mediaPart(path: string): Promise<Part> {- const bytes = await readBytes(path);- const extension = path.toLowerCase().split(".").pop() ?? "";- const mimeType = MIME_TYPES[extension];- assert(mimeType !== undefined, `unsupported media extension for ${path}; supported: ${Object.keys(MIME_TYPES).join(", ")}`);- return { inlineData: { mimeType, data: Buffer.from(bytes).toString("base64") } };+// The Vertex path cannot upload a file and reference it later, so the bytes travel with every+// request. One shot's clip is asked about by its own picture, type and sound observations and by the+// boundary on each side, so reading and encoding it once per invocation is worth the memory.+function mediaParts(): (path: string) => Promise<Part> {+ const encoded = new Map<string, Promise<Part>>();+ return (path) => {+ const held = encoded.get(path);+ if (held !== undefined) return held;+ const part = (async (): Promise<Part> => {+ const bytes = await readBytes(path);+ const extension = path.toLowerCase().split(".").pop() ?? "";+ const mimeType = MIME_TYPES[extension];+ assert(mimeType !== undefined, `unsupported media extension for ${path}; supported: ${Object.keys(MIME_TYPES).join(", ")}`);+ return { inlineData: { mimeType, data: Buffer.from(bytes).toString("base64") } };+ })();+ encoded.set(path, part);+ return part;+ };}function message(error: unknown): string {@@ -224,8 +244,11 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVconst workspaceRoot = resolve(options.workspaceRoot ?? process.cwd());const packageRoot = resolve(options.packageRoot ?? workspaceRoot);const model = options.model ?? process.env.GEMINI_MODEL?.trim() ?? "gemini-3.1-pro-preview";- const concurrency = options.concurrency ?? 2;- const gapMs = options.launchGapMs ?? 6_000;+ // Pacing is deployment policy, not author intent: it depends on the quota behind the credentials,+ // which the calling agent has no way to know. It is settable here and through the environment, and+ // deliberately not through a CLI flag.+ const concurrency = options.concurrency ?? positiveEnv("HYPIT_REFERENCE_CONCURRENCY") ?? 2;+ const gapMs = options.launchGapMs ?? positiveEnv("HYPIT_REFERENCE_LAUNCH_GAP_MS") ?? 6_000;const retryDelayMs = options.retryDelayMs ?? 2_000;let generatorPromise: Promise<GenerateText> | undefined;const generator = async (): Promise<GenerateText> => generatorPromise ??= options.generate === undefined ? defaultGenerate(model) : Promise.resolve(options.generate);@@ -284,6 +307,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVawait writeFile(join(root, "observations.json"), "{}\n", "utf8");}const generate = await generator();+ const mediaPart = mediaParts();const globalParts: Part[] = [await mediaPart(analysisVideo)];const [people, voices, systems] = await Promise.all([state.people_and_product?.status === "complete" && !redoPeople@@ -302,6 +326,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV},async observe_reference(input): Promise<Record<string, unknown>> {+ const mediaPart = mediaParts();const state = await loadState(input.reference_id);const selected = input.shot_ids === undefined ? state.shots : state.shots.filter((shot) => input.shot_ids!.includes(shot.shot_id));assert(selected.length > 0, "no requested shot ids exist");@@ -354,24 +379,20 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVparts.push({ text: `${globalContext}\n\nListen to shot ${shot.index}. Decide who is speaking, whether sound continues from the previous shot, whether visible people actually produce the sound, and whether a silent B-roll person only appears to speak. Handle off-screen, alternating, overlapping, and no-person-visible speech. Return natural language evidence only.` });return await callSafely(retryDelayMs, generate, parts, "Observe sound only. Do not write markup, SVML, JSON plans, or component names.");}}));- const sameTakeTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `same-take:${right.shot_id}`, run: async () => {- const left = state.shots.find((shot) => shot.index === right.index - 1)!;- const parts: Part[] = [await mediaPart(left.clip_ref), await mediaPart(right.clip_ref), await mediaPart(left.tail_frame_ref)];- parts.push({ text: `Compare shots ${left.index} and ${right.index}. Answer only whether they are one continuous camera shot. Explain the visual and sound continuity evidence in natural language.` });- return await callSafely(retryDelayMs, generate, parts, "Analyze camera continuity only. Do not name SVML components or write code.");- }}));- const overlayTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `overlay:${right.shot_id}`, run: async () => {+ // Both boundary questions look at exactly the same three files, so they travel together. Asking+ // them separately uploaded two clips twice to learn two things about one cut.+ const boundaryTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `boundary:${right.shot_id}`, run: async () => {const left = state.shots.find((shot) => shot.index === right.index - 1)!;const parts: Part[] = [await mediaPart(left.clip_ref), await mediaPart(right.clip_ref), await mediaPart(left.tail_frame_ref)];- parts.push({ text: `Compare shots ${left.index} and ${right.index}. Answer only whether the same visible overlay or inserted picture continues across the boundary, changes, or ends. Explain the evidence in natural language.` });- return await callSafely(retryDelayMs, generate, parts, "Analyze overlay continuity only. Do not name SVML components or write code.");+ parts.push({ text: `Compare shots ${left.index} and ${right.index}, which meet at one cut. Answer two questions separately, each under its own heading.\n\nContinuous camera shot: are these one continuous camera shot, or two? Explain the visual and sound continuity evidence.\n\nOverlay continuity: does the same visible overlay or inserted picture continue across the boundary, change, or end? Explain the evidence.\n\nReturn natural language evidence only.` });+ return await callSafely(retryDelayMs, generate, parts, "Analyze what happens at one cut. Do not name SVML components or write code.");}}));- const allTasks = [...visualTasks, ...typeTasks, ...audioTasks, ...sameTakeTasks, ...overlayTasks];+ const allTasks = [...visualTasks, ...typeTasks, ...audioTasks, ...boundaryTasks];const reobserve = input.reobserve === true;const forcedKeys = reobserve ? new Set(allTasks.map((task) => task.key)) : new Set<string>();const observations = await runObservationTasks(state.root, allTasks, forcedKeys, concurrency, gapMs);const unresolved = [...observations].filter(([, value]) => value.status !== "complete").map(([key]) => key);- const boundaryText = boundaryRights.map((right) => `${observations.get(`same-take:${right.shot_id}`)?.text ?? ""}\n${observations.get(`overlay:${right.shot_id}`)?.text ?? ""}`).join("\n");+ const boundaryText = boundaryRights.map((right) => observations.get(`boundary:${right.shot_id}`)?.text ?? "").join("\n");const needsThreeShotReview = selected.length >= 3 && (reobserve ||/uncertain|unknown|possibly|may be|contin(?:ue|ues)|same take|same shot/iu.test(boundaryText)@@ -407,8 +428,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVright_shot_id: right.shot_id,combined_duration_seconds: Number((right.end_seconds - left.start_seconds).toFixed(3)),within_15_seconds: right.end_seconds - left.start_seconds <= 15,- same_take: observations.get(`same-take:${right.shot_id}`)!,- overlay_continuity: observations.get(`overlay:${right.shot_id}`)!,+ continuity: observations.get(`boundary:${right.shot_id}`)!,};});return {@@ -445,6 +465,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV},async compare_reconstruction(input): Promise<Record<string, unknown>> {+ const mediaPart = mediaParts();const state = await loadState(input.reference_id);const shot = state.shots.find((candidate) => candidate.shot_id === input.shot_id);assert(shot !== undefined, `shot ${input.shot_id} does not exist in reference ${input.reference_id}`);diff --git a/packages/reference-video-tools/test/tools.test.ts b/packages/reference-video-tools/test/tools.test.tsindex 9d9c1a9f..9554b242 100644--- a/packages/reference-video-tools/test/tools.test.ts+++ b/packages/reference-video-tools/test/tools.test.ts@@ -79,11 +79,10 @@ test("observation covers picture, drawn type and sound for every shot and cachesconst cache = JSON.parse(await readFile(join(stateRoot, "observations.json"), "utf8")) as Record<string, unknown>;assert.deepEqual(Object.keys(cache).sort(), ["audio:shot-001", "audio:shot-002",- "overlay:shot-002",- "same-take:shot-002",+ "boundary:shot-002","type:shot-001", "type:shot-002","visual:shot-001", "visual:shot-002",- ]);+ ], "one cut is one observation, not a same-take question and an overlay question over the same media");const shots = result["shots"] as readonly Record<string, unknown>[];assert.equal(shots.length, 2);@@ -106,7 +105,7 @@ test("a second observation reuses the cache and only an explicit reobserve runsconst { root } = await workspace(2);const first: Call[] = [];await tools(root, first).observe_reference({ reference_id: REFERENCE });- assert.equal(first.length, 8);+ assert.equal(first.length, 7, "two shots: picture, type and sound each, plus one shared boundary");const second: Call[] = [];await tools(root, second).observe_reference({ reference_id: REFERENCE });@@ -118,7 +117,7 @@ test("a second observation reuses the cache and only an explicit reobserve runsconst forced: Call[] = [];await tools(root, forced).observe_reference({ reference_id: REFERENCE, shot_ids: ["shot-002"], reobserve: true });- assert.equal(forced.length, 5, "reobserve reruns the selected shot's picture, type and sound plus its boundary pair, and nothing else");+ assert.equal(forced.length, 4, "reobserve reruns the selected shot's picture, type and sound plus its one boundary, and nothing else");});test("a narrow question costs one call, answers from the named shots and never re-runs observations", async () => {diff --git a/test/repository-hygiene.test.mjs b/test/repository-hygiene.test.mjsindex e30fbf8d..d9fd4cd7 100644--- a/test/repository-hygiene.test.mjs+++ b/test/repository-hygiene.test.mjs@@ -1,5 +1,5 @@import assert from "node:assert/strict";-import { readdir, readFile } from "node:fs/promises";+import { readdir, readFile, stat } from "node:fs/promises";import { extname, relative } from "node:path";import test from "node:test";import ts from "typescript";@@ -158,3 +158,22 @@ test("production imports belong to each package while the root owns repository t}assert.deepEqual(failures, [], `undeclared package imports found:\n${failures.join("\n")}`);});++test("a declared Surface preview names a picture that exists", async () => {+ // A component that cannot draw itself cannot produce a preview image. Declaring one and shipping+ // nothing is how that goes unnoticed: the declaration reads as evidence the component renders.+ const entries = await repositoryEntries();+ const failures = [];+ for (const entry of entries) {+ const match = /^((?:examples\/[^/]+\/)?packages\/[^/]+)\/src\/.*\.ts$/u.exec(entry.path);+ if (match === null || !entry.isFile) continue;+ const source = await readFile(entry.child, "utf8");+ for (const declared of source.matchAll(/previewImage\(\s*"([^"]+)"\s*\)/gu)) {+ const file = new URL(`./${match[1]}/preview/${declared[1]}`, repositoryRoot);+ const size = await stat(file).then((item) => item.size, () => undefined);+ if (size === undefined) failures.push(`${entry.path} declares preview/${declared[1]}, which does not exist`);+ else if (size === 0) failures.push(`${entry.path} declares preview/${declared[1]}, which is empty`);+ }+ }+ assert.deepEqual(failures, [], `declared previews without a picture:\n${failures.join("\n")}`);+});