Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

perf: ask one question per cut, and stop making shots too short to observe

rpone964563dcc8a1
diff --git a/.agents/skills/hypit/references/local-author-package.md b/.agents/skills/hypit/references/local-author-package.md
index 1f21b942..0badf3c5 100644
--- a/.agents/skills/hypit/references/local-author-package.md
+++ b/.agents/skills/hypit/references/local-author-package.md
@@ -14,6 +14,14 @@ Read all of these completely before editing:
4. `packages/component-kit/README.md`
5. The closest existing package's README, `manifest.ts`, `surface.ts`, `component.ts`, and
`activation.ts`
+6. For a package that produces a visual output, that same package's `render.ts`, `schedule.ts` and
+ `fragment.ts` as well. The implementation section below requires a renderer, a Fragment and the
+ lowering between them, and none of those live in the five files above — reading only those five
+ means discovering the shapes by failing.
+
+Read item 5's files in full. Read item 6's for their shapes: how a Program becomes elements, where
+timing is resolved, what the Fragment declares. They are long, and copying one package's specifics is
+a worse outcome than understanding its structure.
## Package boundary
@@ -67,6 +75,17 @@ nothing.
- A component that cannot render on its own cannot produce the preview image its Surface owes. Treat
a missing preview as evidence of this mistake rather than a step to skip.
+## Freeze the Types before writing in parallel
+
+The nominal Types and the Manifest are what every other file agrees with, so write them first and
+stop changing them. Once they are frozen, `render.ts`, `style.ts` and `schedule.ts` depend on the
+Types rather than on each other and can be written in any order or at the same time. `surface.ts`,
+`component.ts` and `fragment.ts` follow, because they wire what those three produce.
+
+Do not start that parallel stretch while a Type is still in question. Interface drift mid-flight
+costs more than the ordering saves, and the symptom — a Producer rejecting a value that looks right —
+is expensive to read.
+
## Install and validate
Reuse an existing workspace when it already includes `packages/*`; otherwise make only the minimal
diff --git a/packages/reference-video-tools/src/media.ts b/packages/reference-video-tools/src/media.ts
index 0ad60ca8..5322c087 100644
--- a/packages/reference-video-tools/src/media.ts
+++ b/packages/reference-video-tools/src/media.ts
@@ -68,7 +68,18 @@ async function contentBoundaries(path: string, duration: number): Promise<readon
anchor = frames[index]!;
}
}
- return [...starts, duration].filter((value, index, all) => index === 0 || value - all[index - 1]! > 0.3);
+ // A boundary closer than this to the one before it does not describe a shot: it describes a
+ // flicker, a flash frame or a fast pan. Clips that short are also rejected outright by the model,
+ // so admitting them costs a request and returns nothing.
+ const shortest = 1;
+ const kept = [starts[0]!];
+ for (const value of starts.slice(1)) {
+ if (value - kept.at(-1)! >= shortest) kept.push(value);
+ }
+ // The tail is a boundary too, and a final fragment belongs to the shot before it rather than
+ // becoming a shot of its own.
+ if (duration - kept.at(-1)! < shortest && kept.length > 1) kept.pop();
+ return [...kept, duration];
}
type Bounds = { readonly start: number; readonly end: number; readonly group: number; readonly part: number; readonly parts: number };
diff --git a/packages/reference-video-tools/src/tools.ts b/packages/reference-video-tools/src/tools.ts
index 0273b180..05cbc470 100644
--- a/packages/reference-video-tools/src/tools.ts
+++ b/packages/reference-video-tools/src/tools.ts
@@ -61,6 +61,14 @@ type ObservationTask = { readonly key: string; readonly run: () => Promise<Obser
function observation(status: "complete" | "failed", text: string): Observation { return { status, text }; }
+function positiveEnv(name: string): number | undefined {
+ const raw = process.env[name]?.trim();
+ if (raw === undefined || raw.length === 0) return undefined;
+ const value = Number(raw);
+ assert(Number.isSafeInteger(value) && value > 0, `${name} must be a positive integer`);
+ return value;
+}
+
function stateRoot(workspaceRoot: string, reference: string): string {
return join(workspaceRoot, ".hypit", "reference-video-tools", reference);
}
@@ -100,12 +108,24 @@ const MIME_TYPES: Readonly<Record<string, string>> = {
mp4: "video/mp4",
};
-async function mediaPart(path: string): Promise<Part> {
- const bytes = await readBytes(path);
- const extension = path.toLowerCase().split(".").pop() ?? "";
- const mimeType = MIME_TYPES[extension];
- assert(mimeType !== undefined, `unsupported media extension for ${path}; supported: ${Object.keys(MIME_TYPES).join(", ")}`);
- return { inlineData: { mimeType, data: Buffer.from(bytes).toString("base64") } };
+// The Vertex path cannot upload a file and reference it later, so the bytes travel with every
+// request. One shot's clip is asked about by its own picture, type and sound observations and by the
+// boundary on each side, so reading and encoding it once per invocation is worth the memory.
+function mediaParts(): (path: string) => Promise<Part> {
+ const encoded = new Map<string, Promise<Part>>();
+ return (path) => {
+ const held = encoded.get(path);
+ if (held !== undefined) return held;
+ const part = (async (): Promise<Part> => {
+ const bytes = await readBytes(path);
+ const extension = path.toLowerCase().split(".").pop() ?? "";
+ const mimeType = MIME_TYPES[extension];
+ assert(mimeType !== undefined, `unsupported media extension for ${path}; supported: ${Object.keys(MIME_TYPES).join(", ")}`);
+ return { inlineData: { mimeType, data: Buffer.from(bytes).toString("base64") } };
+ })();
+ encoded.set(path, part);
+ return part;
+ };
}
function message(error: unknown): string {
@@ -224,8 +244,11 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
const workspaceRoot = resolve(options.workspaceRoot ?? process.cwd());
const packageRoot = resolve(options.packageRoot ?? workspaceRoot);
const model = options.model ?? process.env.GEMINI_MODEL?.trim() ?? "gemini-3.1-pro-preview";
- const concurrency = options.concurrency ?? 2;
- const gapMs = options.launchGapMs ?? 6_000;
+ // Pacing is deployment policy, not author intent: it depends on the quota behind the credentials,
+ // which the calling agent has no way to know. It is settable here and through the environment, and
+ // deliberately not through a CLI flag.
+ const concurrency = options.concurrency ?? positiveEnv("HYPIT_REFERENCE_CONCURRENCY") ?? 2;
+ const gapMs = options.launchGapMs ?? positiveEnv("HYPIT_REFERENCE_LAUNCH_GAP_MS") ?? 6_000;
const retryDelayMs = options.retryDelayMs ?? 2_000;
let generatorPromise: Promise<GenerateText> | undefined;
const generator = async (): Promise<GenerateText> => generatorPromise ??= options.generate === undefined ? defaultGenerate(model) : Promise.resolve(options.generate);
@@ -284,6 +307,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
await writeFile(join(root, "observations.json"), "{}\n", "utf8");
}
const generate = await generator();
+ const mediaPart = mediaParts();
const globalParts: Part[] = [await mediaPart(analysisVideo)];
const [people, voices, systems] = await Promise.all([
state.people_and_product?.status === "complete" && !redoPeople
@@ -302,6 +326,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
},
async observe_reference(input): Promise<Record<string, unknown>> {
+ const mediaPart = mediaParts();
const state = await loadState(input.reference_id);
const selected = input.shot_ids === undefined ? state.shots : state.shots.filter((shot) => input.shot_ids!.includes(shot.shot_id));
assert(selected.length > 0, "no requested shot ids exist");
@@ -354,24 +379,20 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
parts.push({ text: `${globalContext}\n\nListen to shot ${shot.index}. Decide who is speaking, whether sound continues from the previous shot, whether visible people actually produce the sound, and whether a silent B-roll person only appears to speak. Handle off-screen, alternating, overlapping, and no-person-visible speech. Return natural language evidence only.` });
return await callSafely(retryDelayMs, generate, parts, "Observe sound only. Do not write markup, SVML, JSON plans, or component names.");
}}));
- const sameTakeTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `same-take:${right.shot_id}`, run: async () => {
- const left = state.shots.find((shot) => shot.index === right.index - 1)!;
- const parts: Part[] = [await mediaPart(left.clip_ref), await mediaPart(right.clip_ref), await mediaPart(left.tail_frame_ref)];
- parts.push({ text: `Compare shots ${left.index} and ${right.index}. Answer only whether they are one continuous camera shot. Explain the visual and sound continuity evidence in natural language.` });
- return await callSafely(retryDelayMs, generate, parts, "Analyze camera continuity only. Do not name SVML components or write code.");
- }}));
- const overlayTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `overlay:${right.shot_id}`, run: async () => {
+ // Both boundary questions look at exactly the same three files, so they travel together. Asking
+ // them separately uploaded two clips twice to learn two things about one cut.
+ const boundaryTasks: ObservationTask[] = boundaryRights.map((right) => ({ key: `boundary:${right.shot_id}`, run: async () => {
const left = state.shots.find((shot) => shot.index === right.index - 1)!;
const parts: Part[] = [await mediaPart(left.clip_ref), await mediaPart(right.clip_ref), await mediaPart(left.tail_frame_ref)];
- parts.push({ text: `Compare shots ${left.index} and ${right.index}. Answer only whether the same visible overlay or inserted picture continues across the boundary, changes, or ends. Explain the evidence in natural language.` });
- return await callSafely(retryDelayMs, generate, parts, "Analyze overlay continuity only. Do not name SVML components or write code.");
+ parts.push({ text: `Compare shots ${left.index} and ${right.index}, which meet at one cut. Answer two questions separately, each under its own heading.\n\nContinuous camera shot: are these one continuous camera shot, or two? Explain the visual and sound continuity evidence.\n\nOverlay continuity: does the same visible overlay or inserted picture continue across the boundary, change, or end? Explain the evidence.\n\nReturn natural language evidence only.` });
+ return await callSafely(retryDelayMs, generate, parts, "Analyze what happens at one cut. Do not name SVML components or write code.");
}}));
- const allTasks = [...visualTasks, ...typeTasks, ...audioTasks, ...sameTakeTasks, ...overlayTasks];
+ const allTasks = [...visualTasks, ...typeTasks, ...audioTasks, ...boundaryTasks];
const reobserve = input.reobserve === true;
const forcedKeys = reobserve ? new Set(allTasks.map((task) => task.key)) : new Set<string>();
const observations = await runObservationTasks(state.root, allTasks, forcedKeys, concurrency, gapMs);
const unresolved = [...observations].filter(([, value]) => value.status !== "complete").map(([key]) => key);
- const boundaryText = boundaryRights.map((right) => `${observations.get(`same-take:${right.shot_id}`)?.text ?? ""}\n${observations.get(`overlay:${right.shot_id}`)?.text ?? ""}`).join("\n");
+ const boundaryText = boundaryRights.map((right) => observations.get(`boundary:${right.shot_id}`)?.text ?? "").join("\n");
const needsThreeShotReview = selected.length >= 3 && (
reobserve ||
/uncertain|unknown|possibly|may be|contin(?:ue|ues)|same take|same shot/iu.test(boundaryText)
@@ -407,8 +428,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
right_shot_id: right.shot_id,
combined_duration_seconds: Number((right.end_seconds - left.start_seconds).toFixed(3)),
within_15_seconds: right.end_seconds - left.start_seconds <= 15,
- same_take: observations.get(`same-take:${right.shot_id}`)!,
- overlay_continuity: observations.get(`overlay:${right.shot_id}`)!,
+ continuity: observations.get(`boundary:${right.shot_id}`)!,
};
});
return {
@@ -445,6 +465,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
},
async compare_reconstruction(input): Promise<Record<string, unknown>> {
+ const mediaPart = mediaParts();
const state = await loadState(input.reference_id);
const shot = state.shots.find((candidate) => candidate.shot_id === input.shot_id);
assert(shot !== undefined, `shot ${input.shot_id} does not exist in reference ${input.reference_id}`);
diff --git a/packages/reference-video-tools/test/tools.test.ts b/packages/reference-video-tools/test/tools.test.ts
index 9d9c1a9f..9554b242 100644
--- a/packages/reference-video-tools/test/tools.test.ts
+++ b/packages/reference-video-tools/test/tools.test.ts
@@ -79,11 +79,10 @@ test("observation covers picture, drawn type and sound for every shot and caches
const cache = JSON.parse(await readFile(join(stateRoot, "observations.json"), "utf8")) as Record<string, unknown>;
assert.deepEqual(Object.keys(cache).sort(), [
"audio:shot-001", "audio:shot-002",
- "overlay:shot-002",
- "same-take:shot-002",
+ "boundary:shot-002",
"type:shot-001", "type:shot-002",
"visual:shot-001", "visual:shot-002",
- ]);
+ ], "one cut is one observation, not a same-take question and an overlay question over the same media");
const shots = result["shots"] as readonly Record<string, unknown>[];
assert.equal(shots.length, 2);
@@ -106,7 +105,7 @@ test("a second observation reuses the cache and only an explicit reobserve runs
const { root } = await workspace(2);
const first: Call[] = [];
await tools(root, first).observe_reference({ reference_id: REFERENCE });
- assert.equal(first.length, 8);
+ assert.equal(first.length, 7, "two shots: picture, type and sound each, plus one shared boundary");
const second: Call[] = [];
await tools(root, second).observe_reference({ reference_id: REFERENCE });
@@ -118,7 +117,7 @@ test("a second observation reuses the cache and only an explicit reobserve runs
const forced: Call[] = [];
await tools(root, forced).observe_reference({ reference_id: REFERENCE, shot_ids: ["shot-002"], reobserve: true });
- assert.equal(forced.length, 5, "reobserve reruns the selected shot's picture, type and sound plus its boundary pair, and nothing else");
+ assert.equal(forced.length, 4, "reobserve reruns the selected shot's picture, type and sound plus its one boundary, and nothing else");
});
test("a narrow question costs one call, answers from the named shots and never re-runs observations", async () => {
diff --git a/test/repository-hygiene.test.mjs b/test/repository-hygiene.test.mjs
index e30fbf8d..d9fd4cd7 100644
--- a/test/repository-hygiene.test.mjs
+++ b/test/repository-hygiene.test.mjs
@@ -1,5 +1,5 @@
import assert from "node:assert/strict";
-import { readdir, readFile } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
import { extname, relative } from "node:path";
import test from "node:test";
import ts from "typescript";
@@ -158,3 +158,22 @@ test("production imports belong to each package while the root owns repository t
}
assert.deepEqual(failures, [], `undeclared package imports found:\n${failures.join("\n")}`);
});
+
+test("a declared Surface preview names a picture that exists", async () => {
+ // A component that cannot draw itself cannot produce a preview image. Declaring one and shipping
+ // nothing is how that goes unnoticed: the declaration reads as evidence the component renders.
+ const entries = await repositoryEntries();
+ const failures = [];
+ for (const entry of entries) {
+ const match = /^((?:examples\/[^/]+\/)?packages\/[^/]+)\/src\/.*\.ts$/u.exec(entry.path);
+ if (match === null || !entry.isFile) continue;
+ const source = await readFile(entry.child, "utf8");
+ for (const declared of source.matchAll(/previewImage\(\s*"([^"]+)"\s*\)/gu)) {
+ const file = new URL(`./${match[1]}/preview/${declared[1]}`, repositoryRoot);
+ const size = await stat(file).then((item) => item.size, () => undefined);
+ if (size === undefined) failures.push(`${entry.path} declares preview/${declared[1]}, which does not exist`);
+ else if (size === 0) failures.push(`${entry.path} declares preview/${declared[1]}, which is empty`);
+ }
+ }
+ assert.deepEqual(failures, [], `declared previews without a picture:\n${failures.join("\n")}`);
+});