Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(media-use): record agent-made voice, music and sound effects in the media manifest (#4648)
Miguel Ángele25bc3a80625
diff --git a/packages/cli/src/media-use/lib/candidates.mjs b/packages/cli/src/media-use/lib/candidates.mjsindex c8ecff0559..c58548c25f 100644--- a/packages/cli/src/media-use/lib/candidates.mjs+++ b/packages/cli/src/media-use/lib/candidates.mjs@@ -5,7 +5,7 @@// overlap but never filters a candidate out on zero overlap (that would// pre-empt the agent's judgment); the agent does the semantic call.-import { readManifest } from "./manifest.mjs";+import { currentRecords } from "./manifest.mjs";import { readGlobalManifest } from "./cache.mjs";import { tokenOverlap, typesMatch } from "./match.mjs";@@ -49,7 +49,7 @@ function rankScope(records, scope, type, intent) {// total: { project, global } counts before the cap (machine-readable)// similar: count of candidates with lexical overlap > 0 (drives the nudge)export function listCandidates({ projectDir, type, intent = "", cap = CANDIDATE_CAP }) {- const project = rankScope(readManifest(projectDir), "project", type, intent);+ const project = rankScope(currentRecords(projectDir), "project", type, intent);const global = rankScope(readGlobalManifest(), "global", type, intent);const candidates = [...project.slice(0, cap), ...global.slice(0, cap)];return {diff --git a/packages/cli/src/media-use/lib/index-gen.mjs b/packages/cli/src/media-use/lib/index-gen.mjsindex 65b019a92e..c25ee6d264 100644--- a/packages/cli/src/media-use/lib/index-gen.mjs+++ b/packages/cli/src/media-use/lib/index-gen.mjs@@ -1,6 +1,6 @@import { writeFileSync, mkdirSync } from "node:fs";import { dirname } from "node:path";-import { readManifest, indexPath } from "./manifest.mjs";+import { currentRecords, indexPath } from "./manifest.mjs";function pad(str, len) {return String(str ?? "").padEnd(len);@@ -54,7 +54,7 @@ export function generateIndexContent(records) {}export function regenerateIndex(projectDir) {- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);const content = generateIndexContent(records);const p = indexPath(projectDir);mkdirSync(dirname(p), { recursive: true });diff --git a/packages/cli/src/media-use/lib/manifest.mjs b/packages/cli/src/media-use/lib/manifest.mjsindex 0bd2f7b3da..56b533aaa3 100644--- a/packages/cli/src/media-use/lib/manifest.mjs+++ b/packages/cli/src/media-use/lib/manifest.mjs@@ -10,7 +10,7 @@ import {rmSync,statSync,} from "node:fs";-import { join } from "node:path";+import { basename, join } from "node:path";const MANIFEST_FILE = "manifest.jsonl";const INDEX_FILE = "index.md";@@ -79,6 +79,43 @@ export function appendRecord(projectDir, record) {appendFileSync(p, line);}+/** The record a path has now: the manifest only appends, so the last one for a path wins. */+export function latestRecordFor(projectDir, path) {+ return readManifest(projectDir).findLast((record) => record.path === path);+}++/** Records with every older record for the same path dropped, since the last one for a path is its record. */+export function currentRecords(projectDir) {+ const records = readManifest(projectDir);+ const last = new Map(records.map((record, index) => [record.path, index]));+ return records.filter((record, index) => !record.path || last.get(record.path) === index);+}++/** Records a file already in the project where it is, unless its current record already says the same thing. */+export function recordInPlace(+ projectDir,+ { type, path, source, description, duration, provenance },+) {+ const fields = {+ type,+ path,+ source,+ description: description || basename(path),+ ...(duration != null && { duration: Math.round(duration * 10) / 10 }),+ };+ mkdirSync(mediaDir(projectDir), { recursive: true });+ return withLock(mediaDir(projectDir), () => {+ const latest = latestRecordFor(projectDir, path);+ const same = ["source", "description", "duration"].every(+ (key) => latest?.[key] === fields[key],+ );+ if (latest && same) return latest;+ const record = { id: nextFreeId(projectDir, type), ...fields, provenance };+ appendRecord(projectDir, record);+ return record;+ });+}+// Match prompts forgivingly. Agents rarely re-emit a byte-identical intent, so// keying cache lookups on exact equality meant "Calm piano" and "calm piano"// re-searched and re-downloaded. Normalize (trim, lowercase, collapse internal@@ -93,7 +130,7 @@ export function normalizePrompt(prompt) {export function findByPrompt(projectDir, prompt, type) {const key = normalizePrompt(prompt);if (!key) return null;- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);return (records.find((r) => normalizePrompt(r.provenance?.prompt) === key && (type == null || r.type === type),@@ -103,7 +140,7 @@ export function findByPrompt(projectDir, prompt, type) {export function findByEntity(projectDir, entity) {const lower = entity.toLowerCase();- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null;}@@ -173,24 +210,30 @@ export function allocateId(projectDir, type, ext) {const typeDir = typeDirPath(projectDir, type);mkdirSync(typeDir, { recursive: true });return withLock(mediaDir(projectDir), () => {- const re = new RegExp(`^${type}_(\\d+)`);- let max = 0;- for (const r of readManifest(projectDir)) {- if (r.type !== type) continue;- const m = r.id?.match(re);- if (m) max = Math.max(max, parseInt(m[1], 10));- }- for (const f of readdirSync(typeDir)) {- const m = f.match(re);- if (m) max = Math.max(max, parseInt(m[1], 10)); // skip ids reserved but not yet appended- }- const id = `${type}_${String(max + 1).padStart(3, "0")}`;+ const id = nextFreeId(projectDir, type);const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;writeFileSync(join(projectDir, localPath), "", { flag: "wx" }); // durable reservationreturn { id, localPath };});}+// Call under the lock: counts recorded ids and ids reserved by a file not yet recorded.+function nextFreeId(projectDir, type) {+ const re = new RegExp(`^${type}_(\\d+)`);+ let max = 0;+ for (const r of readManifest(projectDir)) {+ if (r.type !== type) continue;+ const m = r.id?.match(re);+ if (m) max = Math.max(max, parseInt(m[1], 10));+ }+ const typeDir = typeDirPath(projectDir, type);+ for (const f of existsSync(typeDir) ? readdirSync(typeDir) : []) {+ const m = f.match(re);+ if (m) max = Math.max(max, parseInt(m[1], 10));+ }+ return `${type}_${String(max + 1).padStart(3, "0")}`;+}+function reservedFile(projectDir, type, ext) {const allocation = allocateId(projectDir, type, ext);return { ...allocation, fullPath: join(projectDir, allocation.localPath) };diff --git a/packages/cli/src/media-use/lib/manifest.test.mjs b/packages/cli/src/media-use/lib/manifest.test.mjsindex 7422ec36b0..348240fdf3 100644--- a/packages/cli/src/media-use/lib/manifest.test.mjs+++ b/packages/cli/src/media-use/lib/manifest.test.mjs@@ -264,7 +264,13 @@ function runTests() {appendRecord(tmp, makeRecord({ id: "bgm_001" }));appendRecord(tmp,- makeRecord({ id: "sfx_001", type: "sfx", description: "whoosh", duration: 3 }),+ makeRecord({+ id: "sfx_001",+ type: "sfx",+ path: ".media/audio/sfx/sfx_001.mp3",+ description: "whoosh",+ duration: 3,+ }),);regenerateIndex(tmp);const content = readFileSync(join(tmp, ".media", "index.md"), "utf8");diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjsindex 38659996b2..a529fd313d 100644--- a/packages/cli/src/media-use/resolve.mjs+++ b/packages/cli/src/media-use/resolve.mjs@@ -1,11 +1,13 @@#!/usr/bin/env nodeimport { spawnSync } from "node:child_process";-import { existsSync, statSync, writeFileSync, renameSync, rmSync } from "node:fs";-import { resolve, join, extname, basename } from "node:path";+import { existsSync, statSync, writeFileSync, renameSync, rmSync, realpathSync } from "node:fs";+import { resolve, join, extname, basename, relative, isAbsolute, sep } from "node:path";import { parseArgs } from "node:util";import {appendRecord,+ latestRecordFor,+ recordInPlace,findByPrompt,findByEntity,nextId,@@ -23,6 +25,7 @@ import {} from "./lib/registry.mjs";import { freezeUrl, freezeLocalFile, isDirectMediaUrl } from "./lib/freeze.mjs";import { findExistingAsset } from "./lib/adopt.mjs";+import { probe as probeMedia } from "./lib/probe.mjs";import { track } from "./lib/telemetry.mjs";import { recordMiss } from "./lib/misses.mjs";import { buildStats } from "./lib/stats.mjs";@@ -57,6 +60,7 @@ import {} from "./lib/local-media-search.mjs";const INGEST_TYPES = listTypes();+const RECORDED_SOURCES = ["generated", "search", "bundled"];const DEFAULT_EXT = {bgm: ".wav",sfx: ".mp3",@@ -89,6 +93,7 @@ const { values: args } = parseArgs({"dry-run": { type: "boolean", default: false },reuse: { type: "string" },from: { type: "string" },+ source: { type: "string" },params: { type: "string" },for: { type: "string" },analyze: { type: "boolean", default: false },@@ -125,6 +130,8 @@ Options:--reuse <sha> Import a specific global-cache asset (by content sha/prefix,from --candidates) into this project--from <file> Freeze a local file or direct public URL (ingest)+ --source <how> With --from: how the file was made (${RECORDED_SOURCES.join(" | ")}).+ A file already inside the project is then recorded where it is--params <json> Build an explicit parametric LUT (lut/grade only)--for <media> Analyze a local image/video and add measured grade adjustsuggestions (grade only)@@ -208,6 +215,11 @@ if (args.reuse !== undefined) {process.exit(0);}+if (args.source && !args.from) {+ console.error("error: --source goes with --from <file>");+ process.exit(2);+}+// Ingest: freeze a user-supplied local file or direct public URL (no search).if (args.from) {await ingest(args.from);@@ -353,6 +365,8 @@ async function run() {? null: findExistingAsset(projectDir, intent, type);if (existingAsset) {+ const recorded = latestRecordFor(projectDir, existingAsset.relativePath);+ if (recorded) return result(recorded, "cached");const id = nextId(projectDir, type);const record = {id,@@ -904,6 +918,24 @@ async function ingest(src) {console.error(`error: refusing to ingest a 0-byte file: ${src}`);process.exit(2);}+ if (args.source && !RECORDED_SOURCES.includes(args.source)) {+ console.error(`error: --source takes one of: ${RECORDED_SOURCES.join(", ")}`);+ process.exit(2);+ }+ if (args.source && (type === "lut" || type === "grade")) {+ console.error("error: --source records media files; a LUT or grade is ingested without it");+ process.exit(2);+ }+ const real = (path) => (existsSync(path) ? realpathSync(path) : path);+ const inProject = args.source && !isUrl ? relative(real(projectDir), real(resolve(src))) : null;+ if (+ inProject &&+ inProject !== ".." &&+ !inProject.startsWith(`..${sep}`) &&+ !isAbsolute(inProject)+ ) {+ return recordProjectFile(inProject.split(sep).join("/"));+ }const ext = extname(isUrl ? new URL(src).pathname : src) || defaultExt(type);const { id, localPath, fullPath } = await withReservedFile(projectDir,@@ -928,7 +960,7 @@ async function ingest(src) {id,type,path: localPath,- source: "ingested",+ source: args.source || "ingested",description: basename(src.split("?")[0]),provenance: { provider: "local", from: src },};@@ -942,6 +974,30 @@ async function ingest(src) {await result(record, "ingested");}+async function recordProjectFile(path) {+ let duration;+ try {+ duration = probeMedia(join(projectDir, path)).duration;+ } catch (err) {+ exitError(err.message);+ }+ const record = recordInPlace(projectDir, {+ type,+ path,+ source: args.source,+ description: intent,+ duration,+ provenance: {+ provider: args.provider || "local",+ from: path,+ ...(intent && { prompt: intent }),+ },+ });+ regenerateIndex(projectDir);+ // "recorded", not the record's source, so usage counts keep meaning fetches.+ await result(record, "recorded");+}+async function showCandidates() {const projectDir = resolve(args.project);const type = args.type;diff --git a/packages/cli/src/media-use/resolve.test.mjs b/packages/cli/src/media-use/resolve.test.mjsindex c4cb2c25f9..a4b54ff539 100644--- a/packages/cli/src/media-use/resolve.test.mjs+++ b/packages/cli/src/media-use/resolve.test.mjs@@ -13,7 +13,7 @@ import { join } from "node:path";import { tmpdir } from "node:os";import { createServer } from "node:http";import { execFileSync, spawn, spawnSync } from "node:child_process";-import { appendRecord, readManifest } from "./lib/manifest.mjs";+import { appendRecord, findByPrompt, readManifest } from "./lib/manifest.mjs";import { regenerateIndex } from "./lib/index-gen.mjs";import { getProvider } from "./lib/providers.mjs";import { HEYGEN_NOT_FOUND_MESSAGE } from "./lib/heygen-cli.mjs";@@ -566,6 +566,153 @@ test("resolve finds existing unregistered asset before hitting providers", () =>cleanup();});+test("--from records a file already in the project where it is, with how it was made", () => {+ setup();+ mkdirSync(join(tmp, "assets/voice"), { recursive: true });+ writeFileSync(join(tmp, "assets/voice/01.wav"), "fake wav");++ const out = runResolve([+ "--from",+ join(tmp, "assets/voice/01.wav"),+ "--type",+ "voice",+ "--source",+ "generated",+ "--intent",+ "Welcome to the launch",+ "--project",+ tmp,+ "--json",+ ]);++ assert.equal(JSON.parse(out.trim()).path, "assets/voice/01.wav");+ assert.deepEqual(+ readManifest(tmp).map((r) => [r.path, r.source, r.description]),+ [["assets/voice/01.wav", "generated", "Welcome to the launch"]],+ );+ assert.deepEqual(readdirSync(join(tmp, ".media/audio/voice")), []);+ cleanup();+});++test("--from --source adds a record only when what it says changes", () => {+ setup();+ mkdirSync(join(tmp, "assets/bgm"), { recursive: true });+ writeFileSync(join(tmp, "assets/bgm/track.mp3"), "fake mp3");+ const from = join(tmp, "assets/bgm/track.mp3");+ const record = (intent) =>+ runResolve([+ ...["--from", from, "--type", "bgm", "--source", "search", "--intent", intent],+ ...["--project", tmp, "--json"],+ ]);++ record("calm underscore");+ record("calm underscore");+ record("upbeat synth");++ assert.deepEqual(+ readManifest(tmp).map((r) => [r.source, r.description]),+ [+ ["search", "calm underscore"],+ ["search", "upbeat synth"],+ ],+ );+ assert.equal(findByPrompt(tmp, "calm underscore", "bgm"), null);+ const listed = runResolve([+ ...["--candidates", "--type", "bgm", "--intent", "calm underscore"],+ ...["--project", tmp, "--json"],+ ]);+ assert.deepEqual(+ JSON.parse(listed)+ .candidates.filter((c) => c.scope === "project")+ .map((c) => c.description),+ ["upbeat synth"],+ );+ const index = readFileSync(join(tmp, ".media/index.md"), "utf8");+ assert.match(index, /upbeat synth/);+ assert.doesNotMatch(index, /calm underscore/);+ cleanup();+});++test("--from refuses a source it does not know", () => {+ setup();+ mkdirSync(join(tmp, "assets/voice"), { recursive: true });+ writeFileSync(join(tmp, "assets/voice/01.wav"), "fake wav");+ const from = join(tmp, "assets/voice/01.wav");++ const r = spawnResolve(["--from", from, "--type", "voice", "--source", "mine", "--project", tmp]);++ assert.equal(r.status, 2);+ assert.deepEqual(readManifest(tmp), []);+ cleanup();+});++test("resolve does not relabel a recorded file in assets/ as the person's own", () => {+ setup();+ mkdirSync(join(tmp, "assets/bgm"), { recursive: true });+ writeFileSync(join(tmp, "assets/bgm/ambient-track.mp3"), "agent bgm");+ appendRecord(tmp, {+ id: "bgm_001",+ type: "bgm",+ path: "assets/bgm/ambient-track.mp3",+ source: "existing",+ description: "ambient track",+ provenance: { provider: "local", adopted: true },+ });+ appendRecord(tmp, {+ id: "bgm_002",+ type: "bgm",+ path: "assets/bgm/ambient-track.mp3",+ source: "search",+ description: "calm underscore",+ provenance: { provider: "heygen" },+ });++ const out = runResolve([+ "--type",+ "bgm",+ "--intent",+ "ambient track",+ "--project",+ tmp,+ "--json",+ ]);++ const parsed = JSON.parse(out.trim());+ assert.equal(parsed.path, "assets/bgm/ambient-track.mp3");+ assert.equal(parsed.source, "search");+ assert.deepEqual(+ readManifest(tmp).map((r) => r.source),+ ["existing", "search"],+ );+ cleanup();+});++test("--from --source copies a file from outside the project, labelled with that source", () => {+ setup();+ const outside = mkdtempSync(join(tmpdir(), "mu-outside-"));+ writeFileSync(join(outside, "take.wav"), "outside wav");+ const args = ["--from", join(outside, "take.wav"), "--type", "voice", "--source", "generated"];++ const out = runResolve([...args, "--project", tmp, "--json"]);++ const parsed = JSON.parse(out.trim());+ assert.match(parsed.path, /^\.media\/audio\/voice\//);+ assert.equal(parsed.source, "generated");+ rmSync(outside, { recursive: true, force: true });+ cleanup();+});++test("--source is refused for a LUT and without --from", () => {+ setup();+ writeFileSync(join(tmp, "look.cube"), "LUT_3D_SIZE 2\n");+ const lut = ["--from", join(tmp, "look.cube"), "--type", "lut", "--source", "generated"];++ assert.equal(spawnResolve([...lut, "--project", tmp]).status, 2);+ assert.equal(spawnResolve(["--source", "generated", "--project", tmp]).status, 2);+ assert.deepEqual(readManifest(tmp), []);+ cleanup();+});+// --- CLI interface ---test("--help exits 0", () => {diff --git a/packages/core/src/figma/mediaIndex.test.ts b/packages/core/src/figma/mediaIndex.test.tsindex 95aedcb9d6..a77319da08 100644--- a/packages/core/src/figma/mediaIndex.test.ts+++ b/packages/core/src/figma/mediaIndex.test.ts@@ -61,6 +61,24 @@ const FIGMA_ROW = {const JUNK_ROW = { note: "not a media record" };const ALL_ROWS = [MEDIA_USE_ROW, IMAGE_ROW, ICON_ROW, FIGMA_ROW, JUNK_ROW];+// The media-use generator itself, resolved from THIS file (cwd varies per test runner) and+// handed over as a file:// URL so the specifier is valid on windows too.+const genUrl = pathToFileURL(+ join(+ fileURLToPath(new URL(".", import.meta.url)),+ "..",+ "..",+ "..",+ "..",+ "packages",+ "cli",+ "src",+ "media-use",+ "lib",+ "index-gen.mjs",+ ),+).href;+describe("regenerateIndex", () => {it("renders every writer's rows (media-use + figma) into one table", () => {const p = project();@@ -77,24 +95,6 @@ describe("regenerateIndex", () => {// Covers duration, width×height, icon+transparent, no-dims, and junk-row// selection — the full set of branches both generators format.const ours = generateIndexContent(ALL_ROWS as Record<string, unknown>[]);- // Run the actual media-use generator on identical input. Resolve the- // script relative to THIS file (cwd varies per test runner) and hand it- // over as a file:// URL so the specifier is valid on windows too.- const genUrl = pathToFileURL(- join(- fileURLToPath(new URL(".", import.meta.url)),- "..",- "..",- "..",- "..",- "packages",- "cli",- "src",- "media-use",- "lib",- "index-gen.mjs",- ),- ).href;const script = `import { generateIndexContent } from ${JSON.stringify(genUrl)};const rows = ${JSON.stringify(ALL_ROWS)};@@ -105,4 +105,23 @@ describe("regenerateIndex", () => {});expect(ours).toBe(theirs);});++ it("lists only a path's newest record, the same rows media-use's writer picks", () => {+ const p = project();+ const regenerated = { ...MEDIA_USE_ROW, description: "calm underscore" };+ for (const row of [MEDIA_USE_ROW, FIGMA_ROW, regenerated]) {+ appendFileSync(manifestPath(p), JSON.stringify(row) + "\n");+ }+ const ours = regenerateIndex(p);+ const script = `+ import { regenerateIndex } from ${JSON.stringify(genUrl)};+ process.stdout.write(regenerateIndex(${JSON.stringify(p)}));+ `;+ const theirs = execFileSync("node", ["--input-type=module", "-e", script], {+ encoding: "utf8",+ });+ expect(ours).toContain("calm underscore");+ expect(ours).not.toContain("upbeat tech launch");+ expect(ours).toBe(theirs);+ });});diff --git a/packages/core/src/figma/mediaIndex.ts b/packages/core/src/figma/mediaIndex.tsindex 8ec865b35a..6ab33f0cd9 100644--- a/packages/core/src/figma/mediaIndex.ts+++ b/packages/core/src/figma/mediaIndex.ts@@ -1,7 +1,7 @@/*** Regenerate .media/index.md — the agent-readable inventory table — after a* figma import, exactly the way media-use does after a resolve. Both writers- * regenerate the SAME file from the full manifest (all writers' rows), so the+ * regenerate the SAME file from the manifest's current rows (all writers'), so the* output format AND row selection here must stay byte-identical with* skills/media-use/scripts/lib/index-gen.mjs — including rendering every* JSON-parseable row (no shape filtering), or the file would flip-flop@@ -78,9 +78,11 @@ export function generateIndexContent(records: IndexRow[]): string {return lines.join("\n") + "\n";}-/** Rebuild index.md from EVERY writer's manifest rows (media-use + figma). */+/** Rebuild index.md from every writer's rows (media-use + figma), newest record per path. */export function regenerateIndex(projectDir: string): string {- const records = readJsonlValues(manifestPath(projectDir)).filter(isRow);+ const rows = readJsonlValues(manifestPath(projectDir)).filter(isRow);+ const last = new Map(rows.map((row, index) => [row.path, index]));+ const records = rows.filter((row, index) => !row.path || last.get(row.path) === index);const content = generateIndexContent(records);const p = indexPath(projectDir);mkdirSync(dirname(p), { recursive: true });diff --git a/skills-manifest.json b/skills-manifest.jsonindex 030fffedab..27c4b0a9ed 100644--- a/skills-manifest.json+++ b/skills-manifest.json@@ -54,8 +54,8 @@"files": 1},"media-use": {- "hash": "004d60f9660a5f16",- "files": 102+ "hash": "1871ad36628ac54b",+ "files": 104},"motion-graphics": {"hash": "3d3a35ffbe8b7e3e",diff --git a/skills/media-use/audio/references/sfx.md b/skills/media-use/audio/references/sfx.mdindex 6df3268cc3..0fa1d28e77 100644--- a/skills/media-use/audio/references/sfx.md+++ b/skills/media-use/audio/references/sfx.md@@ -16,7 +16,8 @@ Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The"id": "3", // joins the cue to the caller's model (frame / scene / segment)"name": "whoosh","file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root- "source": "heygen" | "local", // which route resolved it+ "source": "heygen" | "local" | "project", // which route resolved it; "project": a file already+ // in assets/sfx under that name that is not the library's copy"offset_s": 0, // delay from the line's start"duration_s": 0.57,"volume": 0.35 // SFX sit UNDER voice + BGMdiff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjsindex a7dc6d93b2..d0437028e7 100644--- a/skills/media-use/audio/scripts/audio.mjs+++ b/skills/media-use/audio/scripts/audio.mjs@@ -55,6 +55,7 @@ import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs"import { resolveSfx } from "./lib/sfx.mjs";import { mapWithConcurrency } from "./lib/concurrency.mjs";import { openAudioMeta } from "./lib/audio-meta.mjs";+import { recordInManifest, writtenAssets } from "./lib/media-record.mjs";const HERE = dirname(fileURLToPath(import.meta.url));const argv = process.argv.slice(2);@@ -289,6 +290,8 @@ const meta = {};mkdirSync(dirname(outPath), { recursive: true });audioMeta.write(meta);+const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });+anomalies.push(...recordInManifest(hyperframesDir, written));console.log(`✓ audio engine → ${outPath}`);console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);diff --git a/skills/media-use/audio/scripts/audio.test.mjs b/skills/media-use/audio/scripts/audio.test.mjsindex b229e40cc3..3d7de20316 100644--- a/skills/media-use/audio/scripts/audio.test.mjs+++ b/skills/media-use/audio/scripts/audio.test.mjs@@ -1,6 +1,6 @@import { strict as assert } from "node:assert";import { test } from "node:test";-import { mkdtempSync, rmSync, existsSync } from "node:fs";+import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs";import { join, dirname } from "node:path";import { tmpdir } from "node:os";import { fileURLToPath } from "node:url";@@ -47,3 +47,27 @@ test("an unknown cue is reported, not fatal", async () => {rmSync(dir, { recursive: true, force: true });}});++test("a person's own file under a bundled name is not labelled as the library's", async () => {+ const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));+ try {+ const resolve = () =>+ resolveSfx({+ cues: [{ id: "1", name: "whoosh" }],+ heygenOK: false,+ hyperframesDir: dir,+ sfxLibDir,+ });+ const copied = await resolve();+ const reused = await resolve();+ writeFileSync(join(dir, copied.sfx[0].file), "the person's own whoosh");+ const own = await resolve();++ assert.deepEqual(+ [copied, reused, own].map(({ sfx }) => sfx[0].source),+ ["local", "local", "project"],+ );+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/skills/media-use/audio/scripts/gemini-pipeline.test.mjs b/skills/media-use/audio/scripts/gemini-pipeline.test.mjsindex 01231d04cf..8658807a89 100644--- a/skills/media-use/audio/scripts/gemini-pipeline.test.mjs+++ b/skills/media-use/audio/scripts/gemini-pipeline.test.mjs@@ -108,6 +108,13 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS},]);assert.equal(meta.total_duration_s, 1.25);+ const manifest = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8").trim().split("\n");+ assert.deepEqual(+ manifest+ .map((line) => JSON.parse(line))+ .map(({ path, type, source }) => [path, type, source]),+ [["assets/voice/intro.wav", "voice", "generated"]],+ );});}}diff --git a/skills/media-use/audio/scripts/lib/media-record.mjs b/skills/media-use/audio/scripts/lib/media-record.mjsnew file mode 100644index 0000000000..ed94a691db--- /dev/null+++ b/skills/media-use/audio/scripts/lib/media-record.mjs@@ -0,0 +1,73 @@+import { recordInPlace } from "../../../scripts/lib/manifest.mjs";+import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs";++const SFX_SOURCES = { heygen: "search", local: "bundled" };++// The files one engine run wrote, each with how it was made, for the project's media manifest.+export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }) {+ const textById = new Map(lines.map((line) => [String(line.id), String(line.text ?? "").trim()]));+ const assets = [];+ if (only.has("tts")) {+ for (const voice of voices) {+ assets.push({+ path: voice.path,+ type: "voice",+ source: "generated",+ intent: textById.get(voice.id),+ duration: voice.duration_s,+ provider: ttsProvider,+ });+ }+ }+ if (only.has("bgm") && bgm && !bgmFields.bgm_pending) {+ assets.push({+ path: bgm.path,+ type: "bgm",+ source: bgmFields.bgm_mode === "retrieve" ? "search" : "generated",+ intent: bgm.query,+ duration: bgm.duration_s,+ provider: bgmFields.bgm_provider,+ });+ }+ if (only.has("sfx")) {+ for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {+ const source = SFX_SOURCES[cue.source];+ if (!source) continue;+ const provider = source === "search" ? "heygen" : "bundled.sfx";+ assets.push({+ path: cue.file,+ type: "sfx",+ source,+ intent: cue.name,+ duration: cue.duration_s,+ provider,+ });+ }+ }+ return assets;+}++/** Records each asset where it lies; returns one anomaly per file left unrecorded. */+export function recordInManifest(hyperframesDir, assets) {+ const anomalies = [];+ for (const { path, type, source, intent, duration, provider } of assets) {+ try {+ recordInPlace(hyperframesDir, {+ type,+ path,+ source,+ description: intent,+ duration,+ provenance: { provider: provider || "local", ...(intent && { prompt: intent }) },+ });+ } catch (error) {+ anomalies.push(`${path}: not recorded in the media manifest (${error.message})`);+ }+ }+ try {+ if (anomalies.length < assets.length) regenerateIndex(hyperframesDir);+ } catch (error) {+ anomalies.push(`.media/index.md: not refreshed (${error.message})`);+ }+ return anomalies;+}diff --git a/skills/media-use/audio/scripts/lib/media-record.test.mjs b/skills/media-use/audio/scripts/lib/media-record.test.mjsnew file mode 100644index 0000000000..b6ef637267--- /dev/null+++ b/skills/media-use/audio/scripts/lib/media-record.test.mjs@@ -0,0 +1,153 @@+import { strict as assert } from "node:assert";+import { test } from "node:test";+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import { tmpdir } from "node:os";+import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";+import { recordInManifest, writtenAssets } from "./media-record.mjs";++const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };++function project(t) {+ const dir = mkdtempSync(join(tmpdir(), "mu-record-"));+ t.after(() => rmSync(dir, { recursive: true, force: true }));+ return dir;+}++test("a voice run leaves a manifest entry marked generated", (t) => {+ const dir = project(t);+ const assets = writtenAssets({+ only: new Set(["tts"]),+ lines: [{ id: "01", text: "Welcome to the launch" }],+ voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2.34 }],+ ttsProvider: "kokoro",+ sfx: [],+ ...noBgm,+ });++ assert.deepEqual(recordInManifest(dir, assets), []);++ assert.deepEqual(+ readManifest(dir).map(({ path, type, source, description, duration, provenance }) => ({+ path,+ type,+ source,+ description,+ duration,+ provenance,+ })),+ [+ {+ path: "assets/voice/01.wav",+ type: "voice",+ source: "generated",+ description: "Welcome to the launch",+ duration: 2.3,+ provenance: { provider: "kokoro", prompt: "Welcome to the launch" },+ },+ ],+ );+});++const voiceLine = (intent, duration) => [+ { path: "assets/voice/01.wav", type: "voice", source: "generated", intent, duration },+];++test("a rerun that writes the same take keeps one record, and a new length is a new take", (t) => {+ const dir = project(t);++ recordInManifest(dir, voiceLine("Hello world", 1.25));+ recordInManifest(dir, voiceLine("Hello world", 1.25));+ assert.equal(readManifest(dir).length, 1);++ recordInManifest(dir, voiceLine("Hello world", 2));+ assert.deepEqual(+ readManifest(dir).map(({ duration }) => duration),+ [1.3, 2],+ );+});++test("a rerun with new text records the new take, and the old text no longer finds the file", (t) => {+ const dir = project(t);++ recordInManifest(dir, voiceLine("Hello world", 1.25));+ recordInManifest(dir, voiceLine("Welcome back to the show", 3.5));++ assert.deepEqual(+ readManifest(dir).map(({ description, duration }) => [description, duration]),+ [+ ["Hello world", 1.3],+ ["Welcome back to the show", 3.5],+ ],+ );+ assert.equal(findByPrompt(dir, "Hello world", "voice"), null);+ const index = readFileSync(join(dir, ".media/index.md"), "utf8");+ assert.match(index, /Welcome back to the show/);+ assert.doesNotMatch(index, /Hello world/);+ assert.equal(findByPrompt(dir, "Welcome back to the show", "voice")?.path, "assets/voice/01.wav");+});++test("music and sound effects are marked by where they came from", () => {+ const assets = writtenAssets({+ only: new Set(["bgm", "sfx"]),+ lines: [],+ voices: [{ id: "01", path: "assets/voice/01.wav" }],+ bgm: { path: "assets/bgm/track.mp3", query: "calm" },+ bgmFields: { bgm_pending: false, bgm_mode: "retrieve" },+ sfx: [+ { file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },+ { file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },+ { file: "assets/sfx/glass.mp3", name: "glass", source: "heygen" },+ { file: "assets/sfx/pop.mp3", name: "pop", source: "project" },+ ],+ });++ assert.deepEqual(+ assets.map(({ path, source, provider }) => [path, source, provider]),+ [+ ["assets/bgm/track.mp3", "search", undefined],+ ["assets/sfx/whoosh.mp3", "bundled", "bundled.sfx"],+ ["assets/sfx/glass.mp3", "search", "heygen"],+ ],+ );+});++test("music made locally is marked generated once it is ready, not while pending", () => {+ const written = (bgm_pending) =>+ writtenAssets({+ only: new Set(["bgm"]),+ lines: [],+ voices: [],+ sfx: [],+ bgm: { path: "assets/bgm/track.wav" },+ bgmFields: { bgm_pending, bgm_mode: "detached-single" },+ }).map(({ source }) => source);++ assert.deepEqual(written(true), []);+ assert.deepEqual(written(false), ["generated"]);+});++test("a file that cannot be recorded becomes an anomaly, not a failure", (t) => {+ const dir = project(t);+ writeFileSync(join(dir, ".media"), "a file where the media folder should be");++ const anomalies = recordInManifest(dir, [+ { path: "assets/voice/01.wav", type: "voice", source: "generated" },+ ]);++ assert.equal(anomalies.length, 1);+ assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/);+});++test("a record never takes the id of a download still in flight", (t) => {+ const dir = project(t);+ mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true });+ writeFileSync(join(dir, ".media/audio/bgm/bgm_001.mp3"), "");++ recordInManifest(dir, [{ path: "assets/bgm/track.wav", type: "bgm", source: "generated" }]);++ assert.deepEqual(+ readManifest(dir).map(({ id }) => id),+ ["bgm_002"],+ );+});diff --git a/skills/media-use/audio/scripts/lib/sfx.mjs b/skills/media-use/audio/scripts/lib/sfx.mjsindex f72baf34ee..7cc3487c34 100644--- a/skills/media-use/audio/scripts/lib/sfx.mjs+++ b/skills/media-use/audio/scripts/lib/sfx.mjs@@ -118,7 +118,8 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL// a file we never copied produces a dangling reference that silently drops// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue// instead, so the audio_meta never references a missing file.- if (!existsSync(dest)) {+ const reused = existsSync(dest);+ if (!reused) {if (!existsSync(src)) {anomalies.push(`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +@@ -129,11 +130,13 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL}copyFileSync(src, dest);}+ // A file already there that is not the library's copy is the person's own.+ const own = reused && !(existsSync(src) && readFileSync(dest).equals(readFileSync(src)));sfx.push({id,name,file: destRel,- source: "local",+ source: own ? "project" : "local",offset_s: 0,duration_s: r3(hit.duration),volume: SFX_VOLUME,diff --git a/skills/media-use/audio/scripts/wait-bgm.mjs b/skills/media-use/audio/scripts/wait-bgm.mjsindex 3e93e6a9fc..02bd1d6ac4 100644--- a/skills/media-use/audio/scripts/wait-bgm.mjs+++ b/skills/media-use/audio/scripts/wait-bgm.mjs@@ -15,6 +15,7 @@import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";import { join, resolve } from "node:path";+import { recordInManifest } from "./lib/media-record.mjs";const argv = process.argv.slice(2);const flag = (name, def) => {@@ -132,6 +133,9 @@ while (Date.now() - started <= timeoutMs) {message: `BGM ready at ${bgmPath}.`,});console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);+ const generated = { path: bgmPath, type: "bgm", source: "generated", provider: base.provider };+ for (const anomaly of recordInManifest(hyperframesDir, [generated]))+ console.error(` ${anomaly}`);process.exit(0);}diff --git a/skills/media-use/audio/scripts/wait-bgm.test.mjs b/skills/media-use/audio/scripts/wait-bgm.test.mjsindex 6355deefc6..eb2fb226b8 100644--- a/skills/media-use/audio/scripts/wait-bgm.test.mjs+++ b/skills/media-use/audio/scripts/wait-bgm.test.mjs@@ -1,5 +1,8 @@import assert from "node:assert/strict";-import { readFileSync } from "node:fs";+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { spawnSync } from "node:child_process";+import { join } from "node:path";+import { tmpdir } from "node:os";import { test } from "node:test";import { runInNewContext } from "node:vm";@@ -63,3 +66,43 @@ test("empty log path does not access the filesystem", () => {"",);});++test("records locally made music as generated once it is ready", () => {+ const dir = mkdtempSync(join(tmpdir(), "mu-wait-bgm-"));+ try {+ mkdirSync(join(dir, "assets/bgm"), { recursive: true });+ writeFileSync(join(dir, "assets/bgm/track.wav"), "generated music");+ writeFileSync(+ join(dir, "audio_meta.json"),+ JSON.stringify({+ bgm: { path: "assets/bgm/track.wav" },+ bgm_pending: true,+ bgm_provider: "musicgen",+ bgm_mode: "detached-single",+ }),+ );+ const script = new URL("./wait-bgm.mjs", import.meta.url).pathname;+ const args = ["--audio-meta", join(dir, "audio_meta.json"), "--hyperframes", dir];++ const run = spawnSync(process.execPath, [script, ...args, "--timeout-ms", "1000"], {+ encoding: "utf8",+ });++ assert.equal(run.status, 0, run.stderr);+ const records = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ assert.deepEqual(+ records.map(({ path, type, source, provenance }) => [+ path,+ type,+ source,+ provenance.provider,+ ]),+ [["assets/bgm/track.wav", "bgm", "generated", "musicgen"]],+ );+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/skills/media-use/references/memory.md b/skills/media-use/references/memory.mdindex 1d1db1c3b5..5c24b861a1 100644--- a/skills/media-use/references/memory.md+++ b/skills/media-use/references/memory.md@@ -26,7 +26,8 @@ The freeze is offered once after the final approval (`hyperframes/references/rev## Files-- `.media/manifest.jsonl`: machine SSOT, one JSON record per line+- `.media/manifest.jsonl`: machine SSOT, one JSON record per line. Records are keyed by path and only appended, so+ the last record for a path is the file's record; a file moved or replaced by hand keeps (or loses) its record by path.- `.media/index.md`: agent-readable table (id, type, dur, dims, path, description)- `.media/preferences.json`: the project's remembered defaults (committed)- `~/.media/`: global cross-project reuse cache (content-addressed, SHA-256)diff --git a/skills/media-use/scripts/lib/index-gen.mjs b/skills/media-use/scripts/lib/index-gen.mjsindex 65b019a92e..c25ee6d264 100644--- a/skills/media-use/scripts/lib/index-gen.mjs+++ b/skills/media-use/scripts/lib/index-gen.mjs@@ -1,6 +1,6 @@import { writeFileSync, mkdirSync } from "node:fs";import { dirname } from "node:path";-import { readManifest, indexPath } from "./manifest.mjs";+import { currentRecords, indexPath } from "./manifest.mjs";function pad(str, len) {return String(str ?? "").padEnd(len);@@ -54,7 +54,7 @@ export function generateIndexContent(records) {}export function regenerateIndex(projectDir) {- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);const content = generateIndexContent(records);const p = indexPath(projectDir);mkdirSync(dirname(p), { recursive: true });diff --git a/skills/media-use/scripts/lib/manifest.mjs b/skills/media-use/scripts/lib/manifest.mjsindex 0bd2f7b3da..56b533aaa3 100644--- a/skills/media-use/scripts/lib/manifest.mjs+++ b/skills/media-use/scripts/lib/manifest.mjs@@ -10,7 +10,7 @@ import {rmSync,statSync,} from "node:fs";-import { join } from "node:path";+import { basename, join } from "node:path";const MANIFEST_FILE = "manifest.jsonl";const INDEX_FILE = "index.md";@@ -79,6 +79,43 @@ export function appendRecord(projectDir, record) {appendFileSync(p, line);}+/** The record a path has now: the manifest only appends, so the last one for a path wins. */+export function latestRecordFor(projectDir, path) {+ return readManifest(projectDir).findLast((record) => record.path === path);+}++/** Records with every older record for the same path dropped, since the last one for a path is its record. */+export function currentRecords(projectDir) {+ const records = readManifest(projectDir);+ const last = new Map(records.map((record, index) => [record.path, index]));+ return records.filter((record, index) => !record.path || last.get(record.path) === index);+}++/** Records a file already in the project where it is, unless its current record already says the same thing. */+export function recordInPlace(+ projectDir,+ { type, path, source, description, duration, provenance },+) {+ const fields = {+ type,+ path,+ source,+ description: description || basename(path),+ ...(duration != null && { duration: Math.round(duration * 10) / 10 }),+ };+ mkdirSync(mediaDir(projectDir), { recursive: true });+ return withLock(mediaDir(projectDir), () => {+ const latest = latestRecordFor(projectDir, path);+ const same = ["source", "description", "duration"].every(+ (key) => latest?.[key] === fields[key],+ );+ if (latest && same) return latest;+ const record = { id: nextFreeId(projectDir, type), ...fields, provenance };+ appendRecord(projectDir, record);+ return record;+ });+}+// Match prompts forgivingly. Agents rarely re-emit a byte-identical intent, so// keying cache lookups on exact equality meant "Calm piano" and "calm piano"// re-searched and re-downloaded. Normalize (trim, lowercase, collapse internal@@ -93,7 +130,7 @@ export function normalizePrompt(prompt) {export function findByPrompt(projectDir, prompt, type) {const key = normalizePrompt(prompt);if (!key) return null;- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);return (records.find((r) => normalizePrompt(r.provenance?.prompt) === key && (type == null || r.type === type),@@ -103,7 +140,7 @@ export function findByPrompt(projectDir, prompt, type) {export function findByEntity(projectDir, entity) {const lower = entity.toLowerCase();- const records = readManifest(projectDir);+ const records = currentRecords(projectDir);return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null;}@@ -173,24 +210,30 @@ export function allocateId(projectDir, type, ext) {const typeDir = typeDirPath(projectDir, type);mkdirSync(typeDir, { recursive: true });return withLock(mediaDir(projectDir), () => {- const re = new RegExp(`^${type}_(\\d+)`);- let max = 0;- for (const r of readManifest(projectDir)) {- if (r.type !== type) continue;- const m = r.id?.match(re);- if (m) max = Math.max(max, parseInt(m[1], 10));- }- for (const f of readdirSync(typeDir)) {- const m = f.match(re);- if (m) max = Math.max(max, parseInt(m[1], 10)); // skip ids reserved but not yet appended- }- const id = `${type}_${String(max + 1).padStart(3, "0")}`;+ const id = nextFreeId(projectDir, type);const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;writeFileSync(join(projectDir, localPath), "", { flag: "wx" }); // durable reservationreturn { id, localPath };});}+// Call under the lock: counts recorded ids and ids reserved by a file not yet recorded.+function nextFreeId(projectDir, type) {+ const re = new RegExp(`^${type}_(\\d+)`);+ let max = 0;+ for (const r of readManifest(projectDir)) {+ if (r.type !== type) continue;+ const m = r.id?.match(re);+ if (m) max = Math.max(max, parseInt(m[1], 10));+ }+ const typeDir = typeDirPath(projectDir, type);+ for (const f of existsSync(typeDir) ? readdirSync(typeDir) : []) {+ const m = f.match(re);+ if (m) max = Math.max(max, parseInt(m[1], 10));+ }+ return `${type}_${String(max + 1).padStart(3, "0")}`;+}+function reservedFile(projectDir, type, ext) {const allocation = allocateId(projectDir, type, ext);return { ...allocation, fullPath: join(projectDir, allocation.localPath) };