Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor: make media execution one body of code, not one Provider's

kashorinc729b1af4169
diff --git a/packages/media-execution/package.json b/packages/media-execution/package.json
new file mode 100644
index 00000000..bdaad203
--- /dev/null
+++ b/packages/media-execution/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@narratage/media-execution",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "dependencies": {
+ "@narratage/media": "workspace:*",
+ "@narratage/media-pipeline": "workspace:*",
+ "@narratage/protocol": "workspace:*",
+ "@narratage/speech": "workspace:*"
+ }
+}
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
new file mode 100644
index 00000000..5eb64e2f
--- /dev/null
+++ b/packages/media-execution/src/execute.ts
@@ -0,0 +1,857 @@
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mediaTypes, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifyTimelineAudio } from "@narratage/media";
+import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, RenderedVisual, TimelineAudio } from "@narratage/media";
+import type { ProgramSpace } from "@narratage/program-space";
+import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary, speechTypes } from "@narratage/speech";
+import type { SpeechEvidenceAudio } from "@narratage/speech";
+import {
+ mediaPipelineCapabilities,
+ verifyAudioProgramPlan,
+ type InspectMediaNeed,
+ type MuxMediaNeed,
+ type NormalizeMediaNeed,
+ type ProjectSpeechEvidenceAudioNeed,
+ type RenderAudioNeed,
+} from "@narratage/media-pipeline";
+import type { AudioProgramClip, AudioProgramPlan } from "@narratage/media-pipeline";
+import {
+ canonicalize,
+ digestOf,
+} from "@narratage/protocol";
+import type { BlobRef, CanonicalValue } from "@narratage/protocol";
+
+import { parseMediaInspection } from "./probe.js";
+
+/**
+ * Where the bytes live and which binaries transform them.
+ *
+ * These five operations are the whole of Narratage's media execution, and they
+ * are written once. A local Provider supplies the Build's own ArtifactStore and
+ * the ffmpeg on its PATH; a Lambda Provider supplies an S3-backed gateway and
+ * the ffmpeg baked into its image. Nothing below knows which it is, so the two
+ * deployments cannot drift into computing different media from one Need.
+ */
+export type MediaArtifactGateway = {
+ get(source: BlobRef): Promise<Uint8Array | undefined>;
+ put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
+};
+
+export type MediaExecutionEnvironment = {
+ readonly artifacts: MediaArtifactGateway;
+ readonly ffmpegPath: string;
+ readonly ffprobePath: string;
+ readonly processTimeoutMs: number;
+ readonly maxProbeOutputBytes: number;
+ /** Recorded in each result's metadata, so an Operation says which Provider ran it. */
+ readonly label: string;
+};
+
+export type MediaOperationResult = {
+ readonly value: CanonicalValue;
+ readonly metadata: CanonicalValue;
+};
+
+/** The request contract each capability answers. Both Providers admit Needs by this table. */
+export const mediaOperationContracts = {
+ inspect: "svml.inspect-media-request@1",
+ normalize: "svml.normalize-media-request@1",
+ projectSpeechEvidenceAudio: "svml.project-speech-evidence-audio-request@1",
+ renderAudio: "svml.render-audio-request@1",
+ mux: "svml.mux-media-request@1",
+} as const;
+
+export function mediaNeedHasContract(value: unknown, contract: string): boolean {
+ return hasContract(value, contract);
+}
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function positiveInteger(value: number, subject: string): number {
+ assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive integer`);
+ return value;
+}
+
+function object(value: unknown, subject: string): Record<string, unknown> {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);
+ return value as Record<string, unknown>;
+}
+
+function hasContract(value: unknown, contract: string): boolean {
+ return value !== null && typeof value === "object" && !Array.isArray(value)
+ && (value as Record<string, unknown>).contract === contract;
+}
+
+async function runProcess(args: {
+ readonly executable: string;
+ readonly argv: readonly string[];
+ readonly timeoutMs: number;
+ readonly maxStdoutBytes: number;
+}): Promise<Uint8Array> {
+ return await new Promise((resolve, reject) => {
+ const child = spawn(args.executable, [...args.argv], {
+ shell: false,
+ stdio: ["ignore", "pipe", "pipe"],
+ env: { PATH: process.env.PATH ?? "" },
+ });
+ const stdout: Buffer[] = [];
+ let stdoutBytes = 0;
+ let stderr = "";
+ let settled = false;
+ const finish = (error?: Error): void => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ if (error !== undefined) reject(error);
+ else resolve(Buffer.concat(stdout));
+ };
+ const timer = setTimeout(() => {
+ child.kill("SIGKILL");
+ finish(new Error(`${args.executable} timed out`));
+ }, args.timeoutMs);
+ child.stdout.on("data", (chunk: Buffer) => {
+ stdoutBytes += chunk.byteLength;
+ if (stdoutBytes > args.maxStdoutBytes) {
+ child.kill("SIGKILL");
+ finish(new Error(`${args.executable} output exceeded the configured limit`));
+ return;
+ }
+ stdout.push(chunk);
+ });
+ child.stderr.on("data", (chunk: Buffer) => {
+ stderr = `${stderr}${chunk.toString()}`.slice(-8_000);
+ });
+ child.on("error", (error) => finish(error));
+ child.on("close", (code) => {
+ if (code !== 0) finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));
+ else finish();
+ });
+ });
+}
+
+async function version(executable: string, timeoutMs: number): Promise<string> {
+ const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024 });
+ const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();
+ assert(line !== undefined && line.length > 0, `${executable} returned no version`);
+ return line;
+}
+
+async function sourceBytes(env: MediaExecutionEnvironment, source: BlobRef): Promise<Uint8Array> {
+ const bytes = await env.artifacts.get(source);
+ assert(bytes !== undefined, `Media source ${source.digest} is unavailable`);
+ assert(bytes.byteLength === source.size, `Media source ${source.digest} size differs`);
+ return bytes;
+}
+
+async function stageArtifact(
+ env: MediaExecutionEnvironment,
+ source: BlobRef,
+ path: string,
+): Promise<void> {
+ await writeFile(path, await sourceBytes(env, source));
+}
+
+function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {
+ return {
+ numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),
+ denominator: BigInt(value.timeBase.denominator),
+ };
+}
+
+function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {
+ const a = timestampFraction(left);
+ const b = timestampFraction(right);
+ const x = a.numerator * b.denominator;
+ const y = b.numerator * a.denominator;
+ return x < y ? -1 : x > y ? 1 : 0;
+}
+
+function roundPositive(numerator: bigint, denominator: bigint): number {
+ assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
+ const value = (numerator * 2n + denominator) / (denominator * 2n);
+ assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
+ return Number(value);
+}
+
+function ceilPositive(numerator: bigint, denominator: bigint): number {
+ assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
+ const value = (numerator + denominator - 1n) / denominator;
+ assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
+ return Number(value);
+}
+
+function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {
+ assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");
+ const start = timestampFraction(value.startPts);
+ const end = timestampFraction(value.endPts);
+ const numerator = end.numerator * start.denominator - start.numerator * end.denominator;
+ const denominator = end.denominator * start.denominator;
+ assert(numerator > 0n, "Selected media stream has an empty presentation interval");
+ return { numerator, denominator };
+}
+
+function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {
+ const a = timestampFraction(origin);
+ const b = timestampFraction(value);
+ const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);
+ const denominator = b.denominator * a.denominator;
+ const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);
+ return numerator < 0n ? -magnitude : magnitude;
+}
+
+function stream<T extends MediaStream["kind"]>(
+ inspection: MediaInspection,
+ index: number | undefined,
+ kind: T,
+): Extract<MediaStream, { readonly kind: T }> | undefined {
+ if (index === undefined) return undefined;
+ const result = inspection.streams.find((item) => item.index === index);
+ assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);
+ assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);
+ return result as Extract<MediaStream, { readonly kind: T }>;
+}
+
+type NormalizationPlan = {
+ readonly video?: MediaVideoStream;
+ readonly audio?: MediaAudioStream;
+ readonly origin: MediaTimestamp;
+ readonly end: MediaTimestamp;
+ readonly frameCount: number;
+ readonly sampleFrames: number;
+ readonly audioTrimStartSamples: number;
+ readonly audioTrimEndSamples: number;
+ readonly audioHeadSamples: number;
+ readonly audioContentSamples: number;
+ readonly audioTailSamples: number;
+};
+
+function normalizationPlan(
+ inspection: MediaInspection,
+ selection: MediaStreamSelection,
+ frameRate: MediaRational,
+): NormalizationPlan {
+ assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0
+ && Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,
+ "Target frame rate is invalid");
+ const video = stream(inspection, selection.videoStreamIndex, "video");
+ const audio = stream(inspection, selection.audioStreamIndex, "audio");
+ if (video !== undefined && selection.spanAuthority !== "video") {
+ throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");
+ }
+ const authority = selection.spanAuthority === "video" ? video : audio;
+ assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");
+ const span = duration(authority);
+ const scaledNumerator = span.numerator * BigInt(frameRate.numerator);
+ const scaledDenominator = span.denominator * BigInt(frameRate.denominator);
+ const frameCount = selection.spanAuthority === "video"
+ ? roundPositive(scaledNumerator, scaledDenominator)
+ : ceilPositive(scaledNumerator, scaledDenominator);
+ assert(frameCount > 0, "Normalized media would contain no frame interval");
+ const sampleFrames = roundPositive(
+ BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),
+ BigInt(frameRate.numerator),
+ );
+ if (audio === undefined) {
+ return {
+ ...(video === undefined ? {} : { video }),
+ origin: authority.startPts,
+ end: authority.endPts,
+ frameCount,
+ sampleFrames,
+ audioTrimStartSamples: 0,
+ audioTrimEndSamples: 0,
+ audioHeadSamples: 0,
+ audioContentSamples: 0,
+ audioTailSamples: 0,
+ };
+ }
+ assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");
+ const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);
+ const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);
+ const contentStart = Math.max(0, audioStart);
+ const contentEnd = Math.min(sampleFrames, audioEnd);
+ assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");
+ return {
+ ...(video === undefined ? {} : { video }),
+ audio,
+ origin: authority.startPts,
+ end: authority.endPts,
+ frameCount,
+ sampleFrames,
+ audioTrimStartSamples: Math.max(0, -audioStart),
+ audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),
+ audioHeadSamples: contentStart,
+ audioContentSamples: contentEnd - contentStart,
+ audioTailSamples: sampleFrames - contentEnd,
+ };
+}
+
+async function inspectFile(args: {
+ readonly source: BlobRef;
+ readonly input: string;
+ readonly ffprobePath: string;
+ readonly timeoutMs: number;
+ readonly maxProbeOutputBytes: number;
+}): Promise<MediaInspection> {
+ const [raw, ffprobeVersion] = await Promise.all([
+ runProcess({
+ executable: args.ffprobePath,
+ argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],
+ timeoutMs: args.timeoutMs,
+ maxStdoutBytes: args.maxProbeOutputBytes,
+ }),
+ version(args.ffprobePath, args.timeoutMs),
+ ]);
+ let value: unknown;
+ try {
+ value = JSON.parse(Buffer.from(raw).toString("utf8"));
+ } catch {
+ throw new Error("ffprobe returned invalid JSON");
+ }
+ return parseMediaInspection({ source: args.source, ffprobeVersion, value });
+}
+
+async function outputInspection(args: {
+ readonly path: string;
+ readonly mediaType: string;
+ readonly ffprobePath: string;
+ readonly timeoutMs: number;
+ readonly maxProbeOutputBytes: number;
+}): Promise<MediaInspection> {
+ const bytes = await readFile(args.path);
+ const source: BlobRef = {
+ kind: "blob",
+ digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` as BlobRef["digest"],
+ size: bytes.byteLength,
+ mediaType: args.mediaType,
+ };
+ return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,
+ timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes });
+}
+
+function result(value: CanonicalValue, metadata: CanonicalValue): MediaOperationResult {
+ return { value, metadata };
+}
+
+function inspectNeed(value: CanonicalValue): InspectMediaNeed {
+ const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;
+ assert(item.contract === "svml.inspect-media-request@1" && item.source?.kind === "blob",
+ "InspectMediaNeed is invalid");
+ return item;
+}
+
+function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {
+ const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;
+ assert(item.contract === "svml.normalize-media-request@1" && item.source?.kind === "blob",
+ "NormalizeMediaNeed is invalid");
+ verifyMediaInspection(item.inspection);
+ verifyMediaStreamSelection(item.selection);
+ assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2
+ && item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",
+ "NormalizeMediaNeed audio profile is unsupported");
+ return item;
+}
+
+function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {
+ const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;
+ assert(item.contract === "svml.project-speech-evidence-audio-request@1" && item.source?.kind === "blob",
+ "ProjectSpeechEvidenceAudioNeed is invalid");
+ assert(item.sourceSampleRate === 48_000 && item.sourceChannels === 2 && item.sourceCodec === "pcm_s16le"
+ && item.evidenceSampleRate === 16_000 && item.evidenceChannels === 1 && item.evidenceCodec === "pcm_s16le",
+ "Speech evidence audio shape is invalid");
+ assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0
+ && Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0
+ && item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),
+ "Speech evidence audio sample map is invalid");
+ return item;
+}
+
+function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {
+ const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;
+ assert(item.contract === "svml.render-audio-request@1", "RenderAudioNeed is invalid");
+ verifyAudioProgramPlan(item.plan);
+ return item;
+}
+
+function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {
+ const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;
+ assert(item.contract === "svml.mux-media-request@1", "MuxMediaNeed is invalid");
+ verifyRenderedVisual(item.visual);
+ verifyTimelineAudio(item.audio);
+ const expectedSamples = roundPositive(
+ BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),
+ BigInt(item.visual.frameRate.numerator),
+ );
+ assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");
+ return item;
+}
+
+function atempo(rate: number): string[] {
+ if (Math.abs(rate - 1) < 1e-12) return [];
+ const filters: string[] = [];
+ let remaining = rate;
+ while (remaining > 2) {
+ filters.push("atempo=2");
+ remaining /= 2;
+ }
+ while (remaining < 0.5) {
+ filters.push("atempo=0.5");
+ remaining /= 0.5;
+ }
+ if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);
+ return filters;
+}
+
+function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number): string {
+ const length = clip.targetEndSampleExclusive - clip.targetStartSample;
+ const filters = [
+ `atrim=start_sample=${clip.sourceStartSample}`,
+ "asetpts=PTS-STARTPTS",
+ ...atempo(clip.playbackRate),
+ `atrim=start_sample=0:end_sample=${length}`,
+ `apad=whole_len=${length}`,
+ `atrim=start_sample=0:end_sample=${length}`,
+ `volume=${clip.gain.toPrecision(15)}`,
+ ...(clip.fadeInSamples === 0 ? [] : [`afade=t=in:start_sample=0:nb_samples=${clip.fadeInSamples}`]),
+ ...(clip.fadeOutSamples === 0 ? [] : [
+ `afade=t=out:start_sample=${length - clip.fadeOutSamples}:nb_samples=${clip.fadeOutSamples}`,
+ ]),
+ `adelay=${clip.targetStartSample}S:all=1`,
+ ];
+ return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;
+}
+
+async function assertCanonicalWav(args: {
+ readonly path: string;
+ readonly source: BlobRef;
+ readonly ffprobePath: string;
+ readonly timeoutMs: number;
+ readonly maxProbeOutputBytes: number;
+}): Promise<MediaAudioStream> {
+ const inspection = await inspectFile({
+ source: args.source,
+ input: args.path,
+ ffprobePath: args.ffprobePath,
+ timeoutMs: args.timeoutMs,
+ maxProbeOutputBytes: args.maxProbeOutputBytes,
+ });
+ const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
+ assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");
+ assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,
+ "Canonical audio must be 48 kHz stereo PCM s16");
+ assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");
+ return audio[0]!;
+}
+
+export async function executeInspectMedia(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const need = inspectNeed(constraints);
+ const work = await mkdtemp(join(tmpdir(), "svml-media-inspect-"));
+ try {
+ const input = join(work, "source.bin");
+ await writeFile(input, await sourceBytes(env, need.source));
+ const inspection = await inspectFile({
+ source: need.source,
+ input,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ return result(canonicalize(inspection), canonicalize({ provider: env.label, operation: "inspect" }));
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+export async function executeNormalizeMedia(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const need = normalizeNeed(constraints);
+ const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);
+ const work = await mkdtemp(join(tmpdir(), "svml-media-normalize-"));
+ try {
+ const input = join(work, "source.bin");
+ await writeFile(input, await sourceBytes(env, need.source));
+ let visualArtifact: BlobRef | undefined;
+ let visualWidth: number | undefined;
+ let visualHeight: number | undefined;
+ if (plan.video !== undefined) {
+ const output = join(work, "visual.mp4");
+ const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;
+ const filter = [
+ "setpts=PTS-STARTPTS",
+ `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
+ `trim=start_frame=0:end_frame=${plan.frameCount}`,
+ `setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,
+ "scale=trunc(iw/2)*2:trunc(ih/2)*2",
+ ].join(",");
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: ["-y", "-i", input, "-map", `0:${plan.video.index}`, "-an", "-vf", filter,
+ "-frames:v", String(plan.frameCount), "-fps_mode", "cfr", "-c:v", "libx264",
+ "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", output],
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ });
+ const inspected = await outputInspection({
+ path: output,
+ mediaType: "video/mp4",
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
+ assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
+ "Normalized visual frame shape differs from its plan");
+ const bytes = await readFile(output);
+ visualArtifact = await env.artifacts.put(bytes, "video/mp4");
+ visualWidth = visual.width;
+ visualHeight = visual.height;
+ }
+ let audioArtifact: BlobRef | undefined;
+ if (plan.audio !== undefined) {
+ const output = join(work, "audio.wav");
+ const filter = [
+ "asetpts=PTS-STARTPTS",
+ "aresample=48000:async=0:first_pts=0",
+ "aformat=sample_rates=48000:channel_layouts=stereo",
+ `atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,
+ "asetpts=N/SR/TB",
+ `adelay=${plan.audioHeadSamples}S:all=1`,
+ `apad=whole_len=${plan.sampleFrames}`,
+ `atrim=start_sample=0:end_sample=${plan.sampleFrames}`,
+ "asetpts=N/SR/TB",
+ ].join(",");
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,
+ "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ });
+ const inspected = await outputInspection({
+ path: output,
+ mediaType: "audio/wav",
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");
+ assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,
+ "Normalized audio sample shape differs from its plan");
+ audioArtifact = await env.artifacts.put(await readFile(output), "audio/wav");
+ }
+ const media = sealSynchronizedMedia({
+ contract: "svml.synchronized-media@1",
+ timeline: {
+ spanAuthority: need.selection.spanAuthority,
+ frameRate: need.frameRate,
+ frameCount: plan.frameCount,
+ sampleRate: 48_000,
+ sampleFrames: plan.sampleFrames,
+ },
+ sourceMap: {
+ sourceOriginPts: plan.origin,
+ sourceEndPts: plan.end,
+ audioTrimStartSamples: plan.audioTrimStartSamples,
+ audioTrimEndSamples: plan.audioTrimEndSamples,
+ audioHeadSamples: plan.audioHeadSamples,
+ audioContentSamples: plan.audioContentSamples,
+ audioTailSamples: plan.audioTailSamples,
+ },
+ ...(visualArtifact === undefined ? {} : {
+ visual: {
+ artifact: visualArtifact,
+ sourceStreamIndex: plan.video!.index,
+ width: visualWidth!,
+ height: visualHeight!,
+ frameRate: need.frameRate,
+ frameCount: plan.frameCount,
+ muted: true as const,
+ },
+ }),
+ ...(audioArtifact === undefined ? {} : {
+ audio: {
+ artifact: audioArtifact,
+ sourceStreamIndex: plan.audio!.index,
+ codec: "pcm_s16le" as const,
+ sampleRate: 48_000 as const,
+ channels: 2 as const,
+ sampleFrames: plan.sampleFrames,
+ loudness: "preserved" as const,
+ },
+ }),
+ });
+ return result(canonicalize(media), canonicalize({ provider: env.label, operation: "normalize",
+ visual: visualArtifact !== undefined, audio: audioArtifact !== undefined }));
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+export async function executeProjectSpeechEvidenceAudio(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const need = evidenceAudioNeed(constraints);
+ const work = await mkdtemp(join(tmpdir(), "svml-media-speech-evidence-"));
+ try {
+ const input = join(work, "speech-master.wav");
+ const output = join(work, "alignment-evidence.wav");
+ await stageArtifact(env, need.source, input);
+ const source = await assertCanonicalWav({
+ path: input,
+ source: need.source,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ assert(source.decodedSampleFrames === need.sourceSampleFrames,
+ "Speech master sample count differs from its ProgramSpace");
+ const filter = [
+ "asetpts=N/SR/TB",
+ "aresample=16000:async=0:first_pts=0",
+ "aformat=sample_rates=16000:channel_layouts=mono",
+ `apad=whole_len=${need.evidenceSampleFrames}`,
+ `atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,
+ "asetpts=N/SR/TB",
+ ].join(",");
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: ["-y", "-i", input, "-vn", "-af", filter,
+ "-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ });
+ const inspected = await outputInspection({
+ path: output,
+ mediaType: "audio/wav",
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
+ assert(streams.length === 1 && inspected.streams.length === 1
+ && streams[0]!.codecName === "pcm_s16le"
+ && streams[0]!.sampleRate === 16_000
+ && streams[0]!.channels === 1
+ && streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,
+ "Alignment evidence must be exact 16 kHz mono PCM s16");
+ const artifact = await env.artifacts.put(await readFile(output), "audio/wav");
+ const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({
+ contract: "svml.speech-evidence-audio@1",
+ artifact,
+ codec: "pcm_s16le",
+ sampleRate: 16_000,
+ channels: 1,
+ sampleFrames: need.evidenceSampleFrames,
+ durationSec: need.durationSec,
+ segments: need.segments,
+ sampleMap: {
+ algorithm: "rational-boundary-round@1",
+ sourceSampleRate: 48_000,
+ evidenceSampleRate: 16_000,
+ sourceSampleFrames: need.sourceSampleFrames,
+ evidenceSampleFrames: need.evidenceSampleFrames,
+ sourceOriginSample: 0,
+ evidenceOriginSample: 0,
+ },
+ });
+ assertSpeechEvidenceAudioIdentity(evidence);
+ return result(canonicalize(evidence), canonicalize({
+ provider: env.label,
+ operation: "project-speech-evidence-audio",
+ sourceSampleFrames: need.sourceSampleFrames,
+ evidenceSampleFrames: need.evidenceSampleFrames,
+ }));
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+export async function executeRenderTimelineAudio(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const need = renderAudioNeed(constraints);
+ const plan: AudioProgramPlan = need.plan;
+ const work = await mkdtemp(join(tmpdir(), "svml-media-audio-"));
+ try {
+ const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number }>();
+ for (const clip of plan.clips) {
+ if (artifacts.has(clip.artifact.digest)) continue;
+ const inputIndex = artifacts.size;
+ const path = join(work, `input-${inputIndex}.wav`);
+ await stageArtifact(env, clip.artifact, path);
+ const audio = await assertCanonicalWav({
+ path,
+ source: clip.artifact,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ assert(audio.decodedSampleFrames > 0, `Audio input ${clip.artifact.digest} is empty`);
+ artifacts.set(clip.artifact.digest, { source: clip.artifact, path, inputIndex });
+ }
+
+ const output = join(work, "program.wav");
+ const argv = ["-y"];
+ for (const item of artifacts.values()) argv.push("-i", item.path);
+ if (plan.clips.length === 0) {
+ argv.push(
+ "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",
+ "-af", `atrim=start_sample=0:end_sample=${plan.sampleFrames},asetpts=N/SR/TB`,
+ );
+ } else {
+ const chains = plan.clips.map((clip, index) => {
+ const inputIndex = artifacts.get(clip.artifact.digest)!.inputIndex;
+ return audioClipFilter(clip, inputIndex, index);
+ });
+ const labels = plan.clips.map((_clip, index) => `[clip${index}]`).join("");
+ chains.push(
+ `${labels}amix=inputs=${plan.clips.length}:duration=longest:dropout_transition=0:normalize=0,`
+ + `apad=whole_len=${plan.sampleFrames},atrim=start_sample=0:end_sample=${plan.sampleFrames},`
+ + "asetpts=N/SR/TB[out]",
+ );
+ argv.push("-filter_complex", chains.join(";"), "-map", "[out]");
+ }
+ argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv,
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ });
+ const bytes = await readFile(output);
+ const artifact = await env.artifacts.put(bytes, "audio/wav");
+ const audio = await assertCanonicalWav({
+ path: output,
+ source: artifact,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ assert(audio.decodedSampleFrames === plan.sampleFrames,
+ "Rendered TimelineAudio sample count differs from its plan");
+ const value: TimelineAudio = sealTimelineAudio({
+ contract: "svml.timeline-audio@1",
+ artifact,
+ codec: "pcm_s16le",
+ sampleRate: 48_000,
+ channels: 2,
+ sampleFrames: plan.sampleFrames,
+ loudness: "planned",
+ });
+ return result(canonicalize(value), canonicalize({
+ provider: env.label,
+ operation: "render-audio",
+ clips: plan.clips.length,
+ }));
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+export async function executeMuxProgramMedia(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const need = muxMediaNeed(constraints);
+ const work = await mkdtemp(join(tmpdir(), "svml-media-mux-"));
+ try {
+ const visualPath = join(work, "visual.mp4");
+ const audioPath = join(work, "audio.wav");
+ const output = join(work, "final.mp4");
+ await Promise.all([
+ stageArtifact(env, need.visual.artifact, visualPath),
+ stageArtifact(env, need.audio.artifact, audioPath),
+ ]);
+ const [visualInspection, audio] = await Promise.all([
+ inspectFile({
+ source: need.visual.artifact,
+ input: visualPath,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ }),
+ assertCanonicalWav({
+ path: audioPath,
+ source: need.audio.artifact,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ }),
+ ]);
+ const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
+ assert(visualStreams.length === 1 && visualInspection.streams.length === 1,
+ "RenderedVisual Artifact must contain exactly one silent video stream");
+ const visual = visualStreams[0]!;
+ assert(visual.role === "moving" && visual.timingStatus === "admissible"
+ && visual.decodedUnitCount === need.visual.frameCount
+ && visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,
+ "RenderedVisual bytes differ from their declared frame domain");
+ assert(audio.decodedSampleFrames === need.audio.sampleFrames,
+ "TimelineAudio bytes differ from their declared sample domain");
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: [
+ "-y", "-i", visualPath, "-i", audioPath,
+ "-map", `0:${visual.index}`, "-map", "1:0",
+ "-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",
+ "-frames:v", String(need.visual.frameCount),
+ "-movflags", "+faststart", output,
+ ],
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ });
+ const finalInspection = await outputInspection({
+ path: output,
+ mediaType: "video/mp4",
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ });
+ const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
+ const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
+ assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,
+ "Final mux must contain exactly one video and one audio stream");
+ assert(finalVideo[0]!.timingStatus === "admissible"
+ && finalVideo[0]!.decodedUnitCount === need.visual.frameCount,
+ "Final mux video frame count differs from RenderedVisual");
+ assert(finalAudio[0]!.timingStatus === "admissible"
+ && finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,
+ "Final mux audio shape differs from TimelineAudio");
+ assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined
+ && compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,
+ "Final mux audio and video do not share one presentation origin");
+ const finalAudioSpan = duration(finalAudio[0]!);
+ assert(roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator)
+ === need.audio.sampleFrames,
+ "Final mux audio presentation span differs from TimelineAudio");
+ const artifact = await env.artifacts.put(await readFile(output), "video/mp4");
+ const value: MuxedMedia = sealMuxedMedia({
+ contract: "svml.muxed-media@1",
+ frameRate: need.visual.frameRate,
+ frameCount: need.visual.frameCount,
+ canvas: need.visual.canvas,
+ presentationSampleFrames: need.audio.sampleFrames,
+ artifact,
+ });
+ return result(canonicalize(value), canonicalize({
+ provider: env.label,
+ operation: "mux",
+ videoStream: finalVideo[0]!.index,
+ audioStream: finalAudio[0]!.index,
+ }));
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.ts
new file mode 100644
index 00000000..64f7fea3
--- /dev/null
+++ b/packages/media-execution/src/index.ts
@@ -0,0 +1,11 @@
+export {
+ executeInspectMedia,
+ executeMuxProgramMedia,
+ executeNormalizeMedia,
+ executeProjectSpeechEvidenceAudio,
+ executeRenderTimelineAudio,
+ mediaNeedHasContract,
+ mediaOperationContracts,
+} from "./execute.js";
+export { parseMediaInspection } from "./probe.js";
+export type * from "./execute.js";
diff --git a/packages/provider-media-local/src/probe.ts b/packages/media-execution/src/probe.ts
similarity index 100%
rename from packages/provider-media-local/src/probe.ts
rename to packages/media-execution/src/probe.ts
diff --git a/packages/provider-media-local/test/probe.test.ts b/packages/media-execution/test/probe.test.ts
similarity index 96%
rename from packages/provider-media-local/test/probe.test.ts
rename to packages/media-execution/test/probe.test.ts
index 11415766..afa377b6 100644
--- a/packages/provider-media-local/test/probe.test.ts
+++ b/packages/media-execution/test/probe.test.ts
@@ -3,7 +3,7 @@ import test from "node:test";
import { digestOf } from "@narratage/protocol";
-import { parseMediaInspection } from "../src/index.js";
+import { parseMediaInspection } from "@narratage/media-execution";
test("probe derives the last decoded-unit duration instead of adding whole-stream duration_ts", () => {
const inspection = parseMediaInspection({
diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.json
index 4122c18c..30016096 100644
--- a/packages/provider-media-local/package.json
+++ b/packages/provider-media-local/package.json
@@ -12,6 +12,7 @@
"dependencies": {
"@narratage/endpoint-kit": "workspace:*",
"@narratage/media": "workspace:*",
+ "@narratage/media-execution": "workspace:*",
"@narratage/media-pipeline": "workspace:*",
"@narratage/program-space": "workspace:*",
"@narratage/protocol": "workspace:*",
diff --git a/packages/provider-media-local/src/index.ts b/packages/provider-media-local/src/index.ts
index faf5c3c0..66994720 100644
--- a/packages/provider-media-local/src/index.ts
+++ b/packages/provider-media-local/src/index.ts
@@ -3,5 +3,4 @@ export {
localMediaProviderImplementationDigest,
localMediaProviderModuleRef,
} from "./provider.js";
-export { parseMediaInspection } from "./probe.js";
export type { CreateLocalMediaProviderOptions } from "./provider.js";
diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.ts
index 261e7bd8..5d12f0a4 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/provider-media-local/src/provider.ts
@@ -1,32 +1,19 @@
-import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { mediaTypes, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifyTimelineAudio } from "@narratage/media";
-import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, RenderedVisual, TimelineAudio } from "@narratage/media";
-import type { ProgramSpace } from "@narratage/program-space";
-import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary, speechTypes } from "@narratage/speech";
-import type { SpeechEvidenceAudio } from "@narratage/speech";
-import type { EndpointInvocationContext, EndpointFulfillment } from "@narratage/endpoint-kit";
+import { mediaTypes } from "@narratage/media";
import {
- mediaPipelineCapabilities,
- verifyAudioProgramPlan,
- type InspectMediaNeed,
- type MuxMediaNeed,
- type NormalizeMediaNeed,
- type ProjectSpeechEvidenceAudioNeed,
- type RenderAudioNeed,
-} from "@narratage/media-pipeline";
-import type { AudioProgramClip, AudioProgramPlan } from "@narratage/media-pipeline";
-import {
- canonicalize,
- digestOf,
-} from "@narratage/protocol";
-import type { BlobRef, CanonicalValue } from "@narratage/protocol";
+ executeInspectMedia,
+ executeMuxProgramMedia,
+ executeNormalizeMedia,
+ executeProjectSpeechEvidenceAudio,
+ executeRenderTimelineAudio,
+ mediaNeedHasContract,
+ mediaOperationContracts,
+} from "@narratage/media-execution";
+import type { MediaExecutionEnvironment, MediaOperationResult } from "@narratage/media-execution";
+import { mediaPipelineCapabilities } from "@narratage/media-pipeline";
+import { canonicalize, digestOf } from "@narratage/protocol";
+import { speechTypes } from "@narratage/speech";
import { defineEndpointPackage } from "@narratage/endpoint-kit";
-
-import { parseMediaInspection } from "./probe.js";
+import type { EndpointFulfillment, EndpointInvocationContext } from "@narratage/endpoint-kit";
export const localMediaProviderModuleRef = { name: "@narratage/provider-media-local", version: "0.0.0-dev" } as const;
export const localMediaProviderImplementationDigest = digestOf("@narratage/provider-media-local/ffmpeg@2");
@@ -50,382 +37,20 @@ function positiveInteger(value: number, subject: string): number {
return value;
}
-function object(value: unknown, subject: string): Record<string, unknown> {
- assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);
- return value as Record<string, unknown>;
-}
-
-function hasContract(value: unknown, contract: string): boolean {
- return value !== null && typeof value === "object" && !Array.isArray(value)
- && (value as Record<string, unknown>).contract === contract;
-}
-
-async function runProcess(args: {
- readonly executable: string;
- readonly argv: readonly string[];
- readonly timeoutMs: number;
- readonly maxStdoutBytes: number;
-}): Promise<Uint8Array> {
- return await new Promise((resolve, reject) => {
- const child = spawn(args.executable, [...args.argv], {
- shell: false,
- stdio: ["ignore", "pipe", "pipe"],
- env: { PATH: process.env.PATH ?? "" },
- });
- const stdout: Buffer[] = [];
- let stdoutBytes = 0;
- let stderr = "";
- let settled = false;
- const finish = (error?: Error): void => {
- if (settled) return;
- settled = true;
- clearTimeout(timer);
- if (error !== undefined) reject(error);
- else resolve(Buffer.concat(stdout));
- };
- const timer = setTimeout(() => {
- child.kill("SIGKILL");
- finish(new Error(`${args.executable} timed out`));
- }, args.timeoutMs);
- child.stdout.on("data", (chunk: Buffer) => {
- stdoutBytes += chunk.byteLength;
- if (stdoutBytes > args.maxStdoutBytes) {
- child.kill("SIGKILL");
- finish(new Error(`${args.executable} output exceeded the configured limit`));
- return;
- }
- stdout.push(chunk);
- });
- child.stderr.on("data", (chunk: Buffer) => {
- stderr = `${stderr}${chunk.toString()}`.slice(-8_000);
- });
- child.on("error", (error) => finish(error));
- child.on("close", (code) => {
- if (code !== 0) finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));
- else finish();
- });
- });
-}
-
-async function version(executable: string, timeoutMs: number): Promise<string> {
- const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024 });
- const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();
- assert(line !== undefined && line.length > 0, `${executable} returned no version`);
- return line;
-}
-
-async function sourceBytes(context: EndpointInvocationContext, source: BlobRef): Promise<Uint8Array> {
- const bytes = await context.artifacts.get(source.digest);
- assert(bytes !== undefined, `Media source ${source.digest} is unavailable`);
- assert(bytes.byteLength === source.size, `Media source ${source.digest} size differs`);
- return bytes;
-}
-
-async function stageArtifact(
- context: EndpointInvocationContext,
- source: BlobRef,
- path: string,
-): Promise<void> {
- await writeFile(path, await sourceBytes(context, source));
-}
-
-function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {
+function fulfillment(result: MediaOperationResult): EndpointFulfillment {
return {
- numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),
- denominator: BigInt(value.timeBase.denominator),
- };
-}
-
-function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {
- const a = timestampFraction(left);
- const b = timestampFraction(right);
- const x = a.numerator * b.denominator;
- const y = b.numerator * a.denominator;
- return x < y ? -1 : x > y ? 1 : 0;
-}
-
-function roundPositive(numerator: bigint, denominator: bigint): number {
- assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
- const value = (numerator * 2n + denominator) / (denominator * 2n);
- assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
- return Number(value);
-}
-
-function ceilPositive(numerator: bigint, denominator: bigint): number {
- assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
- const value = (numerator + denominator - 1n) / denominator;
- assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
- return Number(value);
-}
-
-function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {
- assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");
- const start = timestampFraction(value.startPts);
- const end = timestampFraction(value.endPts);
- const numerator = end.numerator * start.denominator - start.numerator * end.denominator;
- const denominator = end.denominator * start.denominator;
- assert(numerator > 0n, "Selected media stream has an empty presentation interval");
- return { numerator, denominator };
-}
-
-function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {
- const a = timestampFraction(origin);
- const b = timestampFraction(value);
- const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);
- const denominator = b.denominator * a.denominator;
- const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);
- return numerator < 0n ? -magnitude : magnitude;
-}
-
-function stream<T extends MediaStream["kind"]>(
- inspection: MediaInspection,
- index: number | undefined,
- kind: T,
-): Extract<MediaStream, { readonly kind: T }> | undefined {
- if (index === undefined) return undefined;
- const result = inspection.streams.find((item) => item.index === index);
- assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);
- assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);
- return result as Extract<MediaStream, { readonly kind: T }>;
-}
-
-type NormalizationPlan = {
- readonly video?: MediaVideoStream;
- readonly audio?: MediaAudioStream;
- readonly origin: MediaTimestamp;
- readonly end: MediaTimestamp;
- readonly frameCount: number;
- readonly sampleFrames: number;
- readonly audioTrimStartSamples: number;
- readonly audioTrimEndSamples: number;
- readonly audioHeadSamples: number;
- readonly audioContentSamples: number;
- readonly audioTailSamples: number;
-};
-
-function normalizationPlan(
- inspection: MediaInspection,
- selection: MediaStreamSelection,
- frameRate: MediaRational,
-): NormalizationPlan {
- assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0
- && Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,
- "Target frame rate is invalid");
- const video = stream(inspection, selection.videoStreamIndex, "video");
- const audio = stream(inspection, selection.audioStreamIndex, "audio");
- if (video !== undefined && selection.spanAuthority !== "video") {
- throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");
- }
- const authority = selection.spanAuthority === "video" ? video : audio;
- assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");
- const span = duration(authority);
- const scaledNumerator = span.numerator * BigInt(frameRate.numerator);
- const scaledDenominator = span.denominator * BigInt(frameRate.denominator);
- const frameCount = selection.spanAuthority === "video"
- ? roundPositive(scaledNumerator, scaledDenominator)
- : ceilPositive(scaledNumerator, scaledDenominator);
- assert(frameCount > 0, "Normalized media would contain no frame interval");
- const sampleFrames = roundPositive(
- BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),
- BigInt(frameRate.numerator),
- );
- if (audio === undefined) {
- return {
- ...(video === undefined ? {} : { video }),
- origin: authority.startPts,
- end: authority.endPts,
- frameCount,
- sampleFrames,
- audioTrimStartSamples: 0,
- audioTrimEndSamples: 0,
- audioHeadSamples: 0,
- audioContentSamples: 0,
- audioTailSamples: 0,
- };
- }
- assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");
- const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);
- const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);
- const contentStart = Math.max(0, audioStart);
- const contentEnd = Math.min(sampleFrames, audioEnd);
- assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");
- return {
- ...(video === undefined ? {} : { video }),
- audio,
- origin: authority.startPts,
- end: authority.endPts,
- frameCount,
- sampleFrames,
- audioTrimStartSamples: Math.max(0, -audioStart),
- audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),
- audioHeadSamples: contentStart,
- audioContentSamples: contentEnd - contentStart,
- audioTailSamples: sampleFrames - contentEnd,
- };
-}
-
-async function inspectFile(args: {
- readonly source: BlobRef;
- readonly input: string;
- readonly ffprobePath: string;
- readonly timeoutMs: number;
- readonly maxProbeOutputBytes: number;
-}): Promise<MediaInspection> {
- const [raw, ffprobeVersion] = await Promise.all([
- runProcess({
- executable: args.ffprobePath,
- argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],
- timeoutMs: args.timeoutMs,
- maxStdoutBytes: args.maxProbeOutputBytes,
- }),
- version(args.ffprobePath, args.timeoutMs),
- ]);
- let value: unknown;
- try {
- value = JSON.parse(Buffer.from(raw).toString("utf8"));
- } catch {
- throw new Error("ffprobe returned invalid JSON");
- }
- return parseMediaInspection({ source: args.source, ffprobeVersion, value });
-}
-
-async function outputInspection(args: {
- readonly path: string;
- readonly mediaType: string;
- readonly ffprobePath: string;
- readonly timeoutMs: number;
- readonly maxProbeOutputBytes: number;
-}): Promise<MediaInspection> {
- const bytes = await readFile(args.path);
- const source: BlobRef = {
- kind: "blob",
- digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` as BlobRef["digest"],
- size: bytes.byteLength,
- mediaType: args.mediaType,
- };
- return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,
- timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes });
-}
-
-function result(value: CanonicalValue, metadata: CanonicalValue): EndpointFulfillment {
- return {
- value: { kind: "inline", value },
+ value: { kind: "inline", value: result.value },
conformance: "exact",
delivery: "executed",
- metadata,
+ metadata: result.metadata,
};
}
-function inspectNeed(value: CanonicalValue): InspectMediaNeed {
- const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;
- assert(item.contract === "svml.inspect-media-request@1" && item.source?.kind === "blob",
- "InspectMediaNeed is invalid");
- return item;
-}
-
-function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {
- const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;
- assert(item.contract === "svml.normalize-media-request@1" && item.source?.kind === "blob",
- "NormalizeMediaNeed is invalid");
- verifyMediaInspection(item.inspection);
- verifyMediaStreamSelection(item.selection);
- assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2
- && item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",
- "NormalizeMediaNeed audio profile is unsupported");
- return item;
-}
-
-function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {
- const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;
- assert(item.contract === "svml.project-speech-evidence-audio-request@1" && item.source?.kind === "blob",
- "ProjectSpeechEvidenceAudioNeed is invalid");
- assert(item.sourceSampleRate === 48_000 && item.sourceChannels === 2 && item.sourceCodec === "pcm_s16le"
- && item.evidenceSampleRate === 16_000 && item.evidenceChannels === 1 && item.evidenceCodec === "pcm_s16le",
- "Speech evidence audio shape is invalid");
- assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0
- && Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0
- && item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),
- "Speech evidence audio sample map is invalid");
- return item;
-}
-
-function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {
- const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;
- assert(item.contract === "svml.render-audio-request@1", "RenderAudioNeed is invalid");
- verifyAudioProgramPlan(item.plan);
- return item;
-}
-
-function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {
- const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;
- assert(item.contract === "svml.mux-media-request@1", "MuxMediaNeed is invalid");
- verifyRenderedVisual(item.visual);
- verifyTimelineAudio(item.audio);
- const expectedSamples = roundPositive(
- BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),
- BigInt(item.visual.frameRate.numerator),
- );
- assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");
- return item;
-}
-
-function atempo(rate: number): string[] {
- if (Math.abs(rate - 1) < 1e-12) return [];
- const filters: string[] = [];
- let remaining = rate;
- while (remaining > 2) {
- filters.push("atempo=2");
- remaining /= 2;
- }
- while (remaining < 0.5) {
- filters.push("atempo=0.5");
- remaining /= 0.5;
- }
- if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);
- return filters;
-}
-
-function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number): string {
- const length = clip.targetEndSampleExclusive - clip.targetStartSample;
- const filters = [
- `atrim=start_sample=${clip.sourceStartSample}`,
- "asetpts=PTS-STARTPTS",
- ...atempo(clip.playbackRate),
- `atrim=start_sample=0:end_sample=${length}`,
- `apad=whole_len=${length}`,
- `atrim=start_sample=0:end_sample=${length}`,
- `volume=${clip.gain.toPrecision(15)}`,
- ...(clip.fadeInSamples === 0 ? [] : [`afade=t=in:start_sample=0:nb_samples=${clip.fadeInSamples}`]),
- ...(clip.fadeOutSamples === 0 ? [] : [
- `afade=t=out:start_sample=${length - clip.fadeOutSamples}:nb_samples=${clip.fadeOutSamples}`,
- ]),
- `adelay=${clip.targetStartSample}S:all=1`,
- ];
- return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;
-}
-
-async function assertCanonicalWav(args: {
- readonly path: string;
- readonly source: BlobRef;
- readonly ffprobePath: string;
- readonly timeoutMs: number;
- readonly maxProbeOutputBytes: number;
-}): Promise<MediaAudioStream> {
- const inspection = await inspectFile({
- source: args.source,
- input: args.path,
- ffprobePath: args.ffprobePath,
- timeoutMs: args.timeoutMs,
- maxProbeOutputBytes: args.maxProbeOutputBytes,
- });
- const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
- assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");
- assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,
- "Canonical audio must be 48 kHz stereo PCM s16");
- assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");
- return audio[0]!;
-}
-
+/**
+ * The Build's own ArtifactStore, and whatever ffmpeg this machine has. The
+ * operations themselves live in `@narratage/media-execution`, shared with the
+ * AWS Provider so one Need cannot mean two different transforms.
+ */
export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions = {}) {
const ffmpegPath = config.ffmpegPath ?? "ffmpeg";
const ffprobePath = config.ffprobePath ?? "ffprobe";
@@ -433,6 +58,19 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
const maxProbeOutputBytes = positiveInteger(config.maxProbeOutputBytes ?? 256 * 1024 * 1024,
"maxProbeOutputBytes");
const common = { ffmpegPath, ffprobePath, processTimeoutMs, maxProbeOutputBytes };
+ const environment = (context: EndpointInvocationContext): MediaExecutionEnvironment => ({
+ ...common,
+ label: "media.local",
+ artifacts: {
+ get: async (source) => await context.artifacts.get(source.digest),
+ put: async (bytes, mediaType) => await context.artifacts.put(bytes, mediaType),
+ },
+ });
+ const operation = (
+ execute: (env: MediaExecutionEnvironment, constraints: never) => Promise<MediaOperationResult>,
+ ) => async (context: EndpointInvocationContext): Promise<EndpointFulfillment> =>
+ fulfillment(await execute(environment(context), context.need.constraints as never));
+
return defineEndpointPackage({
module: localMediaProviderModuleRef,
facet: "media",
@@ -450,419 +88,37 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.inspect,
returns: mediaTypes.inspection,
- supports: (need) => hasContract(need.constraints, "svml.inspect-media-request@1"),
- handler: async (context) => {
- const need = inspectNeed(context.need.constraints);
- const work = await mkdtemp(join(tmpdir(), "svml-media-inspect-"));
- try {
- const input = join(work, "source.bin");
- await writeFile(input, await sourceBytes(context, need.source));
- const inspection = await inspectFile({
- source: need.source,
- input,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- return result(canonicalize(inspection), canonicalize({ provider: "media.local", operation: "inspect" }));
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.inspect),
+ handler: operation(executeInspectMedia),
},
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.normalize,
returns: mediaTypes.synchronized,
- supports: (need) => hasContract(need.constraints, "svml.normalize-media-request@1"),
- handler: async (context) => {
- const need = normalizeNeed(context.need.constraints);
- const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);
- const work = await mkdtemp(join(tmpdir(), "svml-media-normalize-"));
- try {
- const input = join(work, "source.bin");
- await writeFile(input, await sourceBytes(context, need.source));
- let visualArtifact: BlobRef | undefined;
- let visualWidth: number | undefined;
- let visualHeight: number | undefined;
- if (plan.video !== undefined) {
- const output = join(work, "visual.mp4");
- const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;
- const filter = [
- "setpts=PTS-STARTPTS",
- `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
- `trim=start_frame=0:end_frame=${plan.frameCount}`,
- `setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,
- "scale=trunc(iw/2)*2:trunc(ih/2)*2",
- ].join(",");
- await runProcess({
- executable: ffmpegPath,
- argv: ["-y", "-i", input, "-map", `0:${plan.video.index}`, "-an", "-vf", filter,
- "-frames:v", String(plan.frameCount), "-fps_mode", "cfr", "-c:v", "libx264",
- "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", output],
- timeoutMs: processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- });
- const inspected = await outputInspection({
- path: output,
- mediaType: "video/mp4",
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
- assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
- "Normalized visual frame shape differs from its plan");
- const bytes = await readFile(output);
- visualArtifact = await context.artifacts.put(bytes, "video/mp4");
- visualWidth = visual.width;
- visualHeight = visual.height;
- }
- let audioArtifact: BlobRef | undefined;
- if (plan.audio !== undefined) {
- const output = join(work, "audio.wav");
- const filter = [
- "asetpts=PTS-STARTPTS",
- "aresample=48000:async=0:first_pts=0",
- "aformat=sample_rates=48000:channel_layouts=stereo",
- `atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,
- "asetpts=N/SR/TB",
- `adelay=${plan.audioHeadSamples}S:all=1`,
- `apad=whole_len=${plan.sampleFrames}`,
- `atrim=start_sample=0:end_sample=${plan.sampleFrames}`,
- "asetpts=N/SR/TB",
- ].join(",");
- await runProcess({
- executable: ffmpegPath,
- argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,
- "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],
- timeoutMs: processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- });
- const inspected = await outputInspection({
- path: output,
- mediaType: "audio/wav",
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");
- assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,
- "Normalized audio sample shape differs from its plan");
- audioArtifact = await context.artifacts.put(await readFile(output), "audio/wav");
- }
- const media = sealSynchronizedMedia({
- contract: "svml.synchronized-media@1",
- timeline: {
- spanAuthority: need.selection.spanAuthority,
- frameRate: need.frameRate,
- frameCount: plan.frameCount,
- sampleRate: 48_000,
- sampleFrames: plan.sampleFrames,
- },
- sourceMap: {
- sourceOriginPts: plan.origin,
- sourceEndPts: plan.end,
- audioTrimStartSamples: plan.audioTrimStartSamples,
- audioTrimEndSamples: plan.audioTrimEndSamples,
- audioHeadSamples: plan.audioHeadSamples,
- audioContentSamples: plan.audioContentSamples,
- audioTailSamples: plan.audioTailSamples,
- },
- ...(visualArtifact === undefined ? {} : {
- visual: {
- artifact: visualArtifact,
- sourceStreamIndex: plan.video!.index,
- width: visualWidth!,
- height: visualHeight!,
- frameRate: need.frameRate,
- frameCount: plan.frameCount,
- muted: true as const,
- },
- }),
- ...(audioArtifact === undefined ? {} : {
- audio: {
- artifact: audioArtifact,
- sourceStreamIndex: plan.audio!.index,
- codec: "pcm_s16le" as const,
- sampleRate: 48_000 as const,
- channels: 2 as const,
- sampleFrames: plan.sampleFrames,
- loudness: "preserved" as const,
- },
- }),
- });
- return result(canonicalize(media), canonicalize({ provider: "media.local", operation: "normalize",
- visual: visualArtifact !== undefined, audio: audioArtifact !== undefined }));
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.normalize),
+ handler: operation(executeNormalizeMedia),
},
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,
returns: speechTypes.evidenceAudio,
- supports: (need) => hasContract(need.constraints, "svml.project-speech-evidence-audio-request@1"),
- handler: async (context) => {
- const need = evidenceAudioNeed(context.need.constraints);
- const work = await mkdtemp(join(tmpdir(), "svml-media-speech-evidence-"));
- try {
- const input = join(work, "speech-master.wav");
- const output = join(work, "alignment-evidence.wav");
- await stageArtifact(context, need.source, input);
- const source = await assertCanonicalWav({
- path: input,
- source: need.source,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- assert(source.decodedSampleFrames === need.sourceSampleFrames,
- "Speech master sample count differs from its ProgramSpace");
- const filter = [
- "asetpts=N/SR/TB",
- "aresample=16000:async=0:first_pts=0",
- "aformat=sample_rates=16000:channel_layouts=mono",
- `apad=whole_len=${need.evidenceSampleFrames}`,
- `atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,
- "asetpts=N/SR/TB",
- ].join(",");
- await runProcess({
- executable: ffmpegPath,
- argv: ["-y", "-i", input, "-vn", "-af", filter,
- "-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],
- timeoutMs: processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- });
- const inspected = await outputInspection({
- path: output,
- mediaType: "audio/wav",
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
- assert(streams.length === 1 && inspected.streams.length === 1
- && streams[0]!.codecName === "pcm_s16le"
- && streams[0]!.sampleRate === 16_000
- && streams[0]!.channels === 1
- && streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,
- "Alignment evidence must be exact 16 kHz mono PCM s16");
- const artifact = await context.artifacts.put(await readFile(output), "audio/wav");
- const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({
- contract: "svml.speech-evidence-audio@1",
- artifact,
- codec: "pcm_s16le",
- sampleRate: 16_000,
- channels: 1,
- sampleFrames: need.evidenceSampleFrames,
- durationSec: need.durationSec,
- segments: need.segments,
- sampleMap: {
- algorithm: "rational-boundary-round@1",
- sourceSampleRate: 48_000,
- evidenceSampleRate: 16_000,
- sourceSampleFrames: need.sourceSampleFrames,
- evidenceSampleFrames: need.evidenceSampleFrames,
- sourceOriginSample: 0,
- evidenceOriginSample: 0,
- },
- });
- assertSpeechEvidenceAudioIdentity(evidence);
- return result(canonicalize(evidence), canonicalize({
- provider: "media.local",
- operation: "project-speech-evidence-audio",
- sourceSampleFrames: need.sourceSampleFrames,
- evidenceSampleFrames: need.evidenceSampleFrames,
- }));
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ supports: (need) =>
+ mediaNeedHasContract(need.constraints, mediaOperationContracts.projectSpeechEvidenceAudio),
+ handler: operation(executeProjectSpeechEvidenceAudio),
},
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.renderAudio,
returns: mediaTypes.timelineAudio,
- supports: (need) => hasContract(need.constraints, "svml.render-audio-request@1"),
- handler: async (context) => {
- const need = renderAudioNeed(context.need.constraints);
- const plan: AudioProgramPlan = need.plan;
- const work = await mkdtemp(join(tmpdir(), "svml-media-audio-"));
- try {
- const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number }>();
- for (const clip of plan.clips) {
- if (artifacts.has(clip.artifact.digest)) continue;
- const inputIndex = artifacts.size;
- const path = join(work, `input-${inputIndex}.wav`);
- await stageArtifact(context, clip.artifact, path);
- const audio = await assertCanonicalWav({
- path,
- source: clip.artifact,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- assert(audio.decodedSampleFrames > 0, `Audio input ${clip.artifact.digest} is empty`);
- artifacts.set(clip.artifact.digest, { source: clip.artifact, path, inputIndex });
- }
-
- const output = join(work, "program.wav");
- const argv = ["-y"];
- for (const item of artifacts.values()) argv.push("-i", item.path);
- if (plan.clips.length === 0) {
- argv.push(
- "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",
- "-af", `atrim=start_sample=0:end_sample=${plan.sampleFrames},asetpts=N/SR/TB`,
- );
- } else {
- const chains = plan.clips.map((clip, index) => {
- const inputIndex = artifacts.get(clip.artifact.digest)!.inputIndex;
- return audioClipFilter(clip, inputIndex, index);
- });
- const labels = plan.clips.map((_clip, index) => `[clip${index}]`).join("");
- chains.push(
- `${labels}amix=inputs=${plan.clips.length}:duration=longest:dropout_transition=0:normalize=0,`
- + `apad=whole_len=${plan.sampleFrames},atrim=start_sample=0:end_sample=${plan.sampleFrames},`
- + "asetpts=N/SR/TB[out]",
- );
- argv.push("-filter_complex", chains.join(";"), "-map", "[out]");
- }
- argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);
- await runProcess({
- executable: ffmpegPath,
- argv,
- timeoutMs: processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- });
- const bytes = await readFile(output);
- const artifact = await context.artifacts.put(bytes, "audio/wav");
- const audio = await assertCanonicalWav({
- path: output,
- source: artifact,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- assert(audio.decodedSampleFrames === plan.sampleFrames,
- "Rendered TimelineAudio sample count differs from its plan");
- const value: TimelineAudio = sealTimelineAudio({
- contract: "svml.timeline-audio@1",
- artifact,
- codec: "pcm_s16le",
- sampleRate: 48_000,
- channels: 2,
- sampleFrames: plan.sampleFrames,
- loudness: "planned",
- });
- return result(canonicalize(value), canonicalize({
- provider: "media.local",
- operation: "render-audio",
- clips: plan.clips.length,
- }));
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.renderAudio),
+ handler: operation(executeRenderTimelineAudio),
},
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.mux,
returns: mediaTypes.muxed,
- supports: (need) => hasContract(need.constraints, "svml.mux-media-request@1"),
- handler: async (context) => {
- const need = muxMediaNeed(context.need.constraints);
- const work = await mkdtemp(join(tmpdir(), "svml-media-mux-"));
- try {
- const visualPath = join(work, "visual.mp4");
- const audioPath = join(work, "audio.wav");
- const output = join(work, "final.mp4");
- await Promise.all([
- stageArtifact(context, need.visual.artifact, visualPath),
- stageArtifact(context, need.audio.artifact, audioPath),
- ]);
- const [visualInspection, audio] = await Promise.all([
- inspectFile({
- source: need.visual.artifact,
- input: visualPath,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- }),
- assertCanonicalWav({
- path: audioPath,
- source: need.audio.artifact,
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- }),
- ]);
- const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
- assert(visualStreams.length === 1 && visualInspection.streams.length === 1,
- "RenderedVisual Artifact must contain exactly one silent video stream");
- const visual = visualStreams[0]!;
- assert(visual.role === "moving" && visual.timingStatus === "admissible"
- && visual.decodedUnitCount === need.visual.frameCount
- && visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,
- "RenderedVisual bytes differ from their declared frame domain");
- assert(audio.decodedSampleFrames === need.audio.sampleFrames,
- "TimelineAudio bytes differ from their declared sample domain");
- await runProcess({
- executable: ffmpegPath,
- argv: [
- "-y", "-i", visualPath, "-i", audioPath,
- "-map", `0:${visual.index}`, "-map", "1:0",
- "-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",
- "-frames:v", String(need.visual.frameCount),
- "-movflags", "+faststart", output,
- ],
- timeoutMs: processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- });
- const finalInspection = await outputInspection({
- path: output,
- mediaType: "video/mp4",
- ffprobePath,
- timeoutMs: processTimeoutMs,
- maxProbeOutputBytes,
- });
- const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
- const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
- assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,
- "Final mux must contain exactly one video and one audio stream");
- assert(finalVideo[0]!.timingStatus === "admissible"
- && finalVideo[0]!.decodedUnitCount === need.visual.frameCount,
- "Final mux video frame count differs from RenderedVisual");
- assert(finalAudio[0]!.timingStatus === "admissible"
- && finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,
- "Final mux audio shape differs from TimelineAudio");
- assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined
- && compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,
- "Final mux audio and video do not share one presentation origin");
- const finalAudioSpan = duration(finalAudio[0]!);
- assert(roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator)
- === need.audio.sampleFrames,
- "Final mux audio presentation span differs from TimelineAudio");
- const artifact = await context.artifacts.put(await readFile(output), "video/mp4");
- const value: MuxedMedia = sealMuxedMedia({
- contract: "svml.muxed-media@1",
- frameRate: need.visual.frameRate,
- frameCount: need.visual.frameCount,
- canvas: need.visual.canvas,
- presentationSampleFrames: need.audio.sampleFrames,
- artifact,
- });
- return result(canonicalize(value), canonicalize({
- provider: "media.local",
- operation: "mux",
- videoStream: finalVideo[0]!.index,
- audioStream: finalAudio[0]!.index,
- }));
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.mux),
+ handler: operation(executeMuxProgramMedia),
},
],
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c417f3a4..25a275fb 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -722,6 +722,21 @@ importers:
specifier: workspace:*
version: link:../text
+ packages/media-execution:
+ dependencies:
+ '@narratage/media':
+ specifier: workspace:*
+ version: link:../media
+ '@narratage/media-pipeline':
+ specifier: workspace:*
+ version: link:../media-pipeline
+ '@narratage/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@narratage/speech':
+ specifier: workspace:*
+ version: link:../speech
+
packages/media-pipeline:
dependencies:
'@narratage/artifact':
@@ -1060,6 +1075,9 @@ importers:
'@narratage/media':
specifier: workspace:*
version: link:../media
+ '@narratage/media-execution':
+ specifier: workspace:*
+ version: link:../media-execution
'@narratage/media-pipeline':
specifier: workspace:*
version: link:../media-pipeline
diff --git a/tsconfig.json b/tsconfig.json
index 44690bb9..4132a108 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -141,6 +141,7 @@
"@narratage/provider-kie": [
"packages/provider-kie/src/index.ts"
],
+ "@narratage/media-execution": ["packages/media-execution/src/index.ts"],
"@narratage/media-pipeline": [
"packages/media-pipeline/src/index.ts"
],