Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
refactor: make media execution one body of code, not one Provider's
kashorinc729b1af4169
diff --git a/packages/media-execution/package.json b/packages/media-execution/package.jsonnew file mode 100644index 00000000..bdaad203--- /dev/null+++ b/packages/media-execution/package.json@@ -0,0 +1,15 @@+{+ "name": "@narratage/media-execution",+ "version": "0.0.0-dev",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "dependencies": {+ "@narratage/media": "workspace:*",+ "@narratage/media-pipeline": "workspace:*",+ "@narratage/protocol": "workspace:*",+ "@narratage/speech": "workspace:*"+ }+}diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.tsnew file mode 100644index 00000000..5eb64e2f--- /dev/null+++ b/packages/media-execution/src/execute.ts@@ -0,0 +1,857 @@+import { spawn } from "node:child_process";+import { createHash } from "node:crypto";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { mediaTypes, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifyTimelineAudio } from "@narratage/media";+import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, RenderedVisual, TimelineAudio } from "@narratage/media";+import type { ProgramSpace } from "@narratage/program-space";+import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary, speechTypes } from "@narratage/speech";+import type { SpeechEvidenceAudio } from "@narratage/speech";+import {+ mediaPipelineCapabilities,+ verifyAudioProgramPlan,+ type InspectMediaNeed,+ type MuxMediaNeed,+ type NormalizeMediaNeed,+ type ProjectSpeechEvidenceAudioNeed,+ type RenderAudioNeed,+} from "@narratage/media-pipeline";+import type { AudioProgramClip, AudioProgramPlan } from "@narratage/media-pipeline";+import {+ canonicalize,+ digestOf,+} from "@narratage/protocol";+import type { BlobRef, CanonicalValue } from "@narratage/protocol";++import { parseMediaInspection } from "./probe.js";++/**+ * Where the bytes live and which binaries transform them.+ *+ * These five operations are the whole of Narratage's media execution, and they+ * are written once. A local Provider supplies the Build's own ArtifactStore and+ * the ffmpeg on its PATH; a Lambda Provider supplies an S3-backed gateway and+ * the ffmpeg baked into its image. Nothing below knows which it is, so the two+ * deployments cannot drift into computing different media from one Need.+ */+export type MediaArtifactGateway = {+ get(source: BlobRef): Promise<Uint8Array | undefined>;+ put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;+};++export type MediaExecutionEnvironment = {+ readonly artifacts: MediaArtifactGateway;+ readonly ffmpegPath: string;+ readonly ffprobePath: string;+ readonly processTimeoutMs: number;+ readonly maxProbeOutputBytes: number;+ /** Recorded in each result's metadata, so an Operation says which Provider ran it. */+ readonly label: string;+};++export type MediaOperationResult = {+ readonly value: CanonicalValue;+ readonly metadata: CanonicalValue;+};++/** The request contract each capability answers. Both Providers admit Needs by this table. */+export const mediaOperationContracts = {+ inspect: "svml.inspect-media-request@1",+ normalize: "svml.normalize-media-request@1",+ projectSpeechEvidenceAudio: "svml.project-speech-evidence-audio-request@1",+ renderAudio: "svml.render-audio-request@1",+ mux: "svml.mux-media-request@1",+} as const;++export function mediaNeedHasContract(value: unknown, contract: string): boolean {+ return hasContract(value, contract);+}++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++function positiveInteger(value: number, subject: string): number {+ assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive integer`);+ return value;+}++function object(value: unknown, subject: string): Record<string, unknown> {+ assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);+ return value as Record<string, unknown>;+}++function hasContract(value: unknown, contract: string): boolean {+ return value !== null && typeof value === "object" && !Array.isArray(value)+ && (value as Record<string, unknown>).contract === contract;+}++async function runProcess(args: {+ readonly executable: string;+ readonly argv: readonly string[];+ readonly timeoutMs: number;+ readonly maxStdoutBytes: number;+}): Promise<Uint8Array> {+ return await new Promise((resolve, reject) => {+ const child = spawn(args.executable, [...args.argv], {+ shell: false,+ stdio: ["ignore", "pipe", "pipe"],+ env: { PATH: process.env.PATH ?? "" },+ });+ const stdout: Buffer[] = [];+ let stdoutBytes = 0;+ let stderr = "";+ let settled = false;+ const finish = (error?: Error): void => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ if (error !== undefined) reject(error);+ else resolve(Buffer.concat(stdout));+ };+ const timer = setTimeout(() => {+ child.kill("SIGKILL");+ finish(new Error(`${args.executable} timed out`));+ }, args.timeoutMs);+ child.stdout.on("data", (chunk: Buffer) => {+ stdoutBytes += chunk.byteLength;+ if (stdoutBytes > args.maxStdoutBytes) {+ child.kill("SIGKILL");+ finish(new Error(`${args.executable} output exceeded the configured limit`));+ return;+ }+ stdout.push(chunk);+ });+ child.stderr.on("data", (chunk: Buffer) => {+ stderr = `${stderr}${chunk.toString()}`.slice(-8_000);+ });+ child.on("error", (error) => finish(error));+ child.on("close", (code) => {+ if (code !== 0) finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));+ else finish();+ });+ });+}++async function version(executable: string, timeoutMs: number): Promise<string> {+ const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024 });+ const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();+ assert(line !== undefined && line.length > 0, `${executable} returned no version`);+ return line;+}++async function sourceBytes(env: MediaExecutionEnvironment, source: BlobRef): Promise<Uint8Array> {+ const bytes = await env.artifacts.get(source);+ assert(bytes !== undefined, `Media source ${source.digest} is unavailable`);+ assert(bytes.byteLength === source.size, `Media source ${source.digest} size differs`);+ return bytes;+}++async function stageArtifact(+ env: MediaExecutionEnvironment,+ source: BlobRef,+ path: string,+): Promise<void> {+ await writeFile(path, await sourceBytes(env, source));+}++function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {+ return {+ numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),+ denominator: BigInt(value.timeBase.denominator),+ };+}++function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {+ const a = timestampFraction(left);+ const b = timestampFraction(right);+ const x = a.numerator * b.denominator;+ const y = b.numerator * a.denominator;+ return x < y ? -1 : x > y ? 1 : 0;+}++function roundPositive(numerator: bigint, denominator: bigint): number {+ assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");+ const value = (numerator * 2n + denominator) / (denominator * 2n);+ assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");+ return Number(value);+}++function ceilPositive(numerator: bigint, denominator: bigint): number {+ assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");+ const value = (numerator + denominator - 1n) / denominator;+ assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");+ return Number(value);+}++function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {+ assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");+ const start = timestampFraction(value.startPts);+ const end = timestampFraction(value.endPts);+ const numerator = end.numerator * start.denominator - start.numerator * end.denominator;+ const denominator = end.denominator * start.denominator;+ assert(numerator > 0n, "Selected media stream has an empty presentation interval");+ return { numerator, denominator };+}++function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {+ const a = timestampFraction(origin);+ const b = timestampFraction(value);+ const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);+ const denominator = b.denominator * a.denominator;+ const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);+ return numerator < 0n ? -magnitude : magnitude;+}++function stream<T extends MediaStream["kind"]>(+ inspection: MediaInspection,+ index: number | undefined,+ kind: T,+): Extract<MediaStream, { readonly kind: T }> | undefined {+ if (index === undefined) return undefined;+ const result = inspection.streams.find((item) => item.index === index);+ assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);+ assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);+ return result as Extract<MediaStream, { readonly kind: T }>;+}++type NormalizationPlan = {+ readonly video?: MediaVideoStream;+ readonly audio?: MediaAudioStream;+ readonly origin: MediaTimestamp;+ readonly end: MediaTimestamp;+ readonly frameCount: number;+ readonly sampleFrames: number;+ readonly audioTrimStartSamples: number;+ readonly audioTrimEndSamples: number;+ readonly audioHeadSamples: number;+ readonly audioContentSamples: number;+ readonly audioTailSamples: number;+};++function normalizationPlan(+ inspection: MediaInspection,+ selection: MediaStreamSelection,+ frameRate: MediaRational,+): NormalizationPlan {+ assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0+ && Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,+ "Target frame rate is invalid");+ const video = stream(inspection, selection.videoStreamIndex, "video");+ const audio = stream(inspection, selection.audioStreamIndex, "audio");+ if (video !== undefined && selection.spanAuthority !== "video") {+ throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");+ }+ const authority = selection.spanAuthority === "video" ? video : audio;+ assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");+ const span = duration(authority);+ const scaledNumerator = span.numerator * BigInt(frameRate.numerator);+ const scaledDenominator = span.denominator * BigInt(frameRate.denominator);+ const frameCount = selection.spanAuthority === "video"+ ? roundPositive(scaledNumerator, scaledDenominator)+ : ceilPositive(scaledNumerator, scaledDenominator);+ assert(frameCount > 0, "Normalized media would contain no frame interval");+ const sampleFrames = roundPositive(+ BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),+ BigInt(frameRate.numerator),+ );+ if (audio === undefined) {+ return {+ ...(video === undefined ? {} : { video }),+ origin: authority.startPts,+ end: authority.endPts,+ frameCount,+ sampleFrames,+ audioTrimStartSamples: 0,+ audioTrimEndSamples: 0,+ audioHeadSamples: 0,+ audioContentSamples: 0,+ audioTailSamples: 0,+ };+ }+ assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");+ const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);+ const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);+ const contentStart = Math.max(0, audioStart);+ const contentEnd = Math.min(sampleFrames, audioEnd);+ assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");+ return {+ ...(video === undefined ? {} : { video }),+ audio,+ origin: authority.startPts,+ end: authority.endPts,+ frameCount,+ sampleFrames,+ audioTrimStartSamples: Math.max(0, -audioStart),+ audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),+ audioHeadSamples: contentStart,+ audioContentSamples: contentEnd - contentStart,+ audioTailSamples: sampleFrames - contentEnd,+ };+}++async function inspectFile(args: {+ readonly source: BlobRef;+ readonly input: string;+ readonly ffprobePath: string;+ readonly timeoutMs: number;+ readonly maxProbeOutputBytes: number;+}): Promise<MediaInspection> {+ const [raw, ffprobeVersion] = await Promise.all([+ runProcess({+ executable: args.ffprobePath,+ argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],+ timeoutMs: args.timeoutMs,+ maxStdoutBytes: args.maxProbeOutputBytes,+ }),+ version(args.ffprobePath, args.timeoutMs),+ ]);+ let value: unknown;+ try {+ value = JSON.parse(Buffer.from(raw).toString("utf8"));+ } catch {+ throw new Error("ffprobe returned invalid JSON");+ }+ return parseMediaInspection({ source: args.source, ffprobeVersion, value });+}++async function outputInspection(args: {+ readonly path: string;+ readonly mediaType: string;+ readonly ffprobePath: string;+ readonly timeoutMs: number;+ readonly maxProbeOutputBytes: number;+}): Promise<MediaInspection> {+ const bytes = await readFile(args.path);+ const source: BlobRef = {+ kind: "blob",+ digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` as BlobRef["digest"],+ size: bytes.byteLength,+ mediaType: args.mediaType,+ };+ return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,+ timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes });+}++function result(value: CanonicalValue, metadata: CanonicalValue): MediaOperationResult {+ return { value, metadata };+}++function inspectNeed(value: CanonicalValue): InspectMediaNeed {+ const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;+ assert(item.contract === "svml.inspect-media-request@1" && item.source?.kind === "blob",+ "InspectMediaNeed is invalid");+ return item;+}++function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {+ const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;+ assert(item.contract === "svml.normalize-media-request@1" && item.source?.kind === "blob",+ "NormalizeMediaNeed is invalid");+ verifyMediaInspection(item.inspection);+ verifyMediaStreamSelection(item.selection);+ assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2+ && item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",+ "NormalizeMediaNeed audio profile is unsupported");+ return item;+}++function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {+ const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;+ assert(item.contract === "svml.project-speech-evidence-audio-request@1" && item.source?.kind === "blob",+ "ProjectSpeechEvidenceAudioNeed is invalid");+ assert(item.sourceSampleRate === 48_000 && item.sourceChannels === 2 && item.sourceCodec === "pcm_s16le"+ && item.evidenceSampleRate === 16_000 && item.evidenceChannels === 1 && item.evidenceCodec === "pcm_s16le",+ "Speech evidence audio shape is invalid");+ assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0+ && Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0+ && item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),+ "Speech evidence audio sample map is invalid");+ return item;+}++function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {+ const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;+ assert(item.contract === "svml.render-audio-request@1", "RenderAudioNeed is invalid");+ verifyAudioProgramPlan(item.plan);+ return item;+}++function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {+ const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;+ assert(item.contract === "svml.mux-media-request@1", "MuxMediaNeed is invalid");+ verifyRenderedVisual(item.visual);+ verifyTimelineAudio(item.audio);+ const expectedSamples = roundPositive(+ BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),+ BigInt(item.visual.frameRate.numerator),+ );+ assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");+ return item;+}++function atempo(rate: number): string[] {+ if (Math.abs(rate - 1) < 1e-12) return [];+ const filters: string[] = [];+ let remaining = rate;+ while (remaining > 2) {+ filters.push("atempo=2");+ remaining /= 2;+ }+ while (remaining < 0.5) {+ filters.push("atempo=0.5");+ remaining /= 0.5;+ }+ if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);+ return filters;+}++function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number): string {+ const length = clip.targetEndSampleExclusive - clip.targetStartSample;+ const filters = [+ `atrim=start_sample=${clip.sourceStartSample}`,+ "asetpts=PTS-STARTPTS",+ ...atempo(clip.playbackRate),+ `atrim=start_sample=0:end_sample=${length}`,+ `apad=whole_len=${length}`,+ `atrim=start_sample=0:end_sample=${length}`,+ `volume=${clip.gain.toPrecision(15)}`,+ ...(clip.fadeInSamples === 0 ? [] : [`afade=t=in:start_sample=0:nb_samples=${clip.fadeInSamples}`]),+ ...(clip.fadeOutSamples === 0 ? [] : [+ `afade=t=out:start_sample=${length - clip.fadeOutSamples}:nb_samples=${clip.fadeOutSamples}`,+ ]),+ `adelay=${clip.targetStartSample}S:all=1`,+ ];+ return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;+}++async function assertCanonicalWav(args: {+ readonly path: string;+ readonly source: BlobRef;+ readonly ffprobePath: string;+ readonly timeoutMs: number;+ readonly maxProbeOutputBytes: number;+}): Promise<MediaAudioStream> {+ const inspection = await inspectFile({+ source: args.source,+ input: args.path,+ ffprobePath: args.ffprobePath,+ timeoutMs: args.timeoutMs,+ maxProbeOutputBytes: args.maxProbeOutputBytes,+ });+ const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");+ assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");+ assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,+ "Canonical audio must be 48 kHz stereo PCM s16");+ assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");+ return audio[0]!;+}++export async function executeInspectMedia(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const need = inspectNeed(constraints);+ const work = await mkdtemp(join(tmpdir(), "svml-media-inspect-"));+ try {+ const input = join(work, "source.bin");+ await writeFile(input, await sourceBytes(env, need.source));+ const inspection = await inspectFile({+ source: need.source,+ input,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ return result(canonicalize(inspection), canonicalize({ provider: env.label, operation: "inspect" }));+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}++export async function executeNormalizeMedia(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const need = normalizeNeed(constraints);+ const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);+ const work = await mkdtemp(join(tmpdir(), "svml-media-normalize-"));+ try {+ const input = join(work, "source.bin");+ await writeFile(input, await sourceBytes(env, need.source));+ let visualArtifact: BlobRef | undefined;+ let visualWidth: number | undefined;+ let visualHeight: number | undefined;+ if (plan.video !== undefined) {+ const output = join(work, "visual.mp4");+ const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;+ const filter = [+ "setpts=PTS-STARTPTS",+ `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,+ `trim=start_frame=0:end_frame=${plan.frameCount}`,+ `setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,+ "scale=trunc(iw/2)*2:trunc(ih/2)*2",+ ].join(",");+ await runProcess({+ executable: env.ffmpegPath,+ argv: ["-y", "-i", input, "-map", `0:${plan.video.index}`, "-an", "-vf", filter,+ "-frames:v", String(plan.frameCount), "-fps_mode", "cfr", "-c:v", "libx264",+ "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", output],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ });+ const inspected = await outputInspection({+ path: output,+ mediaType: "video/mp4",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");+ assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",+ "Normalized visual frame shape differs from its plan");+ const bytes = await readFile(output);+ visualArtifact = await env.artifacts.put(bytes, "video/mp4");+ visualWidth = visual.width;+ visualHeight = visual.height;+ }+ let audioArtifact: BlobRef | undefined;+ if (plan.audio !== undefined) {+ const output = join(work, "audio.wav");+ const filter = [+ "asetpts=PTS-STARTPTS",+ "aresample=48000:async=0:first_pts=0",+ "aformat=sample_rates=48000:channel_layouts=stereo",+ `atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,+ "asetpts=N/SR/TB",+ `adelay=${plan.audioHeadSamples}S:all=1`,+ `apad=whole_len=${plan.sampleFrames}`,+ `atrim=start_sample=0:end_sample=${plan.sampleFrames}`,+ "asetpts=N/SR/TB",+ ].join(",");+ await runProcess({+ executable: env.ffmpegPath,+ argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,+ "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ });+ const inspected = await outputInspection({+ path: output,+ mediaType: "audio/wav",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");+ assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,+ "Normalized audio sample shape differs from its plan");+ audioArtifact = await env.artifacts.put(await readFile(output), "audio/wav");+ }+ const media = sealSynchronizedMedia({+ contract: "svml.synchronized-media@1",+ timeline: {+ spanAuthority: need.selection.spanAuthority,+ frameRate: need.frameRate,+ frameCount: plan.frameCount,+ sampleRate: 48_000,+ sampleFrames: plan.sampleFrames,+ },+ sourceMap: {+ sourceOriginPts: plan.origin,+ sourceEndPts: plan.end,+ audioTrimStartSamples: plan.audioTrimStartSamples,+ audioTrimEndSamples: plan.audioTrimEndSamples,+ audioHeadSamples: plan.audioHeadSamples,+ audioContentSamples: plan.audioContentSamples,+ audioTailSamples: plan.audioTailSamples,+ },+ ...(visualArtifact === undefined ? {} : {+ visual: {+ artifact: visualArtifact,+ sourceStreamIndex: plan.video!.index,+ width: visualWidth!,+ height: visualHeight!,+ frameRate: need.frameRate,+ frameCount: plan.frameCount,+ muted: true as const,+ },+ }),+ ...(audioArtifact === undefined ? {} : {+ audio: {+ artifact: audioArtifact,+ sourceStreamIndex: plan.audio!.index,+ codec: "pcm_s16le" as const,+ sampleRate: 48_000 as const,+ channels: 2 as const,+ sampleFrames: plan.sampleFrames,+ loudness: "preserved" as const,+ },+ }),+ });+ return result(canonicalize(media), canonicalize({ provider: env.label, operation: "normalize",+ visual: visualArtifact !== undefined, audio: audioArtifact !== undefined }));+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}++export async function executeProjectSpeechEvidenceAudio(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const need = evidenceAudioNeed(constraints);+ const work = await mkdtemp(join(tmpdir(), "svml-media-speech-evidence-"));+ try {+ const input = join(work, "speech-master.wav");+ const output = join(work, "alignment-evidence.wav");+ await stageArtifact(env, need.source, input);+ const source = await assertCanonicalWav({+ path: input,+ source: need.source,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ assert(source.decodedSampleFrames === need.sourceSampleFrames,+ "Speech master sample count differs from its ProgramSpace");+ const filter = [+ "asetpts=N/SR/TB",+ "aresample=16000:async=0:first_pts=0",+ "aformat=sample_rates=16000:channel_layouts=mono",+ `apad=whole_len=${need.evidenceSampleFrames}`,+ `atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,+ "asetpts=N/SR/TB",+ ].join(",");+ await runProcess({+ executable: env.ffmpegPath,+ argv: ["-y", "-i", input, "-vn", "-af", filter,+ "-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ });+ const inspected = await outputInspection({+ path: output,+ mediaType: "audio/wav",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");+ assert(streams.length === 1 && inspected.streams.length === 1+ && streams[0]!.codecName === "pcm_s16le"+ && streams[0]!.sampleRate === 16_000+ && streams[0]!.channels === 1+ && streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,+ "Alignment evidence must be exact 16 kHz mono PCM s16");+ const artifact = await env.artifacts.put(await readFile(output), "audio/wav");+ const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({+ contract: "svml.speech-evidence-audio@1",+ artifact,+ codec: "pcm_s16le",+ sampleRate: 16_000,+ channels: 1,+ sampleFrames: need.evidenceSampleFrames,+ durationSec: need.durationSec,+ segments: need.segments,+ sampleMap: {+ algorithm: "rational-boundary-round@1",+ sourceSampleRate: 48_000,+ evidenceSampleRate: 16_000,+ sourceSampleFrames: need.sourceSampleFrames,+ evidenceSampleFrames: need.evidenceSampleFrames,+ sourceOriginSample: 0,+ evidenceOriginSample: 0,+ },+ });+ assertSpeechEvidenceAudioIdentity(evidence);+ return result(canonicalize(evidence), canonicalize({+ provider: env.label,+ operation: "project-speech-evidence-audio",+ sourceSampleFrames: need.sourceSampleFrames,+ evidenceSampleFrames: need.evidenceSampleFrames,+ }));+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}++export async function executeRenderTimelineAudio(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const need = renderAudioNeed(constraints);+ const plan: AudioProgramPlan = need.plan;+ const work = await mkdtemp(join(tmpdir(), "svml-media-audio-"));+ try {+ const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number }>();+ for (const clip of plan.clips) {+ if (artifacts.has(clip.artifact.digest)) continue;+ const inputIndex = artifacts.size;+ const path = join(work, `input-${inputIndex}.wav`);+ await stageArtifact(env, clip.artifact, path);+ const audio = await assertCanonicalWav({+ path,+ source: clip.artifact,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ assert(audio.decodedSampleFrames > 0, `Audio input ${clip.artifact.digest} is empty`);+ artifacts.set(clip.artifact.digest, { source: clip.artifact, path, inputIndex });+ }++ const output = join(work, "program.wav");+ const argv = ["-y"];+ for (const item of artifacts.values()) argv.push("-i", item.path);+ if (plan.clips.length === 0) {+ argv.push(+ "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",+ "-af", `atrim=start_sample=0:end_sample=${plan.sampleFrames},asetpts=N/SR/TB`,+ );+ } else {+ const chains = plan.clips.map((clip, index) => {+ const inputIndex = artifacts.get(clip.artifact.digest)!.inputIndex;+ return audioClipFilter(clip, inputIndex, index);+ });+ const labels = plan.clips.map((_clip, index) => `[clip${index}]`).join("");+ chains.push(+ `${labels}amix=inputs=${plan.clips.length}:duration=longest:dropout_transition=0:normalize=0,`+ + `apad=whole_len=${plan.sampleFrames},atrim=start_sample=0:end_sample=${plan.sampleFrames},`+ + "asetpts=N/SR/TB[out]",+ );+ argv.push("-filter_complex", chains.join(";"), "-map", "[out]");+ }+ argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);+ await runProcess({+ executable: env.ffmpegPath,+ argv,+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ });+ const bytes = await readFile(output);+ const artifact = await env.artifacts.put(bytes, "audio/wav");+ const audio = await assertCanonicalWav({+ path: output,+ source: artifact,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ assert(audio.decodedSampleFrames === plan.sampleFrames,+ "Rendered TimelineAudio sample count differs from its plan");+ const value: TimelineAudio = sealTimelineAudio({+ contract: "svml.timeline-audio@1",+ artifact,+ codec: "pcm_s16le",+ sampleRate: 48_000,+ channels: 2,+ sampleFrames: plan.sampleFrames,+ loudness: "planned",+ });+ return result(canonicalize(value), canonicalize({+ provider: env.label,+ operation: "render-audio",+ clips: plan.clips.length,+ }));+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}++export async function executeMuxProgramMedia(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const need = muxMediaNeed(constraints);+ const work = await mkdtemp(join(tmpdir(), "svml-media-mux-"));+ try {+ const visualPath = join(work, "visual.mp4");+ const audioPath = join(work, "audio.wav");+ const output = join(work, "final.mp4");+ await Promise.all([+ stageArtifact(env, need.visual.artifact, visualPath),+ stageArtifact(env, need.audio.artifact, audioPath),+ ]);+ const [visualInspection, audio] = await Promise.all([+ inspectFile({+ source: need.visual.artifact,+ input: visualPath,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ }),+ assertCanonicalWav({+ path: audioPath,+ source: need.audio.artifact,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ }),+ ]);+ const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");+ assert(visualStreams.length === 1 && visualInspection.streams.length === 1,+ "RenderedVisual Artifact must contain exactly one silent video stream");+ const visual = visualStreams[0]!;+ assert(visual.role === "moving" && visual.timingStatus === "admissible"+ && visual.decodedUnitCount === need.visual.frameCount+ && visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,+ "RenderedVisual bytes differ from their declared frame domain");+ assert(audio.decodedSampleFrames === need.audio.sampleFrames,+ "TimelineAudio bytes differ from their declared sample domain");+ await runProcess({+ executable: env.ffmpegPath,+ argv: [+ "-y", "-i", visualPath, "-i", audioPath,+ "-map", `0:${visual.index}`, "-map", "1:0",+ "-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",+ "-frames:v", String(need.visual.frameCount),+ "-movflags", "+faststart", output,+ ],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ });+ const finalInspection = await outputInspection({+ path: output,+ mediaType: "video/mp4",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ });+ const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");+ const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");+ assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,+ "Final mux must contain exactly one video and one audio stream");+ assert(finalVideo[0]!.timingStatus === "admissible"+ && finalVideo[0]!.decodedUnitCount === need.visual.frameCount,+ "Final mux video frame count differs from RenderedVisual");+ assert(finalAudio[0]!.timingStatus === "admissible"+ && finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,+ "Final mux audio shape differs from TimelineAudio");+ assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined+ && compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,+ "Final mux audio and video do not share one presentation origin");+ const finalAudioSpan = duration(finalAudio[0]!);+ assert(roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator)+ === need.audio.sampleFrames,+ "Final mux audio presentation span differs from TimelineAudio");+ const artifact = await env.artifacts.put(await readFile(output), "video/mp4");+ const value: MuxedMedia = sealMuxedMedia({+ contract: "svml.muxed-media@1",+ frameRate: need.visual.frameRate,+ frameCount: need.visual.frameCount,+ canvas: need.visual.canvas,+ presentationSampleFrames: need.audio.sampleFrames,+ artifact,+ });+ return result(canonicalize(value), canonicalize({+ provider: env.label,+ operation: "mux",+ videoStream: finalVideo[0]!.index,+ audioStream: finalAudio[0]!.index,+ }));+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.tsnew file mode 100644index 00000000..64f7fea3--- /dev/null+++ b/packages/media-execution/src/index.ts@@ -0,0 +1,11 @@+export {+ executeInspectMedia,+ executeMuxProgramMedia,+ executeNormalizeMedia,+ executeProjectSpeechEvidenceAudio,+ executeRenderTimelineAudio,+ mediaNeedHasContract,+ mediaOperationContracts,+} from "./execute.js";+export { parseMediaInspection } from "./probe.js";+export type * from "./execute.js";diff --git a/packages/provider-media-local/src/probe.ts b/packages/media-execution/src/probe.tssimilarity index 100%rename from packages/provider-media-local/src/probe.tsrename to packages/media-execution/src/probe.tsdiff --git a/packages/provider-media-local/test/probe.test.ts b/packages/media-execution/test/probe.test.tssimilarity index 96%rename from packages/provider-media-local/test/probe.test.tsrename to packages/media-execution/test/probe.test.tsindex 11415766..afa377b6 100644--- a/packages/provider-media-local/test/probe.test.ts+++ b/packages/media-execution/test/probe.test.ts@@ -3,7 +3,7 @@ import test from "node:test";import { digestOf } from "@narratage/protocol";-import { parseMediaInspection } from "../src/index.js";+import { parseMediaInspection } from "@narratage/media-execution";test("probe derives the last decoded-unit duration instead of adding whole-stream duration_ts", () => {const inspection = parseMediaInspection({diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.jsonindex 4122c18c..30016096 100644--- a/packages/provider-media-local/package.json+++ b/packages/provider-media-local/package.json@@ -12,6 +12,7 @@"dependencies": {"@narratage/endpoint-kit": "workspace:*","@narratage/media": "workspace:*",+ "@narratage/media-execution": "workspace:*","@narratage/media-pipeline": "workspace:*","@narratage/program-space": "workspace:*","@narratage/protocol": "workspace:*",diff --git a/packages/provider-media-local/src/index.ts b/packages/provider-media-local/src/index.tsindex faf5c3c0..66994720 100644--- a/packages/provider-media-local/src/index.ts+++ b/packages/provider-media-local/src/index.ts@@ -3,5 +3,4 @@ export {localMediaProviderImplementationDigest,localMediaProviderModuleRef,} from "./provider.js";-export { parseMediaInspection } from "./probe.js";export type { CreateLocalMediaProviderOptions } from "./provider.js";diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.tsindex 261e7bd8..5d12f0a4 100644--- a/packages/provider-media-local/src/provider.ts+++ b/packages/provider-media-local/src/provider.ts@@ -1,32 +1,19 @@-import { spawn } from "node:child_process";-import { createHash } from "node:crypto";-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";-import { tmpdir } from "node:os";-import { join } from "node:path";-import { mediaTypes, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifyTimelineAudio } from "@narratage/media";-import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, RenderedVisual, TimelineAudio } from "@narratage/media";-import type { ProgramSpace } from "@narratage/program-space";-import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary, speechTypes } from "@narratage/speech";-import type { SpeechEvidenceAudio } from "@narratage/speech";-import type { EndpointInvocationContext, EndpointFulfillment } from "@narratage/endpoint-kit";+import { mediaTypes } from "@narratage/media";import {- mediaPipelineCapabilities,- verifyAudioProgramPlan,- type InspectMediaNeed,- type MuxMediaNeed,- type NormalizeMediaNeed,- type ProjectSpeechEvidenceAudioNeed,- type RenderAudioNeed,-} from "@narratage/media-pipeline";-import type { AudioProgramClip, AudioProgramPlan } from "@narratage/media-pipeline";-import {- canonicalize,- digestOf,-} from "@narratage/protocol";-import type { BlobRef, CanonicalValue } from "@narratage/protocol";+ executeInspectMedia,+ executeMuxProgramMedia,+ executeNormalizeMedia,+ executeProjectSpeechEvidenceAudio,+ executeRenderTimelineAudio,+ mediaNeedHasContract,+ mediaOperationContracts,+} from "@narratage/media-execution";+import type { MediaExecutionEnvironment, MediaOperationResult } from "@narratage/media-execution";+import { mediaPipelineCapabilities } from "@narratage/media-pipeline";+import { canonicalize, digestOf } from "@narratage/protocol";+import { speechTypes } from "@narratage/speech";import { defineEndpointPackage } from "@narratage/endpoint-kit";--import { parseMediaInspection } from "./probe.js";+import type { EndpointFulfillment, EndpointInvocationContext } from "@narratage/endpoint-kit";export const localMediaProviderModuleRef = { name: "@narratage/provider-media-local", version: "0.0.0-dev" } as const;export const localMediaProviderImplementationDigest = digestOf("@narratage/provider-media-local/ffmpeg@2");@@ -50,382 +37,20 @@ function positiveInteger(value: number, subject: string): number {return value;}-function object(value: unknown, subject: string): Record<string, unknown> {- assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);- return value as Record<string, unknown>;-}--function hasContract(value: unknown, contract: string): boolean {- return value !== null && typeof value === "object" && !Array.isArray(value)- && (value as Record<string, unknown>).contract === contract;-}--async function runProcess(args: {- readonly executable: string;- readonly argv: readonly string[];- readonly timeoutMs: number;- readonly maxStdoutBytes: number;-}): Promise<Uint8Array> {- return await new Promise((resolve, reject) => {- const child = spawn(args.executable, [...args.argv], {- shell: false,- stdio: ["ignore", "pipe", "pipe"],- env: { PATH: process.env.PATH ?? "" },- });- const stdout: Buffer[] = [];- let stdoutBytes = 0;- let stderr = "";- let settled = false;- const finish = (error?: Error): void => {- if (settled) return;- settled = true;- clearTimeout(timer);- if (error !== undefined) reject(error);- else resolve(Buffer.concat(stdout));- };- const timer = setTimeout(() => {- child.kill("SIGKILL");- finish(new Error(`${args.executable} timed out`));- }, args.timeoutMs);- child.stdout.on("data", (chunk: Buffer) => {- stdoutBytes += chunk.byteLength;- if (stdoutBytes > args.maxStdoutBytes) {- child.kill("SIGKILL");- finish(new Error(`${args.executable} output exceeded the configured limit`));- return;- }- stdout.push(chunk);- });- child.stderr.on("data", (chunk: Buffer) => {- stderr = `${stderr}${chunk.toString()}`.slice(-8_000);- });- child.on("error", (error) => finish(error));- child.on("close", (code) => {- if (code !== 0) finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));- else finish();- });- });-}--async function version(executable: string, timeoutMs: number): Promise<string> {- const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024 });- const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();- assert(line !== undefined && line.length > 0, `${executable} returned no version`);- return line;-}--async function sourceBytes(context: EndpointInvocationContext, source: BlobRef): Promise<Uint8Array> {- const bytes = await context.artifacts.get(source.digest);- assert(bytes !== undefined, `Media source ${source.digest} is unavailable`);- assert(bytes.byteLength === source.size, `Media source ${source.digest} size differs`);- return bytes;-}--async function stageArtifact(- context: EndpointInvocationContext,- source: BlobRef,- path: string,-): Promise<void> {- await writeFile(path, await sourceBytes(context, source));-}--function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {+function fulfillment(result: MediaOperationResult): EndpointFulfillment {return {- numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),- denominator: BigInt(value.timeBase.denominator),- };-}--function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {- const a = timestampFraction(left);- const b = timestampFraction(right);- const x = a.numerator * b.denominator;- const y = b.numerator * a.denominator;- return x < y ? -1 : x > y ? 1 : 0;-}--function roundPositive(numerator: bigint, denominator: bigint): number {- assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");- const value = (numerator * 2n + denominator) / (denominator * 2n);- assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");- return Number(value);-}--function ceilPositive(numerator: bigint, denominator: bigint): number {- assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");- const value = (numerator + denominator - 1n) / denominator;- assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");- return Number(value);-}--function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {- assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");- const start = timestampFraction(value.startPts);- const end = timestampFraction(value.endPts);- const numerator = end.numerator * start.denominator - start.numerator * end.denominator;- const denominator = end.denominator * start.denominator;- assert(numerator > 0n, "Selected media stream has an empty presentation interval");- return { numerator, denominator };-}--function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {- const a = timestampFraction(origin);- const b = timestampFraction(value);- const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);- const denominator = b.denominator * a.denominator;- const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);- return numerator < 0n ? -magnitude : magnitude;-}--function stream<T extends MediaStream["kind"]>(- inspection: MediaInspection,- index: number | undefined,- kind: T,-): Extract<MediaStream, { readonly kind: T }> | undefined {- if (index === undefined) return undefined;- const result = inspection.streams.find((item) => item.index === index);- assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);- assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);- return result as Extract<MediaStream, { readonly kind: T }>;-}--type NormalizationPlan = {- readonly video?: MediaVideoStream;- readonly audio?: MediaAudioStream;- readonly origin: MediaTimestamp;- readonly end: MediaTimestamp;- readonly frameCount: number;- readonly sampleFrames: number;- readonly audioTrimStartSamples: number;- readonly audioTrimEndSamples: number;- readonly audioHeadSamples: number;- readonly audioContentSamples: number;- readonly audioTailSamples: number;-};--function normalizationPlan(- inspection: MediaInspection,- selection: MediaStreamSelection,- frameRate: MediaRational,-): NormalizationPlan {- assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0- && Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,- "Target frame rate is invalid");- const video = stream(inspection, selection.videoStreamIndex, "video");- const audio = stream(inspection, selection.audioStreamIndex, "audio");- if (video !== undefined && selection.spanAuthority !== "video") {- throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");- }- const authority = selection.spanAuthority === "video" ? video : audio;- assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");- const span = duration(authority);- const scaledNumerator = span.numerator * BigInt(frameRate.numerator);- const scaledDenominator = span.denominator * BigInt(frameRate.denominator);- const frameCount = selection.spanAuthority === "video"- ? roundPositive(scaledNumerator, scaledDenominator)- : ceilPositive(scaledNumerator, scaledDenominator);- assert(frameCount > 0, "Normalized media would contain no frame interval");- const sampleFrames = roundPositive(- BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),- BigInt(frameRate.numerator),- );- if (audio === undefined) {- return {- ...(video === undefined ? {} : { video }),- origin: authority.startPts,- end: authority.endPts,- frameCount,- sampleFrames,- audioTrimStartSamples: 0,- audioTrimEndSamples: 0,- audioHeadSamples: 0,- audioContentSamples: 0,- audioTailSamples: 0,- };- }- assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");- const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);- const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);- const contentStart = Math.max(0, audioStart);- const contentEnd = Math.min(sampleFrames, audioEnd);- assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");- return {- ...(video === undefined ? {} : { video }),- audio,- origin: authority.startPts,- end: authority.endPts,- frameCount,- sampleFrames,- audioTrimStartSamples: Math.max(0, -audioStart),- audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),- audioHeadSamples: contentStart,- audioContentSamples: contentEnd - contentStart,- audioTailSamples: sampleFrames - contentEnd,- };-}--async function inspectFile(args: {- readonly source: BlobRef;- readonly input: string;- readonly ffprobePath: string;- readonly timeoutMs: number;- readonly maxProbeOutputBytes: number;-}): Promise<MediaInspection> {- const [raw, ffprobeVersion] = await Promise.all([- runProcess({- executable: args.ffprobePath,- argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],- timeoutMs: args.timeoutMs,- maxStdoutBytes: args.maxProbeOutputBytes,- }),- version(args.ffprobePath, args.timeoutMs),- ]);- let value: unknown;- try {- value = JSON.parse(Buffer.from(raw).toString("utf8"));- } catch {- throw new Error("ffprobe returned invalid JSON");- }- return parseMediaInspection({ source: args.source, ffprobeVersion, value });-}--async function outputInspection(args: {- readonly path: string;- readonly mediaType: string;- readonly ffprobePath: string;- readonly timeoutMs: number;- readonly maxProbeOutputBytes: number;-}): Promise<MediaInspection> {- const bytes = await readFile(args.path);- const source: BlobRef = {- kind: "blob",- digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` as BlobRef["digest"],- size: bytes.byteLength,- mediaType: args.mediaType,- };- return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,- timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes });-}--function result(value: CanonicalValue, metadata: CanonicalValue): EndpointFulfillment {- return {- value: { kind: "inline", value },+ value: { kind: "inline", value: result.value },conformance: "exact",delivery: "executed",- metadata,+ metadata: result.metadata,};}-function inspectNeed(value: CanonicalValue): InspectMediaNeed {- const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;- assert(item.contract === "svml.inspect-media-request@1" && item.source?.kind === "blob",- "InspectMediaNeed is invalid");- return item;-}--function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {- const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;- assert(item.contract === "svml.normalize-media-request@1" && item.source?.kind === "blob",- "NormalizeMediaNeed is invalid");- verifyMediaInspection(item.inspection);- verifyMediaStreamSelection(item.selection);- assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2- && item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",- "NormalizeMediaNeed audio profile is unsupported");- return item;-}--function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {- const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;- assert(item.contract === "svml.project-speech-evidence-audio-request@1" && item.source?.kind === "blob",- "ProjectSpeechEvidenceAudioNeed is invalid");- assert(item.sourceSampleRate === 48_000 && item.sourceChannels === 2 && item.sourceCodec === "pcm_s16le"- && item.evidenceSampleRate === 16_000 && item.evidenceChannels === 1 && item.evidenceCodec === "pcm_s16le",- "Speech evidence audio shape is invalid");- assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0- && Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0- && item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),- "Speech evidence audio sample map is invalid");- return item;-}--function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {- const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;- assert(item.contract === "svml.render-audio-request@1", "RenderAudioNeed is invalid");- verifyAudioProgramPlan(item.plan);- return item;-}--function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {- const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;- assert(item.contract === "svml.mux-media-request@1", "MuxMediaNeed is invalid");- verifyRenderedVisual(item.visual);- verifyTimelineAudio(item.audio);- const expectedSamples = roundPositive(- BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),- BigInt(item.visual.frameRate.numerator),- );- assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");- return item;-}--function atempo(rate: number): string[] {- if (Math.abs(rate - 1) < 1e-12) return [];- const filters: string[] = [];- let remaining = rate;- while (remaining > 2) {- filters.push("atempo=2");- remaining /= 2;- }- while (remaining < 0.5) {- filters.push("atempo=0.5");- remaining /= 0.5;- }- if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);- return filters;-}--function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number): string {- const length = clip.targetEndSampleExclusive - clip.targetStartSample;- const filters = [- `atrim=start_sample=${clip.sourceStartSample}`,- "asetpts=PTS-STARTPTS",- ...atempo(clip.playbackRate),- `atrim=start_sample=0:end_sample=${length}`,- `apad=whole_len=${length}`,- `atrim=start_sample=0:end_sample=${length}`,- `volume=${clip.gain.toPrecision(15)}`,- ...(clip.fadeInSamples === 0 ? [] : [`afade=t=in:start_sample=0:nb_samples=${clip.fadeInSamples}`]),- ...(clip.fadeOutSamples === 0 ? [] : [- `afade=t=out:start_sample=${length - clip.fadeOutSamples}:nb_samples=${clip.fadeOutSamples}`,- ]),- `adelay=${clip.targetStartSample}S:all=1`,- ];- return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;-}--async function assertCanonicalWav(args: {- readonly path: string;- readonly source: BlobRef;- readonly ffprobePath: string;- readonly timeoutMs: number;- readonly maxProbeOutputBytes: number;-}): Promise<MediaAudioStream> {- const inspection = await inspectFile({- source: args.source,- input: args.path,- ffprobePath: args.ffprobePath,- timeoutMs: args.timeoutMs,- maxProbeOutputBytes: args.maxProbeOutputBytes,- });- const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");- assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");- assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,- "Canonical audio must be 48 kHz stereo PCM s16");- assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");- return audio[0]!;-}-+/**+ * The Build's own ArtifactStore, and whatever ffmpeg this machine has. The+ * operations themselves live in `@narratage/media-execution`, shared with the+ * AWS Provider so one Need cannot mean two different transforms.+ */export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions = {}) {const ffmpegPath = config.ffmpegPath ?? "ffmpeg";const ffprobePath = config.ffprobePath ?? "ffprobe";@@ -433,6 +58,19 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptionsconst maxProbeOutputBytes = positiveInteger(config.maxProbeOutputBytes ?? 256 * 1024 * 1024,"maxProbeOutputBytes");const common = { ffmpegPath, ffprobePath, processTimeoutMs, maxProbeOutputBytes };+ const environment = (context: EndpointInvocationContext): MediaExecutionEnvironment => ({+ ...common,+ label: "media.local",+ artifacts: {+ get: async (source) => await context.artifacts.get(source.digest),+ put: async (bytes, mediaType) => await context.artifacts.put(bytes, mediaType),+ },+ });+ const operation = (+ execute: (env: MediaExecutionEnvironment, constraints: never) => Promise<MediaOperationResult>,+ ) => async (context: EndpointInvocationContext): Promise<EndpointFulfillment> =>+ fulfillment(await execute(environment(context), context.need.constraints as never));+return defineEndpointPackage({module: localMediaProviderModuleRef,facet: "media",@@ -450,419 +88,37 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptionslifecycle: "immediate" as const,capability: mediaPipelineCapabilities.inspect,returns: mediaTypes.inspection,- supports: (need) => hasContract(need.constraints, "svml.inspect-media-request@1"),- handler: async (context) => {- const need = inspectNeed(context.need.constraints);- const work = await mkdtemp(join(tmpdir(), "svml-media-inspect-"));- try {- const input = join(work, "source.bin");- await writeFile(input, await sourceBytes(context, need.source));- const inspection = await inspectFile({- source: need.source,- input,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- return result(canonicalize(inspection), canonicalize({ provider: "media.local", operation: "inspect" }));- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }- },+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.inspect),+ handler: operation(executeInspectMedia),},{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.normalize,returns: mediaTypes.synchronized,- supports: (need) => hasContract(need.constraints, "svml.normalize-media-request@1"),- handler: async (context) => {- const need = normalizeNeed(context.need.constraints);- const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);- const work = await mkdtemp(join(tmpdir(), "svml-media-normalize-"));- try {- const input = join(work, "source.bin");- await writeFile(input, await sourceBytes(context, need.source));- let visualArtifact: BlobRef | undefined;- let visualWidth: number | undefined;- let visualHeight: number | undefined;- if (plan.video !== undefined) {- const output = join(work, "visual.mp4");- const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;- const filter = [- "setpts=PTS-STARTPTS",- `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,- `trim=start_frame=0:end_frame=${plan.frameCount}`,- `setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,- "scale=trunc(iw/2)*2:trunc(ih/2)*2",- ].join(",");- await runProcess({- executable: ffmpegPath,- argv: ["-y", "-i", input, "-map", `0:${plan.video.index}`, "-an", "-vf", filter,- "-frames:v", String(plan.frameCount), "-fps_mode", "cfr", "-c:v", "libx264",- "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", output],- timeoutMs: processTimeoutMs,- maxStdoutBytes: 64 * 1024,- });- const inspected = await outputInspection({- path: output,- mediaType: "video/mp4",- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");- assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",- "Normalized visual frame shape differs from its plan");- const bytes = await readFile(output);- visualArtifact = await context.artifacts.put(bytes, "video/mp4");- visualWidth = visual.width;- visualHeight = visual.height;- }- let audioArtifact: BlobRef | undefined;- if (plan.audio !== undefined) {- const output = join(work, "audio.wav");- const filter = [- "asetpts=PTS-STARTPTS",- "aresample=48000:async=0:first_pts=0",- "aformat=sample_rates=48000:channel_layouts=stereo",- `atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,- "asetpts=N/SR/TB",- `adelay=${plan.audioHeadSamples}S:all=1`,- `apad=whole_len=${plan.sampleFrames}`,- `atrim=start_sample=0:end_sample=${plan.sampleFrames}`,- "asetpts=N/SR/TB",- ].join(",");- await runProcess({- executable: ffmpegPath,- argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,- "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],- timeoutMs: processTimeoutMs,- maxStdoutBytes: 64 * 1024,- });- const inspected = await outputInspection({- path: output,- mediaType: "audio/wav",- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");- assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,- "Normalized audio sample shape differs from its plan");- audioArtifact = await context.artifacts.put(await readFile(output), "audio/wav");- }- const media = sealSynchronizedMedia({- contract: "svml.synchronized-media@1",- timeline: {- spanAuthority: need.selection.spanAuthority,- frameRate: need.frameRate,- frameCount: plan.frameCount,- sampleRate: 48_000,- sampleFrames: plan.sampleFrames,- },- sourceMap: {- sourceOriginPts: plan.origin,- sourceEndPts: plan.end,- audioTrimStartSamples: plan.audioTrimStartSamples,- audioTrimEndSamples: plan.audioTrimEndSamples,- audioHeadSamples: plan.audioHeadSamples,- audioContentSamples: plan.audioContentSamples,- audioTailSamples: plan.audioTailSamples,- },- ...(visualArtifact === undefined ? {} : {- visual: {- artifact: visualArtifact,- sourceStreamIndex: plan.video!.index,- width: visualWidth!,- height: visualHeight!,- frameRate: need.frameRate,- frameCount: plan.frameCount,- muted: true as const,- },- }),- ...(audioArtifact === undefined ? {} : {- audio: {- artifact: audioArtifact,- sourceStreamIndex: plan.audio!.index,- codec: "pcm_s16le" as const,- sampleRate: 48_000 as const,- channels: 2 as const,- sampleFrames: plan.sampleFrames,- loudness: "preserved" as const,- },- }),- });- return result(canonicalize(media), canonicalize({ provider: "media.local", operation: "normalize",- visual: visualArtifact !== undefined, audio: audioArtifact !== undefined }));- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }- },+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.normalize),+ handler: operation(executeNormalizeMedia),},{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,returns: speechTypes.evidenceAudio,- supports: (need) => hasContract(need.constraints, "svml.project-speech-evidence-audio-request@1"),- handler: async (context) => {- const need = evidenceAudioNeed(context.need.constraints);- const work = await mkdtemp(join(tmpdir(), "svml-media-speech-evidence-"));- try {- const input = join(work, "speech-master.wav");- const output = join(work, "alignment-evidence.wav");- await stageArtifact(context, need.source, input);- const source = await assertCanonicalWav({- path: input,- source: need.source,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- assert(source.decodedSampleFrames === need.sourceSampleFrames,- "Speech master sample count differs from its ProgramSpace");- const filter = [- "asetpts=N/SR/TB",- "aresample=16000:async=0:first_pts=0",- "aformat=sample_rates=16000:channel_layouts=mono",- `apad=whole_len=${need.evidenceSampleFrames}`,- `atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,- "asetpts=N/SR/TB",- ].join(",");- await runProcess({- executable: ffmpegPath,- argv: ["-y", "-i", input, "-vn", "-af", filter,- "-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],- timeoutMs: processTimeoutMs,- maxStdoutBytes: 64 * 1024,- });- const inspected = await outputInspection({- path: output,- mediaType: "audio/wav",- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");- assert(streams.length === 1 && inspected.streams.length === 1- && streams[0]!.codecName === "pcm_s16le"- && streams[0]!.sampleRate === 16_000- && streams[0]!.channels === 1- && streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,- "Alignment evidence must be exact 16 kHz mono PCM s16");- const artifact = await context.artifacts.put(await readFile(output), "audio/wav");- const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({- contract: "svml.speech-evidence-audio@1",- artifact,- codec: "pcm_s16le",- sampleRate: 16_000,- channels: 1,- sampleFrames: need.evidenceSampleFrames,- durationSec: need.durationSec,- segments: need.segments,- sampleMap: {- algorithm: "rational-boundary-round@1",- sourceSampleRate: 48_000,- evidenceSampleRate: 16_000,- sourceSampleFrames: need.sourceSampleFrames,- evidenceSampleFrames: need.evidenceSampleFrames,- sourceOriginSample: 0,- evidenceOriginSample: 0,- },- });- assertSpeechEvidenceAudioIdentity(evidence);- return result(canonicalize(evidence), canonicalize({- provider: "media.local",- operation: "project-speech-evidence-audio",- sourceSampleFrames: need.sourceSampleFrames,- evidenceSampleFrames: need.evidenceSampleFrames,- }));- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }- },+ supports: (need) =>+ mediaNeedHasContract(need.constraints, mediaOperationContracts.projectSpeechEvidenceAudio),+ handler: operation(executeProjectSpeechEvidenceAudio),},{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.renderAudio,returns: mediaTypes.timelineAudio,- supports: (need) => hasContract(need.constraints, "svml.render-audio-request@1"),- handler: async (context) => {- const need = renderAudioNeed(context.need.constraints);- const plan: AudioProgramPlan = need.plan;- const work = await mkdtemp(join(tmpdir(), "svml-media-audio-"));- try {- const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number }>();- for (const clip of plan.clips) {- if (artifacts.has(clip.artifact.digest)) continue;- const inputIndex = artifacts.size;- const path = join(work, `input-${inputIndex}.wav`);- await stageArtifact(context, clip.artifact, path);- const audio = await assertCanonicalWav({- path,- source: clip.artifact,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- assert(audio.decodedSampleFrames > 0, `Audio input ${clip.artifact.digest} is empty`);- artifacts.set(clip.artifact.digest, { source: clip.artifact, path, inputIndex });- }-- const output = join(work, "program.wav");- const argv = ["-y"];- for (const item of artifacts.values()) argv.push("-i", item.path);- if (plan.clips.length === 0) {- argv.push(- "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",- "-af", `atrim=start_sample=0:end_sample=${plan.sampleFrames},asetpts=N/SR/TB`,- );- } else {- const chains = plan.clips.map((clip, index) => {- const inputIndex = artifacts.get(clip.artifact.digest)!.inputIndex;- return audioClipFilter(clip, inputIndex, index);- });- const labels = plan.clips.map((_clip, index) => `[clip${index}]`).join("");- chains.push(- `${labels}amix=inputs=${plan.clips.length}:duration=longest:dropout_transition=0:normalize=0,`- + `apad=whole_len=${plan.sampleFrames},atrim=start_sample=0:end_sample=${plan.sampleFrames},`- + "asetpts=N/SR/TB[out]",- );- argv.push("-filter_complex", chains.join(";"), "-map", "[out]");- }- argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);- await runProcess({- executable: ffmpegPath,- argv,- timeoutMs: processTimeoutMs,- maxStdoutBytes: 64 * 1024,- });- const bytes = await readFile(output);- const artifact = await context.artifacts.put(bytes, "audio/wav");- const audio = await assertCanonicalWav({- path: output,- source: artifact,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- assert(audio.decodedSampleFrames === plan.sampleFrames,- "Rendered TimelineAudio sample count differs from its plan");- const value: TimelineAudio = sealTimelineAudio({- contract: "svml.timeline-audio@1",- artifact,- codec: "pcm_s16le",- sampleRate: 48_000,- channels: 2,- sampleFrames: plan.sampleFrames,- loudness: "planned",- });- return result(canonicalize(value), canonicalize({- provider: "media.local",- operation: "render-audio",- clips: plan.clips.length,- }));- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }- },+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.renderAudio),+ handler: operation(executeRenderTimelineAudio),},{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.mux,returns: mediaTypes.muxed,- supports: (need) => hasContract(need.constraints, "svml.mux-media-request@1"),- handler: async (context) => {- const need = muxMediaNeed(context.need.constraints);- const work = await mkdtemp(join(tmpdir(), "svml-media-mux-"));- try {- const visualPath = join(work, "visual.mp4");- const audioPath = join(work, "audio.wav");- const output = join(work, "final.mp4");- await Promise.all([- stageArtifact(context, need.visual.artifact, visualPath),- stageArtifact(context, need.audio.artifact, audioPath),- ]);- const [visualInspection, audio] = await Promise.all([- inspectFile({- source: need.visual.artifact,- input: visualPath,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- }),- assertCanonicalWav({- path: audioPath,- source: need.audio.artifact,- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- }),- ]);- const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");- assert(visualStreams.length === 1 && visualInspection.streams.length === 1,- "RenderedVisual Artifact must contain exactly one silent video stream");- const visual = visualStreams[0]!;- assert(visual.role === "moving" && visual.timingStatus === "admissible"- && visual.decodedUnitCount === need.visual.frameCount- && visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,- "RenderedVisual bytes differ from their declared frame domain");- assert(audio.decodedSampleFrames === need.audio.sampleFrames,- "TimelineAudio bytes differ from their declared sample domain");- await runProcess({- executable: ffmpegPath,- argv: [- "-y", "-i", visualPath, "-i", audioPath,- "-map", `0:${visual.index}`, "-map", "1:0",- "-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",- "-frames:v", String(need.visual.frameCount),- "-movflags", "+faststart", output,- ],- timeoutMs: processTimeoutMs,- maxStdoutBytes: 64 * 1024,- });- const finalInspection = await outputInspection({- path: output,- mediaType: "video/mp4",- ffprobePath,- timeoutMs: processTimeoutMs,- maxProbeOutputBytes,- });- const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");- const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");- assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,- "Final mux must contain exactly one video and one audio stream");- assert(finalVideo[0]!.timingStatus === "admissible"- && finalVideo[0]!.decodedUnitCount === need.visual.frameCount,- "Final mux video frame count differs from RenderedVisual");- assert(finalAudio[0]!.timingStatus === "admissible"- && finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,- "Final mux audio shape differs from TimelineAudio");- assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined- && compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,- "Final mux audio and video do not share one presentation origin");- const finalAudioSpan = duration(finalAudio[0]!);- assert(roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator)- === need.audio.sampleFrames,- "Final mux audio presentation span differs from TimelineAudio");- const artifact = await context.artifacts.put(await readFile(output), "video/mp4");- const value: MuxedMedia = sealMuxedMedia({- contract: "svml.muxed-media@1",- frameRate: need.visual.frameRate,- frameCount: need.visual.frameCount,- canvas: need.visual.canvas,- presentationSampleFrames: need.audio.sampleFrames,- artifact,- });- return result(canonicalize(value), canonicalize({- provider: "media.local",- operation: "mux",- videoStream: finalVideo[0]!.index,- audioStream: finalAudio[0]!.index,- }));- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }- },+ supports: (need) => mediaNeedHasContract(need.constraints, mediaOperationContracts.mux),+ handler: operation(executeMuxProgramMedia),},],});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex c417f3a4..25a275fb 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -722,6 +722,21 @@ importers:specifier: workspace:*version: link:../text+ packages/media-execution:+ dependencies:+ '@narratage/media':+ specifier: workspace:*+ version: link:../media+ '@narratage/media-pipeline':+ specifier: workspace:*+ version: link:../media-pipeline+ '@narratage/protocol':+ specifier: workspace:*+ version: link:../protocol+ '@narratage/speech':+ specifier: workspace:*+ version: link:../speech+packages/media-pipeline:dependencies:'@narratage/artifact':@@ -1060,6 +1075,9 @@ importers:'@narratage/media':specifier: workspace:*version: link:../media+ '@narratage/media-execution':+ specifier: workspace:*+ version: link:../media-execution'@narratage/media-pipeline':specifier: workspace:*version: link:../media-pipelinediff --git a/tsconfig.json b/tsconfig.jsonindex 44690bb9..4132a108 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -141,6 +141,7 @@"@narratage/provider-kie": ["packages/provider-kie/src/index.ts"],+ "@narratage/media-execution": ["packages/media-execution/src/index.ts"],"@narratage/media-pipeline": ["packages/media-pipeline/src/index.ts"],