Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor: consolidate media and image execution owners

kashorincc0350286285
diff --git a/packages/background-removal/README.md b/packages/background-removal/README.md
deleted file mode 100644
index fc8b468c..00000000
--- a/packages/background-removal/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-# @hypit/background-removal
-
-Declares one external image capability: turn an input image into an image with its background removed.
-Use it for a product, portrait or graphic that will be composited over another picture.
-
-```xml
-<import as="remove" from="@hypit/background-removal@1"/>
-<remove:Background id="cutout" source={portrait.image}/>
-```
-
-`source` accepts an image BlobArtifact, including an ordinary image file declaration or a generated
-image Output. `{cutout.image}` publishes one image BlobArtifact with transparency. Keep its actual
-dimensions when placing it as a Visual Clip, or use Image Compose when a flattened still is wanted.
-
-The package declares the visual operation; a project Provider implements the capability
-`@hypit/background-removal@1#remove-background`, accepting `{ source: BlobRef }` and returning one
-image BlobRef with transparency. The Provider owns its service's request mapping and task lifecycle.
-A service-specific author Model is unnecessary when the operation has this same meaning.
-
-For a moving portrait, [Volcengine Matting](../volcengine-matting/README.md) provides
-`<matte:Portrait source={performance.video}/>` through HypiHub. Normalize its processed video
-before using it in a track.
-
-This capability handles still images. Moving-person background removal requires a video-capable
-operation and a result that preserves the changing silhouette and frame correspondence. Removing
-one portrait's background does not remove the background of a subsequently generated performance.
diff --git a/packages/background-removal/package.json b/packages/background-removal/package.json
deleted file mode 100644
index 4f30d2df..00000000
--- a/packages/background-removal/package.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "@hypit/background-removal",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": { ".": "./src/index.ts" },
- "hypit": { "activation": "./src/activation.ts" },
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/component-kit": "workspace:*",
- "@hypit/elaborator": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/markup": "workspace:*"
- }
-}
diff --git a/packages/background-removal/src/activation.ts b/packages/background-removal/src/activation.ts
deleted file mode 100644
index e9a8a677..00000000
--- a/packages/background-removal/src/activation.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { createMarkupSurfaceHostFacet } from "@hypit/markup";
-
-import {
- backgroundRemovalComponent, backgroundRemovalManifest,
- backgroundRemovalModuleRef, decodeBackgroundRemovalSurface,
- backgroundRemovalMarkupSurfaces,
-} from "./index.js";
-
-export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- modules: [{ manifest: backgroundRemovalManifest }],
- components: [backgroundRemovalComponent],
- hostFacets: [{ ...createMarkupSurfaceHostFacet({
- module: backgroundRemovalModuleRef,
- declaration: backgroundRemovalMarkupSurfaces.find((item) => item.name === "background")!, handler: decodeBackgroundRemovalSurface,
- }) }],
-};
-export default hypitPackage;
diff --git a/packages/background-removal/src/component.ts b/packages/background-removal/src/component.ts
deleted file mode 100644
index 2182afa0..00000000
--- a/packages/background-removal/src/component.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import { plannedNeedInputs } from "@hypit/component-kit";
-import type { ComponentPackage, ProducerHandlerContext } from "@hypit/component-kit";
-import type { StoredValue } from "@hypit/protocol";
-
-import { backgroundRemovalCapabilities, backgroundRemovalProducers } from "./manifest.js";
-import { backgroundRemovalRequest } from "./program.js";
-
-function blob(value: StoredValue | undefined) {
- if (value?.kind !== "blob") throw new Error("Background Removal source must be a Blob Artifact.");
- return value;
-}
-
-export const backgroundRemovalComponent = {
- producers: [{
- producer: backgroundRemovalProducers.request,
- handler: ({ inputs }: ProducerHandlerContext) => ({
- outputs: {}, needs: { image: backgroundRemovalRequest(blob(inputs.source?.value)) },
- }),
- }],
- plannedNeeds: [{
- producer: backgroundRemovalProducers.request,
- port: "image",
- capability: backgroundRemovalCapabilities.remove,
- plan: ({ state, step }) => ({
- constraints: {},
- pendingInputs: plannedNeedInputs(state, step, { source: "image" }),
- }),
- present: (specification) => ({ fields: {}, references: {
- image: specification.pendingInputs.filter((input) => input.role === "image").length,
- } }),
- }],
-} satisfies ComponentPackage;
diff --git a/packages/background-removal/src/fragment.ts b/packages/background-removal/src/fragment.ts
deleted file mode 100644
index 6d46460a..00000000
--- a/packages/background-removal/src/fragment.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { artifactTypes } from "@hypit/artifact";
-import { sealGraphFragment } from "@hypit/elaborator";
-
-import { backgroundRemovalProducers } from "./manifest.js";
-
-const input = (name: string) => ({ kind: "fragment-input" as const, name });
-const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
-
-export const backgroundRemovalFragment = sealGraphFragment({
- inputs: [{ name: "source", type: artifactTypes.blob }],
- operations: [{
- id: "image:remove-background", producer: backgroundRemovalProducers.request,
- inputs: { source: input("source") }, result: { kind: "need", name: "image" },
- }],
- exports: [{
- name: "image", type: artifactTypes.blob, root: operation("image:remove-background"),
- }],
-});
diff --git a/packages/background-removal/src/index.ts b/packages/background-removal/src/index.ts
deleted file mode 100644
index 16f9be19..00000000
--- a/packages/background-removal/src/index.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-export { backgroundRemovalComponent } from "./component.js";
-export { backgroundRemovalFragment } from "./fragment.js";
-export { backgroundRemovalCapabilities, backgroundRemovalDependency, backgroundRemovalManifest, backgroundRemovalMarkupSurfaces, backgroundRemovalModuleRef, backgroundRemovalProducers } from "./manifest.js";
-export { assertBackgroundRemovalRequest, backgroundRemovalRequest } from "./program.js";
-export { decodeBackgroundRemovalSurface } from "./surface.js";
-export type * from "./types.js";
diff --git a/packages/background-removal/src/manifest.ts b/packages/background-removal/src/manifest.ts
deleted file mode 100644
index f6e7ecea..00000000
--- a/packages/background-removal/src/manifest.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-import { artifactDependency, artifactTypes } from "@hypit/artifact";
-import type { CapabilityRef, ModuleManifest, ProducerRef } from "@hypit/protocol";
-
-export const backgroundRemovalModuleRef = { name: "@hypit/background-removal", version: "1" } as const;
-export const backgroundRemovalCapabilities = {
- remove: { module: backgroundRemovalModuleRef, name: "remove-background" },
-} satisfies Record<string, CapabilityRef>;
-export const backgroundRemovalProducers = {
- request: { module: backgroundRemovalModuleRef, name: "request-background-removal" },
-} satisfies Record<string, ProducerRef>;
-
-export const backgroundRemovalMarkupSurfaces = [{
- name: "background", tag: "Background", mode: "structured", outputs: [artifactTypes.blob],
- vocabulary: {
- summary: "Removes the background from one image Artifact and publishes the cut-out image.",
- attributes: [
- { name: "id", kind: "identifier", required: true,
- summary: "Names this removal so its image can be referenced elsewhere in the Source." },
- { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],
- summary: "Chooses the image whose background is removed." },
- ],
- ports: [
- { name: "image", type: artifactTypes.blob,
- summary: "The source image with its background removed." },
- ],
- example: `<remove:Background id="cutout" source={portrait.image}/>`,
- notes: [
- "The element is empty; it accepts no children and no text.",
- "The package chooses no model, threshold or storage — the selected Endpoint fulfills the capability.",
- ],
- },
- }] as const;
-
-
-export const backgroundRemovalManifest: ModuleManifest = {
- format: "hypit.module@1", name: backgroundRemovalModuleRef.name, version: backgroundRemovalModuleRef.version,
- dependencies: [artifactDependency], types: [],
- capabilities: [{ name: backgroundRemovalCapabilities.remove.name, returns: artifactTypes.blob }],
- producers: [{
- name: backgroundRemovalProducers.request.name,
- inputs: [{ name: "source", type: artifactTypes.blob }], outputs: [],
- needs: [{ name: "image", capability: backgroundRemovalCapabilities.remove, returns: artifactTypes.blob }],
- }],
-};
-export const backgroundRemovalDependency = { module: backgroundRemovalModuleRef } as const;
diff --git a/packages/background-removal/src/program.ts b/packages/background-removal/src/program.ts
deleted file mode 100644
index f8f7808d..00000000
--- a/packages/background-removal/src/program.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import { canonicalize, isResourceId } from "@hypit/protocol";
-
-import type { BackgroundRemovalRequest } from "./types.js";
-
-function assert(condition: unknown, message: string): asserts condition {
- if (!condition) throw new Error(message);
-}
-
-export function assertBackgroundRemovalRequest(value: BackgroundRemovalRequest): void {
- assert(value.source.kind === "blob" && isResourceId(value.source.resource)
- && Number.isSafeInteger(value.source.size) && value.source.size >= 0
- && value.source.mediaType.startsWith("image/"), "Background Removal source must be an image Blob Artifact.");
-}
-
-export function backgroundRemovalRequest(source: BackgroundRemovalRequest["source"]): BackgroundRemovalRequest {
- const value = { source };
- assertBackgroundRemovalRequest(value);
- return canonicalize(value) as unknown as BackgroundRemovalRequest;
-}
diff --git a/packages/background-removal/src/surface.ts b/packages/background-removal/src/surface.ts
deleted file mode 100644
index 05bed3b1..00000000
--- a/packages/background-removal/src/surface.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { artifactTypes } from "@hypit/artifact";
-import type { StructuredSurfaceHandler, SurfaceResolvedReference, MarkupAttributeValue } from "@hypit/markup";
-
-import { backgroundRemovalFragment } from "./fragment.js";
-
-function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
- return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
-}
-
-export const decodeBackgroundRemovalSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- const unknown = Object.keys(element.attributes).filter((name) => !["id", "source"].includes(name));
- if (unknown.length > 0) throw new Error(`${element.name} does not accept ${unknown[0]}.`);
- if (element.children.some((child) => child.kind === "element" || child.value.trim())) throw new Error(`${element.name} must be empty.`);
- const id = element.attributes.id;
- if (typeof id !== "string" || !id.trim()) throw new Error(`${element.name}.id must be text.`);
- const raw: MarkupAttributeValue | undefined = element.attributes.source;
- if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${element.name}.source must be a reference.`);
- const source = resolveReference(raw.path);
- if (source === undefined || !sameType(source.type, artifactTypes.blob)) throw new Error(`${element.name}.source must be a Blob Artifact.`);
- return {
- records: [], fragments: [backgroundRemovalFragment],
- components: [{
- id: id.trim(), fragment: backgroundRemovalFragment.id, inputs: { source: source.ref },
- outputs: { image: `${id.trim()}.image` }, range: element.range,
- }],
- };
-};
diff --git a/packages/background-removal/src/types.ts b/packages/background-removal/src/types.ts
deleted file mode 100644
index ab4b34e7..00000000
--- a/packages/background-removal/src/types.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-import type { BlobRef } from "@hypit/protocol";
-
-export type BackgroundRemovalRequest = {
- readonly source: BlobRef;
-};
diff --git a/packages/background-removal/test/background-removal.test.ts b/packages/background-removal/test/background-removal.test.ts
deleted file mode 100644
index 0fd85326..00000000
--- a/packages/background-removal/test/background-removal.test.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-import { fixtureResource } from "../../../test/fixture-resource.js";
-
-import { artifactTypes } from "@hypit/artifact";
-import {
- backgroundRemovalCapabilities, backgroundRemovalComponent, backgroundRemovalFragment, backgroundRemovalManifest,
-} from "@hypit/background-removal";
-
-test("Background Removal is one image edge to one exact external Need", async () => {
- assert.deepEqual(backgroundRemovalFragment.inputs.map((input) => input.name), ["source"]);
- assert.deepEqual(backgroundRemovalFragment.exports.map((output) => output.name), ["image"]);
- assert.deepEqual(backgroundRemovalManifest.capabilities, [{ name: "remove-background", returns: artifactTypes.blob }]);
- const source = { kind: "blob" as const, resource: fixtureResource("portrait"), size: 456, mediaType: "image/jpeg" };
- const result = await backgroundRemovalComponent.producers[0]!.handler({ inputs: { source: { value: source } } } as never);
- assert.deepEqual(result.needs.image, { source });
- assert.deepEqual(backgroundRemovalCapabilities.remove.module.version, "1");
-});
diff --git a/packages/image-compose/README.md b/packages/image-compose/README.md
deleted file mode 100644
index 0d2f674c..00000000
--- a/packages/image-compose/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# @hypit/image-compose
-
-Deterministically flatten a fixed arrangement of existing still images into one PNG.
-
-The author graph supplies an explicit Canvas, an ordered list of image Layers, and an explicit Frame for every Layer. The package lowers that meaning to the shared `@hypit/raster` execution contract; an Endpoint such as `@hypit/provider-image-opencv-local` performs the pixels. Core knows nothing about images or compositing.
-
-```xml
-<import as="space" from="@hypit/spatial@1"/>
-<import as="compose" from="@hypit/image-compose@1"/>
-
-<space:Canvas id="comparison-canvas" width="2048" height="1024"/>
-<space:Frame id="before-panel" within={comparison-canvas.bounds}
- left="0%" top="0%" right="50%" bottom="100%"/>
-<space:Frame id="after-panel" within={comparison-canvas.bounds}
- left="50%" top="0%" right="100%" bottom="100%"/>
-
-<compose:Image id="comparison" canvas={comparison-canvas.canvas} background="#EEEAE2FF">
- <compose:Layer source={before.image} frame={before-panel} fit="contain"/>
- <compose:Layer source={after.image} frame={after-panel} fit="contain"/>
-</compose:Image>
-```
-
-Child order is paint order. Frames may extend beyond the Canvas and are clipped. Version 1 always produces a PNG using normal alpha compositing; it deliberately has no implicit base image, layout, or metadata propagation.
-
-This operation does no visual reasoning. It does not reconcile perspective, depth, light, subject scale, scene continuity or a natural seam between its inputs. Use an image-generation or image-editing model when the desired result should become one newly directed camera image. Keep independently timed or editable layers in Tracks and Film. Image Compose is for the narrower case where the intended result is precisely the fixed two-dimensional arrangement itself.
diff --git a/packages/image-compose/package.json b/packages/image-compose/package.json
deleted file mode 100644
index 983bbdc0..00000000
--- a/packages/image-compose/package.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "name": "@hypit/image-compose",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": { ".": "./src/index.ts" },
- "hypit": { "activation": "./src/activation.ts" },
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/component-kit": "workspace:*",
- "@hypit/elaborator": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/raster": "workspace:*",
- "@hypit/spatial": "workspace:*",
- "@hypit/markup": "workspace:*"
- }
-}
diff --git a/packages/image-compose/src/activation.ts b/packages/image-compose/src/activation.ts
deleted file mode 100644
index 7bc500e6..00000000
--- a/packages/image-compose/src/activation.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { createMarkupSurfaceHostFacet } from "@hypit/markup";
-
-import {
- decodeImageComposeSurface,
- imageComposeComponent,
- imageComposeManifest,
- imageComposeModuleRef,
- imageComposeMarkupSurfaces,
-} from "./index.js";
-
-export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- modules: [{ manifest: imageComposeManifest }],
- components: [imageComposeComponent],
- hostFacets: [{ ...createMarkupSurfaceHostFacet({
- module: imageComposeModuleRef,
- declaration: imageComposeMarkupSurfaces.find((item) => item.name === "image")!, handler: decodeImageComposeSurface,
- }) }],
-};
-
-export default hypitPackage;
diff --git a/packages/image-compose/src/index.ts b/packages/image-compose/src/index.ts
deleted file mode 100644
index bfc47d1a..00000000
--- a/packages/image-compose/src/index.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-export { imageComposeComponent } from "./component.js";
-export { createImageComposeFragment } from "./fragment.js";
-export { imageComposeDependency, imageComposeManifest, imageComposeMarkupSurfaces, imageComposeModuleRef, imageComposeProducers, imageComposeTypes } from "./manifest.js";
-export {
- appendImageComposeLayer, assertImageComposeLayerSet, assertImageComposeLayerSpec, assertImageComposeOptions,
- createImageComposeLayerSet,
- sealImageComposeLayerSpec, sealImageComposeOptions,
-} from "./program.js";
-export { decodeImageComposeSurface } from "./surface.js";
-export type * from "./types.js";
diff --git a/packages/image-compose/src/manifest.ts b/packages/image-compose/src/manifest.ts
deleted file mode 100644
index 2d3ba0dc..00000000
--- a/packages/image-compose/src/manifest.ts
+++ /dev/null
@@ -1,119 +0,0 @@
-import { artifactDependency, artifactTypes } from "@hypit/artifact";
-import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/protocol";
-import { rasterCapabilities, rasterDependency } from "@hypit/raster";
-import { spatialDependency, spatialFrameSchema, spatialTypes } from "@hypit/spatial";
-
-export const imageComposeModuleRef = { name: "@hypit/image-compose", version: "1" } as const;
-export const imageComposeTypes = {
- options: { module: imageComposeModuleRef, name: "ImageComposeOptions" },
- layerSpec: { module: imageComposeModuleRef, name: "ImageComposeLayerSpec" },
- layerSet: { module: imageComposeModuleRef, name: "ImageComposeLayerSet" },
-} satisfies Record<string, TypeRef>;
-export const imageComposeProducers = {
- createLayers: { module: imageComposeModuleRef, name: "create-image-compose-layers" },
- appendLayer: { module: imageComposeModuleRef, name: "append-image-compose-layer" },
- request: { module: imageComposeModuleRef, name: "request-image-compose" },
-} satisfies Record<string, ProducerRef>;
-
-const string = { kind: "string", minLength: 1 } as const;
-const number = { kind: "number" } as const;
-const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>): ValueSchema => ({ kind: "object", fields });
-const blob = object({
- kind: { schema: { kind: "literal", value: "blob" } },
- resource: { schema: { kind: "string", minLength: 5, maxLength: 256 } },
- size: { schema: { kind: "number", integer: true, minimum: 0 } },
- mediaType: { schema: string },
-});
-export const imageComposeOptionsSchema: ValueSchema = object({
-
- background: { schema: string },
-});
-export const imageComposeLayerSpecSchema: ValueSchema = object({
-
- fit: { schema: { kind: "string", enum: ["contain", "cover", "stretch"] } },
- interpolation: { schema: { kind: "string", enum: ["nearest", "linear", "cubic", "area", "lanczos"] } },
- opacity: { schema: { kind: "number", minimum: 0, maximum: 1 } },
-});
-const layer = object({
- source: { schema: blob },
- frame: { schema: spatialFrameSchema },
- spec: { schema: imageComposeLayerSpecSchema },
-});
-export const imageComposeLayerSetSchema: ValueSchema = object({
-
- layers: { schema: { kind: "array", maxItems: 64, items: layer } },
-});
-
-export const imageComposeMarkupSurfaces = [{
- name: "image", tag: "Image", mode: "structured",
- outputs: [imageComposeTypes.options, imageComposeTypes.layerSpec, artifactTypes.blob],
- vocabulary: {
- summary: "Flattens ordered still-image Layers at explicit rectangular Frames into one PNG Artifact.",
- attributes: [
- { name: "id", kind: "identifier", required: true,
- summary: "Names this composition so its image can be referenced elsewhere in the Source." },
- { name: "canvas", kind: "reference", required: true, accepts: [spatialTypes.canvas],
- summary: "Chooses the Canvas every Layer is painted onto." },
- { name: "background", kind: "literal", required: false,
- summary: "Sets the color the Canvas is cleared to before the first Layer is painted." },
- ],
- children: [
- { tag: "Layer", cardinality: "many",
- summary: "One image painted into its own Frame, in document order, and empty of children and text.",
- attributes: [
- { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],
- summary: "Chooses the image Artifact this Layer paints." },
- { name: "frame", kind: "reference", required: true, accepts: [spatialTypes.frame],
- summary: "Chooses the Frame on the Canvas the image is painted into." },
- { name: "fit", kind: "literal", required: false, values: ["contain", "cover", "stretch"],
- summary: "Decides how the image is sized to its Frame; defaults to `contain`." },
- { name: "interpolation", kind: "literal", required: false,
- values: ["nearest", "linear", "cubic", "area", "lanczos"],
- summary: "Decides which filter resamples the image while it is scaled; defaults to `lanczos`." },
- { name: "opacity", kind: "literal", required: false,
- summary: "Sets how strongly this Layer covers what is beneath it, from 0 to 1; defaults to 1." },
- ] },
- ],
- ports: [
- { name: "image", type: artifactTypes.blob,
- summary: "The composed picture, a PNG." },
- ],
- example: `<compose:Image id="comparison" canvas={comparison-canvas.canvas} background="#EEEAE2FF">
- <compose:Layer source={before.image} frame={before-panel} fit="contain"/>
- <compose:Layer source={after.image} frame={after-panel} fit="contain"/>
-</compose:Image>`,
- notes: [
- "`background` is written as `#RRGGBBAA` and defaults to `#00000000`.",
- "The composition requires at least one Layer and holds at most 64.",
- "Child order is paint order, and a Frame that extends beyond the Canvas is clipped.",
- "The operation performs rectangular raster placement only; it does not redesign a scene or create timed layers.",
- ],
- },
- }] as const;
-
-
-export const imageComposeManifest: ModuleManifest = {
- format: "hypit.module@1",
- name: imageComposeModuleRef.name,
- version: imageComposeModuleRef.version,
- dependencies: [artifactDependency, spatialDependency, rasterDependency],
- types: [
- { name: imageComposeTypes.options.name },
- { name: imageComposeTypes.layerSpec.name },
- { name: imageComposeTypes.layerSet.name },
- ],
- capabilities: [],
- producers: [
- { name: imageComposeProducers.createLayers.name, inputs: [], outputs: [{ name: "layers", type: imageComposeTypes.layerSet }], needs: [] },
- { name: imageComposeProducers.appendLayer.name, inputs: [
- { name: "layers", type: imageComposeTypes.layerSet }, { name: "source", type: artifactTypes.blob },
- { name: "frame", type: spatialTypes.frame }, { name: "spec", type: imageComposeTypes.layerSpec },
- ], outputs: [{ name: "layers", type: imageComposeTypes.layerSet }], needs: [] },
- { name: imageComposeProducers.request.name, inputs: [
- { name: "canvas", type: spatialTypes.canvas }, { name: "options", type: imageComposeTypes.options },
- { name: "layers", type: imageComposeTypes.layerSet },
- ], outputs: [], needs: [{ name: "image", capability: rasterCapabilities.execute, returns: artifactTypes.blob }] },
- ],
-};
-
-export const imageComposeDependency = { module: imageComposeModuleRef } as const;
diff --git a/packages/image-operations/README.md b/packages/image-operations/README.md
new file mode 100644
index 00000000..9bcb9ec7
--- /dev/null
+++ b/packages/image-operations/README.md
@@ -0,0 +1,33 @@
+# `@hypit/image-operations`
+
+Official deterministic still-image operations for Hypit. One installed package owns two separate
+author meanings and two separate Provider capabilities:
+
+- `Transform`: one image plus an ordered `ImageTransformProgram` becomes one image through
+ `transform-image`;
+- `Compose`: explicit Canvas, Frames and ordered Layers become one PNG through `compose-image`.
+
+They share one package because they have the same owner, installation decision and release
+lifecycle. They do not share one Capability: a Provider may accurately implement either or both.
+The package contains no OpenCV, filesystem, queue or Provider policy.
+
+```xml
+<import as="image" from="@hypit/image-operations@1"/>
+
+<image:Program id="soft-denoise">
+ <image:Denoise/>
+ <image:Encode format="png"/>
+</image:Program>
+
+<image:Transform id="clean-shot" source={shot.image} program={soft-denoise}/>
+
+<image:Compose id="comparison" canvas={comparison-canvas.canvas} background="#EEEAE2FF">
+ <image:Layer source={before.image} frame={before-panel} fit="contain"/>
+ <image:Layer source={after.image} frame={after-panel} fit="contain"/>
+</image:Compose>
+```
+
+Transform operation order is author meaning. Model- or component-specific Programs belong to their
+owning packages and use this package's generic operations. Compose performs rectangular alpha
+composition only; use a generative edit when the desired result is a newly directed scene, and keep
+independently timed or editable layers in Film.
diff --git a/packages/image-operations/package.json b/packages/image-operations/package.json
new file mode 100644
index 00000000..e33c6455
--- /dev/null
+++ b/packages/image-operations/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "@hypit/image-operations",
+ "version": "0.1.0",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "files": [
+ "dist",
+ "README.md"
+ ],
+ "peerDependencies": {
+ "@hypit/hypit": "workspace:^"
+ },
+ "devDependencies": {
+ "@hypit/blob": "workspace:*",
+ "@hypit/producer": "workspace:*",
+ "@hypit/author": "workspace:*",
+ "@hypit/hypit": "workspace:*",
+ "@hypit/markup": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/spatial": "workspace:*"
+ }
+}
diff --git a/packages/image-operations/src/activation.ts b/packages/image-operations/src/activation.ts
new file mode 100644
index 00000000..db1a1f23
--- /dev/null
+++ b/packages/image-operations/src/activation.ts
@@ -0,0 +1,43 @@
+import { createAdmissionPackageFacet } from "@hypit/hypit/admission";
+import { createProducerPackageFacet } from "@hypit/hypit/producer";
+import { createMarkupSurfaceFacet } from "@hypit/hypit/markup";
+
+import {
+ decodeImageComposeSurface,
+ decodeImageTransformProgramSurface,
+ decodeImageTransformSurface,
+ imageComposeComponent,
+ imageTransformComponent,
+ imageOperationsManifest,
+ imageOperationsModuleRef,
+ imageOperationsMarkupSurfaces,
+} from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.package@1" as const,
+ modules: [{
+ manifest: imageOperationsManifest,
+ }],
+ facets: [
+ ...[imageTransformComponent, imageComposeComponent].flatMap((component) => [createProducerPackageFacet(component), createAdmissionPackageFacet(component)]),{
+ ...createMarkupSurfaceFacet({
+ module: imageOperationsModuleRef,
+ declaration: imageOperationsMarkupSurfaces.find((item) => item.name === "program")!,
+ handler: decodeImageTransformProgramSurface,
+ }),
+ }, {
+ ...createMarkupSurfaceFacet({
+ module: imageOperationsModuleRef,
+ declaration: imageOperationsMarkupSurfaces.find((item) => item.name === "transform")!,
+ handler: decodeImageTransformSurface,
+ }),
+ }, {
+ ...createMarkupSurfaceFacet({
+ module: imageOperationsModuleRef,
+ declaration: imageOperationsMarkupSurfaces.find((item) => item.name === "compose")!,
+ handler: decodeImageComposeSurface,
+ }),
+ }],
+};
+
+export default hypitPackage;
diff --git a/packages/image-transform/src/component.ts b/packages/image-operations/src/component.ts
similarity index 78%
rename from packages/image-transform/src/component.ts
rename to packages/image-operations/src/component.ts
index e2442293..31581f7e 100644
--- a/packages/image-transform/src/component.ts
+++ b/packages/image-operations/src/component.ts
@@ -1,11 +1,12 @@
-import { plannedNeedInputs } from "@hypit/component-kit";
-import type { ComponentPackage } from "@hypit/component-kit";
-import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
-import { canonicalize } from "@hypit/protocol";
-import { rasterTransformRequest } from "@hypit/raster";
+import { canonicalize } from "@hypit/hypit/protocol";
+import { plannedNeedInputs } from "@hypit/hypit/producer";
+import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol";
+import type { ProducerPackage } from "@hypit/hypit/producer";
+import type { AdmissionPackage } from "@hypit/hypit/admission";
+import { imageTransformRequest } from "./execution.js";
import { imageTransformProducers, imageTransformTypes } from "./manifest.js";
-import { rasterCapabilities } from "@hypit/raster";
+import { imageOperationsCapabilities } from "./refs.js";
import { verifyImageTransformProgram } from "./program.js";
import type { ImageTransformProgram } from "./types.js";
@@ -15,7 +16,7 @@ function inline(value: StoredValue | undefined, subject: string): ImageTransform
}
function blob(value: StoredValue | undefined, subject: string): BlobRef {
- if (value?.kind !== "blob") throw new Error(`${subject} must be a BlobArtifact`);
+ if (value?.kind !== "blob") throw new Error(`${subject} must be a Blob`);
if (!value.mediaType.startsWith("image/")) throw new Error(`${subject} must be image media`);
return value;
}
@@ -33,13 +34,13 @@ export const imageTransformComponent = {
const source = blob(inputs.source?.value, "ImageTransform source");
const program = inline(inputs.program?.value, "ImageTransformProgram");
verifyImageTransformProgram(program);
- return { outputs: {}, needs: { image: canonicalize(rasterTransformRequest(source, program.operations)) } };
+ return { outputs: {}, needs: { image: canonicalize(imageTransformRequest(source, program.operations)) } };
},
}],
plannedNeeds: [{
producer: imageTransformProducers.request,
port: "image",
- capability: rasterCapabilities.execute,
+ capability: imageOperationsCapabilities.transform,
plan({ state, step }) {
const operation = state.plan.steps.find((item) => item.id === step);
const programRecord = operation?.inputs.program;
@@ -48,7 +49,7 @@ export const imageTransformComponent = {
: state.records.find((record) => record.id === programRecord)?.value;
if (program?.kind !== "inline") return undefined;
return {
- constraints: { kind: "transform", program: program.value as CanonicalValue },
+ constraints: { program: program.value as CanonicalValue },
pendingInputs: plannedNeedInputs(state, step, { source: "image" }),
};
},
@@ -64,4 +65,4 @@ export const imageTransformComponent = {
};
},
}],
-} satisfies ComponentPackage;
+} satisfies ProducerPackage & AdmissionPackage;
diff --git a/packages/image-compose/src/component.ts b/packages/image-operations/src/compose-component.ts
similarity index 82%
rename from packages/image-compose/src/component.ts
rename to packages/image-operations/src/compose-component.ts
index cf7c43dc..04412943 100644
--- a/packages/image-compose/src/component.ts
+++ b/packages/image-operations/src/compose-component.ts
@@ -1,20 +1,20 @@
-import { plannedNeedInputs } from "@hypit/component-kit";
-import type { ComponentPackage, ProducerHandlerContext } from "@hypit/component-kit";
-import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
-import { canonicalize } from "@hypit/protocol";
-import { rasterComposeRequest } from "@hypit/raster";
-import { rasterCapabilities } from "@hypit/raster";
-import type { Canvas, SpatialFrame } from "@hypit/spatial";
+import { plannedNeedInputs } from "@hypit/hypit/producer";
+import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol";
+import type { ProducerPackage, ProducerHandlerContext } from "@hypit/hypit/producer";
+import type { AdmissionPackage } from "@hypit/hypit/admission";
+import { canonicalize } from "@hypit/hypit/protocol";
+import type { Canvas, SpatialFrame } from "@hypit/hypit/spatial";
-import { imageComposeProducers, imageComposeTypes } from "./manifest.js";
+import { imageComposeRequest } from "./execution.js";
+import { imageComposeProducers, imageComposeTypes, imageOperationsCapabilities } from "./refs.js";
import {
appendImageComposeLayer,
assertImageComposeLayerSet,
assertImageComposeLayerSpec,
assertImageComposeOptions,
createImageComposeLayerSet,
-} from "./program.js";
-import type { ImageComposeLayerSet, ImageComposeLayerSpec, ImageComposeOptions } from "./types.js";
+} from "./compose-program.js";
+import type { ImageComposeLayerSet, ImageComposeLayerSpec, ImageComposeOptions } from "./compose-types.js";
function inline<T>(value: StoredValue | undefined, label: string): T {
if (value?.kind !== "inline") throw new Error(`${label} must be inline.`);
@@ -47,7 +47,7 @@ export const imageComposeComponent = {
const canvas = inline<Canvas>(inputs.canvas?.value, "Canvas");
const options = inline<ImageComposeOptions>(inputs.options?.value, "ImageComposeOptions");
const set = inline<ImageComposeLayerSet>(inputs.layers?.value, "ImageComposeLayerSet");
- return { outputs: {}, needs: { image: rasterComposeRequest({
+ return { outputs: {}, needs: { image: imageComposeRequest({
canvas, background: options.background,
layers: set.layers.map((layer) => ({
source: layer.source, frame: layer.frame, fit: layer.spec.fit,
@@ -69,7 +69,7 @@ export const imageComposeComponent = {
plannedNeeds: [{
producer: imageComposeProducers.request,
port: "image",
- capability: rasterCapabilities.execute,
+ capability: imageOperationsCapabilities.compose,
plan({ state, step }) {
const operation = state.plan.steps.find((item) => item.id === step);
if (operation === undefined) return undefined;
@@ -82,7 +82,7 @@ export const imageComposeComponent = {
const options = read("options");
if (canvas === undefined || options === undefined) return undefined;
return {
- constraints: { kind: "compose", canvas, options },
+ constraints: { canvas, options },
pendingInputs: plannedNeedInputs(state, step),
};
},
@@ -97,4 +97,4 @@ export const imageComposeComponent = {
};
},
}],
-} satisfies ComponentPackage;
+} satisfies ProducerPackage & AdmissionPackage;
diff --git a/packages/image-operations/src/compose-declaration.ts b/packages/image-operations/src/compose-declaration.ts
new file mode 100644
index 00000000..693a2a38
--- /dev/null
+++ b/packages/image-operations/src/compose-declaration.ts
@@ -0,0 +1,51 @@
+import { blobTypes } from "@hypit/hypit/blob";
+import { spatialTypes } from "@hypit/hypit/spatial";
+
+import { imageComposeTypes } from "./refs.js";
+
+export const imageComposeMarkupSurface = {
+ name: "compose",
+ tag: "Compose",
+ mode: "structured",
+ outputs: [imageComposeTypes.options, imageComposeTypes.layerSpec, blobTypes.blob],
+ vocabulary: {
+ summary: "Flattens ordered still-image Layers at explicit rectangular Frames into one PNG Artifact.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names this composition so its image can be referenced elsewhere in the Source." },
+ { name: "canvas", kind: "reference", required: true, accepts: [spatialTypes.canvas],
+ summary: "Chooses the Canvas every Layer is painted onto." },
+ { name: "background", kind: "literal", required: false,
+ summary: "Sets the color the Canvas is cleared to before the first Layer is painted." },
+ ],
+ children: [{
+ tag: "Layer", cardinality: "many",
+ summary: "One image painted into its own Frame, in document order, and empty of children and text.",
+ attributes: [
+ { name: "source", kind: "reference", required: true, accepts: [blobTypes.blob],
+ summary: "Chooses the image Artifact this Layer paints." },
+ { name: "frame", kind: "reference", required: true, accepts: [spatialTypes.frame],
+ summary: "Chooses the Frame on the Canvas the image is painted into." },
+ { name: "fit", kind: "literal", required: false, values: ["contain", "cover", "stretch"],
+ summary: "Decides how the image is sized to its Frame; defaults to `contain`." },
+ { name: "interpolation", kind: "literal", required: false,
+ values: ["nearest", "linear", "cubic", "area", "lanczos"],
+ summary: "Decides which filter resamples the image while it is scaled; defaults to `lanczos`." },
+ { name: "opacity", kind: "literal", required: false,
+ summary: "Sets how strongly this Layer covers what is beneath it, from 0 to 1; defaults to 1." },
+ ],
+ }],
+ ports: [{ name: "image", type: blobTypes.blob,
+ summary: "The composed picture, a PNG." }],
+ example: `<image:Compose id="comparison" canvas={comparison-canvas.canvas} background="#EEEAE2FF">
+ <image:Layer source={before.image} frame={before-panel} fit="contain"/>
+ <image:Layer source={after.image} frame={after-panel} fit="contain"/>
+</image:Compose>`,
+ notes: [
+ "`background` is written as `#RRGGBBAA` and defaults to `#00000000`.",
+ "The composition requires at least one Layer and holds at most 64.",
+ "Child order is paint order, and a Frame that extends beyond the Canvas is clipped.",
+ "The operation performs rectangular raster placement only; it does not redesign a scene or create timed layers.",
+ ],
+ },
+} as const;
diff --git a/packages/image-compose/src/fragment.ts b/packages/image-operations/src/compose-fragment.ts
similarity index 79%
rename from packages/image-compose/src/fragment.ts
rename to packages/image-operations/src/compose-fragment.ts
index e53a1145..4e41ad9f 100644
--- a/packages/image-compose/src/fragment.ts
+++ b/packages/image-operations/src/compose-fragment.ts
@@ -1,9 +1,9 @@
-import { artifactTypes } from "@hypit/artifact";
-import { sealGraphFragment } from "@hypit/elaborator";
-import type { FragmentOperation } from "@hypit/elaborator";
-import { spatialTypes } from "@hypit/spatial";
+import { blobTypes } from "@hypit/hypit/blob";
+import { sealGraphFragment } from "@hypit/hypit/author";
+import type { FragmentOperation } from "@hypit/hypit/author";
+import { spatialTypes } from "@hypit/hypit/spatial";
-import { imageComposeProducers, imageComposeTypes } from "./manifest.js";
+import { imageComposeProducers, imageComposeTypes } from "./refs.js";
export type ImageComposeFragmentLayer = {
readonly sourceName: string;
@@ -43,12 +43,12 @@ export function createImageComposeFragment(layers: readonly ImageComposeFragment
{ name: "canvas", type: spatialTypes.canvas },
{ name: "options", type: imageComposeTypes.options },
...layers.flatMap((layer) => [
- { name: layer.sourceName, type: artifactTypes.blob },
+ { name: layer.sourceName, type: blobTypes.blob },
{ name: layer.frameName, type: spatialTypes.frame },
{ name: layer.specName, type: imageComposeTypes.layerSpec },
]),
],
operations,
- exports: [{ name: "image", type: artifactTypes.blob, root: operation("image:compose") }],
+ exports: [{ name: "image", type: blobTypes.blob, root: operation("image:compose") }],
});
}
diff --git a/packages/image-compose/src/program.ts b/packages/image-operations/src/compose-program.ts
similarity index 93%
rename from packages/image-compose/src/program.ts
rename to packages/image-operations/src/compose-program.ts
index 8923e702..8f2e1807 100644
--- a/packages/image-compose/src/program.ts
+++ b/packages/image-operations/src/compose-program.ts
@@ -1,12 +1,12 @@
-import { assertSpatialFrame } from "@hypit/spatial";
-import { canonicalize, isResourceId } from "@hypit/protocol";
+import { assertSpatialFrame } from "@hypit/hypit/spatial";
+import { canonicalize, isResourceId } from "@hypit/hypit/protocol";
import type {
ImageComposeLayer,
ImageComposeLayerSet,
ImageComposeLayerSpec,
ImageComposeOptions,
-} from "./types.js";
+} from "./compose-types.js";
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
@@ -41,7 +41,7 @@ export function sealImageComposeLayerSpec(value: ImageComposeLayerSpec): ImageCo
function assertLayer(value: ImageComposeLayer, label: string): void {
assert(value.source.kind === "blob" && isResourceId(value.source.resource)
&& Number.isSafeInteger(value.source.size) && value.source.size >= 0
- && value.source.mediaType.startsWith("image/"), `${label}.source must be an image BlobArtifact.`);
+ && value.source.mediaType.startsWith("image/"), `${label}.source must be an image Blob.`);
assertSpatialFrame(value.frame);
assertImageComposeLayerSpec(value.spec);
}
diff --git a/packages/image-compose/src/surface.ts b/packages/image-operations/src/compose-surface.ts
similarity index 89%
rename from packages/image-compose/src/surface.ts
rename to packages/image-operations/src/compose-surface.ts
index e700cd4f..07793ba7 100644
--- a/packages/image-compose/src/surface.ts
+++ b/packages/image-operations/src/compose-surface.ts
@@ -1,11 +1,11 @@
-import { artifactTypes } from "@hypit/artifact";
-import type { CanonicalValue } from "@hypit/protocol";
-import { spatialTypes } from "@hypit/spatial";
-import type { StructuredElement, StructuredSurfaceHandler, SurfaceResolvedReference, MarkupAttributeValue } from "@hypit/markup";
+import { blobTypes } from "@hypit/hypit/blob";
+import type { CanonicalValue } from "@hypit/hypit/protocol";
+import type { MarkupAttributeValue, StructuredElement, StructuredSurfaceHandler, SurfaceResolvedReference } from "@hypit/hypit/markup";
+import { spatialTypes } from "@hypit/hypit/spatial";
-import { createImageComposeFragment } from "./fragment.js";
-import { imageComposeTypes } from "./manifest.js";
-import { sealImageComposeLayerSpec, sealImageComposeOptions } from "./program.js";
+import { createImageComposeFragment } from "./compose-fragment.js";
+import { sealImageComposeLayerSpec, sealImageComposeOptions } from "./compose-program.js";
+import { imageComposeTypes } from "./refs.js";
function localName(name: string): string { return name.slice(name.lastIndexOf(":") + 1); }
function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
@@ -58,7 +58,7 @@ export const decodeImageComposeSurface: StructuredSurfaceHandler = ({ element, r
exact(child, ["source", "frame", "fit", "interpolation", "opacity"], ["source", "frame"]);
if (child.children.some((node) => node.kind === "element" || node.value.trim())) throw new Error(`${child.name} must be empty.`);
const suffix = String(layers.length + 1).padStart(4, "0");
- const source = reference(child, "source", artifactTypes.blob, resolveReference);
+ const source = reference(child, "source", blobTypes.blob, resolveReference);
const frame = reference(child, "frame", spatialTypes.frame, resolveReference);
const specId = `${id}.layer.${suffix}.spec`;
records.push({
diff --git a/packages/image-compose/src/types.ts b/packages/image-operations/src/compose-types.ts
similarity index 81%
rename from packages/image-compose/src/types.ts
rename to packages/image-operations/src/compose-types.ts
index f69c532a..cbffb87d 100644
--- a/packages/image-compose/src/types.ts
+++ b/packages/image-operations/src/compose-types.ts
@@ -1,5 +1,5 @@
-import type { BlobRef } from "@hypit/protocol";
-import type { SpatialFrame } from "@hypit/spatial";
+import type { BlobRef } from "@hypit/hypit/protocol";
+import type { SpatialFrame } from "@hypit/hypit/spatial";
export type ImageComposeOptions = {
readonly background: string;
diff --git a/packages/image-operations/src/execution-types.ts b/packages/image-operations/src/execution-types.ts
new file mode 100644
index 00000000..88ac4b9b
--- /dev/null
+++ b/packages/image-operations/src/execution-types.ts
@@ -0,0 +1,58 @@
+import type { BlobRef } from "@hypit/hypit/protocol";
+import type { Canvas, SpatialFrame } from "@hypit/hypit/spatial";
+
+export type ImageInterpolation = "nearest" | "linear" | "cubic" | "area" | "lanczos";
+export type ImageFit = "contain" | "cover" | "stretch";
+
+export type ImageCropOperation = {
+ readonly kind: "crop"; readonly unit: "fraction" | "pixel";
+ readonly x: number; readonly y: number; readonly width: number; readonly height: number;
+};
+export type ImageResizeOperation = {
+ readonly kind: "resize"; readonly width: number; readonly height: number;
+ readonly fit: ImageFit; readonly interpolation: ImageInterpolation; readonly background?: string;
+};
+export type ImageRotateOperation = { readonly kind: "rotate"; readonly degrees: 90 | 180 | 270 };
+export type ImageFlipOperation = { readonly kind: "flip"; readonly axis: "horizontal" | "vertical" | "both" };
+export type ImageDenoiseOperation = {
+ readonly kind: "denoise"; readonly method: "nlm-ycrcb";
+ readonly lumaStrength: number; readonly chromaStrength: number;
+ readonly templateWindow: number; readonly searchWindow: number; readonly saturationRecovery: number;
+};
+export type ImageColorOperation = {
+ readonly kind: "color"; readonly exposureStops: number; readonly contrast: number;
+ readonly saturation: number; readonly temperature: number; readonly tint: number; readonly gamma: number;
+};
+export type ImageSharpenOperation = {
+ readonly kind: "sharpen"; readonly amount: number; readonly radius: number; readonly threshold: number;
+};
+export type ImageBlurOperation = { readonly kind: "blur"; readonly sigma: number };
+export type ImageAlphaOperation = {
+ readonly kind: "alpha"; readonly mode: "preserve" | "flatten"; readonly background?: string;
+};
+export type ImageEncodeOperation = {
+ readonly kind: "encode"; readonly format: "png" | "jpeg" | "webp";
+ readonly quality?: number; readonly background?: string;
+};
+export type ImageTransformOperation =
+ | ImageCropOperation | ImageResizeOperation | ImageRotateOperation | ImageFlipOperation
+ | ImageDenoiseOperation | ImageColorOperation | ImageSharpenOperation | ImageBlurOperation
+ | ImageAlphaOperation | ImageEncodeOperation;
+
+export type ImageComposeExecutionLayer = {
+ readonly source: BlobRef;
+ readonly frame: SpatialFrame;
+ readonly fit: ImageFit;
+ readonly interpolation: ImageInterpolation;
+ readonly opacity: number;
+};
+
+export type ImageTransformRequest = {
+ readonly source: BlobRef;
+ readonly operations: readonly ImageTransformOperation[];
+};
+export type ImageComposeRequest = {
+ readonly canvas: Canvas;
+ readonly background: string;
+ readonly layers: readonly ImageComposeExecutionLayer[];
+};
diff --git a/packages/raster/src/program.ts b/packages/image-operations/src/execution.ts
similarity index 69%
rename from packages/raster/src/program.ts
rename to packages/image-operations/src/execution.ts
index 78a27123..c0024ac5 100644
--- a/packages/raster/src/program.ts
+++ b/packages/image-operations/src/execution.ts
@@ -1,10 +1,10 @@
-import { canonicalize, isResourceId } from "@hypit/protocol";
-import { assertCanvas, assertSpatialFrame } from "@hypit/spatial";
+import { canonicalize, isResourceId } from "@hypit/hypit/protocol";
+import { assertCanvas, assertSpatialFrame } from "@hypit/hypit/spatial";
import type {
- RasterComposeRequest, RasterEncodeOperation, RasterLayer, RasterRequest,
- RasterTransformOperation, RasterTransformRequest,
-} from "./types.js";
+ ImageComposeExecutionLayer, ImageComposeRequest, ImageEncodeOperation,
+ ImageTransformOperation, ImageTransformRequest,
+} from "./execution-types.js";
function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); }
function finite(value: number, minimum: number, maximum: number, subject: string): void {
@@ -20,13 +20,13 @@ function color(value: string | undefined, subject: string, alpha = true): void {
const pattern = alpha ? /^#[0-9a-f]{6}(?:[0-9a-f]{2})?$/iu : /^#[0-9a-f]{8}$/iu;
assert(pattern.test(value), `${subject} must be ${alpha ? "#RRGGBB or #RRGGBBAA" : "#RRGGBBAA"}`);
}
-function image(value: RasterLayer["source"], subject: string): void {
- assert(value.kind === "blob" && isResourceId(value.resource) && Number.isSafeInteger(value.size)
+function image(value: ImageComposeExecutionLayer["source"] | undefined, subject: string): void {
+ assert(value !== undefined && value.kind === "blob" && isResourceId(value.resource) && Number.isSafeInteger(value.size)
&& value.size >= 0 && value.mediaType.startsWith("image/"), `${subject} must be an image Blob Artifact`);
}
-function assertTransformOperation(operation: RasterTransformOperation, index: number): void {
- const subject = `Raster transform operation ${index + 1}`;
+function assertTransformOperation(operation: ImageTransformOperation, index: number): void {
+ const subject = `Image transform operation ${index + 1}`;
switch (operation.kind) {
case "crop":
if (operation.unit === "fraction") {
@@ -83,23 +83,25 @@ function assertTransformOperation(operation: RasterTransformOperation, index: nu
}
}
-export function assertRasterTransformOperations(operations: readonly RasterTransformOperation[]): void {
- assert(Array.isArray(operations) && operations.length > 0, "Raster transform requires at least one operation");
+export function assertImageTransformOperations(operations: readonly ImageTransformOperation[]): void {
+ assert(Array.isArray(operations) && operations.length > 0, "Image transform requires at least one operation");
operations.forEach(assertTransformOperation);
const encodes = operations.filter((operation) => operation.kind === "encode");
- assert(encodes.length <= 1, "Raster transform may encode only once");
- assert(encodes.length === 0 || operations.at(-1)?.kind === "encode", "Raster encode must be final");
+ assert(encodes.length <= 1, "Image transform may encode only once");
+ assert(encodes.length === 0 || operations.at(-1)?.kind === "encode", "Image encode must be final");
}
-export function assertRasterRequest(value: RasterRequest): void {
- if (value.kind === "transform") {
- image(value.source, "Raster transform source");
- assertRasterTransformOperations(value.operations);
- return;
- }
- assert(value.kind === "compose", "RasterRequest.kind is invalid");
- assertCanvas(value.canvas); color(value.background, "Raster compose background", false);
- assert(value.layers.length > 0 && value.layers.length <= 64, "Raster compose requires one to 64 Layers");
+export function assertImageTransformRequest(value: ImageTransformRequest): void {
+ assert(value !== null && typeof value === "object", "Image transform request must be an object");
+ image(value.source, "Image transform source");
+ assertImageTransformOperations(value.operations);
+}
+
+export function assertImageComposeRequest(value: ImageComposeRequest): void {
+ assert(value !== null && typeof value === "object", "Image compose request must be an object");
+ assert(value.canvas !== undefined, "Image compose canvas is required");
+ assertCanvas(value.canvas); color(value.background, "Image compose background", false);
+ assert(value.layers.length > 0 && value.layers.length <= 64, "Image compose requires one to 64 Layers");
value.layers.forEach((layer, index) => {
image(layer.source, `Raster Layer ${index + 1}.source`); assertSpatialFrame(layer.frame);
assert(["contain", "cover", "stretch"].includes(layer.fit), `Raster Layer ${index + 1}.fit is invalid`);
@@ -109,20 +111,22 @@ export function assertRasterRequest(value: RasterRequest): void {
});
}
-export function rasterTransformRequest(source: RasterTransformRequest["source"], operations: readonly RasterTransformOperation[]): RasterTransformRequest {
- const request = canonicalize({ kind: "transform", source, operations }) as unknown as RasterTransformRequest;
- assertRasterRequest(request); return request;
+export function imageTransformRequest(
+ source: ImageTransformRequest["source"],
+ operations: readonly ImageTransformOperation[],
+): ImageTransformRequest {
+ const request = canonicalize({ source, operations }) as unknown as ImageTransformRequest;
+ assertImageTransformRequest(request); return request;
}
-export function rasterComposeRequest(value: Omit<RasterComposeRequest, "kind">): RasterComposeRequest {
- const request = canonicalize({ kind: "compose", ...value }) as unknown as RasterComposeRequest;
- assertRasterRequest(request); return request;
+export function imageComposeRequest(value: ImageComposeRequest): ImageComposeRequest {
+ const request = canonicalize(value) as unknown as ImageComposeRequest;
+ assertImageComposeRequest(request); return request;
}
-export function rasterSources(request: RasterRequest): readonly RasterLayer["source"][] {
- assertRasterRequest(request); return request.kind === "transform" ? [request.source] : request.layers.map((layer) => layer.source);
+export function imageComposeSources(request: ImageComposeRequest): readonly ImageComposeExecutionLayer["source"][] {
+ assertImageComposeRequest(request); return request.layers.map((layer) => layer.source);
}
-export function rasterOutputMediaType(request: RasterRequest): string {
- assertRasterRequest(request);
- if (request.kind === "compose") return "image/png";
- const encode = request.operations.find((operation): operation is RasterEncodeOperation => operation.kind === "encode");
+export function imageTransformOutputMediaType(request: ImageTransformRequest): string {
+ assertImageTransformRequest(request);
+ const encode = request.operations.find((operation): operation is ImageEncodeOperation => operation.kind === "encode");
return encode?.format === "jpeg" ? "image/jpeg" : encode?.format === "webp" ? "image/webp" : "image/png";
}
diff --git a/packages/image-transform/src/fragment.ts b/packages/image-operations/src/fragment.ts
similarity index 78%
rename from packages/image-transform/src/fragment.ts
rename to packages/image-operations/src/fragment.ts
index eb94ee9b..df55bcd2 100644
--- a/packages/image-transform/src/fragment.ts
+++ b/packages/image-operations/src/fragment.ts
@@ -1,5 +1,5 @@
-import { artifactTypes } from "@hypit/artifact";
-import { sealGraphFragment } from "@hypit/elaborator";
+import { blobTypes } from "@hypit/hypit/blob";
+import { sealGraphFragment } from "@hypit/hypit/author";
import { imageTransformProducers, imageTransformTypes } from "./manifest.js";
@@ -8,7 +8,7 @@ const operation = (id: string) => ({ kind: "fragment-operation" as const, operat
export const imageTransformFragment = sealGraphFragment({
inputs: [
- { name: "source", type: artifactTypes.blob },
+ { name: "source", type: blobTypes.blob },
{ name: "program", type: imageTransformTypes.program },
],
operations: [{
@@ -19,7 +19,7 @@ export const imageTransformFragment = sealGraphFragment({
}],
exports: [{
name: "image",
- type: artifactTypes.blob,
+ type: blobTypes.blob,
root: operation("image:transform"),
}],
});
diff --git a/packages/image-operations/src/index.ts b/packages/image-operations/src/index.ts
new file mode 100644
index 00000000..56928a14
--- /dev/null
+++ b/packages/image-operations/src/index.ts
@@ -0,0 +1,47 @@
+export { imageComposeComponent } from "./compose-component.js";
+export { createImageComposeFragment } from "./compose-fragment.js";
+export {
+ appendImageComposeLayer,
+ assertImageComposeLayerSet,
+ assertImageComposeLayerSpec,
+ assertImageComposeOptions,
+ createImageComposeLayerSet,
+ sealImageComposeLayerSpec,
+ sealImageComposeOptions,
+} from "./compose-program.js";
+export { decodeImageComposeSurface } from "./compose-surface.js";
+export type * from "./compose-types.js";
+export { imageTransformComponent } from "./component.js";
+export { imageTransformFragment } from "./fragment.js";
+export {
+ imageComposeProducers,
+ imageComposeTypes,
+ imageOperationsCapabilities,
+ imageOperationsDependency,
+ imageOperationsManifest,
+ imageOperationsMarkupSurfaces,
+ imageOperationsModuleRef,
+ imageTransformProducers,
+ imageTransformProgramSchema,
+ imageTransformTypes,
+} from "./manifest.js";
+export { imageTransformOperationSchema } from "./operation-schema.js";
+export {
+ assertImageComposeRequest,
+ assertImageTransformOperations,
+ assertImageTransformRequest,
+ imageComposeRequest,
+ imageComposeSources,
+ imageTransformOutputMediaType,
+ imageTransformRequest,
+} from "./execution.js";
+export type * from "./execution-types.js";
+export {
+ sealImageTransformProgram,
+ verifyImageTransformProgram,
+} from "./program.js";
+export {
+ decodeImageTransformProgramSurface,
+ decodeImageTransformSurface,
+} from "./surface.js";
+export type * from "./types.js";
diff --git a/packages/image-transform/src/manifest.ts b/packages/image-operations/src/manifest.ts
similarity index 76%
rename from packages/image-transform/src/manifest.ts
rename to packages/image-operations/src/manifest.ts
index cb17cc77..c477c1f0 100644
--- a/packages/image-transform/src/manifest.ts
+++ b/packages/image-operations/src/manifest.ts
@@ -1,25 +1,33 @@
-import { artifactDependency, artifactTypes } from "@hypit/artifact";
-import { rasterCapabilities, rasterDependency, rasterTransformOperationSchema } from "@hypit/raster";
-import type {
- ModuleManifest,
- ProducerRef,
- TypeRef,
- ValueSchema,
-} from "@hypit/protocol";
+import { blobDependency, blobTypes } from "@hypit/hypit/blob";
+import { spatialDependency, spatialTypes } from "@hypit/hypit/spatial";
+import type { ModuleManifest, ValueSchema } from "@hypit/hypit/protocol";
-export const imageTransformModuleRef = { name: "@hypit/image-transform", version: "1" } as const;
-export const imageTransformTypes = {
- program: { module: imageTransformModuleRef, name: "ImageTransformProgram" },
-} satisfies Record<string, TypeRef>;
-export const imageTransformProducers = {
- request: { module: imageTransformModuleRef, name: "request-image-transform" },
-} satisfies Record<string, ProducerRef>;
+import { imageComposeMarkupSurface } from "./compose-declaration.js";
+import { imageTransformOperationSchema } from "./operation-schema.js";
+import {
+ imageComposeProducers,
+ imageComposeTypes,
+ imageOperationsCapabilities,
+ imageOperationsModuleRef,
+ imageTransformProducers,
+ imageTransformTypes,
+} from "./refs.js";
+
+export {
+ imageComposeProducers,
+ imageComposeTypes,
+ imageOperationsCapabilities,
+ imageOperationsDependency,
+ imageOperationsModuleRef,
+ imageTransformProducers,
+ imageTransformTypes,
+} from "./refs.js";
export const imageTransformProgramSchema: ValueSchema = { kind: "object", fields: {
- operations: { schema: { kind: "array", minItems: 1, items: rasterTransformOperationSchema } },
+ operations: { schema: { kind: "array", minItems: 1, items: imageTransformOperationSchema } },
} };
-export const imageTransformMarkupSurfaces = [{
+export const imageOperationsMarkupSurfaces = [{
name: "program",
tag: "Program",
mode: "structured",
@@ -141,7 +149,7 @@ export const imageTransformMarkupSurfaces = [{
] },
],
example: [
- '<image:Program id="clean-gpt-image">',
+ '<image:Program id="soft-denoise">',
" <image:Denoise/>",
' <image:Encode format="png"/>',
"</image:Program>",
@@ -159,51 +167,79 @@ export const imageTransformMarkupSurfaces = [{
name: "transform",
tag: "Transform",
mode: "structured",
- outputs: [artifactTypes.blob],
+ outputs: [blobTypes.blob],
vocabulary: {
summary: "Runs one ImageTransformProgram over a source image Artifact and publishes the transformed image.",
attributes: [
{ name: "id", kind: "identifier", required: true,
summary: "Names this transform so its image can be referenced elsewhere in the Source." },
- { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],
+ { name: "source", kind: "reference", required: true, accepts: [blobTypes.blob],
summary: "Chooses the image the operations are applied to." },
{ name: "program", kind: "reference", required: true, accepts: [imageTransformTypes.program],
summary: "Chooses the ImageTransformProgram whose operations run over the source." },
],
ports: [
- { name: "image", type: artifactTypes.blob,
+ { name: "image", type: blobTypes.blob,
summary: "The transformed image, addressed as `<id>.image`." },
],
- example: `<image:Transform id="clean-shot" source={shot.image} program={clean-gpt-image}/>`,
+ example: `<image:Transform id="clean-shot" source={shot.image} program={soft-denoise}/>`,
notes: [
"The element is empty; it accepts no children and no text.",
"The published image is the transformed image itself, carrying no source digest, Provider name or copied upstream metadata.",
"A Program on its own produces no image; naming it here is what runs it.",
],
},
- }] as const;
+ }, imageComposeMarkupSurface] as const;
-export const imageTransformManifest: ModuleManifest = {
+export const imageOperationsManifest: ModuleManifest = {
format: "hypit.module@1",
- name: imageTransformModuleRef.name,
- version: imageTransformModuleRef.version,
- dependencies: [artifactDependency, rasterDependency],
- types: [{
- name: imageTransformTypes.program.name,
- }],
- capabilities: [],
- producers: [{
- name: imageTransformProducers.request.name,
- inputs: [
- { name: "source", type: artifactTypes.blob },
- { name: "program", type: imageTransformTypes.program },
- ],
- outputs: [],
- needs: [{
- name: "image",
- capability: rasterCapabilities.execute,
- returns: artifactTypes.blob,
- }],
- }],
+ name: imageOperationsModuleRef.name,
+ version: imageOperationsModuleRef.version,
+ dependencies: [blobDependency, spatialDependency],
+ types: [
+ { name: imageTransformTypes.program.name },
+ { name: imageComposeTypes.options.name },
+ { name: imageComposeTypes.layerSpec.name },
+ { name: imageComposeTypes.layerSet.name },
+ ],
+ capabilities: [
+ { name: imageOperationsCapabilities.transform.name, returns: blobTypes.blob },
+ { name: imageOperationsCapabilities.compose.name, returns: blobTypes.blob },
+ ],
+ producers: [
+ {
+ name: imageTransformProducers.request.name,
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "program", type: imageTransformTypes.program },
+ ],
+ outputs: [],
+ needs: [{ name: "image", capability: imageOperationsCapabilities.transform, returns: blobTypes.blob }],
+ },
+ {
+ name: imageComposeProducers.createLayers.name,
+ inputs: [], outputs: [{ name: "layers", type: imageComposeTypes.layerSet }], needs: [],
+ },
+ {
+ name: imageComposeProducers.appendLayer.name,
+ inputs: [
+ { name: "layers", type: imageComposeTypes.layerSet },
+ { name: "source", type: blobTypes.blob },
+ { name: "frame", type: spatialTypes.frame },
+ { name: "spec", type: imageComposeTypes.layerSpec },
+ ],
+ outputs: [{ name: "layers", type: imageComposeTypes.layerSet }], needs: [],
+ },
+ {
+ name: imageComposeProducers.request.name,
+ inputs: [
+ { name: "canvas", type: spatialTypes.canvas },
+ { name: "options", type: imageComposeTypes.options },
+ { name: "layers", type: imageComposeTypes.layerSet },
+ ],
+ outputs: [],
+ needs: [{ name: "image", capability: imageOperationsCapabilities.compose, returns: blobTypes.blob }],
+ },
+ ],
};
diff --git a/packages/raster/src/schema.ts b/packages/image-operations/src/operation-schema.ts
similarity index 96%
rename from packages/raster/src/schema.ts
rename to packages/image-operations/src/operation-schema.ts
index 6bbb5e04..04211d47 100644
--- a/packages/raster/src/schema.ts
+++ b/packages/image-operations/src/operation-schema.ts
@@ -1,4 +1,4 @@
-import type { ValueSchema } from "@hypit/protocol";
+import type { ValueSchema } from "@hypit/hypit/protocol";
const number = (minimum?: number, maximum?: number): ValueSchema => ({
kind: "number", ...(minimum === undefined ? {} : { minimum }), ...(maximum === undefined ? {} : { maximum }),
@@ -9,7 +9,7 @@ const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema;
const color: ValueSchema = { kind: "string", minLength: 7, maxLength: 9 };
/** Declarative mirror of the closed transform vocabulary, shared by every author Surface. */
-export const rasterTransformOperationSchema: ValueSchema = {
+export const imageTransformOperationSchema: ValueSchema = {
kind: "oneOf",
variants: [
object({
diff --git a/packages/image-transform/src/program.ts b/packages/image-operations/src/program.ts
similarity index 53%
rename from packages/image-transform/src/program.ts
rename to packages/image-operations/src/program.ts
index 85b153de..c9930be4 100644
--- a/packages/image-transform/src/program.ts
+++ b/packages/image-operations/src/program.ts
@@ -1,12 +1,12 @@
-import { canonicalize } from "@hypit/protocol";
-import { assertRasterTransformOperations } from "@hypit/raster";
+import { canonicalize } from "@hypit/hypit/protocol";
+import { assertImageTransformOperations } from "./execution.js";
import type { ImageTransformProgram } from "./types.js";
function assert(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); }
export function verifyImageTransformProgram(program: ImageTransformProgram): void {
- assertRasterTransformOperations(program.operations);
+ assertImageTransformOperations(program.operations);
}
export function sealImageTransformProgram(program: ImageTransformProgram): ImageTransformProgram {
@@ -14,10 +14,3 @@ export function sealImageTransformProgram(program: ImageTransformProgram): Image
verifyImageTransformProgram(sealed);
return sealed;
}
-
-export const gptImageDenoiseV1 = sealImageTransformProgram({
- operations: [{
- kind: "denoise", method: "nlm-ycrcb", lumaStrength: 2, chromaStrength: 10,
- templateWindow: 7, searchWindow: 21, saturationRecovery: 1.02,
- }, { kind: "encode", format: "png" }],
-});
diff --git a/packages/image-operations/src/refs.ts b/packages/image-operations/src/refs.ts
new file mode 100644
index 00000000..c0206084
--- /dev/null
+++ b/packages/image-operations/src/refs.ts
@@ -0,0 +1,30 @@
+import type { CapabilityRef, ProducerRef, TypeRef } from "@hypit/hypit/protocol";
+
+export const imageOperationsModuleRef = { name: "@hypit/image-operations", version: "1" } as const;
+
+export const imageOperationsCapabilities = {
+ transform: { module: imageOperationsModuleRef, name: "transform-image" },
+ compose: { module: imageOperationsModuleRef, name: "compose-image" },
+} satisfies Record<string, CapabilityRef>;
+
+export const imageTransformTypes = {
+ program: { module: imageOperationsModuleRef, name: "ImageTransformProgram" },
+} satisfies Record<string, TypeRef>;
+
+export const imageComposeTypes = {
+ options: { module: imageOperationsModuleRef, name: "ImageComposeOptions" },
+ layerSpec: { module: imageOperationsModuleRef, name: "ImageComposeLayerSpec" },
+ layerSet: { module: imageOperationsModuleRef, name: "ImageComposeLayerSet" },
+} satisfies Record<string, TypeRef>;
+
+export const imageTransformProducers = {
+ request: { module: imageOperationsModuleRef, name: "request-image-transform" },
+} satisfies Record<string, ProducerRef>;
+
+export const imageComposeProducers = {
+ createLayers: { module: imageOperationsModuleRef, name: "create-image-compose-layers" },
+ appendLayer: { module: imageOperationsModuleRef, name: "append-image-compose-layer" },
+ request: { module: imageOperationsModuleRef, name: "request-image-compose" },
+} satisfies Record<string, ProducerRef>;
+
+export const imageOperationsDependency = { module: imageOperationsModuleRef } as const;
diff --git a/packages/image-transform/src/surface.ts b/packages/image-operations/src/surface.ts
similarity index 96%
rename from packages/image-transform/src/surface.ts
rename to packages/image-operations/src/surface.ts
index 5231f0af..4a94c1a2 100644
--- a/packages/image-transform/src/surface.ts
+++ b/packages/image-operations/src/surface.ts
@@ -1,11 +1,6 @@
-import { artifactTypes } from "@hypit/artifact";
-import type {
- StructuredElement,
- StructuredSurfaceHandler,
- SurfaceResolvedReference,
- MarkupAttributeValue,
-} from "@hypit/markup";
-import type { CanonicalValue } from "@hypit/protocol";
+import { blobTypes } from "@hypit/hypit/blob";
+import type { StructuredElement, StructuredSurfaceHandler, SurfaceResolvedReference, MarkupAttributeValue } from "@hypit/hypit/markup";
+import type { CanonicalValue } from "@hypit/hypit/protocol";
import { imageTransformFragment } from "./fragment.js";
import { imageTransformTypes } from "./manifest.js";
@@ -200,7 +195,7 @@ export const decodeImageTransformSurface: StructuredSurfaceHandler = ({ element,
exact(element, ["id", "source", "program"], ["id", "source", "program"]);
onlyTrivia(element);
const id = text(element, "id");
- const source = ref(element, "source", artifactTypes.blob, resolveReference);
+ const source = ref(element, "source", blobTypes.blob, resolveReference);
const program = ref(element, "program", imageTransformTypes.program, resolveReference);
return {
records: [],
diff --git a/packages/image-operations/src/types.ts b/packages/image-operations/src/types.ts
new file mode 100644
index 00000000..9b60622c
--- /dev/null
+++ b/packages/image-operations/src/types.ts
@@ -0,0 +1,8 @@
+import type { ImageTransformOperation } from "./execution-types.js";
+
+export type * from "./execution-types.js";
+
+/** Authored transformation intent. Operation order is author meaning. */
+export type ImageTransformProgram = {
+ readonly operations: readonly ImageTransformOperation[];
+};
diff --git a/packages/image-operations/test/execution.test.ts b/packages/image-operations/test/execution.test.ts
new file mode 100644
index 00000000..cb6618fd
--- /dev/null
+++ b/packages/image-operations/test/execution.test.ts
@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { fixtureResource } from "../../../test/fixture-resource.js";
+
+import {
+ assertImageComposeRequest,
+ assertImageTransformRequest,
+ imageComposeRequest,
+ imageComposeSources,
+ imageOperationsCapabilities,
+ imageTransformOutputMediaType,
+ imageTransformRequest,
+} from "@hypit/image-operations";
+
+const image = {
+ kind: "blob" as const,
+ resource: fixtureResource("image-operations-source"),
+ size: 12,
+ mediaType: "image/png",
+};
+
+test("Transform and Compose are separate capabilities in one Image Operations module", () => {
+ const transform = imageTransformRequest(image, [{ kind: "encode", format: "webp", quality: 90 }]);
+ const compose = imageComposeRequest({
+ canvas: { widthPx: 100, heightPx: 200 },
+ background: "#00000000",
+ layers: [{
+ source: image,
+ frame: { xPx: 0, yPx: 0, widthPx: 100, heightPx: 200 },
+ fit: "contain", interpolation: "lanczos", opacity: 1,
+ }],
+ });
+ assert.equal(imageOperationsCapabilities.transform.name, "transform-image");
+ assert.equal(imageOperationsCapabilities.compose.name, "compose-image");
+ assert.equal(imageTransformOutputMediaType(transform), "image/webp");
+ assert.deepEqual(imageComposeSources(compose), [image]);
+});
+
+test("each capability validates only the request meaning it promises", () => {
+ assert.throws(() => assertImageTransformRequest({ canvas: {} } as never), /source/u);
+ assert.throws(() => assertImageComposeRequest({ source: image } as never), /Canvas|canvas/u);
+});
diff --git a/packages/image-compose/test/image-compose.test.ts b/packages/image-operations/test/image-compose.test.ts
similarity index 84%
rename from packages/image-compose/test/image-compose.test.ts
rename to packages/image-operations/test/image-compose.test.ts
index fd904a0c..5706f556 100644
--- a/packages/image-compose/test/image-compose.test.ts
+++ b/packages/image-operations/test/image-compose.test.ts
@@ -2,17 +2,17 @@ import assert from "node:assert/strict";
import test from "node:test";
import { fixtureResource } from "../../../test/fixture-resource.js";
-import { artifactTypes } from "@hypit/artifact";
+import { blobTypes } from "@hypit/blob";
import {
createImageComposeFragment,
decodeImageComposeSurface,
imageComposeComponent,
- imageComposeManifest,
+ imageOperationsCapabilities,
+ imageOperationsManifest,
imageComposeTypes,
sealImageComposeLayerSpec,
sealImageComposeOptions,
-} from "@hypit/image-compose";
-import { rasterCapabilities } from "@hypit/raster";
+} from "@hypit/image-operations";
import type { CanonicalValue, StoredValue, TypeRef, TypedRecord } from "@hypit/protocol";
import { sealCanvas, sealSpatialFrame, spatialTypes } from "@hypit/spatial";
@@ -40,8 +40,8 @@ test("Image Compose is an ordered Layer graph with no privileged base image", ()
"image:layers:append:0001", "image:layers:append:0002", "image:layers:empty",
].sort());
assert.deepEqual(fragment.exports.map((output) => output.name), ["image"]);
- assert.equal(imageComposeManifest.capabilities.length, 0);
- assert.deepEqual(imageComposeManifest.producers.at(-1)?.needs[0]?.capability, rasterCapabilities.execute);
+ assert.deepEqual(imageOperationsManifest.producers.at(-1)?.needs[0]?.capability,
+ imageOperationsCapabilities.compose);
});
test("component data flow preserves explicit Canvas, Frame and paint order", async () => {
@@ -50,7 +50,7 @@ test("component data flow preserves explicit Canvas, Frame and paint order", asy
assert("layers" in empty.outputs);
const appended = await append!.handler({ inputs: {
layers: record("layers", imageComposeTypes.layerSet, empty.outputs.layers!),
- source: record("source", artifactTypes.blob, source),
+ source: record("source", blobTypes.blob, source),
frame: record("frame", spatialTypes.frame, { kind: "inline", value: frame as unknown as CanonicalValue }),
spec: record("spec", imageComposeTypes.layerSpec, { kind: "inline", value: spec as unknown as CanonicalValue }),
} } as never);
@@ -62,22 +62,22 @@ test("component data flow preserves explicit Canvas, Frame and paint order", asy
} } as never);
assert("image" in result.needs);
assert.deepEqual(result.needs.image, {
- kind: "compose", canvas, background: options.background,
+ canvas, background: options.background,
layers: [{ source, frame, fit: spec.fit, interpolation: spec.interpolation, opacity: spec.opacity }],
});
});
test("the Surface declares once and paints child Layers in document order", async () => {
const references: Record<string, { type: TypeRef; record?: TypedRecord }> = {
- portrait: { type: spatialTypes.canvas }, hero: { type: artifactTypes.blob }, heroFrame: { type: spatialTypes.frame },
+ portrait: { type: spatialTypes.canvas }, hero: { type: blobTypes.blob }, heroFrame: { type: spatialTypes.frame },
};
const output = await decodeImageComposeSurface({
sourceName: "main.svml",
element: {
- kind: "element", name: "compose:Image", range: { start: 0, end: 200 },
+ kind: "element", name: "image:Compose", range: { start: 0, end: 200 },
attributes: { id: "poster", canvas: { kind: "reference", path: "portrait" }, background: "#00000000" },
children: [{
- kind: "element", name: "compose:Layer", range: { start: 20, end: 100 }, children: [],
+ kind: "element", name: "image:Layer", range: { start: 20, end: 100 }, children: [],
attributes: {
source: { kind: "reference", path: "hero" }, frame: { kind: "reference", path: "heroFrame" },
fit: "cover", opacity: "0.8",
diff --git a/packages/image-transform/test/image-transform.test.ts b/packages/image-operations/test/image-transform.test.ts
similarity index 70%
rename from packages/image-transform/test/image-transform.test.ts
rename to packages/image-operations/test/image-transform.test.ts
index 32e6d9e6..56ff5c5d 100644
--- a/packages/image-transform/test/image-transform.test.ts
+++ b/packages/image-operations/test/image-transform.test.ts
@@ -4,19 +4,30 @@ import { fixtureResource } from "../../../test/fixture-resource.js";
import {
decodeImageTransformProgramSurface,
- gptImageDenoiseV1,
imageTransformComponent,
imageTransformFragment,
- imageTransformManifest,
+ imageOperationsCapabilities,
+ imageOperationsManifest,
imageTransformTypes,
sealImageTransformProgram,
-} from "@hypit/image-transform";
-import { artifactTypes } from "@hypit/artifact";
-import { rasterCapabilities } from "@hypit/raster";
+} from "@hypit/image-operations";
+import { blobTypes } from "@hypit/blob";
import type { CanonicalValue, TypedRecord } from "@hypit/protocol";
-test("the GPT Image cleanup is one explicit reusable Program", () => {
- assert.deepEqual(gptImageDenoiseV1.operations, [{
+const denoiseProgram = sealImageTransformProgram({
+ operations: [{
+ kind: "denoise",
+ method: "nlm-ycrcb",
+ lumaStrength: 2,
+ chromaStrength: 10,
+ templateWindow: 7,
+ searchWindow: 21,
+ saturationRecovery: 1.02,
+ }, { kind: "encode", format: "png" }],
+});
+
+test("Program sealing validates ordered generic image operations", () => {
+ assert.deepEqual(denoiseProgram.operations, [{
kind: "denoise",
method: "nlm-ycrcb",
lumaStrength: 2,
@@ -36,7 +47,7 @@ test("the official Surface separates Program declaration from Transform use", as
element: {
kind: "element",
name: "image:Program",
- attributes: { id: "clean-gpt-image" },
+ attributes: { id: "soft-denoise" },
range: { start: 0, end: 100 },
children: [{
kind: "element",
@@ -59,15 +70,18 @@ test("the official Surface separates Program declaration from Transform use", as
assert.deepEqual(output.records[0]?.type, imageTransformTypes.program);
assert.deepEqual(output.records[0]?.value, {
kind: "inline",
- value: gptImageDenoiseV1 as unknown as CanonicalValue,
+ value: denoiseProgram as unknown as CanonicalValue,
});
});
test("the graph contract is exactly source plus Program to one image Need", async () => {
assert.deepEqual(imageTransformFragment.inputs.map((input) => input.name).sort(), ["program", "source"]);
assert.deepEqual(imageTransformFragment.exports.map((output) => output.name), ["image"]);
- assert.equal(imageTransformManifest.capabilities.length, 0);
- assert.deepEqual(imageTransformManifest.producers[0]?.needs[0]?.capability, rasterCapabilities.execute);
+ assert.deepEqual(imageOperationsManifest.capabilities.map((capability) => capability.name), [
+ "transform-image", "compose-image",
+ ]);
+ assert.deepEqual(imageOperationsManifest.producers[0]?.needs[0]?.capability,
+ imageOperationsCapabilities.transform);
const producer = imageTransformComponent.producers[0]!;
const source = {
kind: "blob" as const,
@@ -82,15 +96,14 @@ test("the graph contract is exactly source plus Program to one image Need", asyn
});
const result = await producer.handler({
inputs: {
- source: record("source", artifactTypes.blob, source),
+ source: record("source", blobTypes.blob, source),
program: record("program", imageTransformTypes.program,
- { kind: "inline", value: gptImageDenoiseV1 as unknown as CanonicalValue }),
+ { kind: "inline", value: denoiseProgram as unknown as CanonicalValue }),
},
} as never);
assert.deepEqual(result.outputs, {});
assert.deepEqual(result.needs.image, {
- kind: "transform",
source,
- operations: gptImageDenoiseV1.operations,
+ operations: denoiseProgram.operations,
});
});
diff --git a/packages/image-transform/README.md b/packages/image-transform/README.md
deleted file mode 100644
index f9b44e8c..00000000
--- a/packages/image-transform/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-# `@hypit/image-transform`
-
-An ordinary graph component with two inputs and one result:
-
-```text
-BlobArtifact(image) + ImageTransformProgram -> Need -> BlobArtifact(image)
-```
-
-The result is the transformed image itself. It contains no source digest, Provider name,
-post-processing report or copied upstream metadata. The graph already contains the source and
-program edges; the Runtime selects the endpoint that fulfills the explicit Need.
-
-`gptImageDenoiseV1` applies the GPT Image cleanup profile: YCrCb NLM with luma 2,
-chroma 10, 7/21 windows and 1.02 saturation recovery, followed by PNG encoding. It is now an
-explicit reusable Program rather than hidden behavior inside GPT Image generation.
-
-The official Markup Surface separates declaration from use:
-
-```xml
-<image:Program id="clean-gpt-image">
- <image:Denoise/>
- <image:Encode format="png"/>
-</image:Program>
-
-<image:Transform id="clean-shot" source={shot.image} program={clean-gpt-image}/>
-```
-
-Operation order is author meaning. Geometry, NLM denoise, color, sharpen/blur, alpha handling and
-encoding are bounded closed data. OpenCV is only one Runtime realization.
diff --git a/packages/image-transform/package.json b/packages/image-transform/package.json
deleted file mode 100644
index b0f8b7ba..00000000
--- a/packages/image-transform/package.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "name": "@hypit/image-transform",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": {
- ".": "./src/index.ts"
- },
- "hypit": {
- "activation": "./src/activation.ts"
- },
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/component-kit": "workspace:*",
- "@hypit/elaborator": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/raster": "workspace:*",
- "@hypit/markup": "workspace:*"
- }
-}
diff --git a/packages/image-transform/src/activation.ts b/packages/image-transform/src/activation.ts
deleted file mode 100644
index 8949c663..00000000
--- a/packages/image-transform/src/activation.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import { createMarkupSurfaceHostFacet } from "@hypit/markup";
-
-import {
- decodeImageTransformProgramSurface,
- decodeImageTransformSurface,
- imageTransformComponent,
- imageTransformManifest,
- imageTransformModuleRef,
- imageTransformMarkupSurfaces,
-} from "./index.js";
-
-export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- modules: [{
- manifest: imageTransformManifest,
- }],
- components: [imageTransformComponent],
- hostFacets: [{
- ...createMarkupSurfaceHostFacet({
- module: imageTransformModuleRef,
- declaration: imageTransformMarkupSurfaces.find((item) => item.name === "program")!,
- handler: decodeImageTransformProgramSurface,
- }),
- }, {
- ...createMarkupSurfaceHostFacet({
- module: imageTransformModuleRef,
- declaration: imageTransformMarkupSurfaces.find((item) => item.name === "transform")!,
- handler: decodeImageTransformSurface,
- }),
- }],
-};
-
-export default hypitPackage;
diff --git a/packages/image-transform/src/index.ts b/packages/image-transform/src/index.ts
deleted file mode 100644
index 7017af4a..00000000
--- a/packages/image-transform/src/index.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-export { imageTransformComponent } from "./component.js";
-export { imageTransformFragment } from "./fragment.js";
-export { imageTransformManifest, imageTransformMarkupSurfaces, imageTransformModuleRef, imageTransformProducers, imageTransformProgramSchema, imageTransformTypes } from "./manifest.js";
-export {
- gptImageDenoiseV1,
- sealImageTransformProgram,
- verifyImageTransformProgram,
-} from "./program.js";
-export {
- decodeImageTransformProgramSurface,
- decodeImageTransformSurface,
-} from "./surface.js";
-export type * from "./types.js";
diff --git a/packages/image-transform/src/types.ts b/packages/image-transform/src/types.ts
deleted file mode 100644
index a846afe7..00000000
--- a/packages/image-transform/src/types.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import type { RasterTransformOperation } from "@hypit/raster";
-
-export type {
- RasterAlphaOperation as ImageAlphaOperation,
- RasterBlurOperation as ImageBlurOperation,
- RasterColorOperation as ImageColorOperation,
- RasterCropOperation as ImageCropOperation,
- RasterDenoiseOperation as ImageDenoiseOperation,
- RasterEncodeOperation as ImageEncodeOperation,
- RasterFlipOperation as ImageFlipOperation,
- RasterResizeOperation as ImageResizeOperation,
- RasterRotateOperation as ImageRotateOperation,
- RasterSharpenOperation as ImageSharpenOperation,
- RasterTransformOperation as ImageTransformOperation,
-} from "@hypit/raster";
-
-/** Authored transformation intent. Operation order is author meaning. */
-export type ImageTransformProgram = {
- readonly operations: readonly RasterTransformOperation[];
-};
diff --git a/packages/media-execution/README.md b/packages/media-execution/README.md
deleted file mode 100644
index d1a29756..00000000
--- a/packages/media-execution/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# `@hypit/media-execution`
-
-Shared FFmpeg execution body for Hypit media Providers.
-
-Timeline-audio requests accept an optional `range: { startFrame, endFrameExclusive }` on the
-original programme clock. Only overlapping clips are read. Loop phase, tempo, gain and fades
-are evaluated in their original clip coordinates before trimming and rebasing the selected mix.
-The output remains 48 kHz stereo PCM. At rational frame rates, its sample count is rounded from
-the selected frame count; reconciling that zero-based clock can trim or pad one boundary sample.
-
-`@hypit/media-pipeline` owns provider-neutral Needs, plans and result contracts. This package
-owns the byte-level implementations of the eight current operations:
-
-- inspect every media stream;
-- normalize selected audio/video streams;
-- transform synchronized media with ordered trim and pitch-preserving retime operations;
-- extract one selected audio stream as an ordinary 48 kHz stereo WAV Artifact;
-- extract a first, last, indexed or time-selected frame as PNG;
-- project canonical 16 kHz mono speech-evidence audio;
-- render an exact sample-domain audio program;
-- mux rendered visual and audio products.
-
-The local Provider calls these functions with its Artifact I/O and FFmpeg launch environment. Other
-deployments can reuse the same functions to preserve stream selection, timing, codec and validation behavior.
-
-This is not an author package, Provider, queue or Core extension. It performs no endpoint selection,
-credential lookup or SVML parsing.
-
-`verifyCompositableSurfaceFile({ surface, path, ... })` inspects a file already staged by an execution
-owner. That owner must keep it unchanged until the call settles, including cancellation; inspection
-neither copies nor removes it. `verifyCompositableSurfaceBytes({ surface, bytes, ... })` remains a
-convenience wrapper for callers that have bytes, creates its own temporary file, and removes that
-file only after probing has settled. Both use the same decoded Surface checks and return no extra
-result facts. Filesystem paths belong to this Node execution boundary, not to Surface or Blob refs.
-
-Audio gain envelopes and audible subranges are evaluated per sample after tempo/looping and Clip
-fades, before the requested range is cropped. Original source and envelope progress therefore survive
-local silence and range rendering. This is generic AudioProgramPlan execution, independent of author
-component or Style names.
diff --git a/packages/media-execution/package.json b/packages/media-execution/package.json
deleted file mode 100644
index 04d14721..00000000
--- a/packages/media-execution/package.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "name": "@hypit/media-execution",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": {
- ".": "./src/index.ts"
- },
- "dependencies": {
- "@hypit/media": "workspace:*",
- "@hypit/media-pipeline": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/speech-evidence": "workspace:*",
- "@hypit/timeline": "workspace:*"
- }
-}
diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.ts
deleted file mode 100644
index 6aa6796b..00000000
--- a/packages/media-execution/src/index.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-export {
- executeInspectMedia,
- executeExtractAudio,
- executeExtractFrame,
- executeMuxProgramMedia,
- executeNormalizeMedia,
- executeProjectSpeechEvidenceAudio,
- executeRenderTimelineAudio,
- executeRenderStillVideo,
- executeTransformMedia,
-} from "./execute.js";
-export { parseMediaInspection } from "./probe.js";
-export { verifyCompositableSurfaceBytes, verifyCompositableSurfaceFile } from "./surface.js";
-export { probeMediaToolchain } from "./toolchain.js";
-export type { MediaToolchainState } from "./toolchain.js";
-export type * from "./execute.js";
diff --git a/packages/media-execution/src/surface.ts b/packages/media-execution/src/surface.ts
deleted file mode 100644
index 0ca6c49e..00000000
--- a/packages/media-execution/src/surface.ts
+++ /dev/null
@@ -1,293 +0,0 @@
-import { spawn } from "node:child_process";
-import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { assertCompositableSurfaceRef } from "@hypit/media";
-import { mediaProcessEnv } from "./process-env.js";
-import type { CompositableSurfaceRef } from "@hypit/media";
-
-type JsonObject = Record<string, unknown>;
-
-type ProbeStream = {
- readonly codec_type?: unknown;
- readonly codec_name?: unknown;
- readonly width?: unknown;
- readonly height?: unknown;
- readonly pix_fmt?: unknown;
- readonly sample_aspect_ratio?: unknown;
- readonly avg_frame_rate?: unknown;
- readonly r_frame_rate?: unknown;
- readonly nb_read_frames?: unknown;
- readonly nb_frames?: unknown;
- readonly color_range?: unknown;
- readonly color_space?: unknown;
- readonly color_transfer?: unknown;
- readonly color_primaries?: unknown;
- readonly field_order?: unknown;
- readonly disposition?: { readonly attached_pic?: unknown };
- readonly tags?: { readonly rotate?: unknown; readonly alpha_mode?: unknown };
- readonly side_data_list?: readonly unknown[];
-};
-
-type PixelFormat = {
- readonly name?: unknown;
- readonly flags?: { readonly alpha?: unknown };
-};
-
-function assert(condition: unknown, message: string): asserts condition {
- if (!condition) throw new Error(message);
-}
-
-function object(value: unknown): JsonObject | undefined {
- return value !== null && typeof value === "object" && !Array.isArray(value)
- ? value as JsonObject
- : undefined;
-}
-
-async function runProcess(args: {
- readonly executable: string;
- readonly argv: readonly string[];
- readonly timeoutMs: number;
- readonly maxOutputBytes: number;
- readonly signal?: AbortSignal;
-}): Promise<Uint8Array> {
- args.signal?.throwIfAborted();
- return await new Promise((resolve, reject) => {
- const child = spawn(args.executable, [...args.argv], {
- shell: false,
- windowsHide: true,
- stdio: ["ignore", "pipe", "pipe"],
- env: mediaProcessEnv(),
- });
- const stdout: Buffer[] = [];
- let bytes = 0;
- let stderr = "";
- let settled = false;
- let failure: Error | undefined;
- const stop = (error: Error) => {
- failure ??= error;
- child.kill("SIGKILL");
- };
- const finish = (error?: Error): void => {
- if (settled) return;
- settled = true;
- clearTimeout(timer);
- args.signal?.removeEventListener("abort", abort);
- if (error === undefined) resolve(Buffer.concat(stdout));
- else reject(error);
- };
- const abort = () => stop(args.signal?.reason ?? new Error("Process aborted"));
- args.signal?.addEventListener("abort", abort, { once: true });
- const timer = setTimeout(() => {
- stop(new Error(`${args.executable} timed out`));
- }, args.timeoutMs);
- child.stdout.on("data", (chunk: Buffer) => {
- bytes += chunk.byteLength;
- if (bytes > args.maxOutputBytes) {
- stop(new Error(`${args.executable} output exceeded the configured limit`));
- return;
- }
- stdout.push(chunk);
- });
- child.stderr.on("data", (chunk: Buffer) => {
- bytes += chunk.byteLength;
- stderr = `${stderr}${chunk.toString()}`.slice(-8_000);
- if (bytes > args.maxOutputBytes) {
- stop(new Error(`${args.executable} output exceeded the configured limit`));
- }
- });
- child.on("error", (error) => finish(error));
- if (args.signal?.aborted) abort();
- child.on("close", (code) => {
- if (failure !== undefined) finish(failure);
- else if (code === 0) finish();
- else finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));
- });
- });
-}
-
-function json(bytes: Uint8Array, subject: string): JsonObject {
- let value: unknown;
- try {
- value = JSON.parse(Buffer.from(bytes).toString("utf8"));
- } catch {
- throw new Error(`${subject} returned invalid JSON`);
- }
- const result = object(value);
- assert(result !== undefined, `${subject} returned an invalid object`);
- return result;
-}
-
-function positiveInteger(value: unknown, subject: string): number {
- const parsed = typeof value === "number"
- ? value
- : typeof value === "string" && /^\d+$/u.test(value) ? Number(value) : NaN;
- assert(Number.isSafeInteger(parsed) && parsed > 0, `${subject} is not a positive integer`);
- return parsed;
-}
-
-function rational(value: unknown, subject: string): { readonly numerator: number; readonly denominator: number } {
- assert(typeof value === "string" && /^\d+\/\d+$/u.test(value), `${subject} is not a rational`);
- const [rawNumerator, rawDenominator] = value.split("/");
- const numerator = Number(rawNumerator);
- const denominator = Number(rawDenominator);
- assert(Number.isSafeInteger(numerator) && numerator > 0
- && Number.isSafeInteger(denominator) && denominator > 0, `${subject} is invalid`);
- return { numerator, denominator };
-}
-
-function sameRational(
- left: { readonly numerator: number; readonly denominator: number },
- right: { readonly numerator: number; readonly denominator: number },
-): boolean {
- return BigInt(left.numerator) * BigInt(right.denominator)
- === BigInt(right.numerator) * BigInt(left.denominator);
-}
-
-function rotation(stream: ProbeStream): number {
- const side = stream.side_data_list?.map(object).find((item) => item?.side_data_type === "Display Matrix");
- const value = side?.rotation ?? stream.tags?.rotate ?? 0;
- const parsed = typeof value === "number" ? value : Number(value);
- assert(Number.isFinite(parsed), "Surface rotation metadata is invalid");
- return ((parsed % 360) + 360) % 360;
-}
-
-function assertSrgb(stream: ProbeStream): void {
- const allowed = {
- color_space: new Set([undefined, "unknown", "gbr", "rgb", "bt709"]),
- color_transfer: new Set([undefined, "unknown", "iec61966-2-1", "bt709"]),
- color_primaries: new Set([undefined, "unknown", "bt709"]),
- color_range: new Set([undefined, "unknown", "pc", "tv"]),
- } as const;
- for (const [name, values] of Object.entries(allowed)) {
- const value = stream[name as keyof typeof allowed];
- assert(typeof value === "string" || value === undefined, `Surface ${name} metadata is invalid`);
- assert(values.has(value), `Surface ${name} ${String(value)} is not SDR sRGB-compatible`);
- }
-}
-
-const pixelFormats = new Map<string, Map<string, boolean>>();
-
-async function pixelFormatAlpha(
- ffprobePath: string,
- timeoutMs: number,
- maxOutputBytes: number,
- signal: AbortSignal,
-): Promise<Map<string, boolean>> {
- signal.throwIfAborted();
- const cached = pixelFormats.get(ffprobePath);
- if (cached !== undefined) return cached;
- const root = json(await runProcess({ executable: ffprobePath,
- argv: ["-v", "error", "-print_format", "json", "-show_pixel_formats"],
- timeoutMs, maxOutputBytes, signal,
- }), "ffprobe pixel-format query");
- assert(Array.isArray(root.pixel_formats), "ffprobe returned no pixel-format table");
- const result = new Map<string, boolean>();
- for (const raw of root.pixel_formats) {
- const format = object(raw) as PixelFormat | undefined;
- if (typeof format?.name === "string") result.set(format.name, format.flags?.alpha === 1);
- }
- // Cache completed facts only; cancelling one caller cannot cancel another's probe.
- pixelFormats.set(ffprobePath, result);
- return result;
-}
-
-type SurfaceInspectionOptions = {
- readonly surface: CompositableSurfaceRef;
- readonly ffprobePath?: string;
- readonly processTimeoutMs?: number;
- readonly maxProbeOutputBytes?: number;
- readonly signal?: AbortSignal;
-};
-
-/** Inspect bytes supplied by callers that do not already own a staged file. */
-export async function verifyCompositableSurfaceBytes(options: SurfaceInspectionOptions & {
- readonly bytes: Uint8Array;
-}): Promise<void> {
- options.signal?.throwIfAborted();
- assertCompositableSurfaceRef(options.surface);
- assert(options.bytes.byteLength === options.surface.artifact.size,
- `Surface ${options.surface.artifact.resource} byte size differs`);
- const directory = await mkdtemp(join(tmpdir(), "hypit-surface-verify-"));
- try {
- // FFprobe detects the encoded media itself; both entry points inspect the same formats.
- const path = join(directory, "surface");
- await writeFile(path, options.bytes, { signal: options.signal });
- await verifyCompositableSurfaceFile({ ...options, path });
- } finally {
- await rm(directory, { recursive: true, force: true }).catch(() => {});
- }
-}
-
-/**
- * Inspect an existing execution file without copying it. The caller owns the file
- * and keeps it unchanged until this call settles. Success returns no side-channel facts.
- */
-export async function verifyCompositableSurfaceFile(options: SurfaceInspectionOptions & {
- readonly path: string;
-}): Promise<void> {
- const controller = new AbortController();
- const signal = options.signal === undefined ? controller.signal : AbortSignal.any([controller.signal, options.signal]);
- signal.throwIfAborted();
- assertCompositableSurfaceRef(options.surface);
- assert((await stat(options.path)).size === options.surface.artifact.size,
- `Surface ${options.surface.artifact.resource} byte size differs`);
- signal.throwIfAborted();
- const ffprobePath = options.ffprobePath ?? "ffprobe";
- const timeoutMs = options.processTimeoutMs ?? 120_000;
- const maxOutputBytes = options.maxProbeOutputBytes ?? 8 * 1024 * 1024;
- const pending = [
- runProcess({
- executable: ffprobePath,
- argv: ["-v", "error", "-print_format", "json", "-show_streams", "-count_frames", options.path],
- timeoutMs,
- maxOutputBytes, signal,
- }),
- pixelFormatAlpha(ffprobePath, timeoutMs, maxOutputBytes, signal),
- ] as const;
- await Promise.allSettled(pending.map(async (job) => {
- try { return await job; } catch (error) { controller.abort(error); throw error; }
- }));
- signal.throwIfAborted();
- const [probe, formats] = await Promise.all(pending);
- const root = json(probe, "ffprobe Surface query");
- assert(Array.isArray(root.streams) && root.streams.length === 1,
- "Surface must contain exactly one visual stream and no audio or auxiliary streams");
- const stream = root.streams[0] as ProbeStream;
- assert(stream.codec_type === "video", "Surface stream is not visual");
- assert(stream.disposition?.attached_pic !== 1, "Surface cannot be an attached-picture stream");
- const width = positiveInteger(stream.width, "Surface width");
- const height = positiveInteger(stream.height, "Surface height");
- assert(width === options.surface.width && height === options.surface.height,
- "Surface decoded dimensions differ from its declaration");
- assert(stream.sample_aspect_ratio === undefined || stream.sample_aspect_ratio === "1:1",
- "Surface must use square pixels");
- assert(rotation(stream) === 0, "Surface must not depend on display rotation metadata");
- assert(stream.field_order === undefined || stream.field_order === "progressive" || stream.field_order === "unknown",
- "Surface must be progressive");
- assertSrgb(stream);
-
- assert(typeof stream.pix_fmt === "string" && stream.pix_fmt.length > 0,
- "Surface pixel format is absent");
- const pixelAlpha = formats.get(stream.pix_fmt);
- assert(pixelAlpha !== undefined, `Surface pixel format ${stream.pix_fmt} is unknown to ffprobe`);
- const taggedAlpha = stream.tags?.alpha_mode === "1" || stream.tags?.alpha_mode === "straight";
- const hasAlpha = pixelAlpha || taggedAlpha;
- if (options.surface.alphaMode === "straight") {
- assert(hasAlpha, "Surface declares straight alpha but its bytes carry no alpha channel");
- } else {
- assert(!hasAlpha, "Surface declares opaque pixels but its encoded format carries alpha");
- }
-
- const frameCount = positiveInteger(stream.nb_read_frames ?? stream.nb_frames, "Surface frame count");
- if (options.surface.timing.kind === "still") {
- assert(frameCount === 1, "Still Surface must decode to exactly one frame");
- } else {
- assert(frameCount === options.surface.timing.frameCount,
- "Surface decoded frame count differs from its declaration");
- const rate = rational(stream.avg_frame_rate ?? stream.r_frame_rate, "Surface frame rate");
- assert(sameRational(rate, options.surface.timing.frameRate),
- "Surface decoded frame rate differs from its declaration");
- }
-}
diff --git a/packages/media-execution/test/surface.test.ts b/packages/media-execution/test/surface.test.ts
deleted file mode 100644
index 82785b49..00000000
--- a/packages/media-execution/test/surface.test.ts
+++ /dev/null
@@ -1,153 +0,0 @@
-import { spawn, spawnSync } from "node:child_process";
-import { chmod, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import assert from "node:assert/strict";
-import test from "node:test";
-import { setTimeout as delay } from "node:timers/promises";
-
-import { verifyCompositableSurfaceBytes, verifyCompositableSurfaceFile } from "@hypit/media-execution";
-import type { CompositableSurfaceRef } from "@hypit/media";
-
-const hasMediaTools = spawnSync("ffmpeg", ["-version"], { stdio: "ignore", windowsHide: true }).status === 0
- && spawnSync("ffprobe", ["-version"], { stdio: "ignore", windowsHide: true }).status === 0;
-
-async function ffmpeg(argv: readonly string[]): Promise<void> {
- await new Promise<void>((resolve, reject) => {
- const child = spawn("ffmpeg", ["-hide_banner", "-loglevel", "error", ...argv], {
- shell: false,
- windowsHide: true,
- stdio: ["ignore", "ignore", "pipe"],
- });
- let stderr = "";
- child.stderr.on("data", (chunk: Buffer) => { stderr += chunk.toString(); });
- child.on("error", reject);
- child.on("close", (code) => code === 0 ? resolve() : reject(new Error(stderr)));
- });
-}
-
-let surfaceId = 0;
-
-function surface(bytes: Uint8Array, options: {
- readonly mediaType: string;
- readonly width: number;
- readonly height: number;
- readonly alphaMode: "opaque" | "straight";
- readonly timing: CompositableSurfaceRef["timing"];
-}): CompositableSurfaceRef {
- const resource = `res_surface-${surfaceId += 1}` as const;
- return {
- artifact: { kind: "blob", resource, size: bytes.byteLength, mediaType: options.mediaType },
- width: options.width,
- height: options.height,
- colorSpace: "srgb",
- alphaMode: options.alphaMode,
- timing: options.timing,
- };
-}
-
-test("cancelling staged Surface inspection stops both probes and preserves the caller's file", {
- skip: process.platform === "win32",
-}, async () => {
- const directory = await mkdtemp(join(tmpdir(), "hypit-surface-cancel-"));
- const program = join(directory, "ffprobe");
- const marker = join(directory, "running");
- const bytes = new Uint8Array([0]);
- const value = surface(bytes, { mediaType: "image/png", width: 1, height: 1,
- alphaMode: "opaque", timing: { kind: "still" } });
- try {
- await writeFile(program, `#!${process.execPath}
-const fs = require('node:fs');
-fs.writeFileSync(${JSON.stringify(marker)} + '-' + process.pid, '');
-setInterval(() => {}, 1000);
-`);
- await chmod(program, 0o755);
- const path = join(directory, "staged.png");
- await writeFile(path, bytes);
- const controller = new AbortController();
- const checking = verifyCompositableSurfaceFile({ surface: value, path, ffprobePath: program,
- signal: controller.signal });
- const rejected = assert.rejects(checking, /stop probes/u);
- let pids: number[] = [];
- for (let i = 0; i < 200; i++) {
- pids = (await readdir(directory)).filter((name) => name.startsWith("running-"))
- .map((name) => Number(name.slice("running-".length)));
- if (pids.length === 2) break;
- await delay(10);
- }
- controller.abort(new Error("stop probes"));
- await rejected;
- assert.equal(pids.length, 2);
- for (const pid of pids) assert.throws(() => process.kill(pid, 0), { code: "ESRCH" });
- assert.deepEqual([...await readFile(path)], [...bytes]);
- await writeFile(program, `#!${process.execPath}
-console.log(JSON.stringify(process.argv.includes('-show_pixel_formats')
- ? { pixel_formats: [{ name: 'rgb24', flags: { alpha: 0 } }] }
- : { streams: [{ codec_type: 'video', width: 1, height: 1, pix_fmt: 'rgb24', nb_read_frames: '1' }] }));
-`);
- await verifyCompositableSurfaceBytes({ surface: value, bytes, ffprobePath: program });
- } finally { await rm(directory, { recursive: true, force: true }); }
-});
-
-test("Surface byte admission accepts matching still/video bytes and rejects contradictions", {
- skip: !hasMediaTools,
-}, async () => {
- const directory = await mkdtemp(join(tmpdir(), "hypit-surface-test-"));
- try {
- const opaquePath = join(directory, "opaque.png");
- const alphaPath = join(directory, "alpha.png");
- const videoPath = join(directory, "video.mp4");
- await ffmpeg(["-f", "lavfi", "-i", "color=c=red:s=8x6:d=1", "-frames:v", "1", opaquePath]);
- await ffmpeg(["-f", "lavfi", "-i", "[email protected]:s=8x6:d=1,format=rgba", "-frames:v", "1", alphaPath]);
- await ffmpeg([
- "-f", "lavfi", "-i", "color=c=blue:s=8x6:r=4:d=1",
- "-frames:v", "4", "-c:v", "libx264", "-pix_fmt", "yuv420p", videoPath,
- ]);
- const [opaque, alpha, video] = await Promise.all([
- readFile(opaquePath), readFile(alphaPath), readFile(videoPath),
- ]);
- const opaqueSurface = surface(opaque, {
- mediaType: "image/png", width: 8, height: 6, alphaMode: "opaque", timing: { kind: "still" },
- });
- const alphaSurface = surface(alpha, {
- mediaType: "image/png", width: 8, height: 6, alphaMode: "straight", timing: { kind: "still" },
- });
- const videoSurface = surface(video, {
- mediaType: "video/mp4", width: 8, height: 6, alphaMode: "opaque",
- timing: { kind: "frames", frameRate: { numerator: 4, denominator: 1 }, frameCount: 4 },
- });
-
- await verifyCompositableSurfaceBytes({ surface: opaqueSurface, bytes: opaque });
- await verifyCompositableSurfaceBytes({ surface: alphaSurface, bytes: alpha });
- await verifyCompositableSurfaceBytes({ surface: videoSurface, bytes: video });
- await verifyCompositableSurfaceFile({ surface: videoSurface, path: videoPath });
- await assert.rejects(verifyCompositableSurfaceFile({ surface: {
- ...videoSurface, artifact: { ...videoSurface.artifact, size: video.byteLength + 1 },
- }, path: videoPath }), /byte size differs/u);
- await assert.rejects(verifyCompositableSurfaceFile({ surface: {
- ...videoSurface, width: 10,
- }, path: videoPath }), /dimensions differ/u);
- assert.deepEqual(await readFile(videoPath), video, "inspection never rewrites or deletes the caller's file");
-
- await assert.rejects(
- verifyCompositableSurfaceBytes({ surface: { ...opaqueSurface, width: 9 }, bytes: opaque }),
- /dimensions differ/u,
- );
- await assert.rejects(
- verifyCompositableSurfaceBytes({ surface: { ...alphaSurface, alphaMode: "opaque" }, bytes: alpha }),
- /encoded format carries alpha/u,
- );
- await assert.rejects(
- verifyCompositableSurfaceBytes({
- surface: {
- ...videoSurface,
- timing: { kind: "frames", frameRate: { numerator: 4, denominator: 1 }, frameCount: 3 },
- },
- bytes: video,
- }),
- /frame count differs/u,
- );
- } finally {
- await rm(directory, { recursive: true, force: true });
- }
-});
diff --git a/packages/provider-media-local/README.md b/packages/media-local/README.md
similarity index 51%
rename from packages/provider-media-local/README.md
rename to packages/media-local/README.md
index 05e0b838..5fba56d3 100644
--- a/packages/provider-media-local/README.md
+++ b/packages/media-local/README.md
@@ -1,10 +1,50 @@
-# `@hypit/provider-media-local`
+# `@hypit/media-local`
-Trusted local ffprobe/ffmpeg implementation of `@hypit/media-pipeline`'s nine exact byte-operation
-capabilities. It is Runtime configuration and is never imported by author `.svml`.
+Trusted local ffprobe/ffmpeg owner. It exposes the same concrete local-media behavior through three
+orthogonal ports:
+
+- a Provider port implements `@hypit/media-operations` Needs against a Build's ResourceStore;
+- a file port inspects or creates files explicitly named by an authoring tool;
+- `@hypit/media-local/cli` presents that file port as `hypit media`.
+
+The Provider is Runtime configuration and is never imported by author `.svml`. The file and CLI ports
+do not manufacture a Graph, Build, Result or Runtime state.
+
+This Provider owns its FFmpeg commands, media inspection, animated-WebP handling and audio-program
+execution. They are implementation details of this machine executor, not a shared domain package or
+a root `@hypit/hypit` facade. Another Provider implements the same Needs against the public operation
+contracts rather than importing this Provider's process code.
+
+The local process runner is shared by the Provider and file ports, so bounds, shell avoidance and
+failure reporting do not fork into two executors. Domain contracts remain in `@hypit/media-operations`;
+paths, subprocesses and ffprobe output remain here.
+
+## File authoring and inspection
+
+```bash
+hypit media probe reference.mp4
+hypit media cut reference.mp4 --start 12 --end 19.5 --to assets/hook.mp4
+hypit media frames reference.mp4 --at 12.4,13.1 --to notes/hook-frames
+hypit media tile reference.mp4 --start 12 --end 19.5 --to notes/hook-grid.jpg
+hypit media tiles reference.mp4 --ranges notes/ranges.json --to notes/grids
+hypit media boundaries reference.mp4
+```
+
+Network acquisition is not a media-execution operation. `@hypit/yt-dlp/cli` owns
+`hypit download`; the resulting file becomes an ordinary input to these commands.
+
+Frame extraction, native-PTS sequences and labelled grids are evidence about a source media clock.
+They therefore live beside the decoder that establishes that clock. `HtmlProgram` frames remain an
+`html-program` concern, and capturing Studio's current `HtmlProgram` remains a Studio concern; the word
+“frame” does not create a central package.
+
+`tile` and `tiles` use the same grid-layout helper exported for Studio snapshot presentation. Reusing
+that pure layout does not transfer frame acquisition or Runtime authority. Transcript-aware ranges
+consume `hypit.transcript@1`; parsing and phrase matching are owned by `@hypit/speech-evidence`, while
+this package only joins those ranges to file decoding.
```ts
-import { createLocalMediaProvider } from "@hypit/provider-media-local";
+import { createLocalMediaProvider } from "@hypit/media-local";
const media = createLocalMediaProvider({ instance: "media.local", defaultConcurrency: 1 });
```
@@ -22,7 +62,8 @@ The Endpoint:
- preserves source A/V offset through deterministic trim, delay, pad and crop operations;
- emits a silent CFR visual and, when selected, an exact-length 48 kHz stereo PCM WAV;
- transforms synchronized A/V, extracts generic reference audio and extracts exact source frames;
-- encodes one authored image frame as an ordinary finite video-only MP4 before normalization;
+- encodes authored image frames as ordinary finite video-only MP4 with explicit BT.709 conversion and
+ colour metadata before normalization;
- renders an explicit `AudioProgramPlan` into one exact-length 48 kHz stereo PCM `TimelineAudio`;
- muxes exactly one verified silent visual stream and one verified program-audio stream into MP4;
- distinguishes AAC coding-frame padding from the authoritative packet presentation span.
@@ -33,13 +74,16 @@ Animated WebP is also normalized through an alpha-capable output. Transparent VP
decoders so FFmpeg retains the alpha sidecar, including during PNG frame extraction.
Transparent normalization requires an FFmpeg build with the `libvpx-vp9` encoder; the normal
process error reports a missing codec if a custom deployment does not provide it.
+The lossless alpha encoding uses libvpx row threading and its realtime effort profile. Those options
+change encoder work, not the decoded frames or the public SynchronizedMedia value; no hidden media
+cache or content identity is introduced.
These are this FFmpeg implementation's intermediate encodings; Source and Track inputs continue
to use ordinary SynchronizedMedia. Transform currently emits opaque MP4, so perform trim/retime
before matting when the resulting clip needs transparency.
The result is ordinary SynchronizedMedia. Timeline construction consumes only its local domain;
Visual Clip or a Visual Clip consumes its transparent picture according to the authored
-relationship. Local HyperFrames extracts alpha-preserving PNGs and composites them against the authored
+relationship. The local HTML Provider extracts alpha-preserving PNGs and composites them against the authored
Canvas and lower visual layers before encoding the final MP4.
The Runtime Adapter declares the selected `ffmpeg`/`ffprobe` pair as an external, non-daemon Program.
@@ -50,7 +94,7 @@ system package manager.
The reusable display-materialization capabilities—inspection, normalization, transform, extraction,
StillVideo—also opt into transient authoring execution. Speech-evidence projection,
-programme-audio rendering and final muxing remain Build-only. This is declared per capability; the
+timeline-audio rendering and final muxing remain Build-only. This is declared per capability; the
Runtime and Studio contain no media capability allowlist.
Video-backed normalization is video-authoritative so an AAC packet tail cannot extend the program
diff --git a/packages/media-local/package.json b/packages/media-local/package.json
new file mode 100644
index 00000000..e73dbcfa
--- /dev/null
+++ b/packages/media-local/package.json
@@ -0,0 +1,35 @@
+{
+ "name": "@hypit/media-local",
+ "version": "0.1.0",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts",
+ "./cli": "./src/cli.ts",
+ "./files": "./src/files.ts"
+ },
+ "files": ["dist", "README.md"],
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@hypit/media-operations": "workspace:^",
+ "sharp": "0.35.4"
+ },
+ "peerDependencies": {
+ "@hypit/hypit": "workspace:^",
+ "@hypit/runtime-local": "workspace:^"
+ },
+ "devDependencies": {
+ "@hypit/blob": "workspace:*",
+ "@hypit/endpoint": "workspace:*",
+ "@hypit/media": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/runtime": "workspace:*",
+ "@hypit/runtime-local": "workspace:*",
+ "@hypit/speech-evidence": "workspace:*",
+ "@hypit/hypit": "workspace:*",
+ "@hypit/executor": "workspace:*"
+ }
+}
diff --git a/packages/provider-media-local/src/activation.ts b/packages/media-local/src/activation.ts
similarity index 92%
rename from packages/provider-media-local/src/activation.ts
rename to packages/media-local/src/activation.ts
index 45e540c1..c3aa58a6 100644
--- a/packages/provider-media-local/src/activation.ts
+++ b/packages/media-local/src/activation.ts
@@ -4,17 +4,17 @@ import {
runtimeConfigObject,
runtimeConfigPositiveInteger,
runtimeConfigString,
-} from "@hypit/runtime-kit";
+} from "@hypit/runtime-local/extension";
import {
diagnoseRuntimeExecutable,
resolveRuntimeExecutable,
-} from "@hypit/runtime-host-node";
+} from "@hypit/runtime-local/node";
import { createLocalMediaProvider } from "./provider.js";
import { localMediaToolchainProgram } from "./program.js";
const localMediaRuntimeAdapter = createRuntimeEndpointAdapterFacet({
- use: "@hypit/provider-media-local",
+ use: "@hypit/media-local",
activate(context) {
if (context.pool === undefined) throw new Error("local media Provider Pool is required");
const config = runtimeConfigObject(context.config, "local media");
@@ -58,8 +58,8 @@ const localMediaRuntimeAdapter = createRuntimeEndpointAdapterFacet({
});
export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- hostFacets: [localMediaRuntimeAdapter],
+ format: "hypit.package@1" as const,
+ facets: [localMediaRuntimeAdapter],
};
export default hypitPackage;
diff --git a/packages/media-execution/src/audio-level-automation.ts b/packages/media-local/src/audio-level-automation.ts
similarity index 96%
rename from packages/media-execution/src/audio-level-automation.ts
rename to packages/media-local/src/audio-level-automation.ts
index 4e7b7445..8b86db50 100644
--- a/packages/media-execution/src/audio-level-automation.ts
+++ b/packages/media-local/src/audio-level-automation.ts
@@ -1,4 +1,4 @@
-import type { AudioProgramClip } from "@hypit/media-pipeline";
+import type { AudioProgramClip } from "@hypit/media-operations";
/** Per-sample level automation after source playback/fades, before any requested render-range crop. */
export function audioLevelAutomationFilter(clip: AudioProgramClip): string[] {
diff --git a/packages/media-local/src/cli.ts b/packages/media-local/src/cli.ts
new file mode 100644
index 00000000..daa80dd1
--- /dev/null
+++ b/packages/media-local/src/cli.ts
@@ -0,0 +1,22 @@
+import type { CliCommandModule } from "@hypit/hypit/cli";
+
+import { isMediaCommand, mediaCommands, runMediaCli, writeMediaHelp } from "./files.js";
+
+export const cliCommandModules = [{
+ format: "hypit.cli-command@1",
+ id: "@hypit/media-local",
+ commands: ["media"],
+ writeRootHelp(io) {
+ io.write(`\nLocal media\n media ${mediaCommands.join(" | ")}\n hypit media --help for exact options\n`);
+ },
+ writeHelp(argv, io) {
+ const subcommand = argv[0] === "help" ? argv[2] : argv[1];
+ writeMediaHelp(io, isMediaCommand(subcommand) ? subcommand : undefined);
+ },
+ async run(argv, io, context) {
+ await runMediaCli(argv, io, context.cwd);
+ },
+}] as const satisfies readonly CliCommandModule[];
+
+export { isMediaCommand, mediaCommands, runMediaCli, writeMediaHelp } from "./files.js";
+export type { MediaCommand, MediaProbe } from "./files.js";
diff --git a/packages/media-execution/src/execute.ts b/packages/media-local/src/execute.ts
similarity index 95%
rename from packages/media-execution/src/execute.ts
rename to packages/media-local/src/execute.ts
index 74389def..2dc77ac3 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-local/src/execute.ts
@@ -1,17 +1,15 @@
import { audioLevelAutomationFilter } from "./audio-level-automation.js";
-import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdtemp, open, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { verifyMediaFrameRange, mediaFrameRangeSamples, mediaTypes, sealMediaInspection, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyRenderedVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
-import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, RenderedVisual, SynchronizedMedia, TimelineAudio } from "@hypit/media";
-import type { Timeline } from "@hypit/timeline";
-import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "@hypit/speech-evidence";
-import type { SpeechEvidenceAudio } from "@hypit/speech-evidence";
+import { verifyMediaFrameRange, mediaFrameRangeSamples, mediaTypes, sealMediaInspection, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyTimelineVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/hypit/media";
+import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, TimelineVisual, SynchronizedMedia, TimelineAudio } from "@hypit/hypit/media";
+import type { Timeline } from "@hypit/hypit/timeline";
+import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "@hypit/hypit/speech-evidence";
+import type { ProjectSpeechEvidenceAudioNeed, SpeechEvidenceAudio } from "@hypit/hypit/speech-evidence";
import {
- mediaPipelineCapabilities,
verifyAudioExtractionRequest,
verifyAudioProgramPlan,
verifyFrameExtractionRequest,
@@ -21,19 +19,18 @@ import {
type InspectMediaNeed,
type MuxMediaNeed,
type NormalizeMediaNeed,
- type ProjectSpeechEvidenceAudioNeed,
type RenderAudioNeed,
type RenderStillVideoNeed,
type TransformMediaNeed,
verifyStillVideoRequest,
-} from "@hypit/media-pipeline";
-import type { AudioProgramClip, AudioProgramPlan, MediaTransformOperation } from "@hypit/media-pipeline";
+} from "@hypit/media-operations";
+import type { AudioProgramClip, AudioProgramPlan, MediaTransformOperation } from "@hypit/media-operations";
import {
canonicalize,
- } from "@hypit/protocol";
-import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
+ } from "@hypit/hypit/protocol";
+import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol";
-import { mediaProcessEnv } from "./process-env.js";
+import { runBoundedMediaProcess } from "./process.js";
import { parseMediaInspection } from "./probe.js";
import {
compositeAnimatedWebpFrame,
@@ -46,11 +43,9 @@ import type { AnimatedWebp } from "./webp.js";
/**
* Where the bytes live and which binaries transform them.
*
- * These byte operations are the whole of Hypit's media execution, and they
- * are written once. A local Provider supplies the Build's own ResourceStore and
- * the ffmpeg on its PATH; a Lambda Provider supplies an S3-backed gateway and
- * the ffmpeg carried by its deployment. Nothing below knows which it is, so the two
- * deployments cannot drift into computing different media from one Need.
+ * These byte operations are written once behind an Artifact gateway. The local Provider supplies
+ * the Build's ResourceStore and the ffmpeg on its PATH. Another execution owner may reuse the same
+ * algorithms with a different gateway and tool bundle; nothing here claims a Provider or deployment.
*/
export type MediaArtifactGateway = {
get(source: BlobRef): Promise<Uint8Array | undefined>;
@@ -94,46 +89,7 @@ async function runProcess(args: {
readonly maxStdoutBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<Uint8Array> {
- return await new Promise((resolve, reject) => {
- const child = spawn(args.executable, [...args.argv], {
- shell: false,
- windowsHide: true,
- stdio: ["ignore", "pipe", "pipe"],
- env: mediaProcessEnv(args.sharedLibraryPath === undefined ? undefined : { LD_LIBRARY_PATH: args.sharedLibraryPath }),
- });
- const stdout: Buffer[] = [];
- let stdoutBytes = 0;
- let stderr = "";
- let settled = false;
- const finish = (error?: Error): void => {
- if (settled) return;
- settled = true;
- clearTimeout(timer);
- if (error !== undefined) reject(error);
- else resolve(Buffer.concat(stdout));
- };
- const timer = setTimeout(() => {
- child.kill("SIGKILL");
- finish(new Error(`${args.executable} timed out`));
- }, args.timeoutMs);
- child.stdout.on("data", (chunk: Buffer) => {
- stdoutBytes += chunk.byteLength;
- if (stdoutBytes > args.maxStdoutBytes) {
- child.kill("SIGKILL");
- finish(new Error(`${args.executable} output exceeded the configured limit`));
- return;
- }
- stdout.push(chunk);
- });
- child.stderr.on("data", (chunk: Buffer) => {
- stderr = `${stderr}${chunk.toString()}`.slice(-8_000);
- });
- child.on("error", (error) => finish(error));
- child.on("close", (code) => {
- if (code !== 0) finish(new Error(`${args.executable} exited ${String(code)}: ${stderr}`));
- else finish();
- });
- });
+ return await runBoundedMediaProcess(args);
}
async function version(executable: string, timeoutMs: number, sharedLibraryPath?: string): Promise<string> {
@@ -507,7 +463,7 @@ function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {
function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {
const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;
- verifyRenderedVisual(item.visual);
+ verifyTimelineVisual(item.visual);
verifyTimelineAudio(item.audio);
const expectedSamples = roundPositive(
BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),
@@ -681,7 +637,10 @@ export async function executeNormalizeMedia(
// The execution format must retain the source's alpha while materializing
// the program clock. Both encodings publish the same SynchronizedMedia type.
const encoderArgs = encoding.alpha
- ? ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-lossless", "1", "-auto-alt-ref", "0"]
+ ? [
+ "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-lossless", "1", "-auto-alt-ref", "0",
+ "-row-mt", "1", "-deadline", "realtime", "-cpu-used", "8",
+ ]
: ["-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart"];
await runProcess({
executable: env.ffmpegPath,
@@ -802,9 +761,14 @@ export async function executeRenderStillVideo(
`settb=expr=${denominator}/${numerator}`,
`setpts=N*${denominator}/(${numerator}*TB)`,
].join(",");
+ const bt709Output = [
+ "scale=out_color_matrix=bt709:out_range=tv",
+ "format=yuv420p",
+ "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
+ ].join(",");
let argv: string[];
if (need.request.segments.length === 1) {
- const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`;
+ const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,${bt709Output}`;
argv = ["-y", "-i", inputs[0]!, "-map", "0:v:0", "-an", "-vf", filter];
} else {
const first = await outputInspection({
@@ -821,8 +785,8 @@ export async function executeRenderStillVideo(
const height = Math.ceil(picture.height / 2) * 2;
const chains = need.request.segments.map((segment, index) =>
`[${index}:v]${hold(segment.endFrameExclusive - segment.startFrame)},`
- + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p[s${index}]`);
- const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[v]`;
+ + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[s${index}]`);
+ const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[joined];[joined]${bt709Output}[v]`;
argv = ["-y", ...inputs.flatMap((path) => ["-i", path]), "-filter_complex", `${chains.join(";")};${concat}`, "-map", "[v]", "-an"];
}
await runProcess({
@@ -1263,12 +1227,12 @@ export async function executeMuxProgramMedia(
]);
const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
assert(visualStreams.length === 1 && visualInspection.streams.length === 1,
- "RenderedVisual Artifact must contain exactly one silent video stream");
+ "TimelineVisual Artifact must contain exactly one silent video stream");
const visual = visualStreams[0]!;
assert(visual.role === "moving" && visual.timingStatus === "admissible"
&& visual.decodedUnitCount === need.visual.frameCount
&& visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,
- "RenderedVisual bytes differ from their declared frame domain");
+ "TimelineVisual bytes differ from their declared frame domain");
assert(audio.decodedSampleFrames === need.audio.sampleFrames,
"TimelineAudio bytes differ from their declared sample domain");
await runProcess({
@@ -1297,7 +1261,7 @@ export async function executeMuxProgramMedia(
"Final mux must contain exactly one video and one audio stream");
assert(finalVideo[0]!.timingStatus === "admissible"
&& finalVideo[0]!.decodedUnitCount === need.visual.frameCount,
- "Final mux video frame count differs from RenderedVisual");
+ "Final mux video frame count differs from TimelineVisual");
assert(finalAudio[0]!.timingStatus === "admissible"
&& finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,
"Final mux audio shape differs from TimelineAudio");
diff --git a/packages/media-local/src/file-frames.ts b/packages/media-local/src/file-frames.ts
new file mode 100644
index 00000000..c20ad5e4
--- /dev/null
+++ b/packages/media-local/src/file-frames.ts
@@ -0,0 +1,25 @@
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { runProcessOutput } from "./process.js";
+
+export type DecodedFrame = { readonly at: number; readonly path: string };
+
+/** Decode a source interval once, retaining each decoded frame and its actual source timestamp. */
+export async function decodeMediaFrames(source: string, start: number, end: number, directory: string): Promise<readonly DecodedFrame[]> {
+ if (!Number.isFinite(start) || start < 0 || !Number.isFinite(end) || end <= start) throw new Error("Native frames need 0 <= start < end");
+ await mkdir(directory, { recursive: true });
+ const times: number[] = [];
+ let numerator = 0, denominator = 0;
+ await runProcessOutput("ffmpeg", ["-hide_banner", "-loglevel", "info", "-nostdin", "-copyts", "-start_at_zero",
+ ...(start > 0 ? ["-ss", String(start)] : []), "-i", source,
+ "-map", "0:v:0", "-an", "-vf", `trim=start=${start}:end=${end},showinfo=checksum=0`,
+ "-fps_mode", "passthrough", "-pix_fmt", "yuvj420p", "-q:v", "3", "-start_number", "0", join(directory, "%09d.jpg"),
+ ], 300_000, line => {
+ const base = /config in time_base:\s*(\d+)\/(\d+)/u.exec(line);
+ if (base !== null) { numerator = Number(base[1]); denominator = Number(base[2]); }
+ const pts = /\bn:\s*\d+\s+pts:\s*(-?\d+)/u.exec(line);
+ if (pts !== null && denominator > 0) times.push(Number(pts[1]) * numerator / denominator);
+ });
+ if (times.length === 0) throw new Error(`No video frames in [${start}, ${end}) s`);
+ return times.map((at, index) => ({ at, path: join(directory, `${String(index).padStart(9, "0")}.jpg`) }));
+}
diff --git a/packages/media-local/src/files.ts b/packages/media-local/src/files.ts
new file mode 100644
index 00000000..279e0885
--- /dev/null
+++ b/packages/media-local/src/files.ts
@@ -0,0 +1,843 @@
+import { copyFile, link, mkdir, mkdtemp, readFile, rename, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, extname, join, resolve } from "node:path";
+
+import type { CliIo } from "@hypit/hypit/cli";
+import sharp from "sharp";
+
+import { decodeMediaFrames } from "./file-frames.js";
+import { writeFrameGrid } from "./frame-grid.js";
+import { runProcess, runProcessOutput, runProcessWithInput } from "./process.js";
+import { parseTranscriptWords, phraseRanges, wordsAt } from "@hypit/hypit/speech-evidence";
+import type { FrameWords, TranscriptWord } from "@hypit/hypit/speech-evidence";
+
+/**
+ * Local views of media at chosen times and scales. Transcript annotations use the same seconds as
+ * the input media. These commands expose evidence; editorial interpretation belongs to the author.
+ */
+
+export const mediaCommands = ["probe", "cut", "frames", "tile", "tiles", "boundaries"] as const;
+export type MediaCommand = typeof mediaCommands[number];
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+async function readTranscript(path: string): Promise<readonly TranscriptWord[]> {
+ return parseTranscriptWords(JSON.parse(await readFile(path, "utf8")), path);
+}
+
+export function isMediaCommand(value: string | undefined): value is MediaCommand {
+ return (mediaCommands as readonly string[]).includes(value ?? "");
+}
+
+function round(value: number): number { return Number(value.toFixed(3)); }
+function clamp(value: number, low: number, high: number): number { return Math.max(low, Math.min(high, value)); }
+
+// ---------------------------------------------------------------------------------------------------
+// Arguments
+
+type Parsed = {
+ readonly positionals: readonly string[];
+ readonly options: ReadonlyMap<string, string>;
+ readonly repeated: ReadonlyMap<string, readonly string[]>;
+ readonly flags: ReadonlySet<string>;
+ readonly json: boolean;
+};
+
+function parseArguments(
+ argv: readonly string[], allowed: readonly string[], allowedFlags: readonly string[] = [], repeatable: readonly string[] = [],
+): Parsed {
+ const positionals: string[] = [];
+ const options = new Map<string, string>();
+ const repeated = new Map<string, string[]>();
+ const flags = new Set<string>();
+ let json = false;
+ for (let index = 0; index < argv.length; index += 1) {
+ const item = argv[index]!;
+ if (item === "--json") { json = true; continue; }
+ if (item === "--debug" || item === "--verbose" || item === "--no-color") continue;
+ if (item === "--color") { index += 1; continue; }
+ if (!item.startsWith("--")) { positionals.push(item); continue; }
+ if (allowedFlags.includes(item)) {
+ if (flags.has(item)) throw new Error(`${item} cannot be repeated`);
+ flags.add(item);
+ continue;
+ }
+ if (!allowed.includes(item)) throw new Error(`unknown option ${item}`);
+ if (!repeatable.includes(item) && options.has(item)) throw new Error(`${item} cannot be repeated`);
+ const value = argv[index + 1];
+ if (value === undefined || (value.startsWith("--") && value.length > 2)) throw new Error(`${item} requires a value`);
+ if (repeatable.includes(item)) repeated.set(item, [...(repeated.get(item) ?? []), value]);
+ else options.set(item, value);
+ index += 1;
+ }
+ return { positionals, options, repeated, flags, json };
+}
+
+function required(parsed: Parsed, option: string, hint: string): string {
+ const value = parsed.options.get(option);
+ if (value === undefined) throw new Error(`${option} is required: ${hint}`);
+ return value;
+}
+
+function secondsOption(parsed: Parsed, option: string, fallback?: number): number {
+ const raw = parsed.options.get(option);
+ if (raw === undefined) {
+ if (fallback !== undefined) return fallback;
+ throw new Error(`${option} is required: a time in seconds`);
+ }
+ const value = Number(raw);
+ assert(Number.isFinite(value) && value >= 0, `${option} must be a non-negative number of seconds, got ${raw}`);
+ return value;
+}
+
+function integerOption(parsed: Parsed, option: string, fallback: number, low: number): number {
+ const raw = parsed.options.get(option);
+ if (raw === undefined) return fallback;
+ const value = Number(raw);
+ assert(Number.isSafeInteger(value) && value >= low, `${option} must be a whole number of at least ${low}, got ${raw}`);
+ return value;
+}
+
+function numberOption(parsed: Parsed, option: string, fallback: number, low: number, high: number): number {
+ const raw = parsed.options.get(option);
+ if (raw === undefined) return fallback;
+ const value = Number(raw);
+ assert(Number.isFinite(value) && value >= low && value <= high, `${option} must be between ${low} and ${high}, got ${raw}`);
+ return value;
+}
+
+function secondsList(raw: string, option: string, minimum: number): number[] {
+ const parts = raw.split(",").map((item) => item.trim());
+ assert(parts.length >= minimum && parts.every((item) => item.length > 0), `${option} must list at least ${minimum} seconds, got ${raw}`);
+ const values = parts.map(Number);
+ assert(values.every((value) => Number.isFinite(value) && value >= 0), `${option} must list non-negative seconds, got ${raw}`);
+ return values;
+}
+
+async function sourceFile(parsed: Parsed, cwd: string): Promise<string> {
+ const [source] = parsed.positionals;
+ assert(source !== undefined, "name the media file first");
+ assert(parsed.positionals.length === 1, "media commands take exactly one media file");
+ const path = resolve(cwd, source);
+ assert(await stat(path).then((item) => item.isFile(), () => false), `cannot read ${path}`);
+ return path;
+}
+
+async function destination(parsed: Parsed, cwd: string, hint: string): Promise<string> {
+ const to = resolve(cwd, required(parsed, "--to", hint));
+ assert(!(await stat(to).then(() => true, () => false)), `Destination ${to} already exists`);
+ await mkdir(dirname(to), { recursive: true });
+ return to;
+}
+
+async function destinationDirectory(parsed: Parsed, cwd: string, hint: string): Promise<string> {
+ const to = resolve(cwd, required(parsed, "--to", hint));
+ assert(!(await stat(to).then(() => true, () => false)), `Destination ${to} already exists`);
+ await mkdir(dirname(to), { recursive: true });
+ return to;
+}
+
+// ---------------------------------------------------------------------------------------------------
+// ffmpeg
+
+type VideoProbe = {
+ readonly hasVideo: true;
+ readonly width: number;
+ readonly height: number;
+ readonly frameRate: number;
+};
+
+export type MediaProbe = {
+ readonly duration: number;
+ readonly hasAudio: boolean;
+} & (VideoProbe | { readonly hasVideo: false });
+
+function requireVideo(info: MediaProbe, path: string): asserts info is MediaProbe & VideoProbe {
+ assert(info.hasVideo, `${path}: no video stream`);
+}
+
+export async function probeMedia(path: string, options: { readonly ffprobePath?: string } = {}): Promise<MediaProbe> {
+ const raw = await runProcess(options.ffprobePath ?? "ffprobe", [
+ "-v", "error", "-show_entries", "format=duration:stream=codec_type,width,height,r_frame_rate:stream_disposition=attached_pic", "-of", "json", path,
+ ]);
+ const parsed = JSON.parse(raw.toString("utf8")) as {
+ format?: { duration?: string };
+ streams?: readonly {
+ codec_type?: string; width?: number; height?: number; r_frame_rate?: string;
+ disposition?: { attached_pic?: number };
+ }[];
+ };
+ const video = parsed.streams?.find((item) => item.codec_type === "video" && item.disposition?.attached_pic !== 1);
+ const duration = Number(parsed.format?.duration);
+ assert(Number.isFinite(duration) && duration > 0, `${path}: duration is unavailable`);
+ const hasAudio = parsed.streams?.some((item) => item.codec_type === "audio") ?? false;
+ assert(video !== undefined || hasAudio, `${path}: no audio or video stream`);
+ if (video === undefined) return { duration: round(duration), hasVideo: false, hasAudio };
+ assert(video.width !== undefined && video.height !== undefined, `${path}: video dimensions are unavailable`);
+ // ffprobe reports the rate as a ratio; a still image reports none.
+ const ratio = (video.r_frame_rate ?? "").split("/");
+ const rate = Number(ratio[0]) / Number(ratio[1] ?? 1);
+ return {
+ duration: round(duration),
+ hasVideo: true,
+ width: video.width,
+ height: video.height,
+ frameRate: Number.isFinite(rate) && rate > 0 ? round(rate) : 0,
+ hasAudio,
+ };
+}
+
+/**
+ * Open `source` on the frame at `at` seconds.
+ *
+ * `-ss` after `-i` alone decodes every frame up to the mark; `-ss` before `-i` alone lands on the
+ * keyframe before it. Jumping to two seconds early and decoding the short run-up lands on the exact
+ * frame: measured on a 1080p source nine minutes in, the same JPEG to the byte, 11.6 s to 0.2 s.
+ */
+const SEEK_RUN_UP = 2;
+function openAt(source: string, at: number): readonly string[] {
+ const jump = Math.max(0, at - SEEK_RUN_UP);
+ const runUp = at - jump;
+ return [...(jump > 0 ? ["-ss", String(round(jump))] : []), "-i", source, ...(runUp > 0 ? ["-ss", String(round(runUp))] : [])];
+}
+
+// JPEG is a full-range format; web video is routinely full-range YUV and the encoder refuses it
+// unless told which it is writing (`Non full-range YUV is non-standard`).
+const JPEG = ["-pix_fmt", "yuvj420p", "-q:v", "3"] as const;
+
+const GLYPHS: Readonly<Record<string, readonly string[]>> = {
+ "0": ["11111", "10001", "10011", "10101", "11001", "10001", "11111"],
+ "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"],
+ "2": ["11110", "00001", "00001", "11110", "10000", "10000", "11111"],
+ "3": ["11110", "00001", "00001", "01110", "00001", "00001", "11110"],
+ "4": ["10010", "10010", "10010", "11111", "00010", "00010", "00010"],
+ "5": ["11111", "10000", "10000", "11110", "00001", "00001", "11110"],
+ "6": ["01111", "10000", "10000", "11110", "10001", "10001", "01110"],
+ "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"],
+ "8": ["01110", "10001", "10001", "01110", "10001", "10001", "01110"],
+ "9": ["01110", "10001", "10001", "01111", "00001", "00001", "11110"],
+ ":": ["00000", "00100", "00100", "00000", "00100", "00100", "00000"],
+ ".": ["00000", "00000", "00000", "00000", "00000", "00100", "00100"],
+};
+
+function timecode(seconds: number): string {
+ const milliseconds = Math.max(0, Math.round(seconds * 1_000));
+ const hours = Math.floor(milliseconds / 3_600_000);
+ const minutes = Math.floor(milliseconds / 60_000) % 60;
+ const wholeSeconds = Math.floor(milliseconds / 1_000) % 60;
+ const remainder = milliseconds % 1_000;
+ return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(remainder).padStart(3, "0")}`;
+}
+
+type RasterLabel = {
+ readonly width: number;
+ readonly height: number;
+ readonly bytes: Buffer;
+};
+
+/** A tiny built-in numeric face keeps evidence labels available even when ffmpeg lacks drawtext. */
+function rasterLabel(text: string, requestedWidth?: number): RasterLabel {
+ const scale = requestedWidth === undefined
+ ? 3
+ : clamp(Math.min(Math.floor((requestedWidth - 8) / (text.length * 6)), Math.floor(requestedWidth / 240) + 1), 1, 3);
+ const padding = scale * 2;
+ const naturalWidth = padding * 2 + text.length * 6 * scale - scale;
+ const width = Math.max(requestedWidth ?? naturalWidth, naturalWidth);
+ const rawHeight = padding * 2 + 7 * scale;
+ const height = rawHeight + rawHeight % 2;
+ const bytes = Buffer.alloc(width * height * 3, 12);
+ let left = padding;
+ for (const character of text) {
+ const glyph = GLYPHS[character];
+ assert(glyph !== undefined, `cannot draw time label character ${character}`);
+ for (let row = 0; row < glyph.length; row += 1) {
+ for (let column = 0; column < glyph[row]!.length; column += 1) {
+ if (glyph[row]![column] !== "1") continue;
+ for (let y = 0; y < scale; y += 1) {
+ for (let x = 0; x < scale; x += 1) {
+ const pixel = ((padding + row * scale + y) * width + left + column * scale + x) * 3;
+ bytes[pixel] = 245;
+ bytes[pixel + 1] = 245;
+ bytes[pixel + 2] = 245;
+ }
+ }
+ }
+ }
+ left += 6 * scale;
+ }
+ return { width, height, bytes };
+}
+
+function rawVideoInput(label: RasterLabel, frameRate: number): readonly string[] {
+ return [
+ "-f", "rawvideo", "-pixel_format", "rgb24", "-video_size", `${label.width}x${label.height}`,
+ "-framerate", String(frameRate), "-i", "pipe:0",
+ ];
+}
+
+export async function cutClip(
+ source: string,
+ start: number,
+ end: number,
+ target: string,
+ labelInfo?: Pick<VideoProbe, "frameRate">,
+): Promise<void> {
+ assert(end > start, `the clip must end after it starts (${start} to ${end})`);
+ const seconds = round(end - start);
+ const common = ["-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart"];
+ if (labelInfo === undefined) {
+ await runProcess("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y", ...openAt(source, start), "-t", String(seconds), ...common, target,
+ ]);
+ return;
+ }
+ const temporary = await mkdtemp(join(tmpdir(), "hypit-labeled-clip-"));
+ const clean = join(temporary, "clean.mp4");
+ try {
+ await cutClip(source, start, end, clean);
+ const rate = clamp(Math.ceil(labelInfo.frameRate || 10), 1, 30);
+ const count = Math.max(1, Math.ceil(seconds * rate));
+ const first = rasterLabel(timecode(start));
+ function* labels(): Iterable<Uint8Array> {
+ yield first.bytes;
+ for (let index = 1; index < count; index += 1) yield rasterLabel(timecode(start + index / rate)).bytes;
+ }
+ await runProcessWithInput("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y", "-i", clean, ...rawVideoInput(first, rate),
+ "-t", String(seconds), "-filter_complex", "[0:v][1:v]overlay=8:8:eof_action=repeat[v]",
+ "-map", "[v]", "-map", "0:a?", ...common, target,
+ ], labels());
+ } finally {
+ await rm(temporary, { recursive: true, force: true });
+ }
+}
+
+type CutSpan = { readonly start: number; readonly end: number };
+
+function keptSpan(raw: string, duration: number): CutSpan {
+ const parts = raw.split(":");
+ assert(parts.length === 2 && parts.every((part) => part.trim().length > 0), `--keep needs start:end in seconds, got ${raw}`);
+ const [start, end] = parts.map(Number);
+ assert(Number.isFinite(start) && Number.isFinite(end) && start! >= 0 && end! > start! && end! <= duration + 0.001,
+ `--keep ${raw} must satisfy 0 <= start < end <= ${duration} s`);
+ return { start: start!, end: end! };
+}
+
+/** Decode the selected parts once, join them on a new local clock, and preserve only existing streams. */
+async function cutProduction(source: string, spans: readonly CutSpan[], target: string, info: MediaProbe): Promise<void> {
+ const seek = Math.max(0, spans[0]!.start - SEEK_RUN_UP);
+ const labels: string[] = [];
+ const filters: string[] = [];
+ const videoInputs = spans.map((_, index) => `vsrc${index}`);
+ const audioInputs = spans.map((_, index) => `asrc${index}`);
+ if (info.hasVideo && spans.length > 1) filters.push(`[0:v:0]split=${spans.length}${videoInputs.map((item) => `[${item}]`).join("")}`);
+ if (info.hasAudio && spans.length > 1) filters.push(`[0:a:0]asplit=${spans.length}${audioInputs.map((item) => `[${item}]`).join("")}`);
+ for (const [index, span] of spans.entries()) {
+ const start = round(span.start - seek);
+ const end = round(span.end - seek);
+ if (info.hasVideo) filters.push(`[${spans.length === 1 ? "0:v:0" : videoInputs[index]}]trim=start=${start}:end=${end},setpts=PTS-STARTPTS[v${index}]`);
+ if (info.hasAudio) filters.push(`[${spans.length === 1 ? "0:a:0" : audioInputs[index]}]atrim=start=${start}:end=${end},asetpts=PTS-STARTPTS[a${index}]`);
+ labels.push(`${info.hasVideo ? `[v${index}]` : ""}${info.hasAudio ? `[a${index}]` : ""}`);
+ }
+ if (spans.length > 1) filters.push(`${labels.join("")}concat=n=${spans.length}:v=${info.hasVideo ? 1 : 0}:a=${info.hasAudio ? 1 : 0}${info.hasVideo ? "[v]" : ""}${info.hasAudio ? "[a]" : ""}`);
+ const videoLabel = spans.length === 1 ? "[v0]" : "[v]";
+ const audioLabel = spans.length === 1 ? "[a0]" : "[a]";
+ await runProcess("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y",
+ ...(seek > 0 ? ["-ss", String(round(seek))] : []), "-i", source,
+ "-filter_complex", filters.join(";"),
+ ...(info.hasVideo ? ["-map", videoLabel, "-c:v", "libx264", "-preset", "medium", "-crf", "17", "-pix_fmt", "yuv420p", "-fps_mode", "vfr",
+ ...([".mp4", ".mov"].includes(extname(target).toLowerCase()) ? ["-movflags", "+faststart"] : [])] : []),
+ ...(info.hasAudio ? ["-map", audioLabel, "-c:a", ...(info.hasVideo ? ["aac", "-b:a", "192k"] : ["pcm_s24le"])] : []),
+ target,
+ ]);
+}
+
+/** Publish a completed cut without overwriting a file another process may have created meanwhile. */
+async function writeCut(
+ source: string, spans: readonly CutSpan[], target: string, info: MediaProbe, labeled: boolean,
+): Promise<void> {
+ const work = await mkdtemp(join(dirname(target), ".hypit-cut-"));
+ const staged = join(work, `cut${extname(target)}`);
+ try {
+ if (labeled) {
+ requireVideo(info, source);
+ await cutClip(source, spans[0]!.start, spans[0]!.end, staged, info);
+ } else await cutProduction(source, spans, staged, info);
+ await link(staged, target);
+ } finally {
+ await rm(work, { recursive: true, force: true });
+ }
+}
+
+export async function cutFrame(source: string, at: number, target: string, labelTime = false): Promise<number> {
+ if (!labelTime) {
+ const jump = Math.max(0, at - SEEK_RUN_UP);
+ const output = await runProcessOutput("ffmpeg", [
+ "-hide_banner", "-loglevel", "info", "-y", "-copyts", "-start_at_zero",
+ ...(jump > 0 ? ["-ss", String(jump)] : []), "-i", source,
+ "-vf", `trim=start=${at},showinfo`, "-frames:v", "1", "-fps_mode", "passthrough", ...JPEG, target,
+ ]);
+ const base = /config in time_base:\s*(\d+)\/(\d+)/u.exec(output.stderr);
+ const pts = /\bn:\s*0\s+pts:\s*(-?\d+)/u.exec(output.stderr);
+ assert(base !== null && pts !== null, `no video frame exists at or after ${at} s`);
+ return Number(pts[1]) * Number(base[1]) / Number(base[2]);
+ }
+ const temporary = await mkdtemp(join(tmpdir(), "hypit-labeled-frame-"));
+ const clean = join(temporary, "clean.jpg");
+ try {
+ const actual = await cutFrame(source, at, clean);
+ const label = rasterLabel(timecode(actual));
+ await runProcessWithInput("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y", "-i", clean, ...rawVideoInput(label, 1),
+ "-filter_complex", "[0:v][1:v]overlay=8:8[out]", "-map", "[out]", "-frames:v", "1", ...JPEG, target,
+ ], label.bytes);
+ return actual;
+ } finally {
+ await rm(temporary, { recursive: true, force: true });
+ }
+}
+
+const TILE_COLUMNS = 3;
+const TILE_CELL_WIDTH = 480;
+/** A compact overview default. Callers choose denser samples for fast changes. */
+export function tileFrameCount(seconds: number): number { return clamp(Math.round(seconds * 1.5), 4, 9); }
+
+export function tileSampleTimes(start: number, end: number, frameCount: number): readonly number[] {
+ assert(end > start, `the stretch must end after it starts (${start} to ${end})`);
+ assert(Number.isSafeInteger(frameCount) && frameCount >= 2, "a grid needs at least two frames");
+ return Array.from({ length: frameCount }, (_, index) => round(start + (index + 0.5) * (end - start) / frameCount));
+}
+
+function escapeMarkup(value: string): string {
+ return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
+}
+
+async function frameLabel(at: number, width: number, words?: FrameWords): Promise<Buffer> {
+ const lines = [timecode(at)];
+ if (words !== undefined) {
+ lines.push(words.active.length === 0 ? "No timed word at this frame" : words.active.map((word) =>
+ `<b>${escapeMarkup(word.text)}</b> ${word.start!.toFixed(3)}–${word.end!.toFixed(3)} s`).join("\n"));
+ if (words.context.length > 0) lines.push(words.context.map((word) => words.active.includes(word)
+ ? `<span foreground="#ffdc80"><b>${escapeMarkup(word.text)}</b></span>` : escapeMarkup(word.text)).join(" "));
+ }
+ const text = await sharp({ text: {
+ text: `<span foreground="#f5f5f5">${lines.join("\n")}</span>`,
+ font: `sans ${Math.max(12, Math.round(width / 28))}`, width: width - 16, rgba: true, wrap: "word-char",
+ } }).png().toBuffer({ resolveWithObject: true });
+ return sharp({ create: { width, height: text.info.height + 16, channels: 3, background: "#0c0c0c" } })
+ .composite([{ input: text.data, left: 8, top: 8 }]).png().toBuffer();
+}
+
+type SampledFrame = { readonly requestedAt: number; readonly at: number; readonly words?: FrameWords };
+
+export async function tileFrames(
+ source: string,
+ times: readonly number[],
+ target: string,
+ cellWidth: number,
+ columns: number,
+ words?: readonly TranscriptWord[],
+): Promise<readonly SampledFrame[]> {
+ assert(times.length >= 1, "a grid needs at least one frame");
+ const temporary = await mkdtemp(join(tmpdir(), "hypit-tile-"));
+ try {
+ const frames: SampledFrame[] = [];
+ const cells: { path: string; label: Buffer }[] = [];
+ for (let index = 0; index < times.length; index += 1) {
+ const at = times[index]!;
+ const path = join(temporary, `${index}.jpg`);
+ const actual = await cutFrame(source, at, path);
+ const frameWords = words === undefined ? undefined : wordsAt(words, actual);
+ frames.push({ requestedAt: at, at: actual, ...(frameWords === undefined ? {} : { words: frameWords }) });
+ cells.push({ path, label: await frameLabel(actual, cellWidth, frameWords) });
+ }
+ await writeFrameGrid(cells, target, cellWidth, columns);
+ return frames;
+ } finally {
+ await rm(temporary, { recursive: true, force: true });
+ }
+}
+
+/**
+ * Adjacent-frame visual change candidates. Scores are mechanical evidence, not editorial shots.
+ */
+const DEFAULT_BOUNDARY_RATE = 12;
+const DEFAULT_BOUNDARY_THRESHOLD = 0.1;
+export type VisualBoundary = { readonly at: number; readonly score: number };
+export async function visualBoundaries(
+ source: string,
+ sampleRate = DEFAULT_BOUNDARY_RATE,
+ threshold = DEFAULT_BOUNDARY_THRESHOLD,
+): Promise<readonly VisualBoundary[]> {
+ const bytes = await runProcess("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-i", source, "-vf", `fps=${sampleRate},scale=32:32,format=rgb24`, "-f", "rawvideo", "-",
+ ]);
+ const stride = 32 * 32 * 3;
+ const frames: Uint8Array[] = [];
+ for (let index = 0; index + stride <= bytes.byteLength; index += stride) frames.push(bytes.subarray(index, index + stride));
+ const candidates: VisualBoundary[] = [];
+ for (let index = 1; index < frames.length; index += 1) {
+ const before = frames[index - 1]!;
+ const after = frames[index]!;
+ let total = 0;
+ for (let byte = 0; byte < after.length; byte += 1) total += Math.abs(after[byte]! - before[byte]!);
+ const score = total / after.length / 255;
+ if (score >= threshold) candidates.push({ at: round(index / sampleRate), score: round(score) });
+ }
+ return candidates;
+}
+
+// ---------------------------------------------------------------------------------------------------
+// Commands
+
+const SAMPLE_OPTIONS = ["--start", "--end", "--at", "--every", "--transcript", "--around", "--occurrence", "--padding"];
+
+async function transcriptOption(parsed: Parsed, cwd: string): Promise<readonly TranscriptWord[] | undefined> {
+ const path = parsed.options.get("--transcript");
+ return path === undefined ? undefined : readTranscript(resolve(cwd, path));
+}
+
+export function intervalSamples(start: number, end: number, every: number): readonly number[] {
+ assert(Number.isFinite(start) && start >= 0 && Number.isFinite(end) && end > start, "a sample range needs 0 <= start < end");
+ assert(Number.isFinite(every) && every >= 0.001, "--every must be at least 0.001 seconds");
+ return Array.from({ length: Math.ceil((end - start) / every) }, (_, index) => round(start + index * every))
+ .filter((time) => time < end);
+}
+
+function sampleRange(parsed: Parsed, duration: number, words?: readonly TranscriptWord[]): { start: number; end: number } {
+ const around = parsed.options.get("--around");
+ if (around !== undefined) {
+ assert(words !== undefined, "--around requires --transcript");
+ assert(!parsed.options.has("--start") && !parsed.options.has("--end"), "choose --around or --start/--end");
+ const matches = phraseRanges(words, around);
+ assert(matches.length > 0, `No timed phrase matches ${JSON.stringify(around)}`);
+ assert(matches.length === 1 || parsed.options.has("--occurrence"),
+ `The phrase occurs ${matches.length} times: ${matches.map((match, index) => `${index + 1}: ${match.start}–${match.end}s`).join(", ")}. Choose --occurrence <n> or --start/--end.`);
+ const occurrence = integerOption(parsed, "--occurrence", 1, 1);
+ const match = matches[occurrence - 1];
+ assert(match !== undefined, `--occurrence ${occurrence} is beyond the ${matches.length} matches`);
+ const padding = secondsOption(parsed, "--padding", 0.3);
+ const start = Math.max(0, round(match.start - padding));
+ const end = Math.min(duration, round(match.end + padding));
+ assert(end > start, "the phrase lies outside the input media; use its matching transcript");
+ return { start, end };
+ }
+ assert(!parsed.options.has("--occurrence") && !parsed.options.has("--padding"), "--occurrence and --padding apply to --around");
+ const start = secondsOption(parsed, "--start", 0);
+ const end = secondsOption(parsed, "--end", duration);
+ assert(end > start && end <= duration, `range must satisfy 0 <= start < end <= ${duration} s`);
+ return { start, end };
+}
+
+function sampleTimes(parsed: Parsed, duration: number, words?: readonly TranscriptWord[], requireEvery = false): readonly number[] {
+ const at = parsed.options.get("--at");
+ if (at !== undefined) {
+ for (const key of ["--start", "--end", "--every", "--frames", "--around", "--occurrence", "--padding"]) {
+ assert(!parsed.options.has(key), `--at cannot be combined with ${key}`);
+ }
+ const times = secondsList(at, "--at", 1);
+ assert(times.every((time) => time < duration), `a frame time must be before the end of the file (${duration} s)`);
+ assert(times.every((time, index) => index === 0 || time > times[index - 1]!), "sample times must be strictly increasing");
+ return times;
+ }
+ const { start, end } = sampleRange(parsed, duration, words);
+ const every = parsed.options.get("--every");
+ assert(every === undefined || !parsed.options.has("--frames"), "choose --every or --frames");
+ assert(!requireEvery || every !== undefined, "frames requires --at <s,s,…> or --every <seconds>");
+ const times = every === undefined
+ ? tileSampleTimes(start, end, integerOption(parsed, "--frames", tileFrameCount(end - start), 2))
+ : intervalSamples(start, end, Number(every));
+ assert(times.length > 0 && times.every((time) => time < duration), "the range needs sample times before the media ends");
+ assert(times.every((time, index) => index === 0 || time > times[index - 1]!), "sample times must be distinct to the nearest millisecond");
+ return times;
+}
+
+async function probe(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, []);
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ if (parsed.json) { io.write(`${JSON.stringify({ path: source, ...info }, null, 2)}\n`); return; }
+ io.write(`${source}\n ${info.duration} s ${info.hasVideo
+ ? `${info.width}×${info.height} ${info.frameRate > 0 ? `${info.frameRate} fps` : "still"} ${info.hasAudio ? "with audio" : "no audio"}`
+ : "audio only"}\n`);
+}
+
+async function cut(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, ["--start", "--end", "--keep", "--to"], ["--label-time"], ["--keep"]);
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ const kept = parsed.repeated.get("--keep") ?? [];
+ assert(kept.length === 0 || (!parsed.options.has("--start") && !parsed.options.has("--end")),
+ "choose --keep intervals or --start/--end");
+ const spans = kept.length > 0 ? kept.map((item) => keptSpan(item, info.duration)) : [{
+ start: secondsOption(parsed, "--start", 0), end: secondsOption(parsed, "--end", info.duration),
+ }];
+ for (const [index, span] of spans.entries()) {
+ assert(span.end > span.start && span.end <= info.duration + 0.001,
+ `range must satisfy 0 <= start < end <= ${info.duration} s`);
+ if (index > 0) assert(span.start >= spans[index - 1]!.end, "--keep intervals must be ordered and non-overlapping");
+ }
+ const labeled = parsed.flags.has("--label-time");
+ assert(!labeled || (info.hasVideo && spans.length === 1), "--label-time needs one video interval");
+ const target = await destination(parsed, cwd, "the clip to write, for example notes/hook.mp4");
+ if (!info.hasVideo) assert(extname(target).toLowerCase() === ".wav", "--to must end in .wav for audio-only media");
+ await writeCut(source, spans, target, info, labeled);
+ const output = await probeMedia(target);
+ const requestedSeconds = round(spans.reduce((total, span) => total + span.end - span.start, 0));
+ if (parsed.json) {
+ let outputAt = 0;
+ const mapping = spans.map((span) => {
+ const item = { sourceStart: span.start, sourceEnd: span.end,
+ nominalOutputStart: round(outputAt), nominalOutputEnd: round(outputAt + span.end - span.start) };
+ outputAt += span.end - span.start;
+ return item;
+ });
+ io.write(`${JSON.stringify({ path: target, ...(spans.length === 1 ? { start: spans[0]!.start, end: spans[0]!.end } : {}),
+ spans, mapping, seconds: requestedSeconds, actualSeconds: output.duration, hasVideo: output.hasVideo, hasAudio: output.hasAudio, labeled }, null, 2)}\n`);
+ return;
+ }
+ io.write(`${target}\n ${spans.map((span) => `${span.start}–${span.end} s`).join(" + ")} (${output.duration} s output)${labeled ? ", source time visible" : ""}\n`);
+}
+
+async function frames(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, [...SAMPLE_OPTIONS, "--to"], ["--label-time", "--every-frame"]);
+ if (parsed.flags.has("--every-frame")) { await nativeFrames(parsed, io, cwd, false); return; }
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ requireVideo(info, source);
+ const words = await transcriptOption(parsed, cwd);
+ const times = sampleTimes(parsed, info.duration, words, true);
+ const to = await destinationDirectory(parsed, cwd, "the new directory to write the frames into");
+ const staging = await mkdtemp(join(dirname(to), ".hypit-frames-"));
+ const labeled = parsed.flags.has("--label-time") || words !== undefined;
+ const written: (SampledFrame & { readonly path: string })[] = [];
+ try {
+ for (const time of times) {
+ const name = `frame-${time.toFixed(3).replace(".", "_")}s.jpg`;
+ const frame = words === undefined
+ ? { requestedAt: time, at: await cutFrame(source, time, join(staging, name), labeled) }
+ : (await tileFrames(source, [time], join(staging, name), info.width, 1, words))[0]!;
+ written.push({ ...frame, path: join(to, name) });
+ }
+ await rename(staging, to);
+ } catch (error) {
+ await rm(staging, { recursive: true, force: true });
+ throw error;
+ }
+ if (parsed.json) { io.write(`${JSON.stringify({ directory: to, labeled, frames: written }, null, 2)}\n`); return; }
+ io.write(`${to}\n ${written.length} ${labeled ? "time-labeled " : ""}${written.length === 1 ? "frame" : "frames"} from ${written[0]!.at} s to ${written.at(-1)!.at} s\n`);
+}
+
+async function tile(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, [...SAMPLE_OPTIONS, "--frames", "--cell", "--columns", "--to"]);
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ requireVideo(info, source);
+ const words = await transcriptOption(parsed, cwd);
+ const times = sampleTimes(parsed, info.duration, words);
+ // Preserve source width unless an exceptionally tiny input needs room for a legible timecode.
+ const cell = integerOption(parsed, "--cell", Math.max(80, Math.min(TILE_CELL_WIDTH, info.width)), 80);
+ const columns = integerOption(parsed, "--columns", TILE_COLUMNS, 1);
+ const target = await destination(parsed, cwd, "the grid image to write, for example notes/hook-grid.jpg");
+ const frames = await tileFrames(source, times, target, cell, columns, words);
+ const rows = Math.ceil(times.length / columns);
+ if (parsed.json) { io.write(`${JSON.stringify({ path: target, samples: times, columns, rows, cellWidth: cell, frames }, null, 2)}\n`); return; }
+ io.write(`${target}\n ${times.length} labeled frames, ${columns}×${rows}, ${cell} px cells\n ${times.map((at) => `${at} s`).join(" ")}\n`);
+}
+
+type TileRange = {
+ readonly id?: string;
+ readonly start: number;
+ readonly end: number;
+ readonly frames?: number;
+ readonly every?: number;
+};
+
+function tileRange(value: unknown, index: number, duration: number): TileRange {
+ assert(typeof value === "object" && value !== null && !Array.isArray(value), `range ${index + 1} must be an object`);
+ const raw = value as Record<string, unknown>;
+ assert(typeof raw.start === "number" && Number.isFinite(raw.start) && raw.start >= 0, `range ${index + 1}.start must be non-negative seconds`);
+ assert(typeof raw.end === "number" && Number.isFinite(raw.end) && raw.end > raw.start, `range ${index + 1}.end must be after its start`);
+ assert(raw.end <= duration, `range ${index + 1}.end is past the end of the file (${duration} s)`);
+ assert(raw.id === undefined || (typeof raw.id === "string" && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(raw.id)), `range ${index + 1}.id must be a safe file name`);
+ assert(raw.frames === undefined || (Number.isSafeInteger(raw.frames) && (raw.frames as number) >= 2), `range ${index + 1}.frames must be a whole number of at least 2`);
+ assert(raw.every === undefined || (typeof raw.every === "number" && Number.isFinite(raw.every) && raw.every >= 0.001), `range ${index + 1}.every must be at least 0.001 seconds`);
+ assert(raw.frames === undefined || raw.every === undefined, `range ${index + 1} must choose frames or every`);
+ return { start: raw.start, end: raw.end, ...(raw.id === undefined ? {} : { id: raw.id as string }),
+ ...(raw.frames === undefined ? {} : { frames: raw.frames as number }), ...(raw.every === undefined ? {} : { every: raw.every as number }) };
+}
+
+async function tiles(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, [...SAMPLE_OPTIONS, "--ranges", "--frames", "--cell", "--columns", "--rows", "--to"], ["--every-frame"]);
+ if (parsed.flags.has("--every-frame")) { await nativeFrames(parsed, io, cwd, true); return; }
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ requireVideo(info, source);
+ const words = await transcriptOption(parsed, cwd);
+ const rangesPath = parsed.options.get("--ranges");
+ let ranges: { id?: string; start: number; end: number; times: readonly number[] }[];
+ if (rangesPath === undefined) {
+ const times = sampleTimes(parsed, info.duration, words);
+ const range = parsed.options.has("--at")
+ ? { start: times[0]!, end: times.at(-1)! }
+ : sampleRange(parsed, info.duration, words);
+ ranges = [{ ...range, times }];
+ } else {
+ for (const key of ["--start", "--end", "--at", "--around", "--padding", "--occurrence"]) {
+ assert(!parsed.options.has(key), `--ranges cannot be combined with ${key}`);
+ }
+ assert(!parsed.options.has("--frames") || !parsed.options.has("--every"), "choose --frames or --every");
+ const raw = JSON.parse(await readFile(resolve(cwd, rangesPath), "utf8")) as unknown;
+ assert(Array.isArray(raw) && raw.length > 0, `${rangesPath} must contain a non-empty JSON array`);
+ ranges = raw.map((value, index) => {
+ const range = tileRange(value, index, info.duration);
+ const options = new Map(parsed.options);
+ options.set("--start", String(range.start)); options.set("--end", String(range.end));
+ if (range.frames !== undefined) { options.delete("--every"); options.set("--frames", String(range.frames)); }
+ if (range.every !== undefined) { options.delete("--frames"); options.set("--every", String(range.every)); }
+ return { ...range, times: sampleTimes({ ...parsed, options }, info.duration, words) };
+ });
+ const ids = ranges.flatMap((range) => range.id === undefined ? [] : [range.id]);
+ assert(new Set(ids).size === ids.length, "range ids must be unique");
+ }
+ const cell = integerOption(parsed, "--cell", Math.max(80, Math.min(TILE_CELL_WIDTH, info.width)), 80);
+ const columns = integerOption(parsed, "--columns", TILE_COLUMNS, 1);
+ const rows = integerOption(parsed, "--rows", 3, 1);
+ const perPage = columns * rows;
+ const to = await destinationDirectory(parsed, cwd, "the new directory to write the grids into");
+ const staging = await mkdtemp(join(dirname(to), ".hypit-tiles-"));
+ const written: { id?: string; start: number; end: number; samples: readonly number[]; path: string; frames: readonly SampledFrame[] }[] = [];
+ try {
+ for (const [index, range] of ranges.entries()) {
+ const stem = range.id ?? `${range.start.toFixed(3).replace(".", "_")}s-${range.end.toFixed(3).replace(".", "_")}s`;
+ for (let offset = 0; offset < range.times.length; offset += perPage) {
+ const times = range.times.slice(offset, offset + perPage);
+ const page = range.times.length > perPage ? `-p${String(offset / perPage + 1).padStart(3, "0")}` : "";
+ const name = `${String(index + 1).padStart(3, "0")}-${stem}${page}.jpg`;
+ const frames = await tileFrames(source, times, join(staging, name), cell, columns, words);
+ written.push({ ...(range.id === undefined ? {} : { id: range.id }), start: range.start, end: range.end,
+ samples: times, path: join(to, name), frames });
+ }
+ }
+ await rename(staging, to);
+ } catch (error) {
+ await rm(staging, { recursive: true, force: true });
+ throw error;
+ }
+ if (parsed.json) { io.write(`${JSON.stringify({ directory: to, columns, rows, cellWidth: cell, grids: written }, null, 2)}\n`); return; }
+ io.write(`${to}\n ${written.length} time-labeled ${written.length === 1 ? "grid" : "grids"}\n`);
+}
+
+/** Native-frame observation: each requested interval is decoded once, then paginated. */
+async function nativeFrames(parsed: Parsed, io: CliIo, cwd: string, grid: boolean): Promise<void> {
+ for (const key of ["--at", "--every", "--frames"]) assert(!parsed.options.has(key), `--every-frame cannot be combined with ${key}`);
+ const source = await sourceFile(parsed, cwd);
+ const info = await probeMedia(source);
+ requireVideo(info, source);
+ const words = await transcriptOption(parsed, cwd);
+ let ranges: TileRange[];
+ const rangeFile = parsed.options.get("--ranges");
+ if (rangeFile === undefined) ranges = [sampleRange(parsed, info.duration, words)];
+ else {
+ for (const key of ["--start", "--end", "--around", "--padding", "--occurrence"]) assert(!parsed.options.has(key), `--ranges cannot be combined with ${key}`);
+ const raw = JSON.parse(await readFile(resolve(cwd, rangeFile), "utf8"));
+ assert(Array.isArray(raw) && raw.length > 0, "--ranges needs a non-empty JSON array");
+ ranges = raw.map((value, index) => tileRange(value, index, info.duration));
+ assert(ranges.every(range => range.frames === undefined && range.every === undefined), "Native ranges cannot specify frames or every");
+ }
+ const cell = integerOption(parsed, "--cell", Math.max(80, Math.min(TILE_CELL_WIDTH, info.width)), 80);
+ const columns = integerOption(parsed, "--columns", TILE_COLUMNS, 1);
+ const rows = integerOption(parsed, "--rows", 3, 1);
+ const to = await destinationDirectory(parsed, cwd, "the new directory for native frame evidence");
+ const staging = await mkdtemp(join(dirname(to), ".hypit-native-"));
+ const written: { at: number; path: string }[] = [];
+ const grids: { path: string; frames: readonly { at: number }[] }[] = [];
+ try {
+ for (const [rangeIndex, range] of ranges.entries()) {
+ const decodedDirectory = join(staging, "decoded");
+ const decoded = await decodeMediaFrames(source, range.start, range.end, decodedDirectory);
+ if (grid) {
+ const perPage = columns * rows;
+ for (let offset = 0; offset < decoded.length; offset += perPage) {
+ const page = decoded.slice(offset, offset + perPage);
+ const cells = [];
+ for (const frame of page) cells.push({ path: frame.path, label: await frameLabel(frame.at, cell,
+ words === undefined ? undefined : wordsAt(words, frame.at)) });
+ const name = `${String(rangeIndex + 1).padStart(3, "0")}-${range.id ?? "frames"}-p${String(offset / perPage + 1).padStart(3, "0")}.jpg`;
+ await writeFrameGrid(cells, join(staging, name), cell, columns);
+ grids.push({ path: join(to, name), frames: page.map(frame => ({ at: frame.at })) });
+ }
+ } else for (const [index, frame] of decoded.entries()) {
+ const name = `frame-${String(index).padStart(9, "0")}.jpg`;
+ const target = join(staging, name);
+ if (parsed.flags.has("--label-time") || words !== undefined) await writeFrameGrid([{ path: frame.path,
+ label: await frameLabel(frame.at, info.width, words === undefined ? undefined : wordsAt(words, frame.at)) }], target, info.width, 1);
+ else await copyFile(frame.path, target);
+ written.push({ at: frame.at, path: join(to, name) });
+ }
+ await rm(decodedDirectory, { recursive: true, force: true });
+ }
+ await rename(staging, to);
+ } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; }
+ if (parsed.json) io.write(`${JSON.stringify({ directory: to, ...(grid ? { grids } : { frames: written }) }, null, 2)}\n`);
+ else io.write(`${to}\n ${grid ? `${grids.length} grids` : `${written.length} frames`} from continuous source decoding\n`);
+}
+
+async function boundaries(argv: readonly string[], io: CliIo, cwd: string): Promise<void> {
+ const parsed = parseArguments(argv, ["--rate", "--threshold"]);
+ const source = await sourceFile(parsed, cwd);
+ requireVideo(await probeMedia(source), source);
+ const rate = numberOption(parsed, "--rate", DEFAULT_BOUNDARY_RATE, 1, 120);
+ const threshold = numberOption(parsed, "--threshold", DEFAULT_BOUNDARY_THRESHOLD, 0, 1);
+ const candidates = await visualBoundaries(source, rate, threshold);
+ if (parsed.json) { io.write(`${JSON.stringify({ path: source, sampleRate: rate, threshold, candidates }, null, 2)}\n`); return; }
+ const shown = candidates.slice(0, 40);
+ const remaining = candidates.length - shown.length;
+ io.write(`${source}\n ${candidates.length} visual-change ${candidates.length === 1 ? "candidate" : "candidates"} at ${rate} samples/s; scores are not shot labels\n${
+ shown.map((candidate) => ` ${candidate.at} s score ${candidate.score}`).join("\n")}${remaining > 0 ? `\n … ${remaining} more; --json returns all candidates` : ""}\n`);
+}
+
+// ---------------------------------------------------------------------------------------------------
+// Entry
+
+export function writeMediaHelp(io: CliIo, topic?: MediaCommand): void {
+ const sections: Record<MediaCommand, readonly string[]> = {
+ probe: [" hypit media probe <file>", " Duration and available audio/video streams; video adds size and frame rate."],
+ cut: [" hypit media cut <file> [--start <s> --end <s> | --keep <start:end> ...] [--label-time] --to <clip.mp4|clip.wav>",
+ " Keep one stretch or join ordered stretches from one source. Audio-only outputs WAV; MP4 is a useful video target.",
+ " --label-time overlays source time on a single video evidence copy."],
+ frames: [" hypit media frames <file> (--at <s,s,…> | --every <s> | --every-frame) [--start <s> --end <s>] [--label-time] [--transcript <json>] --to <dir>",
+ " Individual frames named by source time; --transcript adds word times and context below each picture."],
+ tile: [" hypit media tile <file> [--start <s> --end <s> | --at <s,s,…>] [--every <s> | --frames <n>] [--transcript <json>] [--cell <px>] [--columns <n>] --to <grid.jpg>",
+ " One grid. --every samples from start, excluding end; --frames chooses evenly spaced midpoints."],
+ tiles: [" hypit media tiles <file> [--start <s> --end <s> | --at <s,s,…> | --ranges <json>] [--every <s> | --frames <n> | --every-frame] [--transcript <json>] [--cell <px>] [--columns <n>] [--rows <n>] --to <dir>",
+ " Paginated grids (3 rows by default). --every-frame decodes every original frame once per interval, preserving actual timestamps. Range files contain { start, end, id?, frames?, every? }."],
+ boundaries: [" hypit media boundaries <file> [--rate <samples/s>] [--threshold <0..1>]",
+ " Mechanical adjacent-frame change candidates with scores; never editorial shot labels."],
+ };
+ const chosen = topic === undefined ? mediaCommands : [topic];
+ io.write(`hypit media\nInspect and prepare media locally. No Runtime Profile, network request or project state.\n\n${
+ chosen.map((item) => sections[item].join("\n")).join("\n\n")}\n\nFrames and grids accept --around <phrase> with --transcript instead of --start/--end.\n`
+ + "Use --occurrence <n> for a repeated phrase; --padding <s> adds time on each side (default 0.3).\n"
+ + "Transcript times must share the input media's clock. An untimed gap is labeled as such, without inferring silence.\n"
+ + "Commands write only what --to names and refuse to overwrite. Add --json for the complete machine view.\n");
+}
+
+export async function runMediaCli(argv: readonly string[], io: CliIo, cwd = process.cwd()): Promise<void> {
+ const command = argv[1];
+ if (command === undefined || command === "--help" || argv.includes("--help")) {
+ writeMediaHelp(io, isMediaCommand(command) ? command : undefined);
+ return;
+ }
+ assert(isMediaCommand(command), `unknown media command ${command}; one of ${mediaCommands.join(", ")}`);
+ const rest = argv.slice(2);
+ if (command === "probe") await probe(rest, io, cwd);
+ else if (command === "cut") await cut(rest, io, cwd);
+ else if (command === "frames") await frames(rest, io, cwd);
+ else if (command === "tile") await tile(rest, io, cwd);
+ else if (command === "tiles") await tiles(rest, io, cwd);
+ else await boundaries(rest, io, cwd);
+}
diff --git a/packages/media-local/src/frame-grid.ts b/packages/media-local/src/frame-grid.ts
new file mode 100644
index 00000000..6c997606
--- /dev/null
+++ b/packages/media-local/src/frame-grid.ts
@@ -0,0 +1,34 @@
+import sharp from "sharp";
+
+/** Arrange already captured evidence. Decoding and time selection belong to the caller. */
+export async function writeFrameGrid(
+ frames: readonly { readonly path: string; readonly label: Buffer }[],
+ target: string, cellWidth: number, columns: number,
+): Promise<void> {
+ if (frames.length === 0) throw new Error("A frame grid needs at least one image");
+ const cells = [];
+ for (const frame of frames) {
+ const picture = await sharp(frame.path).resize({ width: cellWidth }).png().toBuffer({ resolveWithObject: true });
+ const labelHeight = (await sharp(frame.label).metadata()).height!;
+ cells.push({ picture: picture.data, height: picture.info.height, label: frame.label, labelHeight });
+ }
+ const pictureHeight = Math.max(...cells.map(cell => cell.height));
+ const rowHeight = pictureHeight + Math.max(...cells.map(cell => cell.labelHeight));
+ const rows = Math.ceil(cells.length / columns);
+ await sharp({ create: {
+ width: columns * (cellWidth + 8) + 8, height: rows * (rowHeight + 8) + 8,
+ channels: 3, background: "#0c0c0c",
+ } }).composite(cells.flatMap((cell, index) => {
+ const left = 8 + index % columns * (cellWidth + 8);
+ const top = 8 + Math.floor(index / columns) * (rowHeight + 8);
+ return [{ input: cell.picture, left, top }, { input: cell.label, left, top: top + pictureHeight }];
+ })).toFile(target);
+}
+
+export async function snapshotFrameLabel(frame: number, seconds: number, width: number): Promise<Buffer> {
+ const text = await sharp({ text: { text: `<span foreground="#f5f5f5">Frame ${frame} · ${seconds.toFixed(6)} s</span>`,
+ font: `sans ${Math.max(12, Math.round(width / 28))}`, width: width - 16, rgba: true,
+ } }).png().toBuffer({ resolveWithObject: true });
+ return sharp({ create: { width, height: text.info.height + 16, channels: 3, background: "#0c0c0c" } })
+ .composite([{ input: text.data, left: 8, top: 8 }]).png().toBuffer();
+}
diff --git a/packages/media-local/src/index.ts b/packages/media-local/src/index.ts
new file mode 100644
index 00000000..8429d7b5
--- /dev/null
+++ b/packages/media-local/src/index.ts
@@ -0,0 +1,8 @@
+export { createLocalMediaProvider, localMediaProviderModuleRef } from "./provider.js";
+export type { CreateLocalMediaProviderOptions } from "./provider.js";
+export { decodeMediaFrames } from "./file-frames.js";
+export type { DecodedFrame } from "./file-frames.js";
+export { snapshotFrameLabel, writeFrameGrid } from "./frame-grid.js";
+export { probeMedia } from "./files.js";
+export type { MediaProbe } from "./files.js";
+export { speechEvidenceFile } from "./speech.js";
diff --git a/packages/media-execution/src/probe.ts b/packages/media-local/src/probe.ts
similarity index 98%
rename from packages/media-execution/src/probe.ts
rename to packages/media-local/src/probe.ts
index 60cf55a9..da29a052 100644
--- a/packages/media-execution/src/probe.ts
+++ b/packages/media-local/src/probe.ts
@@ -1,6 +1,6 @@
-import { sealMediaInspection } from "@hypit/media";
-import type { MediaAudioStream, MediaInspection, MediaOtherStream, MediaRational, MediaStream, MediaTimestamp, MediaVideoStream } from "@hypit/media";
-import type { BlobRef } from "@hypit/protocol";
+import { sealMediaInspection } from "@hypit/hypit/media";
+import type { MediaAudioStream, MediaInspection, MediaOtherStream, MediaRational, MediaStream, MediaTimestamp, MediaVideoStream } from "@hypit/hypit/media";
+import type { BlobRef } from "@hypit/hypit/protocol";
type JsonObject = Record<string, unknown>;
diff --git a/packages/media-execution/src/process-env.ts b/packages/media-local/src/process-env.ts
similarity index 100%
rename from packages/media-execution/src/process-env.ts
rename to packages/media-local/src/process-env.ts
diff --git a/packages/media-local/src/process.ts b/packages/media-local/src/process.ts
new file mode 100644
index 00000000..16484149
--- /dev/null
+++ b/packages/media-local/src/process.ts
@@ -0,0 +1,112 @@
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { mediaProcessEnv } from "./process-env.js";
+
+type ProcessInput = Uint8Array | Iterable<Uint8Array> | AsyncIterable<Uint8Array>;
+type ProcessOutput = { readonly stdout: Buffer; readonly stderr: string };
+
+function run(
+ executable: string,
+ args: readonly string[],
+ input: ProcessInput | undefined,
+ timeoutMs: number,
+ onStderrLine?: (line: string) => void,
+ maxStdoutBytes = Number.POSITIVE_INFINITY,
+ environment?: NodeJS.ProcessEnv,
+): Promise<ProcessOutput> {
+ return new Promise((resolveRun, reject) => {
+ const child = spawn(executable, [...args], {
+ stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
+ windowsHide: true,
+ ...(environment === undefined ? {} : { env: environment }),
+ });
+ const out: Buffer[] = [];
+ let stdoutBytes = 0;
+ let err = "";
+ let settled = false;
+ const finish = (error?: Error): void => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ if (error === undefined) resolveRun({ stdout: Buffer.concat(out), stderr: err });
+ else reject(error);
+ };
+ const timer = setTimeout(() => { child.kill("SIGKILL"); finish(new Error(`${executable} timed out after ${timeoutMs} ms`)); }, timeoutMs);
+ child.stdout?.on("data", (chunk: Buffer) => {
+ stdoutBytes += chunk.byteLength;
+ if (stdoutBytes > maxStdoutBytes) {
+ child.kill("SIGKILL");
+ finish(new Error(`${executable} output exceeded the configured limit`));
+ return;
+ }
+ out.push(chunk);
+ });
+ let pendingLine = "";
+ child.stderr?.on("data", (chunk: Buffer) => {
+ const text = chunk.toString("utf8");
+ err = `${err}${text}`.slice(-100_000);
+ if (onStderrLine !== undefined) {
+ const lines = `${pendingLine}${text}`.split(/\r?\n/u);
+ pendingLine = lines.pop()!;
+ for (const line of lines) onStderrLine(line);
+ }
+ });
+ child.on("error", (error) => finish(new Error(`${executable} could not start: ${error.message}`)));
+ child.on("close", (code) => {
+ if (code === 0) finish();
+ else finish(new Error(`${executable} exited with ${code}: ${err.trim()}`));
+ });
+ if (input !== undefined && child.stdin !== null) {
+ child.stdin.on("error", (error: NodeJS.ErrnoException) => {
+ if (error.code !== "EPIPE") finish(new Error(`${executable} input failed: ${error.message}`));
+ });
+ void (async () => {
+ const chunks: Iterable<Uint8Array> | AsyncIterable<Uint8Array> = input instanceof Uint8Array ? [input] : input;
+ for await (const chunk of chunks) {
+ if (!child.stdin!.write(chunk)) await once(child.stdin!, "drain");
+ }
+ child.stdin!.end();
+ })().catch((error: unknown) => {
+ if (!(error instanceof Error && "code" in error && error.code === "EPIPE")) {
+ child.kill("SIGKILL");
+ finish(error instanceof Error ? error : new Error(String(error)));
+ }
+ });
+ }
+ });
+}
+
+/** Shared bounded process boundary for the Provider and direct-file ports of local media. */
+export async function runBoundedMediaProcess(options: {
+ readonly executable: string;
+ readonly argv: readonly string[];
+ readonly timeoutMs: number;
+ readonly maxStdoutBytes: number;
+ readonly sharedLibraryPath?: string;
+}): Promise<Uint8Array> {
+ return (await run(options.executable, options.argv, undefined, options.timeoutMs, undefined,
+ options.maxStdoutBytes,
+ mediaProcessEnv(options.sharedLibraryPath === undefined ? undefined : {
+ LD_LIBRARY_PATH: options.sharedLibraryPath,
+ }))).stdout;
+}
+
+/** Run one external program to completion and return its stdout; stderr becomes the error text. */
+export async function runProcess(executable: string, args: readonly string[], timeoutMs = 300_000): Promise<Buffer> {
+ return (await run(executable, args, undefined, timeoutMs)).stdout;
+}
+
+/** Successful stderr can carry tool metadata, such as the timestamp of an extracted frame. */
+export function runProcessOutput(executable: string, args: readonly string[], timeoutMs = 300_000, onStderrLine?: (line: string) => void): Promise<ProcessOutput> {
+ return run(executable, args, undefined, timeoutMs, onStderrLine);
+}
+
+/** The same process boundary when a deterministic byte stream is one of the program's inputs. */
+export async function runProcessWithInput(
+ executable: string,
+ args: readonly string[],
+ input: ProcessInput,
+ timeoutMs = 300_000,
+): Promise<Buffer> {
+ return (await run(executable, args, input, timeoutMs)).stdout;
+}
diff --git a/packages/provider-media-local/src/program.ts b/packages/media-local/src/program.ts
similarity index 82%
rename from packages/provider-media-local/src/program.ts
rename to packages/media-local/src/program.ts
index 1c2fb6ff..c5d3490c 100644
--- a/packages/provider-media-local/src/program.ts
+++ b/packages/media-local/src/program.ts
@@ -1,5 +1,5 @@
-import { probeMediaToolchain } from "@hypit/media-execution";
-import type { ManagedProgram } from "@hypit/runtime-kit";
+import { probeMediaToolchain } from "./toolchain.js";
+import type { ManagedProgram } from "@hypit/runtime-local/extension";
/** A system/custom FFmpeg deployment is external: programs up probes it but never mutates a package manager. */
export function localMediaToolchainProgram(input: {
diff --git a/packages/provider-media-local/src/provider.ts b/packages/media-local/src/provider.ts
similarity index 74%
rename from packages/provider-media-local/src/provider.ts
rename to packages/media-local/src/provider.ts
index 1a4ba86d..0e8bd170 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/media-local/src/provider.ts
@@ -1,5 +1,5 @@
-import { mediaTypes } from "@hypit/media";
-import { artifactTypes } from "@hypit/artifact";
+import { mediaTypes } from "@hypit/hypit/media";
+import { blobTypes } from "@hypit/hypit/blob";
import {
executeExtractAudio,
executeExtractFrame,
@@ -10,15 +10,15 @@ import {
executeRenderTimelineAudio,
executeRenderStillVideo,
executeTransformMedia,
-} from "@hypit/media-execution";
-import type { MediaExecutionEnvironment, MediaOperationResult } from "@hypit/media-execution";
-import { mediaPipelineCapabilities } from "@hypit/media-pipeline";
-import { isStreamingResourceStore } from "@hypit/runtime";
-import { speechEvidenceTypes } from "@hypit/speech-evidence";
-import { defineEndpointPackage } from "@hypit/endpoint-kit";
-import type { EndpointFulfillment, EndpointInvocationContext } from "@hypit/endpoint-kit";
+} from "./execute.js";
+import type { MediaExecutionEnvironment, MediaOperationResult } from "./execute.js";
+import { mediaOperationsCapabilities } from "@hypit/media-operations";
+import { isStreamingResourceStore } from "@hypit/hypit/endpoint";
+import { speechEvidenceCapabilities, speechEvidenceTypes } from "@hypit/hypit/speech-evidence";
+import { defineEndpoint } from "@hypit/hypit/endpoint";
+import type { EndpointFulfillment, EndpointInvocationContext } from "@hypit/hypit/endpoint";
-export const localMediaProviderModuleRef = { name: "@hypit/provider-media-local", version: "1" } as const;
+export const localMediaProviderModuleRef = { name: "@hypit/media-local", version: "1" } as const;
export type CreateLocalMediaProviderOptions = {
readonly instance?: string;
@@ -44,9 +44,8 @@ function fulfillment(result: MediaOperationResult): EndpointFulfillment {
}
/**
- * The Build's own ResourceStore, and whatever ffmpeg this machine has. The
- * operations themselves live in `@hypit/media-execution`, shared with the
- * AWS Provider so one Need cannot mean two different transforms.
+ * Execute media-operation Needs against the Build's ResourceStore and the
+ * FFmpeg toolchain selected for this local Provider.
*/
export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions) {
const ffmpegPath = config.ffmpegPath ?? "ffmpeg";
@@ -75,9 +74,7 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
) => async (context: EndpointInvocationContext): Promise<EndpointFulfillment> =>
fulfillment(await execute(environment(context), context.need.constraints as never));
- return defineEndpointPackage({
- module: localMediaProviderModuleRef,
- facet: "media",
+ return defineEndpoint({
instance: config.instance ?? "media.local",
pool: config.pool ?? config.instance ?? "media.local",
pricing: { kind: "local" },
@@ -86,60 +83,60 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.inspect,
+ capability: mediaOperationsCapabilities.inspect,
returns: mediaTypes.inspection,
handler: operation(executeInspectMedia),
},
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.normalize,
+ capability: mediaOperationsCapabilities.normalize,
returns: mediaTypes.synchronized,
handler: operation(executeNormalizeMedia),
},
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.transform,
- returns: artifactTypes.blob,
+ capability: mediaOperationsCapabilities.transform,
+ returns: blobTypes.blob,
handler: operation(executeTransformMedia),
},
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.extractAudio,
- returns: artifactTypes.blob,
+ capability: mediaOperationsCapabilities.extractAudio,
+ returns: blobTypes.blob,
handler: operation(executeExtractAudio),
},
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.extractFrame,
- returns: artifactTypes.blob,
+ capability: mediaOperationsCapabilities.extractFrame,
+ returns: blobTypes.blob,
handler: operation(executeExtractFrame),
},
{
lifecycle: "immediate" as const,
transient: true,
- capability: mediaPipelineCapabilities.renderStill,
- returns: artifactTypes.blob,
+ capability: mediaOperationsCapabilities.renderStill,
+ returns: blobTypes.blob,
handler: operation(executeRenderStillVideo),
},
{
lifecycle: "immediate" as const,
- capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,
+ capability: speechEvidenceCapabilities.projectAudio,
returns: speechEvidenceTypes.audio,
handler: operation(executeProjectSpeechEvidenceAudio),
},
{
lifecycle: "immediate" as const,
- capability: mediaPipelineCapabilities.renderAudio,
+ capability: mediaOperationsCapabilities.renderAudio,
returns: mediaTypes.timelineAudio,
handler: operation(executeRenderTimelineAudio),
},
{
lifecycle: "immediate" as const,
- capability: mediaPipelineCapabilities.mux,
+ capability: mediaOperationsCapabilities.mux,
returns: mediaTypes.muxed,
handler: operation(executeMuxProgramMedia),
},
diff --git a/packages/media-local/src/speech.ts b/packages/media-local/src/speech.ts
new file mode 100644
index 00000000..772a5744
--- /dev/null
+++ b/packages/media-local/src/speech.ts
@@ -0,0 +1,65 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { runProcess } from "./process.js";
+
+const EVIDENCE_SAMPLE_RATE = 16_000;
+
+type WavShape = {
+ readonly sampleRate: number;
+ readonly channels: number;
+ readonly bits: number;
+ readonly codec: number;
+ readonly dataBytes: number;
+};
+
+function wavShape(bytes: Uint8Array): WavShape | undefined {
+ const fourCc = (offset: number) => String.fromCharCode(...bytes.subarray(offset, offset + 4));
+ if (bytes.byteLength < 44 || fourCc(0) !== "RIFF" || fourCc(8) !== "WAVE") return undefined;
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ let offset = 12;
+ let format: Omit<WavShape, "dataBytes"> | undefined;
+ let dataBytes: number | undefined;
+ while (offset + 8 <= bytes.byteLength) {
+ const name = fourCc(offset);
+ const size = view.getUint32(offset + 4, true);
+ const body = offset + 8;
+ if (body + size > bytes.byteLength) return undefined;
+ if (name === "fmt " && size >= 16) {
+ format = { codec: view.getUint16(body, true), channels: view.getUint16(body + 2, true),
+ sampleRate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) };
+ } else if (name === "data") dataBytes = size;
+ offset = body + size + (size % 2);
+ }
+ return format === undefined || dataBytes === undefined ? undefined : { ...format, dataBytes };
+}
+
+function canonicalEvidence(shape: WavShape | undefined): shape is WavShape {
+ return shape !== undefined && shape.codec === 1 && shape.channels === 1
+ && shape.sampleRate === EVIDENCE_SAMPLE_RATE && shape.bits === 16;
+}
+
+/** Project one named local media file into WhisperX's canonical 16 kHz mono PCM evidence. */
+export async function speechEvidenceFile(path: string): Promise<{
+ readonly bytes: Uint8Array;
+ readonly sampleFrames: number;
+ readonly extracted: boolean;
+}> {
+ const original = new Uint8Array(await readFile(path));
+ const shape = wavShape(original);
+ if (canonicalEvidence(shape)) return { bytes: original, sampleFrames: shape.dataBytes / 2, extracted: false };
+ const scratch = await mkdtemp(join(tmpdir(), "hypit-speech-evidence-"));
+ try {
+ const target = join(scratch, "speech.wav");
+ await runProcess("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", path,
+ "-map", "0:a:0", "-vn", "-ac", "1", "-ar", String(EVIDENCE_SAMPLE_RATE),
+ "-c:a", "pcm_s16le", "-bitexact", target]);
+ const bytes = new Uint8Array(await readFile(target));
+ const extracted = wavShape(bytes);
+ if (!canonicalEvidence(extracted)) throw new Error("ffmpeg did not produce 16 kHz mono PCM audio");
+ return { bytes, sampleFrames: extracted.dataBytes / 2, extracted: true };
+ } finally {
+ await rm(scratch, { recursive: true, force: true });
+ }
+}
diff --git a/packages/media-execution/src/toolchain.ts b/packages/media-local/src/toolchain.ts
similarity index 100%
rename from packages/media-execution/src/toolchain.ts
rename to packages/media-local/src/toolchain.ts
diff --git a/packages/media-execution/src/webp.ts b/packages/media-local/src/webp.ts
similarity index 100%
rename from packages/media-execution/src/webp.ts
rename to packages/media-local/src/webp.ts
diff --git a/packages/media-execution/test/alpha.test.ts b/packages/media-local/test/alpha.test.ts
similarity index 98%
rename from packages/media-execution/test/alpha.test.ts
rename to packages/media-local/test/alpha.test.ts
index 6563aeb7..228a0078 100644
--- a/packages/media-execution/test/alpha.test.ts
+++ b/packages/media-local/test/alpha.test.ts
@@ -4,7 +4,7 @@ import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { MemoryResourceStore, alphaSize, ffmpegBytes, hasMediaBinaries, normalizeTestVideo, testMediaEnvironment, transparentVideoFixture, writeTestArtifact } from "../../../test/alpha-video-fixture.js";
-import { executeExtractFrame } from "../src/index.js";
+import { executeExtractFrame } from "../src/execute.js";
import { canonicalize } from "@hypit/protocol";
for (const format of ["webm", "webm-vp8", "mov"] as const) {
diff --git a/packages/media-local/test/file-frames.test.ts b/packages/media-local/test/file-frames.test.ts
new file mode 100644
index 00000000..eed61dfa
--- /dev/null
+++ b/packages/media-local/test/file-frames.test.ts
@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { decodeMediaFrames } from "../src/file-frames.js";
+import { runMediaCli } from "../src/files.js";
+
+test("native evidence preserves variable-rate timestamps and half-open bounds across grid pages", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-native-frames-test-"));
+ try {
+ const source = join(directory, "vfr.mkv");
+ const encoded = spawnSync("ffmpeg", ["-v", "error", "-f", "lavfi", "-i", "testsrc2=size=96x64:rate=10:duration=0.4",
+ "-vf", "setpts=if(lt(N\\,2)\\,N\\,N*2-1)/(10*TB)", "-fps_mode", "vfr", "-c:v", "libx264", source], { encoding: "utf8" });
+ assert.equal(encoded.status, 0, encoded.stderr);
+ const all = await decodeMediaFrames(source, 0, 0.6, join(directory, "all"));
+ assert.deepEqual(all.map(frame => frame.at), [0, 0.1, 0.3, 0.5]);
+ const selected = await decodeMediaFrames(source, 0.05, 0.5, join(directory, "selected"));
+ assert.deepEqual(selected.map(frame => frame.at), [0.1, 0.3]);
+ assert.equal((await readdir(join(directory, "selected"))).length, 2);
+ let output = "";
+ await runMediaCli(["media", "tiles", source, "--start", "0", "--end", "0.6", "--every-frame", "--columns", "2", "--rows", "1",
+ "--to", "grids", "--json"], { write: text => { output += text; } }, directory);
+ const grids = JSON.parse(output).grids;
+ assert.deepEqual(grids.map((grid: { frames: { at: number }[] }) => grid.frames.map(frame => frame.at)), [[0, 0.1], [0.3, 0.5]]);
+ assert.equal((await readdir(join(directory, "grids"))).length, 2);
+ output = "";
+ await runMediaCli(["media", "frames", source, "--start", "0.05", "--end", "0.5", "--every-frame", "--to", "files", "--json"],
+ { write: text => { output += text; } }, directory);
+ assert.deepEqual(JSON.parse(output).frames.map((frame: { at: number }) => frame.at), [0.1, 0.3]);
+ await assert.rejects(runMediaCli(["media", "tiles", source, "--every-frame", "--every", "0.1", "--to", "bad"], { write() {} }, directory), /cannot be combined/u);
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
diff --git a/packages/media-local/test/files.test.ts b/packages/media-local/test/files.test.ts
new file mode 100644
index 00000000..b4b4fbf8
--- /dev/null
+++ b/packages/media-local/test/files.test.ts
@@ -0,0 +1,282 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import sharp from "sharp";
+
+import type { CliIo } from "@hypit/hypit/cli";
+
+import { cutFrame, probeMedia, runMediaCli, tileFrameCount, tileSampleTimes, visualBoundaries } from "../src/files.js";
+
+const ffmpeg = spawnSync("ffmpeg", ["-version"], { stdio: "ignore" }).status === 0
+ && spawnSync("ffprobe", ["-version"], { stdio: "ignore" }).status === 0;
+
+function io(): { readonly io: CliIo; text: () => string } {
+ let out = "";
+ return {
+ io: {
+ write: (chunk) => { out += chunk; },
+ setExitCode: () => {},
+ readSecret: async () => "",
+ terminal: { isTTY: false, color: false, unicode: false, columns: 100 },
+ },
+ text: () => out,
+ };
+}
+
+/** Three seconds with three sub-second cuts, 24 fps and no audio. */
+async function sample(directory: string): Promise<string> {
+ const path = join(directory, "sample.mp4");
+ const result = spawnSync("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y",
+ "-f", "lavfi", "-i", "color=c=red:s=320x240:r=24:d=0.5",
+ "-f", "lavfi", "-i", "color=c=blue:s=320x240:r=24:d=0.5",
+ "-f", "lavfi", "-i", "color=c=green:s=320x240:r=24:d=0.5",
+ "-f", "lavfi", "-i", "color=c=white:s=320x240:r=24:d=1.5",
+ "-filter_complex", "[0:v][1:v][2:v][3:v]concat=n=4:v=1:a=0",
+ "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", path,
+ ], { encoding: "utf8" });
+ assert.equal(result.status, 0, result.stderr);
+ return path;
+}
+
+test("tile frame counts follow the measured bounds", () => {
+ assert.equal(tileFrameCount(0.5), 4);
+ assert.equal(tileFrameCount(4), 6);
+ assert.equal(tileFrameCount(30), 9);
+ assert.deepEqual(tileSampleTimes(1, 2, 4), [1.125, 1.375, 1.625, 1.875]);
+});
+
+test("media help names the available commands and both cut forms", () => {
+ const out = io();
+ runMediaCli(["media"], out.io);
+ for (const name of ["probe", "cut", "frames", "tile", "tiles", "boundaries"]) assert.match(out.text(), new RegExp(`hypit media ${name}`));
+ assert.doesNotMatch(out.text(), /fetch/u);
+ assert.match(out.text(), /--start <s> --end <s> \| --keep <start:end>/);
+ assert.match(out.text(), /--label-time/);
+});
+
+test("media evidence keeps source times and sub-second visual changes", { skip: !ffmpeg && "ffmpeg is not installed" }, async () => {
+ const work = await mkdtemp(join(tmpdir(), "hypit-media-"));
+ try {
+ const source = await sample(work);
+ const info = await probeMedia(source);
+ assert.equal(info.hasVideo, true);
+ if (!info.hasVideo) throw new Error("sample has no video stream");
+ assert.equal(info.width, 320);
+ assert.equal(info.height, 240);
+ assert.equal(info.hasAudio, false);
+ assert.ok(Math.abs(info.duration - 3) < 0.1, `duration ${info.duration}`);
+
+ const probe = io();
+ await runMediaCli(["media", "probe", source, "--json"], probe.io, work);
+ assert.equal((JSON.parse(probe.text()) as { width: number }).width, 320);
+
+ const cut = io();
+ await runMediaCli(["media", "cut", source, "--start", "0.5", "--end", "2", "--to", "out/piece.mp4"], cut.io, work);
+ const piece = await probeMedia(join(work, "out", "piece.mp4"));
+ assert.ok(Math.abs(piece.duration - 1.5) < 0.15, `cut duration ${piece.duration}`);
+ await assert.rejects(
+ runMediaCli(["media", "cut", source, "--start", "0", "--end", "1", "--to", "out/piece.mp4"], io().io, work),
+ /already exists/,
+ );
+ const labeledCut = io();
+ await runMediaCli(["media", "cut", source, "--start", "0.5", "--end", "1", "--label-time", "--to", "out/labeled.mp4", "--json"], labeledCut.io, work);
+ assert.equal((JSON.parse(labeledCut.text()) as { labeled: boolean }).labeled, true);
+
+ const frames = io();
+ await runMediaCli(["media", "frames", source, "--at", "0.2,2.5", "--to", "out/frames"], frames.io, work);
+ const written = (await readdir(join(work, "out", "frames"))).sort();
+ assert.deepEqual(written, ["frame-0_200s.jpg", "frame-2_500s.jpg"]);
+
+ const every = io();
+ await runMediaCli(["media", "frames", source, "--every", "1", "--label-time", "--to", "out/every", "--json"], every.io, work);
+ const everyView = JSON.parse(every.text()) as { labeled: boolean; frames: unknown[] };
+ assert.equal(everyView.frames.length, 3);
+ assert.equal(everyView.labeled, true);
+
+ const tile = io();
+ await runMediaCli(["media", "tile", source, "--at", "0.2,1.1,2.5", "--columns", "2", "--to", "out/grid.jpg", "--json"], tile.io, work);
+ const grid = JSON.parse(tile.text()) as { samples: number[]; columns: number; rows: number; cellWidth: number };
+ assert.deepEqual(grid.samples, [0.2, 1.1, 2.5]);
+ assert.equal(grid.columns, 2);
+ assert.equal(grid.rows, 2);
+ assert.equal(grid.cellWidth, 320, "a grid never enlarges a small source");
+ assert.ok((await stat(join(work, "out", "grid.jpg"))).size > 0);
+
+ const ranges = join(work, "ranges.json");
+ await writeFile(ranges, JSON.stringify([
+ { id: "opening", start: 0, end: 0.8, frames: 3 },
+ { start: 1.5, end: 3 },
+ ]));
+ const tiles = io();
+ await runMediaCli(["media", "tiles", source, "--ranges", ranges, "--to", "out/grids", "--json"], tiles.io, work);
+ const grids = (JSON.parse(tiles.text()) as { grids: { samples: number[]; path: string }[] }).grids;
+ assert.equal(grids.length, 2);
+ assert.equal(grids[0]!.samples.length, 3);
+ assert.deepEqual((await readdir(join(work, "out", "grids"))).sort(), ["001-opening.jpg", "002-1_500s-3_000s.jpg"]);
+
+ const changes = await visualBoundaries(source);
+ assert.ok(changes.some((item) => Math.abs(item.at - 0.5) < 0.1), JSON.stringify(changes));
+ assert.ok(changes.some((item) => Math.abs(item.at - 1) < 0.1), JSON.stringify(changes));
+ assert.ok(changes.some((item) => Math.abs(item.at - 1.5) < 0.1), JSON.stringify(changes));
+ const boundaries = io();
+ await runMediaCli(["media", "boundaries", source], boundaries.io, work);
+ assert.match(boundaries.text(), /visual-change candidates/);
+ assert.match(boundaries.text(), /scores are not shot labels/);
+ } finally {
+ await rm(work, { recursive: true, force: true });
+ }
+});
+
+test("album artwork does not turn an audio cut into video", { skip: !ffmpeg && "ffmpeg is not installed" }, async () => {
+ const work = await mkdtemp(join(tmpdir(), "hypit-covered-audio-"));
+ try {
+ const audio = join(work, "plain.mp3");
+ const artwork = join(work, "cover.jpg");
+ const covered = join(work, "covered.mp3");
+ const tone = spawnSync("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i",
+ "sine=frequency=440:duration=1", "-c:a", "libmp3lame", audio,
+ ], { encoding: "utf8" });
+ assert.equal(tone.status, 0, tone.stderr);
+ await sharp({ create: { width: 32, height: 32, channels: 3, background: "#ee3344" } }).jpeg().toFile(artwork);
+ const attach = spawnSync("ffmpeg", [
+ "-hide_banner", "-loglevel", "error", "-y", "-i", audio, "-i", artwork,
+ "-map", "0:a:0", "-map", "1:v:0", "-c", "copy", "-id3v2_version", "3",
+ "-disposition:v:0", "attached_pic", covered,
+ ], { encoding: "utf8" });
+ assert.equal(attach.status, 0, attach.stderr);
+
+ for (const [index, source] of [audio, covered].entries()) {
+ const info = await probeMedia(source);
+ assert.equal(info.hasAudio, true);
+ assert.equal(info.hasVideo, false, "album artwork is not a timed video stream");
+ const output = join(work, `cut-${index}.wav`);
+ const cut = io();
+ await runMediaCli(["media", "cut", source, "--start", "0", "--end", "0.5", "--to", output, "--json"], cut.io, work);
+ const result = JSON.parse(cut.text()) as { hasVideo: boolean; hasAudio: boolean; actualSeconds: number };
+ assert.equal(result.hasVideo, false);
+ assert.equal(result.hasAudio, true);
+ assert.ok(Math.abs(result.actualSeconds - 0.5) < 0.01);
+ const probe = spawnSync("ffprobe", [
+ "-v", "error", "-show_entries", "stream=codec_type,codec_name", "-of", "json", output,
+ ], { encoding: "utf8" });
+ assert.equal(probe.status, 0, probe.stderr);
+ assert.deepEqual(JSON.parse(probe.stdout).streams, [{ codec_name: "pcm_s24le", codec_type: "audio" }]);
+ }
+ } finally {
+ await rm(work, { recursive: true, force: true });
+ }
+});
+
+test("cut prepares recorded audio and joined video without changing the existing evidence cut", { skip: !ffmpeg && "ffmpeg is not installed" }, async () => {
+ const work = await mkdtemp(join(tmpdir(), "hypit-recorded-cut-"));
+ try {
+ const silentVideo = await sample(work);
+ const audio = join(work, "source.wav");
+ const tone = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=4", "-c:a", "pcm_s16le", audio], { encoding: "utf8" });
+ assert.equal(tone.status, 0, tone.stderr);
+ const audioProbe = await probeMedia(audio);
+ assert.equal(audioProbe.hasVideo, false);
+ assert.equal(audioProbe.hasAudio, true);
+ const probe = io();
+ await runMediaCli(["media", "probe", audio], probe.io, work);
+ assert.match(probe.text(), /audio only/);
+
+ const audioCut = io();
+ await runMediaCli(["media", "cut", audio, "--keep", "0.25:1", "--keep", "2.5:3", "--to", "spoken.wav", "--json"], audioCut.io, work);
+ const audioResult = JSON.parse(audioCut.text()) as { spans: { start: number; end: number }[];
+ mapping: { nominalOutputStart: number; nominalOutputEnd: number }[]; actualSeconds: number; hasVideo: boolean; hasAudio: boolean };
+ assert.deepEqual(audioResult.spans, [{ start: 0.25, end: 1 }, { start: 2.5, end: 3 }]);
+ assert.deepEqual(audioResult.mapping.map((item) => [item.nominalOutputStart, item.nominalOutputEnd]), [[0, 0.75], [0.75, 1.25]]);
+ assert.ok(Math.abs(audioResult.actualSeconds - 1.25) < 0.03, `audio duration ${audioResult.actualSeconds}`);
+ assert.equal(audioResult.hasVideo, false);
+ assert.equal(audioResult.hasAudio, true);
+ await runMediaCli(["media", "cut", audio, "--start", "2.5", "--end", "3", "--to", "late.wav"], io().io, work);
+ assert.ok(Math.abs((await probeMedia(join(work, "late.wav"))).duration - 0.5) < 0.03,
+ "a late single interval lands on the requested audio time");
+ await assert.rejects(runMediaCli(["media", "cut", audio, "--start", "0", "--end", "1", "--to", "spoken.wav"], io().io, work), /already exists/);
+ await assert.rejects(runMediaCli(["media", "cut", audio, "--start", "0", "--end", "1", "--label-time", "--to", "labeled.wav"], io().io, work), /needs one video interval/);
+
+ const withAudio = join(work, "source-av.mp4");
+ const mux = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", silentVideo, "-i", audio,
+ "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac", "-shortest", withAudio], { encoding: "utf8" });
+ assert.equal(mux.status, 0, mux.stderr);
+ const videoCut = io();
+ await runMediaCli(["media", "cut", withAudio, "--keep", "0:0.5", "--keep", "1:1.5", "--to", "joined.mp4", "--json"], videoCut.io, work);
+ const joined = await probeMedia(join(work, "joined.mp4"));
+ assert.equal(joined.hasVideo, true);
+ assert.equal(joined.hasAudio, true);
+ assert.ok(Math.abs(joined.duration - 1) < 0.08, `joined duration ${joined.duration}`);
+ const first = join(work, "first.jpg");
+ const second = join(work, "second.jpg");
+ await cutFrame(join(work, "joined.mp4"), 0.2, first);
+ await cutFrame(join(work, "joined.mp4"), 0.7, second);
+ const firstPixel = await sharp(first).extract({ left: 160, top: 120, width: 1, height: 1 }).raw().toBuffer();
+ const secondPixel = await sharp(second).extract({ left: 160, top: 120, width: 1, height: 1 }).raw().toBuffer();
+ assert.ok(firstPixel[0]! > firstPixel[1]! * 2, `first frame should be red: ${firstPixel}`);
+ assert.ok(secondPixel[1]! > secondPixel[0]! * 2, `second frame should be green: ${secondPixel}`);
+
+ await runMediaCli(["media", "cut", withAudio, "--start", "2.1", "--end", "2.8", "--to", "late.mp4"], io().io, work);
+ const lateVideo = await probeMedia(join(work, "late.mp4"));
+ assert.equal(lateVideo.hasAudio, true);
+ assert.ok(Math.abs(lateVideo.duration - 0.7) < 0.08, `late video duration ${lateVideo.duration}`);
+ await runMediaCli(["media", "cut", withAudio, "--start", "0", "--end", "0.5", "--to", "clip.mov"], io().io, work);
+ assert.equal((await probeMedia(join(work, "clip.mov"))).hasAudio, true);
+
+ const silentCut = join(work, "silent-cut.mp4");
+ await runMediaCli(["media", "cut", silentVideo, "--keep", "0:0.5", "--keep", "1:1.5", "--to", silentCut], io().io, work);
+ assert.equal((await probeMedia(silentCut)).hasAudio, false);
+ } finally {
+ await rm(work, { recursive: true, force: true });
+ }
+});
+
+test("word-located grids paginate dense samples without covering the source picture", { skip: !ffmpeg && "ffmpeg is not installed" }, async () => {
+ const work = await mkdtemp(join(tmpdir(), "hypit-word-grids-"));
+ try {
+ const source = await sample(work);
+ const transcript = join(work, "words.json");
+ await writeFile(transcript, JSON.stringify({ format: "hypit.transcript@1", passages: [{ words: [
+ { text: "Hello", start_seconds: 0.1, end_seconds: 0.4 },
+ { text: "world!", start_seconds: 0.4, end_seconds: 0.8 },
+ { text: "<你好 & café>" },
+ { text: "Hello", start_seconds: 2.1, end_seconds: 2.4 },
+ { text: "world!", start_seconds: 2.4, end_seconds: 2.8 },
+ ] }] }));
+ await assert.rejects(runMediaCli(["media", "tile", source, "--around", "hello world", "--transcript", transcript,
+ "--to", "ambiguous.jpg"], io().io, work), /occurs 2 times.*--occurrence/);
+ const around = io();
+ await runMediaCli(["media", "tile", source, "--around", "hello world", "--occurrence", "2", "--padding", "0.1",
+ "--every", "0.2", "--transcript", transcript, "--to", "around.jpg", "--json"], around.io, work);
+ const located = JSON.parse(around.text());
+ assert.deepEqual(located.samples, [2, 2.2, 2.4, 2.6, 2.8]);
+ assert.deepEqual(located.frames[0].words.active, []);
+ assert.equal(located.frames[1].words.active[0].text, "Hello");
+ assert.equal(located.frames[1].requestedAt, 2.2);
+ assert.ok(Math.abs(located.frames[1].at - 53 / 24) < 0.000001, "labels use the decoded frame time, not the requested seek time");
+ assert.equal(located.frames[2].words.active[0].text, "world!");
+
+ const pages = io();
+ await runMediaCli(["media", "tiles", source, "--start", "0", "--end", "3", "--every", "0.6",
+ "--transcript", transcript, "--columns", "2", "--rows", "1", "--to", "pages", "--json"], pages.io, work);
+ const grids = JSON.parse(pages.text()).grids;
+ assert.deepEqual(grids.map((grid: { samples: number[] }) => grid.samples), [[0, 0.6], [1.2, 1.8], [2.4]]);
+ assert.ok(grids.every((grid: { start: number; end: number }) => grid.start === 0 && grid.end === 3));
+ assert.equal((await readdir(join(work, "pages"))).length, 3);
+ const image = await sharp(grids[0].path).raw().toBuffer({ resolveWithObject: true });
+ const center = (128 * image.info.width + 168) * image.info.channels;
+ assert.ok(image.data[center]! > 220 && image.data[center + 1]! < 20, "the first source frame stays red and unobscured");
+ assert.ok(image.info.height > 240 + 32, "word annotations have their own space beneath the picture");
+
+ const frames = io();
+ await runMediaCli(["media", "frames", source, "--start", "0.5", "--end", "1.5", "--every", "0.4",
+ "--transcript", transcript, "--to", "frames", "--json"], frames.io, work);
+ assert.deepEqual(JSON.parse(frames.text()).frames.map((frame: { requestedAt: number }) => frame.requestedAt), [0.5, 0.9, 1.3]);
+ } finally {
+ await rm(work, { recursive: true, force: true });
+ }
+});
diff --git a/packages/media-execution/test/probe.test.ts b/packages/media-local/test/probe.test.ts
similarity index 98%
rename from packages/media-execution/test/probe.test.ts
rename to packages/media-local/test/probe.test.ts
index 2739b84d..2e1e6fea 100644
--- a/packages/media-execution/test/probe.test.ts
+++ b/packages/media-local/test/probe.test.ts
@@ -3,7 +3,7 @@ import test from "node:test";
import { fixtureResource } from "../../../test/fixture-resource.js";
-import { parseMediaInspection } from "@hypit/media-execution";
+import { parseMediaInspection } from "../src/probe.js";
test("probe derives the last decoded-unit duration instead of adding whole-stream duration_ts", () => {
const inspection = parseMediaInspection({
diff --git a/packages/media-execution/test/process-env.test.ts b/packages/media-local/test/process-env.test.ts
similarity index 100%
rename from packages/media-execution/test/process-env.test.ts
rename to packages/media-local/test/process-env.test.ts
diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/media-local/test/provider.test.ts
similarity index 89%
rename from packages/provider-media-local/test/provider.test.ts
rename to packages/media-local/test/provider.test.ts
index 2808223a..6fb3869f 100644
--- a/packages/provider-media-local/test/provider.test.ts
+++ b/packages/media-local/test/provider.test.ts
@@ -3,24 +3,24 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
-import { artifactTypes } from "@hypit/artifact";
-import { mediaTypes, sealRenderedVisual, sealTimelineAudio, synchronizedMediaSampleFrames, verifyMediaInspection, verifyMuxedMedia, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
+import { blobTypes } from "@hypit/blob";
+import { mediaTypes, sealTimelineVisual, sealTimelineAudio, synchronizedMediaSampleFrames, verifyMediaInspection, verifyMuxedMedia, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
import type { MediaAudioStream, MediaInspection, MuxedMedia, SynchronizedMedia, TimelineAudio } from "@hypit/media";
-import { assertSpeechEvidenceAudioIdentity, speechEvidenceTypes } from "@hypit/speech-evidence";
+import { assertSpeechEvidenceAudioIdentity, speechEvidenceCapabilities, speechEvidenceTypes } from "@hypit/speech-evidence";
import type { SpeechEvidenceAudio } from "@hypit/speech-evidence";
import assert from "node:assert/strict";
import {
MemoryResourceStore,
EndpointRegistry,
-} from "@hypit/driver-node";
-import type { EndpointRegistration } from "@hypit/driver-node";
-import type { ImmediateEndpointHandler } from "@hypit/endpoint-kit";
+} from "@hypit/executor";
+import type { EndpointRegistration } from "@hypit/executor";
+import type { ImmediateEndpointHandler } from "@hypit/endpoint";
import {
- mediaPipelineCapabilities,
+ mediaOperationsCapabilities,
sealAudioProgramPlan,
sealMediaSelectionRequest,
selectMediaStreams,
-} from "@hypit/media-pipeline";
+} from "@hypit/media-operations";
import { canonicalize } from "@hypit/protocol";
import type { CapabilityRef, CanonicalValue, Need, TypeRef } from "@hypit/protocol";
@@ -59,6 +59,16 @@ async function run(executable: string, args: readonly string[]): Promise<void> {
});
}
+function runBytes(executable: string, args: readonly string[], input?: Uint8Array): Buffer {
+ const result = spawnSync(executable, [...args], {
+ ...(input === undefined ? {} : { input }),
+ maxBuffer: 32 * 1024 * 1024,
+ windowsHide: true,
+ });
+ assert.equal(result.status, 0, Buffer.from(result.stderr ?? []).toString());
+ return Buffer.from(result.stdout ?? []);
+}
+
function rampWav(sampleFrames: number): Buffer {
const dataBytes = sampleFrames * 2 * 2;
const bytes = Buffer.alloc(44 + dataBytes);
@@ -207,7 +217,7 @@ test("local media opts only reusable display materialization into transient exec
async function inspectArtifact(resources: MemoryResourceStore, source: Awaited<ReturnType<MemoryResourceStore["put"]>>) {
const constraints = canonicalize({ source });
- const request = need("need:media-inspect", mediaPipelineCapabilities.inspect,
+ const request = need("need:media-inspect", mediaOperationsCapabilities.inspect,
mediaTypes.inspection, constraints);
const provider = await handlerFor(request);
const result = await provider.handler({
@@ -236,7 +246,7 @@ async function normalizeArtifact(args: {
frameRate: args.frameRate,
audio: { sampleRate: 48_000, channels: 2, codec: "pcm_s16le", loudness: "preserve" },
});
- const request = need("need:media-normalize", mediaPipelineCapabilities.normalize,
+ const request = need("need:media-normalize", mediaOperationsCapabilities.normalize,
mediaTypes.synchronized, constraints);
const provider = await handlerFor(request);
const result = await provider.handler({
@@ -254,7 +264,7 @@ async function normalizeArtifact(args: {
test("local media Provider enumerates attached pictures and jointly normalizes 32k AAC without inventing speech", {
skip: !hasMediaBinaries,
}, async () => {
- const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-local-"));
+ const root = await mkdtemp(join(tmpdir(), "hypit-media-local-"));
try {
const sourcePath = await fixture(root);
const resources = new MemoryResourceStore();
@@ -447,7 +457,7 @@ test("local media Provider derives one exact 16 kHz mono WhisperX evidence artif
});
const request = need(
"need:speech-evidence-audio",
- mediaPipelineCapabilities.projectSpeechEvidenceAudio,
+ speechEvidenceCapabilities.projectAudio,
speechEvidenceTypes.audio,
constraints,
);
@@ -582,13 +592,13 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
credentials: {},
});
assert.equal(result.value.kind, "blob");
- return result.value.kind === "blob" ? result.value : (() => { throw new Error("expected BlobArtifact"); })();
+ return result.value.kind === "blob" ? result.value : (() => { throw new Error("expected Blob"); })();
};
const transformed = await executeArtifact(need(
"need:transform",
- mediaPipelineCapabilities.transform,
- artifactTypes.blob,
+ mediaOperationsCapabilities.transform,
+ blobTypes.blob,
canonicalize({
media: normalized,
program: {
@@ -605,8 +615,8 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
const audioIndex = inspection.streams.find((item) => item.kind === "audio")!.index;
const extractedAudio = await executeArtifact(need(
"need:extract-audio",
- mediaPipelineCapabilities.extractAudio,
- artifactTypes.blob,
+ mediaOperationsCapabilities.extractAudio,
+ blobTypes.blob,
canonicalize({
source,
streamIndex: audioIndex,
@@ -621,8 +631,8 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
const video = inspection.streams.find((item) => item.kind === "video" && item.role === "moving")!;
const extractedFrame = await executeArtifact(need(
"need:extract-frame",
- mediaPipelineCapabilities.extractFrame,
- artifactTypes.blob,
+ mediaOperationsCapabilities.extractFrame,
+ blobTypes.blob,
canonicalize({
source,
streamIndex: video.index,
@@ -637,8 +647,8 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
const stillVideo = await executeArtifact(need(
"need:render-still-video",
- mediaPipelineCapabilities.renderStill,
- artifactTypes.blob,
+ mediaOperationsCapabilities.renderStill,
+ blobTypes.blob,
canonicalize({
request: {
frameRate: { numerator: 30, denominator: 1 },
@@ -656,8 +666,8 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
const spread = await executeArtifact(need(
"need:render-still-spread",
- mediaPipelineCapabilities.renderStill,
- artifactTypes.blob,
+ mediaOperationsCapabilities.renderStill,
+ blobTypes.blob,
canonicalize({
request: {
frameRate: { numerator: 30, denominator: 1 },
@@ -675,8 +685,8 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
assert.equal(spreadInspection.streams[0]?.decodedUnitCount, 15, "two pictures spread over the same frame count");
const twentyFourSpread = await executeArtifact(need(
"need:render-still-24fps-spread",
- mediaPipelineCapabilities.renderStill,
- artifactTypes.blob,
+ mediaOperationsCapabilities.renderStill,
+ blobTypes.blob,
canonicalize({
request: {
frameRate: { numerator: 24, denominator: 1 },
@@ -713,6 +723,97 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
}
});
+test("StillVideo converts authored RGB pictures to tagged BT.709 in both render paths", {
+ skip: !hasMediaBinaries,
+}, async () => {
+ const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-bt709-"));
+ try {
+ const width = 1280;
+ const height = 720;
+ const expected = [0x20, 0xc0, 0x40] as const;
+ const rgb = Buffer.alloc(width * height * 3);
+ for (let offset = 0; offset < rgb.length; offset += 3) {
+ rgb[offset] = expected[0];
+ rgb[offset + 1] = expected[1];
+ rgb[offset + 2] = expected[2];
+ }
+ const png = runBytes("ffmpeg", [
+ "-v", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", `${width}x${height}`,
+ "-i", "pipe:0", "-frames:v", "1", "-f", "image2pipe", "-c:v", "png", "pipe:1",
+ ], rgb);
+ const resources = new MemoryResourceStore();
+ const source = await resources.put(png, "image/png");
+ const render = async (id: string, segments: readonly {
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+ readonly source: typeof source;
+ }[]) => {
+ const request = need(
+ `need:${id}`,
+ mediaOperationsCapabilities.renderStill,
+ blobTypes.blob,
+ canonicalize({
+ request: {
+ frameRate: { numerator: 30, denominator: 1 },
+ frameCount: 4,
+ output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments,
+ },
+ }),
+ );
+ const provider = await handlerFor(request);
+ const result = await provider.handler({
+ command: { kind: "fulfill-need", id: `command:${id}`, need: request },
+ need: request,
+ resources,
+ credentials: {},
+ });
+ assert.equal(result.value.kind, "blob");
+ assert.equal(result.value.kind === "blob" && result.value.mediaType, "video/mp4");
+ const artifact = result.value.kind === "blob" ? result.value : undefined;
+ assert(artifact !== undefined);
+ const bytes = await resources.get(artifact.resource);
+ assert(bytes !== undefined);
+ const path = join(root, `${id}.mp4`);
+ await writeFile(path, bytes);
+ return path;
+ };
+ const paths = await Promise.all([
+ render("single", [{ startFrame: 0, endFrameExclusive: 4, source }]),
+ render("spread", [
+ { startFrame: 0, endFrameExclusive: 2, source },
+ { startFrame: 2, endFrameExclusive: 4, source },
+ ]),
+ ]);
+ for (const path of paths) {
+ const probe = spawnSync("ffprobe", [
+ "-v", "error", "-select_streams", "v:0",
+ "-show_entries", "stream=color_range,color_space,color_transfer,color_primaries",
+ "-of", "json", path,
+ ], { encoding: "utf8", windowsHide: true });
+ assert.equal(probe.status, 0, probe.stderr);
+ const stream = (JSON.parse(probe.stdout) as { streams: Record<string, string>[] }).streams[0]!;
+ assert.equal(stream.color_range, "tv");
+ assert.equal(stream.color_space, "bt709");
+ assert.equal(stream.color_transfer, "bt709");
+ assert.equal(stream.color_primaries, "bt709");
+
+ const decoded = runBytes("ffmpeg", [
+ "-v", "error", "-i", path, "-frames:v", "1",
+ "-vf", "scale=in_color_matrix=bt709:in_range=tv:out_range=pc,format=rgb24",
+ "-f", "rawvideo", "pipe:1",
+ ]);
+ const center = (Math.floor(height / 2) * width + Math.floor(width / 2)) * 3;
+ expected.forEach((channel, index) => {
+ assert.ok(Math.abs(decoded[center + index]! - channel) <= 3,
+ `${path} channel ${index} decoded as ${decoded[center + index]}, expected ${channel}`);
+ });
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true }).catch(() => {});
+ }
+});
+
test("local media Provider renders one frame-domain audio plan and muxes exactly one video-only visual with it", {
skip: !hasMediaBinaries,
}, async () => {
@@ -789,7 +890,7 @@ test("local media Provider renders one frame-domain audio plan and muxes exactly
});
const audioRequest = need(
"need:render-program-audio",
- mediaPipelineCapabilities.renderAudio,
+ mediaOperationsCapabilities.renderAudio,
mediaTypes.timelineAudio,
canonicalize({ plan }),
);
@@ -799,7 +900,7 @@ test("local media Provider renders one frame-domain audio plan and muxes exactly
assert.equal(audio.sampleFrames, 48_000);
assert.equal(await resources.has(audio.artifact.resource), true);
- const visual = sealRenderedVisual({
+ const visual = sealTimelineVisual({
frameRate: { numerator: 30, denominator: 1 },
frameCount: 30,
canvas: { width: 160, height: 96 },
@@ -807,7 +908,7 @@ test("local media Provider renders one frame-domain audio plan and muxes exactly
});
const muxRequest = need(
"need:mux-program-media",
- mediaPipelineCapabilities.mux,
+ mediaOperationsCapabilities.mux,
mediaTypes.muxed,
canonicalize({ visual, audio }),
);
@@ -860,7 +961,7 @@ test("local media Provider preserves the audio tail for a 30000/1001 frame domai
resources.put(await readFile(visualPath), "video/mp4"),
]);
const audio = sealTimelineAudio({ artifact: audioArtifact, sampleFrames });
- const visual = sealRenderedVisual({
+ const visual = sealTimelineVisual({
frameRate,
frameCount,
canvas: { width: 160, height: 96 },
@@ -868,7 +969,7 @@ test("local media Provider preserves the audio tail for a 30000/1001 frame domai
});
const muxValue = await fulfillInline(resources, need(
"need:mux-ntsc-program-media",
- mediaPipelineCapabilities.mux,
+ mediaOperationsCapabilities.mux,
mediaTypes.muxed,
canonicalize({ visual, audio }),
));
@@ -920,7 +1021,7 @@ test("local media Provider executes an end-aligned loop from the exact authored
});
const value = await fulfillInline(resources, need(
"need:render-loop-audio",
- mediaPipelineCapabilities.renderAudio,
+ mediaOperationsCapabilities.renderAudio,
mediaTypes.timelineAudio,
canonicalize({ plan }),
));
@@ -959,7 +1060,7 @@ test("audio range preserves loop phase, tempo and intersected fades from the ful
fadeInSamples: 48000, fadeOutSamples: 30000 }], mix: { normalize: false, limiter: "none" },
});
const pcm = async (name: string, range?: { startFrame: number; endFrameExclusive: number }) => {
- const value = await fulfillInline(resources, need(`need:${name}`, mediaPipelineCapabilities.renderAudio,
+ const value = await fulfillInline(resources, need(`need:${name}`, mediaOperationsCapabilities.renderAudio,
mediaTypes.timelineAudio, canonicalize({ plan, ...(range === undefined ? {} : { range }) })));
verifyTimelineAudio(value);
const audio = value as unknown as TimelineAudio;
@@ -999,7 +1100,7 @@ test("audio envelopes and audible regions preserve sample phase through gaps and
}], mix: { normalize: false, limiter: "none" },
});
const pcm = async (name: string, range?: { startFrame: number; endFrameExclusive: number }) => {
- const value = await fulfillInline(resources, need(`need:${name}`, mediaPipelineCapabilities.renderAudio,
+ const value = await fulfillInline(resources, need(`need:${name}`, mediaOperationsCapabilities.renderAudio,
mediaTypes.timelineAudio, canonicalize({ plan, ...(range === undefined ? {} : { range }) })));
verifyTimelineAudio(value);
const path = join(root, `${name}.wav`), raw = join(root, `${name}.raw`);
diff --git a/packages/media-execution/test/toolchain.test.ts b/packages/media-local/test/toolchain.test.ts
similarity index 100%
rename from packages/media-execution/test/toolchain.test.ts
rename to packages/media-local/test/toolchain.test.ts
diff --git a/packages/media-execution/test/webp.test.ts b/packages/media-local/test/webp.test.ts
similarity index 100%
rename from packages/media-execution/test/webp.test.ts
rename to packages/media-local/test/webp.test.ts
diff --git a/packages/media-operations/README.md b/packages/media-operations/README.md
new file mode 100644
index 00000000..d2e65d0b
--- /dev/null
+++ b/packages/media-operations/README.md
@@ -0,0 +1,122 @@
+# `@hypit/media-operations`
+
+Provider-neutral media operation vocabulary. Ingestion contributes one ordinary finite Fragment:
+
+```text
+Blob
+ -> inspect-media Need
+ -> MediaInspection
+ -> deterministic stream selection
+ -> MediaStreamSelection
+ -> normalize-media Need
+ -> SynchronizedMedia
+```
+
+`MediaInspection` enumerates every container stream. The default moving-video policy excludes
+`attached_pic` streams, prefers one declared default, accepts one unambiguous remaining candidate,
+and otherwise fails closed. An explicit stream index is available when the author module genuinely
+knows which alternate stream it wants.
+
+Selecting embedded audio is a media fact only. This package never assigns Script meaning. A later
+speech alignment component may bind one normalized local domain to one authored Segment.
+
+The current normalization profile uses one source presentation origin, a requested rational video frame
+rate, and a 48 kHz stereo PCM render stem. It preserves input level: loudness/mastering remains a
+separate author policy. Its public `SynchronizedMedia` waist carries only the common frame domain,
+optional visual Artifact plus intrinsic extent, and optional audio Artifact. Stream indexes,
+authority choice and the trim/pad ledger remain in Selection and execution instead of travelling
+through every consumer. The field is named `frameDomain`, not `timeline`, because it describes the
+normalized source before any program placement.
+
+Normalization preserves transparency in the selected visual stream. The executing Provider owns
+the intermediate encoding; both opaque and transparent pictures use the same SynchronizedMedia
+value. Normalization also publishes the media's neutral local temporal domain. Speech alignment may
+describe Narrative boundaries on that domain; Timeline authoring may consume its Extent; a domain
+projector may map its boundaries through an equal-length Window. None of those relationships is embedded
+back into SynchronizedMedia.
+
+This package is the shared execution vocabulary. Speech authoring makes the boundary explicit:
+`<mediaop:Normalize>` produces `SynchronizedMedia + LocalTemporalDomain`; a speech provider and
+deterministic alignment produce `NarrativeAlignment`; Timeline authoring separately produces absolute
+Windows. Visual and Audio Tracks consume ordinary media plus the explicit Windows and sampling choices
+they declare; neither owns an implicit normalization policy.
+
+The frame clock is a separate program fact and can be shared by every Normalize and Timeline assembly:
+
+```svml
+<time:Clock id="clock" frame-rate="30"/>
+<mediaop:Normalize id="shot-media" source={shot.video}
+ recipe={recipes.media.aroll} clock={clock}/>
+```
+
+An SVS recipe may centralize the stream policy (`video`, `audio` and `span-authority`); it does not
+hide the Normalize graph node or turn multiple sources into one opaque batch operation.
+
+A still image becomes ordinary video before it enters that waist:
+
+```svml
+<mediaop:StillVideo id="opening-still" source={opening-head.image}
+ duration="6s" clock={clock}/>
+<mediaop:Normalize id="opening-media" source={opening-still.video}
+ video="primary-moving" audio="none" span-authority="video" clock={clock}/>
+```
+
+Several images spread over one literal duration the same way, each held for its share of the frames:
+
+```svml
+<mediaop:StillVideo id="kitchen-stills" duration="6s" clock={clock}>
+ <mediaop:Still source={counter.image}/>
+ <mediaop:Still source={basil.image} weight="2"/>
+ <mediaop:Still source={board.image}/>
+</mediaop:StillVideo>
+```
+
+`StillVideo` returns a video-only MP4 `Blob`, not `SynchronizedMedia`. Its duration is an exact
+`TemporalDuration`; it must land on a positive whole frame count for the selected Clock and is never
+rounded. The Surface publishes that duration and the weights as Records; `plan-still-video` divides the whole frame count among the
+pictures (every picture holds at least one frame, the rest go by weight with leftovers to the largest
+remainders, earlier first), `bind-still-video-source` attaches each picture in authored order, and
+encoding is one `render-still-video` Need. Pictures of different sizes are fitted into the first one's
+frame and letterboxed. Inspection and normalization remain the same explicit steps used by imported
+or generated moving video. The resulting Blob is ordinary time-bearing visual media. Its later role
+comes entirely from the downstream Source relationships; StillVideo itself owns only the authored
+images, duration and frame clock.
+
+The package also exposes the one-picture Run Fragment `still-video`. It takes `duration`, `clock`,
+`layout` and `source-0`. A Run can select its `video` export as a Candidate through `satisfy`.
+
+Four ordinary author operations reuse that same inspection/execution boundary:
+
+```xml
+<mediaop:Normalize id="shot-media" source={shot.video}
+ video="primary-moving" audio="default" span-authority="video" clock={clock}/>
+<mediaop:Transform id="prepared" source={shot-media.media}>
+ <mediaop:Trim tail="0.25s"/>
+ <mediaop:Retime rate="1.05"/>
+</mediaop:Transform>
+
+<mediaop:ExtractAudio id="voice-reference" source={prepared.video} audio="default"/>
+<mediaop:ExtractFrame id="continuity" source={prepared.video} video="primary-moving" at="last"/>
+```
+
+Every result is an ordinary `Blob`. Audio extraction emits a deterministic 48 kHz stereo PCM
+WAV but makes no NarrativeAlignment, speaker or semantic claim; it can therefore feed a later model
+reference port directly. Frame extraction supports `first`, `last`, `frame:<index>` and
+`time:<seconds>`. Transform operations are ordered author meaning and never an arbitrary FFmpeg string.
+
+The same package also owns two provider-neutral media completion plans/capabilities:
+
+```text
+Composition -> pure AudioProgramPlan -> render-timeline-audio Need -> TimelineAudio
+TimelineVisual + TimelineAudio -> mux-program-media Need -> MuxedMedia
+```
+
+Compiling an `AudioProgramPlan` is ordinary deterministic code. Reading Artifact bytes, decoding,
+resampling, mixing, encoding or muxing is never a Producer shortcut: it is an explicit Provider
+Need. The plan fixes exact 48 kHz sample boundaries, source-time-derived piece rates, gain/fades and the absence
+of hidden normalization/limiting. The local FFmpeg Provider consumes the shared execution body. Other Providers implement the same
+public request and result semantics.
+
+AudioProgramPlan preserves AudioClip `gainEnvelope` and `audibility` on the full program sample
+clock. The [Composition definition](../composition/README.md#audio-presentation-on-the-program-clock)
+owns their semantics; the plan does not reinterpret source choice or presentation precedence.
diff --git a/packages/media-operations/package.json b/packages/media-operations/package.json
new file mode 100644
index 00000000..7430b9d5
--- /dev/null
+++ b/packages/media-operations/package.json
@@ -0,0 +1,34 @@
+{
+ "name": "@hypit/media-operations",
+ "version": "0.1.0",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "files": [
+ "dist",
+ "README.md"
+ ],
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "peerDependencies": {
+ "@hypit/hypit": "workspace:^"
+ },
+ "devDependencies": {
+ "@hypit/blob": "workspace:*",
+ "@hypit/producer": "workspace:*",
+ "@hypit/composition": "workspace:*",
+ "@hypit/author": "workspace:*",
+ "@hypit/media": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/run": "workspace:*",
+ "@hypit/recipe": "workspace:*",
+ "@hypit/markup": "workspace:*",
+ "@hypit/timeline": "workspace:*",
+ "@hypit/temporal": "workspace:*",
+ "@hypit/hypit": "workspace:*"
+ }
+}
diff --git a/packages/media-operations/src/activation.ts b/packages/media-operations/src/activation.ts
new file mode 100644
index 00000000..447640a8
--- /dev/null
+++ b/packages/media-operations/src/activation.ts
@@ -0,0 +1,49 @@
+import { createAdmissionPackageFacet } from "@hypit/hypit/admission";
+import { createProducerPackageFacet } from "@hypit/hypit/producer";
+import { createMarkupSurfaceFacet } from "@hypit/hypit/markup";
+import { createRunFragmentFacet } from "@hypit/hypit/run";
+import {
+ decodeSynchronizedMediaSurface,
+ decodeStillVideoSurface,
+ decodeExtractAudioSurface,
+ decodeExtractFrameSurface,
+ decodeTransformMediaSurface,
+ mediaOperationsComponent,
+ mediaOperationsManifest,
+ mediaOperationsModuleRef,
+ mediaOperationsMarkupSurfaces,
+ stillVideoFragment,
+} from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.package@1" as const,
+ modules: [{ manifest: mediaOperationsManifest }],
+ facets: [
+ ...[mediaOperationsComponent].flatMap((component) => [createProducerPackageFacet(component), createAdmissionPackageFacet(component)]),createRunFragmentFacet({
+ name: "@hypit/media-operations@1",
+ fragments: {
+ "still-video": stillVideoFragment,
+ },
+ }), createMarkupSurfaceFacet({
+ module: mediaOperationsModuleRef,
+ declaration: mediaOperationsMarkupSurfaces.find((item) => item.name === "synchronized-media")!,
+ handler: decodeSynchronizedMediaSurface,
+ }), createMarkupSurfaceFacet({
+ module: mediaOperationsModuleRef,
+ declaration: mediaOperationsMarkupSurfaces.find((item) => item.name === "still-video")!,
+ handler: decodeStillVideoSurface,
+ }), createMarkupSurfaceFacet({
+ module: mediaOperationsModuleRef,
+ declaration: mediaOperationsMarkupSurfaces.find((item) => item.name === "transform-media")!,
+ handler: decodeTransformMediaSurface,
+ }), createMarkupSurfaceFacet({
+ module: mediaOperationsModuleRef,
+ declaration: mediaOperationsMarkupSurfaces.find((item) => item.name === "extract-audio")!,
+ handler: decodeExtractAudioSurface,
+ }), createMarkupSurfaceFacet({
+ module: mediaOperationsModuleRef,
+ declaration: mediaOperationsMarkupSurfaces.find((item) => item.name === "extract-frame")!,
+ handler: decodeExtractFrameSurface,
+ })],
+};
+export default hypitPackage;
diff --git a/packages/media-operations/src/audio-plan.ts b/packages/media-operations/src/audio-plan.ts
new file mode 100644
index 00000000..1fd5f76f
--- /dev/null
+++ b/packages/media-operations/src/audio-plan.ts
@@ -0,0 +1,146 @@
+import { clockFrameSampleBoundary, timelineFrameCount, timelineFrameSampleBoundary } from "@hypit/hypit/timeline";
+import type { Timeline } from "@hypit/hypit/timeline";
+import { assertAudioLevelAutomation, assertCompositionIdentity } from "@hypit/hypit/composition";
+import type { Composition } from "@hypit/hypit/composition";
+import { canonicalize, isResourceId } from "@hypit/hypit/protocol";
+
+import type { AudioProgramPlan } from "./types.js";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function exactInteger(value: { readonly numerator: number; readonly denominator: number }, label: string): number {
+ assert(Number.isSafeInteger(value.numerator) && Number.isSafeInteger(value.denominator) && value.denominator > 0
+ && value.numerator % value.denominator === 0, `${label} must resolve to an exact source-sample boundary`);
+ const result = value.numerator / value.denominator;
+ assert(Number.isSafeInteger(result) && result >= 0, `${label} is invalid`);
+ return result;
+}
+
+function exactSourceEnd(sourceAtStart: { readonly numerator: number; readonly denominator: number },
+ rate: { readonly numerator: number; readonly denominator: number }, targetLength: number, label: string): number {
+ const numerator = BigInt(sourceAtStart.numerator) * BigInt(rate.denominator)
+ + BigInt(targetLength) * BigInt(rate.numerator) * BigInt(sourceAtStart.denominator);
+ const denominator = BigInt(sourceAtStart.denominator) * BigInt(rate.denominator);
+ assert(numerator % denominator === 0n, `${label} must resolve to an exact source-sample boundary`);
+ const result = numerator / denominator;
+ assert(result >= 0n && result <= BigInt(Number.MAX_SAFE_INTEGER), `${label} is invalid`);
+ return Number(result);
+}
+
+export function sealAudioProgramPlan(value: AudioProgramPlan): AudioProgramPlan {
+ return canonicalize(value) as unknown as AudioProgramPlan;
+}
+
+export function verifyAudioProgramPlan(value: unknown): asserts value is AudioProgramPlan {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "AudioProgramPlan must be an object");
+ const item = value as AudioProgramPlan;
+ assert(Number.isSafeInteger(item.frameRate?.numerator) && item.frameRate.numerator > 0
+ && Number.isSafeInteger(item.frameRate?.denominator) && item.frameRate.denominator > 0,
+ "AudioProgramPlan frame rate is invalid");
+ assert(Number.isSafeInteger(item.frameCount) && item.frameCount > 0, "AudioProgramPlan frame count is invalid");
+ assert(item.sampleRate === 48_000, "AudioProgramPlan sample rate must be 48000");
+ assert(item.sampleFrames === clockFrameSampleBoundary({ frameRate: item.frameRate }, item.frameCount, 48_000),
+ "AudioProgramPlan sample count differs from its frame domain");
+ assert(Array.isArray(item.clips), "AudioProgramPlan clips are invalid");
+ const ids = new Set<string>();
+ for (const clip of item.clips) {
+ assert(typeof clip.id === "string" && clip.id.length > 0 && !ids.has(clip.id),
+ "AudioProgramPlan clip id is empty or repeated");
+ ids.add(clip.id);
+ assert(clip.artifact?.kind === "blob" && isResourceId(clip.artifact.resource)
+ && Number.isSafeInteger(clip.artifact.size) && clip.artifact.size >= 0
+ && clip.artifact.mediaType === "audio/wav",
+ `AudioProgramPlan clip ${clip.id} must reference canonical WAV`);
+ assert(Number.isSafeInteger(clip.targetStartSample) && clip.targetStartSample >= 0
+ && Number.isSafeInteger(clip.targetEndSampleExclusive)
+ && clip.targetEndSampleExclusive > clip.targetStartSample
+ && clip.targetEndSampleExclusive <= item.sampleFrames,
+ `AudioProgramPlan clip ${clip.id} target interval is invalid`);
+ assert(Number.isSafeInteger(clip.sourceStartSample) && clip.sourceStartSample >= 0,
+ `AudioProgramPlan clip ${clip.id} source start is invalid`);
+ assert(Number.isSafeInteger(clip.sourceSampleFrames) && clip.sourceSampleFrames > 0
+ && Number.isSafeInteger(clip.sourceEndSampleExclusive)
+ && clip.sourceEndSampleExclusive > clip.sourceStartSample
+ && clip.sourceEndSampleExclusive <= clip.sourceSampleFrames,
+ `AudioProgramPlan clip ${clip.id} source interval is invalid`);
+ const sourceLength = clip.sourceEndSampleExclusive - clip.sourceStartSample;
+ assert(typeof clip.sourceLoop === "boolean"
+ && Number.isSafeInteger(clip.sourcePhaseSample) && clip.sourcePhaseSample >= 0
+ && clip.sourcePhaseSample < sourceLength
+ && (clip.sourceLoop || clip.sourcePhaseSample === 0),
+ `AudioProgramPlan clip ${clip.id} loop phase is invalid`);
+ assert(Number.isFinite(clip.playbackRate) && clip.playbackRate > 0,
+ `AudioProgramPlan clip ${clip.id} playback rate is invalid`);
+ assert(Number.isSafeInteger(clip.mixStartSample) && clip.mixStartSample >= 0
+ && Number.isSafeInteger(clip.mixEndSampleExclusive) && clip.mixEndSampleExclusive > clip.mixStartSample
+ && clip.targetStartSample >= clip.mixStartSample && clip.targetEndSampleExclusive <= clip.mixEndSampleExclusive,
+ `AudioProgramPlan clip ${clip.id} mix interval is invalid`);
+ assert(Number.isFinite(clip.gain) && clip.gain >= 0 && clip.gain <= 64,
+ `AudioProgramPlan clip ${clip.id} gain is invalid`);
+ assertAudioLevelAutomation(clip, { startSample: clip.targetStartSample, endSampleExclusive: clip.targetEndSampleExclusive }, item.sampleFrames);
+ const mixLength = clip.mixEndSampleExclusive - clip.mixStartSample;
+ assert(Number.isSafeInteger(clip.fadeInSamples) && clip.fadeInSamples >= 0 && clip.fadeInSamples <= mixLength
+ && Number.isSafeInteger(clip.fadeOutSamples) && clip.fadeOutSamples >= 0 && clip.fadeOutSamples <= mixLength,
+ `AudioProgramPlan clip ${clip.id} fade is invalid`);
+ }
+ assert(item.mix?.normalize === false && item.mix?.limiter === "none",
+ "AudioProgramPlan cannot hide normalization or limiting");
+}
+
+export function compileAudioProgramPlan(composition: Composition, timeline: Timeline): AudioProgramPlan {
+ assertCompositionIdentity(composition, timeline);
+ const frameCount = timelineFrameCount(timeline);
+ const clips = composition.tracks
+ .filter((track) => track.kind === "audio")
+ .flatMap((track) => track.clips.flatMap((clip) => {
+ if (clip.artifact.mediaType !== "audio/wav") {
+ throw new Error(`Audio clip ${track.id}.${clip.id} must be normalized to canonical WAV before mixing`);
+ }
+ return clip.sourceTime.pieces.map((piece, pieceIndex) => {
+ const targetLength = piece.target.endSampleExclusive - piece.target.startSample;
+ const sourceAtStart = exactInteger(piece.sourceAtStart,
+ `${track.id}.${clip.id} piece ${pieceIndex} source start`);
+ const wrap = piece.wrap;
+ return {
+ id: `${track.id}:${clip.id}:${String(pieceIndex).padStart(4, "0")}`,
+ artifact: {
+ kind: "blob" as const,
+ resource: clip.artifact.resource,
+ size: clip.artifact.size,
+ mediaType: clip.artifact.mediaType,
+ },
+ targetStartSample: clip.target.startSample + piece.target.startSample,
+ targetEndSampleExclusive: clip.target.startSample + piece.target.endSampleExclusive,
+ sourceSampleFrames: clip.sourceTime.sourceSampleFrames,
+ sourceStartSample: wrap?.startSample ?? sourceAtStart,
+ sourceEndSampleExclusive: wrap?.endSampleExclusive
+ ?? exactSourceEnd(piece.sourceAtStart, piece.rate, targetLength,
+ `${track.id}.${clip.id} piece ${pieceIndex} source end`),
+ sourceLoop: wrap !== undefined,
+ sourcePhaseSample: wrap === undefined ? 0 : sourceAtStart - wrap.startSample,
+ playbackRate: piece.rate.numerator / piece.rate.denominator,
+ mixStartSample: clip.target.startSample,
+ mixEndSampleExclusive: clip.target.endSampleExclusive,
+ gain: clip.gain,
+ fadeInSamples: clip.fadeInSamples,
+ fadeOutSamples: clip.fadeOutSamples,
+ ...(clip.gainEnvelope === undefined ? {} : { gainEnvelope: clip.gainEnvelope }),
+ ...(clip.audibility === undefined ? {} : { audibility: clip.audibility.flatMap((span) => {
+ const startSample = Math.max(span.startSample, clip.target.startSample + piece.target.startSample);
+ const endSampleExclusive = Math.min(span.endSampleExclusive, clip.target.startSample + piece.target.endSampleExclusive);
+ return endSampleExclusive <= startSample ? [] : [{ startSample, endSampleExclusive }];
+ }) }),
+ };
+ });
+ }));
+ return sealAudioProgramPlan({
+ frameRate: { ...timeline.frameRate },
+ frameCount,
+ sampleRate: 48_000,
+ sampleFrames: timelineFrameSampleBoundary(timeline, frameCount, 48_000),
+ clips,
+ mix: { normalize: false, limiter: "none" },
+ });
+}
diff --git a/packages/media-operations/src/component.ts b/packages/media-operations/src/component.ts
new file mode 100644
index 00000000..a8644c38
--- /dev/null
+++ b/packages/media-operations/src/component.ts
@@ -0,0 +1,364 @@
+import { mediaFrameRangeSamples, verifyMediaFrameRange } from "@hypit/hypit/media";
+import { mediaComponent } from "@hypit/hypit/media";
+import { plannedNeedInputs } from "@hypit/hypit/producer";
+import type { ProducerPackage, PlannedNeedFacet } from "@hypit/hypit/producer";
+import type { AdmissionPackage } from "@hypit/hypit/admission";
+import { verifyMediaInspection, verifyMediaStreamSelection, verifyMuxedMedia, verifyTimelineVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/hypit/media";
+import type { MediaInspection, MediaStreamSelection, MuxedMedia, TimelineVisual, TimelineAudio } from "@hypit/hypit/media";
+import { assertClockIdentity } from "@hypit/hypit/timeline";
+import type { Clock, Timeline } from "@hypit/hypit/timeline";
+import { durationInFrames } from "@hypit/hypit/temporal";
+import type { TemporalDuration } from "@hypit/hypit/temporal";
+import type { Composition } from "@hypit/hypit/composition";
+import type { BlobRef, CanonicalValue, CapabilityRef, ProducerRef, StoredValue } from "@hypit/hypit/protocol";
+import { canonicalize } from "@hypit/hypit/protocol";
+
+import {
+ compileAudioProgramPlan,
+ verifyAudioProgramPlan,
+} from "./audio-plan.js";
+import { mediaOperationsCapabilities, mediaOperationsProducers, mediaOperationsTypes } from "./manifest.js";
+import {
+ selectMediaStreams,
+ verifyMediaSelectionRequest,
+} from "./selection.js";
+import {
+ selectAudioStream,
+ selectVideoStream,
+ bindStillVideoSource,
+ planStillVideoSegments,
+ sealStillVideoRequest,
+ verifyStillVideoLayout,
+ verifyAudioExtractionRequest,
+ verifyFrameExtractionRequest,
+ verifyMediaTransformProgram,
+ verifyStillVideoRequest,
+} from "./operations.js";
+import type {
+ AudioExtractionRequest,
+ ExtractAudioNeed,
+ ExtractFrameNeed,
+ FrameExtractionRequest,
+ InspectMediaNeed,
+ MuxMediaNeed,
+ MediaSelectionRequest,
+ NormalizeMediaNeed,
+ RenderAudioNeed,
+ RenderStillVideoNeed,
+ StillVideoLayout,
+ StillVideoRequest,
+ TransformMediaNeed,
+} from "./types.js";
+
+function inline(value: StoredValue, subject: string): CanonicalValue {
+ if (value.kind !== "inline") throw new Error(`${subject} must be inline`);
+ return value.value;
+}
+
+function blob(value: StoredValue, subject: string): BlobRef {
+ if (value.kind !== "blob") throw new Error(`${subject} must be a Blob`);
+ return value;
+}
+
+/** Preserve unresolved graph inputs without pretending a structured value is a file. */
+function plannedMediaNeed(
+ producer: ProducerRef,
+ port: string,
+ capability: CapabilityRef,
+ roles: Readonly<Record<string, string>> = {},
+ present?: PlannedNeedFacet["present"],
+): PlannedNeedFacet {
+ return {
+ producer,
+ port,
+ capability,
+ plan({ state, step }) {
+ return { constraints: {}, pendingInputs: plannedNeedInputs(state, step, roles) };
+ },
+ present(specification) {
+ if (present !== undefined) return present(specification);
+ const references: Record<string, number> = {};
+ for (const item of specification.pendingInputs) {
+ const role = item.role;
+ if (role === undefined) continue;
+ references[role] = (references[role] ?? 0) + 1;
+ }
+ return { fields: {}, references };
+ },
+ };
+}
+
+const presentAudioRender: NonNullable<PlannedNeedFacet["present"]> = (specification) => {
+ const request = specification.constraints as Partial<RenderAudioNeed>;
+ if (request.plan === undefined) return { fields: {}, references: {} };
+ const { plan, range } = request;
+ return { fields: {
+ startFrame: [range?.startFrame ?? 0],
+ endFrameExclusive: [range?.endFrameExclusive ?? plan.frameCount],
+ frameRate: [`${plan.frameRate.numerator}/${plan.frameRate.denominator}`],
+ sampleRate: [plan.sampleRate],
+ }, references: {} };
+};
+
+export const mediaOperationsComponent = {
+ validators: [
+ {
+ type: mediaOperationsTypes.selectionRequest,
+ handler: ({ value }) => {
+ verifyMediaSelectionRequest(inline(value, "MediaSelectionRequest"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.audioProgramPlan,
+ handler: ({ value }) => {
+ verifyAudioProgramPlan(inline(value, "AudioProgramPlan"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.transformProgram,
+ handler: ({ value }) => {
+ verifyMediaTransformProgram(inline(value, "MediaTransformProgram"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.audioExtractionRequest,
+ handler: ({ value }) => {
+ verifyAudioExtractionRequest(inline(value, "AudioExtractionRequest"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.frameExtractionRequest,
+ handler: ({ value }) => {
+ verifyFrameExtractionRequest(inline(value, "FrameExtractionRequest"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.stillVideoLayout,
+ handler: ({ value }) => {
+ verifyStillVideoLayout(inline(value, "StillVideoLayout"));
+ },
+ },
+ {
+ type: mediaOperationsTypes.stillVideoRequest,
+ handler: ({ value }) => {
+ verifyStillVideoRequest(inline(value, "StillVideoRequest"));
+ },
+ },
+ ],
+ producers: [
+ {
+ producer: mediaOperationsProducers.inspect,
+ handler: ({ inputs }) => {
+ const source = blob(inputs.source!.value, "Media inspection source");
+ const need: InspectMediaNeed = { source };
+ return { outputs: {}, needs: { inspection: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.select,
+ handler: ({ inputs }) => {
+ const inspection = inline(inputs.inspection!.value, "MediaInspection");
+ const request = inline(inputs.request!.value, "MediaSelectionRequest");
+ verifyMediaInspection(inspection);
+ verifyMediaSelectionRequest(request);
+ return {
+ outputs: {
+ selection: { kind: "inline", value: canonicalize(selectMediaStreams(inspection, request)) },
+ },
+ needs: {},
+ };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.normalize,
+ handler: ({ inputs }) => {
+ const source = blob(inputs.source!.value, "Media normalization source");
+ const inspection = inline(inputs.inspection!.value, "MediaInspection");
+ const selection = inline(inputs.selection!.value, "MediaStreamSelection");
+ const request = inline(inputs.request!.value, "MediaSelectionRequest");
+ verifyMediaInspection(inspection);
+ verifyMediaStreamSelection(selection);
+ verifyMediaSelectionRequest(request);
+ const need: NormalizeMediaNeed = {
+ source,
+ inspection,
+ selection,
+ frameRate: request.frameRate,
+ audio: { sampleRate: 48_000, channels: 2, codec: "pcm_s16le", loudness: "preserve" },
+ };
+ return { outputs: {}, needs: { media: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.transform,
+ handler: ({ inputs }) => {
+ const media = inline(inputs.media!.value, "SynchronizedMedia");
+ const program = inline(inputs.program!.value, "MediaTransformProgram");
+ verifySynchronizedMedia(media);
+ verifyMediaTransformProgram(program);
+ if (media.visual === undefined) throw new Error("Media transform requires a visual stream");
+ const need: TransformMediaNeed = {
+ media,
+ program,
+ };
+ return { outputs: {}, needs: { video: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.extractAudio,
+ handler: ({ inputs }) => {
+ const source = blob(inputs.source!.value, "Audio extraction source");
+ const inspection = inline(inputs.inspection!.value, "MediaInspection");
+ const request = inline(inputs.request!.value, "AudioExtractionRequest") as unknown as AudioExtractionRequest;
+ verifyMediaInspection(inspection);
+ verifyAudioExtractionRequest(request);
+ const selected = selectAudioStream(inspection, request.audio);
+ const need: ExtractAudioNeed = {
+ source,
+ streamIndex: selected.index,
+ output: request.output,
+ };
+ return { outputs: {}, needs: { audio: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.extractFrame,
+ handler: ({ inputs }) => {
+ const source = blob(inputs.source!.value, "Frame extraction source");
+ const inspection = inline(inputs.inspection!.value, "MediaInspection");
+ const request = inline(inputs.request!.value, "FrameExtractionRequest") as unknown as FrameExtractionRequest;
+ verifyMediaInspection(inspection);
+ verifyFrameExtractionRequest(request);
+ const selected = selectVideoStream(inspection, request.video);
+ if (request.at.kind === "frame" && request.at.index >= selected.decodedUnitCount) {
+ throw new Error(`Frame ${request.at.index} is outside the selected stream (${selected.decodedUnitCount} frames)`);
+ }
+ const need: ExtractFrameNeed = {
+ source,
+ streamIndex: selected.index,
+ sourceFrameCount: selected.decodedUnitCount,
+ at: request.at,
+ output: request.output,
+ };
+ return { outputs: {}, needs: { image: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.planStill,
+ handler: ({ inputs }) => {
+ const duration = inline(inputs.duration!.value, "TemporalDuration") as unknown as TemporalDuration;
+ const clock = inline(inputs.clock!.value, "Clock") as unknown as Clock;
+ assertClockIdentity(clock);
+ const layout = inline(inputs.layout!.value, "StillVideoLayout") as unknown as StillVideoLayout;
+ verifyStillVideoLayout(layout);
+ const exactFrames = durationInFrames(duration, clock);
+ if (exactFrames.numerator <= 0n || exactFrames.denominator !== 1n
+ || exactFrames.numerator > BigInt(Number.MAX_SAFE_INTEGER)) {
+ throw new Error("Still video duration must resolve to a positive whole frame count");
+ }
+ const frames = Number(exactFrames.numerator);
+ const request = sealStillVideoRequest({
+ frameRate: clock.frameRate,
+ frameCount: frames,
+ output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: planStillVideoSegments(frames, layout.weights),
+ });
+ return { outputs: { request: { kind: "inline", value: canonicalize(request) } }, needs: {} };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.bindStill,
+ handler: ({ inputs }) => {
+ const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;
+ const source = blob(inputs.source!.value, "Still video source");
+ const bound = bindStillVideoSource(request, source);
+ return { outputs: { request: { kind: "inline", value: canonicalize(bound) } }, needs: {} };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.renderStill,
+ handler: ({ inputs }) => {
+ const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;
+ verifyStillVideoRequest(request);
+ if (request.segments.some((segment) => segment.source === undefined)) {
+ throw new Error("Still video has a segment without a picture");
+ }
+ const need: RenderStillVideoNeed = { request };
+ return { outputs: {}, needs: { video: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.planAudio,
+ handler: ({ inputs }) => {
+ const composition = inline(inputs.composition!.value, "Composition") as unknown as Composition;
+ const timeline = inline(inputs.timeline!.value, "Timeline") as unknown as Timeline;
+ return {
+ outputs: { plan: { kind: "inline", value: canonicalize(compileAudioProgramPlan(composition, timeline)) } },
+ needs: {},
+ };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.renderAudio,
+ handler: ({ inputs }) => {
+ const plan = inline(inputs.plan!.value, "AudioProgramPlan");
+ verifyAudioProgramPlan(plan);
+ const need: RenderAudioNeed = { plan };
+ return { outputs: {}, needs: { audio: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.renderAudioRange,
+ handler: ({ inputs }) => {
+ const plan = inline(inputs.plan!.value, "AudioProgramPlan");
+ verifyAudioProgramPlan(plan);
+ const range = inline(inputs.range!.value, "MediaFrameRange");
+ verifyMediaFrameRange(range, plan.frameCount);
+ const need: RenderAudioNeed = { plan, range };
+ return { outputs: {}, needs: { audio: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.mux,
+ handler: ({ inputs }) => {
+ const visual = inline(inputs.visual!.value, "TimelineVisual");
+ const audio = inline(inputs.audio!.value, "TimelineAudio");
+ verifyTimelineVisual(visual);
+ verifyTimelineAudio(audio);
+ const { sampleFrames } = mediaFrameRangeSamples(
+ { startFrame: 0, endFrameExclusive: visual.frameCount }, visual.frameRate,
+ );
+ if (audio.sampleFrames !== sampleFrames) {
+ throw new Error("Rendered visual and TimelineAudio have different presentation durations");
+ }
+ const need: MuxMediaNeed = { visual, audio };
+ return { outputs: {}, needs: { media: canonicalize(need) } };
+ },
+ },
+ {
+ producer: mediaOperationsProducers.projectMuxed,
+ handler: ({ inputs }) => {
+ const media = inline(inputs.media!.value, "MuxedMedia");
+ verifyMuxedMedia(media);
+ return {
+ outputs: { video: media.artifact },
+ needs: {},
+ };
+ },
+ },
+ ],
+ plannedNeeds: [
+ plannedMediaNeed(mediaOperationsProducers.inspect, "inspection", mediaOperationsCapabilities.inspect, { source: "media" }),
+ plannedMediaNeed(mediaOperationsProducers.normalize, "media", mediaOperationsCapabilities.normalize, { source: "media" }),
+ plannedMediaNeed(mediaOperationsProducers.transform, "video", mediaOperationsCapabilities.transform, { media: "video" }),
+ plannedMediaNeed(mediaOperationsProducers.extractAudio, "audio", mediaOperationsCapabilities.extractAudio, { source: "audio" }),
+ plannedMediaNeed(mediaOperationsProducers.extractFrame, "image", mediaOperationsCapabilities.extractFrame, { source: "video" }),
+ plannedMediaNeed(mediaOperationsProducers.renderStill, "video", mediaOperationsCapabilities.renderStill, { request: "image" }),
+ plannedMediaNeed(mediaOperationsProducers.renderAudio, "audio", mediaOperationsCapabilities.renderAudio, {}, presentAudioRender),
+ plannedMediaNeed(mediaOperationsProducers.renderAudioRange, "audio", mediaOperationsCapabilities.renderAudio, {}, presentAudioRender),
+ plannedMediaNeed(mediaOperationsProducers.mux, "media", mediaOperationsCapabilities.mux, { visual: "video", audio: "audio" }),
+ ],
+} satisfies ProducerPackage & AdmissionPackage;
+
+/** Convenience set: public media validators must accompany the operation Producers. */
+export const mediaOperationsComponents = [mediaComponent, mediaOperationsComponent] as const;
diff --git a/packages/media-operations/src/fragment.ts b/packages/media-operations/src/fragment.ts
new file mode 100644
index 00000000..9ba7feb1
--- /dev/null
+++ b/packages/media-operations/src/fragment.ts
@@ -0,0 +1,188 @@
+import { mediaProducers, mediaTypes } from "@hypit/hypit/media";
+import { temporalProducers, temporalTypes } from "@hypit/hypit/temporal";
+import { blobTypes } from "@hypit/hypit/blob";
+import { timelineTypes } from "@hypit/hypit/timeline";
+import { sealGraphFragment } from "@hypit/hypit/author";
+
+import {
+ mediaOperationsProducers,
+ mediaOperationsTypes,
+} from "./manifest.js";
+
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+
+export const synchronizedMediaFragment = sealGraphFragment({
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "request", type: mediaOperationsTypes.selectionRequest },
+ { name: "domain", type: mediaTypes.domainSpec },
+ ],
+ operations: [
+ {
+ id: "inspect",
+ producer: mediaOperationsProducers.inspect,
+ inputs: { source: input("source") },
+ result: { kind: "need", name: "inspection" },
+ },
+ {
+ id: "select",
+ producer: mediaOperationsProducers.select,
+ inputs: { inspection: operation("inspect"), request: input("request") },
+ result: { kind: "output", name: "selection" },
+ },
+ {
+ id: "normalize",
+ producer: mediaOperationsProducers.normalize,
+ inputs: {
+ source: input("source"),
+ inspection: operation("inspect"),
+ selection: operation("select"),
+ request: input("request"),
+ },
+ result: { kind: "need", name: "media" },
+ },
+ {
+ id: "domain",
+ producer: mediaProducers.localDomain,
+ inputs: { media: operation("normalize"), spec: input("domain") },
+ result: { kind: "output", name: "domain" },
+ },
+ {
+ id: "extent",
+ producer: temporalProducers.extentFromDomain,
+ inputs: { domain: operation("domain") },
+ result: { kind: "output", name: "extent" },
+ },
+ ],
+ exports: [{
+ name: "media",
+ type: mediaTypes.synchronized,
+ root: operation("normalize"),
+ }, {
+ name: "domain",
+ type: temporalTypes.localDomain,
+ root: operation("domain"),
+ }, {
+ name: "extent",
+ type: temporalTypes.extent,
+ root: operation("extent"),
+ }],
+});
+
+export const transformMediaFragment = sealGraphFragment({
+ inputs: [
+ { name: "media", type: mediaTypes.synchronized },
+ { name: "program", type: mediaOperationsTypes.transformProgram },
+ ],
+ operations: [
+ {
+ id: "transform",
+ producer: mediaOperationsProducers.transform,
+ inputs: { media: input("media"), program: input("program") },
+ result: { kind: "need", name: "video" },
+ },
+ ],
+ exports: [{
+ name: "video",
+ type: blobTypes.blob,
+ root: operation("transform"),
+ }],
+});
+
+export const extractAudioFragment = sealGraphFragment({
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "request", type: mediaOperationsTypes.audioExtractionRequest },
+ ],
+ operations: [
+ {
+ id: "inspect",
+ producer: mediaOperationsProducers.inspect,
+ inputs: { source: input("source") },
+ result: { kind: "need", name: "inspection" },
+ },
+ {
+ id: "extract",
+ producer: mediaOperationsProducers.extractAudio,
+ inputs: { source: input("source"), inspection: operation("inspect"), request: input("request") },
+ result: { kind: "need", name: "audio" },
+ },
+ ],
+ exports: [{
+ name: "audio",
+ type: blobTypes.blob,
+ root: operation("extract"),
+ }],
+});
+
+export const extractFrameFragment = sealGraphFragment({
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "request", type: mediaOperationsTypes.frameExtractionRequest },
+ ],
+ operations: [
+ {
+ id: "inspect",
+ producer: mediaOperationsProducers.inspect,
+ inputs: { source: input("source") },
+ result: { kind: "need", name: "inspection" },
+ },
+ {
+ id: "extract",
+ producer: mediaOperationsProducers.extractFrame,
+ inputs: { source: input("source"), inspection: operation("inspect"), request: input("request") },
+ result: { kind: "need", name: "image" },
+ },
+ ],
+ exports: [{
+ name: "image",
+ type: blobTypes.blob,
+ root: operation("extract"),
+ }],
+});
+
+/**
+ * A still video over `count` pictures: plan the frame split, bind each picture to its segment in
+ * authored order, render once. The shape depends only on the count, so two elements with the same
+ * number of pictures share one Fragment identity.
+ */
+export function createStillVideoFragment(count: number) {
+ if (!Number.isSafeInteger(count) || count < 1) throw new Error("A still video needs at least one picture");
+ const sources = Array.from({ length: count }, (_, index) => `source-${index}`);
+ return sealGraphFragment({
+ inputs: [
+ { name: "duration", type: temporalTypes.duration },
+ { name: "clock", type: timelineTypes.clock },
+ { name: "layout", type: mediaOperationsTypes.stillVideoLayout },
+ ...sources.map((name) => ({ name, type: blobTypes.blob })),
+ ],
+ operations: [
+ {
+ id: "plan",
+ producer: mediaOperationsProducers.planStill,
+ inputs: { duration: input("duration"), clock: input("clock"), layout: input("layout") },
+ result: { kind: "output", name: "request" },
+ },
+ ...sources.map((name, index) => ({
+ id: `bind-${index}`,
+ producer: mediaOperationsProducers.bindStill,
+ inputs: { request: operation(index === 0 ? "plan" : `bind-${index - 1}`), source: input(name) },
+ result: { kind: "output" as const, name: "request" },
+ })),
+ {
+ id: "render",
+ producer: mediaOperationsProducers.renderStill,
+ inputs: { request: operation(`bind-${count - 1}`) },
+ result: { kind: "need", name: "video" },
+ },
+ ],
+ exports: [{
+ name: "video",
+ type: blobTypes.blob,
+ root: operation("render"),
+ }],
+ });
+}
+
+export const stillVideoFragment = createStillVideoFragment(1);
diff --git a/packages/media-operations/src/index.ts b/packages/media-operations/src/index.ts
new file mode 100644
index 00000000..44bd3a0a
--- /dev/null
+++ b/packages/media-operations/src/index.ts
@@ -0,0 +1,46 @@
+export {
+ mediaOperationsComponent,
+ mediaOperationsComponents,
+} from "./component.js";
+export {
+ extractAudioFragment,
+ extractFrameFragment,
+ synchronizedMediaFragment,
+ stillVideoFragment,
+ createStillVideoFragment,
+ transformMediaFragment,
+} from "./fragment.js";
+export {
+ compileAudioProgramPlan,
+ sealAudioProgramPlan,
+ verifyAudioProgramPlan,
+} from "./audio-plan.js";
+export { mediaOperationsCapabilities, mediaOperationsManifest, mediaOperationsMarkupSurfaces, mediaOperationsModuleRef, mediaOperationsProducers, mediaOperationsTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, stillVideoLayoutSchema, mediaTransformProgramSchema } from "./manifest.js";
+export {
+ sealMediaSelectionRequest,
+ selectMediaStreams,
+ verifyMediaSelectionRequest,
+} from "./selection.js";
+export { decodeSynchronizedMediaSurface, decodeStillVideoSurface } from "./surface.js";
+export {
+ decodeExtractAudioSurface,
+ decodeExtractFrameSurface,
+ decodeTransformMediaSurface,
+} from "./surface.js";
+export {
+ sealAudioExtractionRequest,
+ sealFrameExtractionRequest,
+ sealMediaTransformProgram,
+ sealStillVideoLayout,
+ sealStillVideoRequest,
+ planStillVideoSegments,
+ bindStillVideoSource,
+ selectAudioStream,
+ selectVideoStream,
+ verifyAudioExtractionRequest,
+ verifyFrameExtractionRequest,
+ verifyMediaTransformProgram,
+ verifyStillVideoLayout,
+ verifyStillVideoRequest,
+} from "./operations.js";
+export type * from "./types.js";
diff --git a/packages/media-operations/src/manifest.ts b/packages/media-operations/src/manifest.ts
new file mode 100644
index 00000000..c201f256
--- /dev/null
+++ b/packages/media-operations/src/manifest.ts
@@ -0,0 +1,636 @@
+import { blobDependency } from "@hypit/hypit/blob";
+import { mediaDependency, mediaTypes } from "@hypit/hypit/media";
+import { timelineDependency, timelineTypes } from "@hypit/hypit/timeline";
+import { recipeModuleRef, recipeType } from "@hypit/hypit/recipe";
+import { compositionDependency, compositionTypes, audioSampleSpanSchema, audioGainEnvelopeSchema } from "@hypit/hypit/composition";
+import { temporalDependency, temporalTypes } from "@hypit/hypit/temporal";
+import { blobTypes } from "@hypit/hypit/blob";
+import type { CapabilityRef, ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/hypit/protocol";
+
+export const mediaOperationsModuleRef = { name: "@hypit/media-operations", version: "1" } as const;
+export const mediaOperationsTypes = {
+ selectionRequest: { module: mediaOperationsModuleRef, name: "MediaSelectionRequest" },
+ audioProgramPlan: { module: mediaOperationsModuleRef, name: "AudioProgramPlan" },
+ transformProgram: { module: mediaOperationsModuleRef, name: "MediaTransformProgram" },
+ audioExtractionRequest: { module: mediaOperationsModuleRef, name: "AudioExtractionRequest" },
+ frameExtractionRequest: { module: mediaOperationsModuleRef, name: "FrameExtractionRequest" },
+ stillVideoRequest: { module: mediaOperationsModuleRef, name: "StillVideoRequest" },
+ stillVideoLayout: { module: mediaOperationsModuleRef, name: "StillVideoLayout" },
+} satisfies Record<string, TypeRef>;
+export const mediaOperationsCapabilities = {
+ inspect: { module: mediaOperationsModuleRef, name: "inspect-media" },
+ normalize: { module: mediaOperationsModuleRef, name: "normalize-media" },
+ transform: { module: mediaOperationsModuleRef, name: "transform-media" },
+ extractAudio: { module: mediaOperationsModuleRef, name: "extract-media-audio" },
+ extractFrame: { module: mediaOperationsModuleRef, name: "extract-media-frame" },
+ renderStill: { module: mediaOperationsModuleRef, name: "render-still-video" },
+ renderAudio: { module: mediaOperationsModuleRef, name: "render-timeline-audio" },
+ mux: { module: mediaOperationsModuleRef, name: "mux-program-media" },
+} satisfies Record<string, CapabilityRef>;
+export const mediaOperationsProducers = {
+ inspect: { module: mediaOperationsModuleRef, name: "request-media-inspection" },
+ select: { module: mediaOperationsModuleRef, name: "select-media-streams" },
+ normalize: { module: mediaOperationsModuleRef, name: "request-media-normalization" },
+ transform: { module: mediaOperationsModuleRef, name: "request-media-transform" },
+ extractAudio: { module: mediaOperationsModuleRef, name: "request-audio-extraction" },
+ extractFrame: { module: mediaOperationsModuleRef, name: "request-frame-extraction" },
+ planStill: { module: mediaOperationsModuleRef, name: "plan-still-video" },
+ renderStill: { module: mediaOperationsModuleRef, name: "request-still-video" },
+ bindStill: { module: mediaOperationsModuleRef, name: "bind-still-video-source" },
+ planAudio: { module: mediaOperationsModuleRef, name: "compile-audio-program" },
+ renderAudio: { module: mediaOperationsModuleRef, name: "request-audio-render" },
+ renderAudioRange: { module: mediaOperationsModuleRef, name: "request-audio-range" },
+ mux: { module: mediaOperationsModuleRef, name: "request-media-mux" },
+ projectMuxed: { module: mediaOperationsModuleRef, name: "project-muxed-media" },
+} satisfies Record<string, ProducerRef>;
+
+const integer = { kind: "number", integer: true, minimum: 0 } as const;
+const mode = (name: string): ValueSchema => ({
+ kind: "object",
+ fields: { mode: { schema: { kind: "literal", value: name } } },
+});
+const streamMode: ValueSchema = {
+ kind: "object",
+ fields: {
+ mode: { schema: { kind: "literal", value: "stream-index" } },
+ streamIndex: { schema: integer },
+ },
+};
+export const mediaSelectionRequestSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ video: { schema: { kind: "oneOf", variants: [mode("primary-moving"), streamMode, mode("none")] } },
+ audio: { schema: { kind: "oneOf", variants: [mode("default"), streamMode, mode("none")] } },
+ spanAuthority: { schema: { kind: "string", enum: ["video", "audio"] } },
+ frameRate: { schema: {
+ kind: "object",
+ fields: {
+ numerator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ denominator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ },
+ } },
+ },
+};
+
+const videoSelectorSchema: ValueSchema = {
+ kind: "oneOf",
+ variants: [mode("primary-moving"), streamMode],
+};
+const audioSelectorSchema: ValueSchema = {
+ kind: "oneOf",
+ variants: [mode("default"), streamMode],
+};
+const nonNegativeNumber = { kind: "number", minimum: 0 } as const;
+const trimOperationSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ kind: { schema: { kind: "literal", value: "trim" } },
+ startSec: { schema: nonNegativeNumber, optional: true },
+ endSec: { schema: nonNegativeNumber, optional: true },
+ tailSec: { schema: nonNegativeNumber, optional: true },
+ },
+};
+const retimeOperationSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ kind: { schema: { kind: "literal", value: "retime" } },
+ rate: { schema: { kind: "number", minimum: 0.000001, maximum: 100 } },
+ },
+};
+export const mediaTransformProgramSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ operations: { schema: { kind: "array", minItems: 1, items: {
+ kind: "oneOf", variants: [trimOperationSchema, retimeOperationSchema],
+ } } },
+ },
+};
+export const audioExtractionRequestSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ audio: { schema: audioSelectorSchema },
+ output: { schema: {
+ kind: "object",
+ fields: {
+ container: { schema: { kind: "literal", value: "wav" } },
+ codec: { schema: { kind: "literal", value: "pcm_s16le" } },
+ sampleRate: { schema: { kind: "literal", value: 48_000 } },
+ channels: { schema: { kind: "literal", value: 2 } },
+ },
+ } },
+ },
+};
+export const frameExtractionRequestSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ video: { schema: videoSelectorSchema },
+ at: { schema: { kind: "oneOf", variants: [
+ { kind: "object", fields: { kind: { schema: { kind: "literal", value: "first" } } } },
+ { kind: "object", fields: { kind: { schema: { kind: "literal", value: "last" } } } },
+ { kind: "object", fields: {
+ kind: { schema: { kind: "literal", value: "frame" } },
+ index: { schema: integer },
+ } },
+ { kind: "object", fields: {
+ kind: { schema: { kind: "literal", value: "time" } },
+ seconds: { schema: nonNegativeNumber },
+ } },
+ ] } },
+ output: { schema: {
+ kind: "object",
+ fields: { format: { schema: { kind: "literal", value: "png" } } },
+ } },
+ },
+};
+
+export const stillVideoRequestSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ frameRate: { schema: {
+ kind: "object",
+ fields: {
+ numerator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ denominator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ },
+ } },
+ frameCount: { schema: { kind: "number", integer: true, minimum: 1 } },
+ output: { schema: {
+ kind: "object",
+ fields: {
+ container: { schema: { kind: "literal", value: "mp4" } },
+ codec: { schema: { kind: "literal", value: "h264" } },
+ pixelFormat: { schema: { kind: "literal", value: "yuv420p" } },
+ },
+ } },
+ segments: { schema: { kind: "array", items: {
+ kind: "object",
+ fields: {
+ startFrame: { schema: { kind: "number", integer: true, minimum: 0 } },
+ endFrameExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ source: { optional: true, schema: {
+ kind: "object",
+ fields: {
+ kind: { schema: { kind: "literal", value: "blob" } },
+ resource: { schema: { kind: "string", minLength: 5, maxLength: 256 } },
+ size: { schema: { kind: "number", integer: true, minimum: 0 } },
+ mediaType: { schema: { kind: "string", minLength: 6 } },
+ },
+ } },
+ },
+ } } },
+ },
+};
+
+export const stillVideoLayoutSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ weights: { schema: { kind: "array", items: { kind: "number", minimum: 0 } } },
+ },
+};
+
+const blobRefSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ kind: { schema: { kind: "literal", value: "blob" } },
+ resource: { schema: { kind: "string", minLength: 5, maxLength: 256 } },
+ size: { schema: { kind: "number", integer: true, minimum: 0 } },
+ mediaType: { schema: { kind: "literal", value: "audio/wav" } },
+ },
+};
+
+export const audioProgramPlanSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ frameRate: { schema: {
+ kind: "object",
+ fields: {
+ numerator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ denominator: { schema: { kind: "number", integer: true, minimum: 1 } },
+ },
+ } },
+ frameCount: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sampleRate: { schema: { kind: "literal", value: 48_000 } },
+ sampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } },
+ clips: { schema: { kind: "array", items: {
+ kind: "object",
+ fields: {
+ id: { schema: { kind: "string", minLength: 1 } },
+ artifact: { schema: blobRefSchema },
+ targetStartSample: { schema: integer },
+ targetEndSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sourceSampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sourceStartSample: { schema: integer },
+ sourceEndSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ sourceLoop: { schema: { kind: "boolean" } },
+ sourcePhaseSample: { schema: integer },
+ playbackRate: { schema: { kind: "number" } },
+ mixStartSample: { schema: integer },
+ mixEndSampleExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ gain: { schema: { kind: "number", minimum: 0, maximum: 64 } },
+ fadeInSamples: { schema: integer },
+ fadeOutSamples: { schema: integer },
+ gainEnvelope: { schema: audioGainEnvelopeSchema, optional: true },
+ audibility: { schema: { kind: "array", items: audioSampleSpanSchema }, optional: true },
+ },
+ } } },
+ mix: { schema: {
+ kind: "object",
+ fields: {
+ normalize: { schema: { kind: "literal", value: false } },
+ limiter: { schema: { kind: "literal", value: "none" } },
+ },
+ } },
+ },
+};
+
+export const mediaOperationsMarkupSurfaces = [
+ {
+ name: "synchronized-media", tag: "Normalize", mode: "structured",
+ outputs: [mediaOperationsTypes.selectionRequest, mediaTypes.domainSpec, mediaTypes.synchronized,
+ temporalTypes.localDomain, temporalTypes.extent],
+ vocabulary: {
+ summary:
+ "Inspects one Blob, selects its video and audio streams and normalizes them into SynchronizedMedia on one frame domain.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names the selection Record and the normalized media this element publishes." },
+ { name: "source", kind: "reference", required: true,
+ accepts: [blobTypes.blob],
+ summary: "Selects the media Artifact this element inspects and normalizes." },
+ { name: "recipe", kind: "reference", required: false, accepts: [recipeType],
+ summary: "Selects a Recipe containing video, audio and span-authority stream policy." },
+ { name: "video", kind: "literal", required: false,
+ summary: "Decides which moving-image stream is carried: `primary-moving`, `none`, or `stream:<index>`." },
+ { name: "audio", kind: "literal", required: false,
+ summary: "Decides which audio stream is carried: `default`, `none`, or `stream:<index>`." },
+ { name: "span-authority", kind: "literal", required: false,
+ values: ["video", "audio"],
+ summary: "Decides which selected stream defines the extent the other is trimmed or padded to." },
+ { name: "clock", kind: "reference", required: true, accepts: [timelineTypes.clock],
+ summary: "Selects the authored frame clock shared with the programme." },
+ ],
+ ports: [
+ { name: "media", type: mediaTypes.synchronized,
+ summary: "The normalized SynchronizedMedia, addressed as `<id>.media`." },
+ { name: "domain", type: temporalTypes.localDomain,
+ summary: "Its source-local frame domain, addressed as `<id>.domain`." },
+ { name: "extent", type: temporalTypes.extent,
+ summary: "Its unpositioned normalized length, addressed as `<id>.extent`." },
+ ],
+ example: `<mediaop:Normalize id="music-media" source={music}
+ recipe={recipes.media.audio} clock={clock}/>`,
+ notes: [
+ "Write exactly one of recipe or the direct video/audio/span-authority attributes. Clock is always an explicit reference.",
+ "`primary-moving` excludes attached-picture streams, prefers one declared default and fails closed on an ambiguous container; `stream:<index>` is for a container the author genuinely knows.",
+ "Selecting embedded audio is a media fact only and makes no speaker or alignment claim.",
+ ],
+ },
+ },
+ {
+ name: "still-video", tag: "StillVideo", mode: "structured",
+ outputs: [temporalTypes.duration, mediaOperationsTypes.stillVideoLayout, mediaOperationsTypes.stillVideoRequest, blobTypes.blob],
+ vocabulary: {
+ summary: "Spreads one or more authored images over an explicit duration as an ordinary video-only MP4 Blob on a frame clock.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names the still-video operation and the MP4 Artifact it publishes." },
+ { name: "source", kind: "reference", required: false, accepts: [blobTypes.blob],
+ summary: "Selects the one authored image held for the full video; write Still children instead for several images." },
+ { name: "duration", kind: "literal", required: true,
+ summary: "Sets the video's exact temporal length, such as 6s or 150f; it must resolve to a whole frame on the selected clock." },
+ { name: "clock", kind: "reference", required: true, accepts: [timelineTypes.clock],
+ summary: "Selects the frame clock used by the generated MP4." },
+ ],
+ children: [
+ { tag: "Still", cardinality: "many",
+ summary: "One image in authored order; the duration is divided among the Still children by weight, equally unless weights say otherwise.",
+ attributes: [
+ { name: "source", kind: "reference", required: true, accepts: [blobTypes.blob],
+ summary: "Selects the authored image whose first decoded frame is held for this share." },
+ { name: "weight", kind: "literal", required: false,
+ summary: "Sets this image's share of the duration relative to its siblings, such as 2; every Still weighs 1 unless written." },
+ ] },
+ ],
+ ports: [{ name: "video", type: blobTypes.blob,
+ summary: "The ordinary video-only MP4 Artifact, addressed as `<id>.video`." }],
+ example: `<mediaop:StillVideo id="kitchen-stills" duration="6s" clock={clock}>
+ <mediaop:Still source={counter}/>
+ <mediaop:Still source={basil} weight="2"/>
+ <mediaop:Still source={board}/>
+</mediaop:StillVideo>`,
+ notes: [
+ "Write exactly one of source or Still children. Frames are whole: each image gets the floor of its share and the leftover frames go to the largest remainders, so the split is deterministic and every image holds at least one frame.",
+ "Images of different sizes are fitted into the first image's frame, letterboxed on black.",
+ "The result is a normal video Blob, not SynchronizedMedia or semantic evidence.",
+ "Use Normalize afterward exactly as for generated or imported moving video.",
+ "Encoding is a render-still-video Need fulfilled by the selected media Provider; this Surface never invokes FFmpeg itself.",
+ ],
+ },
+ },
+ {
+ name: "transform-media", tag: "Transform", mode: "structured",
+ outputs: [mediaOperationsTypes.transformProgram, blobTypes.blob],
+ vocabulary: {
+ summary:
+ "Runs an ordered trim and retime program over one already normalized SynchronizedMedia value.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names the transform program Record and the transformed video this element publishes." },
+ { name: "source", kind: "reference", required: true,
+ accepts: [mediaTypes.synchronized],
+ summary: "Selects the prepared SynchronizedMedia this element transforms." },
+ ],
+ children: [
+ { tag: "Trim", cardinality: "many",
+ summary: "Removes time from the head or the tail, taking at least one of `start`, `end` or `tail` in seconds.",
+ attributes: [
+ { name: "start", kind: "literal", required: false,
+ summary: "Decides where the kept span begins, as seconds from the current start, such as `0.25s` or `2`." },
+ { name: "end", kind: "literal", required: false,
+ summary: "Decides where the kept span ends, as positive seconds from the current start." },
+ { name: "tail", kind: "literal", required: false,
+ summary: "Decides how many seconds are removed from the current end." },
+ ] },
+ { tag: "Retime", cardinality: "many",
+ summary: "Changes tempo by `rate` in the range (0, 100] while preserving pitch.",
+ attributes: [
+ { name: "rate", kind: "literal", required: true,
+ summary: "Decides the playback speed multiplier, above `0` and at most `100`." },
+ ] },
+ ],
+ ports: [
+ { name: "video", type: blobTypes.blob,
+ summary: "The transformed media Artifact, addressed as `<id>.video`." },
+ ],
+ example: `<mediaop:Transform id="prepared" source={shot-media.media}>
+ <mediaop:Trim tail="0.25s"/>
+ <mediaop:Retime rate="1.05"/>
+</mediaop:Transform>`,
+ notes: [
+ "At least one `Trim` or `Retime` child is required, and children run in the order they are written.",
+ "`Trim` cannot combine `end` and `tail`, and both children are written empty.",
+ "Normalization and stream selection are explicit upstream operations rather than hidden Transform policy.",
+ ],
+ },
+ },
+ {
+ name: "extract-audio", tag: "ExtractAudio", mode: "structured",
+ outputs: [mediaOperationsTypes.audioExtractionRequest, blobTypes.blob],
+ vocabulary: {
+ summary:
+ "Extracts one audio stream from a Blob as a deterministic 48 kHz stereo PCM WAV Artifact.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names the extraction request Record and the extracted audio this element publishes." },
+ { name: "source", kind: "reference", required: true,
+ accepts: [blobTypes.blob],
+ summary: "Selects the media Artifact this element extracts audio from." },
+ { name: "audio", kind: "literal", required: true,
+ summary: "Decides which audio stream is extracted: `default` or `stream:<index>`." },
+ ],
+ ports: [
+ { name: "audio", type: blobTypes.blob,
+ summary: "The extracted WAV Artifact, addressed as `<id>.audio`." },
+ ],
+ example: '<mediaop:ExtractAudio id="voice-reference" source={prepared.video} audio="default"/>',
+ notes: [
+ "The element accepts no children and no text content, and the output container, codec, sample rate and channel count are fixed.",
+ "The result makes no speaker or alignment claim, so it can feed a model reference port directly.",
+ ],
+ },
+ },
+ {
+ name: "extract-frame", tag: "ExtractFrame", mode: "structured",
+ outputs: [mediaOperationsTypes.frameExtractionRequest, blobTypes.blob],
+ vocabulary: {
+ summary: "Extracts one still frame from a Blob as a PNG Artifact.",
+ attributes: [
+ { name: "id", kind: "identifier", required: true,
+ summary: "Names the extraction request Record and the extracted image this element publishes." },
+ { name: "source", kind: "reference", required: true,
+ accepts: [blobTypes.blob],
+ summary: "Selects the media Artifact this element extracts a frame from." },
+ { name: "video", kind: "literal", required: true,
+ summary: "Decides which moving-image stream the frame is taken from: `primary-moving` or `stream:<index>`." },
+ { name: "at", kind: "literal", required: true,
+ summary: "Decides which frame is taken: `first`, `last`, `frame:<index>` or `time:<seconds>`." },
+ ],
+ ports: [
+ { name: "image", type: blobTypes.blob,
+ summary: "The extracted PNG Artifact, addressed as `<id>.image`." },
+ ],
+ example: '<mediaop:ExtractFrame id="continuity" source={prepared.video} video="primary-moving" at="last"/>',
+ notes: [
+ "The element accepts no children and no text content, and the output format is fixed to PNG.",
+ "`time:<seconds>` is written as a non-negative number with an optional `s`, such as `time:0.25s`.",
+ ],
+ },
+ },
+ ] as const;
+
+
+export const mediaOperationsManifest: ModuleManifest = {
+ format: "hypit.module@1",
+ name: mediaOperationsModuleRef.name,
+ version: mediaOperationsModuleRef.version,
+ dependencies: [
+ blobDependency,
+ mediaDependency,
+ timelineDependency,
+ compositionDependency,
+ temporalDependency,
+ { module: recipeModuleRef },
+ ],
+ types: [
+ {
+ name: mediaOperationsTypes.selectionRequest.name,
+ },
+ {
+ name: mediaOperationsTypes.audioProgramPlan.name,
+ },
+ {
+ name: mediaOperationsTypes.transformProgram.name,
+ },
+ {
+ name: mediaOperationsTypes.audioExtractionRequest.name,
+ },
+ {
+ name: mediaOperationsTypes.frameExtractionRequest.name,
+ },
+ {
+ name: mediaOperationsTypes.stillVideoRequest.name,
+ },
+ {
+ name: mediaOperationsTypes.stillVideoLayout.name,
+ },
+ ],
+ capabilities: [
+ { name: mediaOperationsCapabilities.inspect.name, returns: mediaTypes.inspection },
+ { name: mediaOperationsCapabilities.normalize.name, returns: mediaTypes.synchronized },
+ { name: mediaOperationsCapabilities.transform.name, returns: blobTypes.blob },
+ { name: mediaOperationsCapabilities.extractAudio.name, returns: blobTypes.blob },
+ { name: mediaOperationsCapabilities.extractFrame.name, returns: blobTypes.blob },
+ { name: mediaOperationsCapabilities.renderStill.name, returns: blobTypes.blob },
+ { name: mediaOperationsCapabilities.renderAudio.name, returns: mediaTypes.timelineAudio },
+ { name: mediaOperationsCapabilities.mux.name, returns: mediaTypes.muxed },
+ ],
+ producers: [
+ {
+ name: mediaOperationsProducers.inspect.name,
+ inputs: [{ name: "source", type: blobTypes.blob }],
+ outputs: [],
+ needs: [{
+ name: "inspection",
+ capability: mediaOperationsCapabilities.inspect,
+ returns: mediaTypes.inspection,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.select.name,
+ inputs: [
+ { name: "inspection", type: mediaTypes.inspection },
+ { name: "request", type: mediaOperationsTypes.selectionRequest },
+ ],
+ outputs: [{
+ name: "selection",
+ type: mediaTypes.streamSelection,
+ }],
+ needs: [],
+ },
+ {
+ name: mediaOperationsProducers.normalize.name,
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "inspection", type: mediaTypes.inspection },
+ { name: "selection", type: mediaTypes.streamSelection },
+ { name: "request", type: mediaOperationsTypes.selectionRequest },
+ ],
+ outputs: [],
+ needs: [{
+ name: "media",
+ capability: mediaOperationsCapabilities.normalize,
+ returns: mediaTypes.synchronized,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.transform.name,
+ inputs: [
+ { name: "media", type: mediaTypes.synchronized },
+ { name: "program", type: mediaOperationsTypes.transformProgram },
+ ],
+ outputs: [],
+ needs: [{
+ name: "video",
+ capability: mediaOperationsCapabilities.transform,
+ returns: blobTypes.blob,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.extractAudio.name,
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "inspection", type: mediaTypes.inspection },
+ { name: "request", type: mediaOperationsTypes.audioExtractionRequest },
+ ],
+ outputs: [],
+ needs: [{
+ name: "audio",
+ capability: mediaOperationsCapabilities.extractAudio,
+ returns: blobTypes.blob,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.extractFrame.name,
+ inputs: [
+ { name: "source", type: blobTypes.blob },
+ { name: "inspection", type: mediaTypes.inspection },
+ { name: "request", type: mediaOperationsTypes.frameExtractionRequest },
+ ],
+ outputs: [],
+ needs: [{
+ name: "image",
+ capability: mediaOperationsCapabilities.extractFrame,
+ returns: blobTypes.blob,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.planStill.name,
+ inputs: [
+ { name: "duration", type: temporalTypes.duration },
+ { name: "clock", type: timelineTypes.clock },
+ { name: "layout", type: mediaOperationsTypes.stillVideoLayout },
+ ],
+ outputs: [{ name: "request", type: mediaOperationsTypes.stillVideoRequest }],
+ needs: [],
+ },
+ {
+ name: mediaOperationsProducers.bindStill.name,
+ inputs: [
+ { name: "request", type: mediaOperationsTypes.stillVideoRequest },
+ { name: "source", type: blobTypes.blob },
+ ],
+ outputs: [{ name: "request", type: mediaOperationsTypes.stillVideoRequest }],
+ needs: [],
+ },
+ {
+ name: mediaOperationsProducers.renderStill.name,
+ inputs: [
+ { name: "request", type: mediaOperationsTypes.stillVideoRequest },
+ ],
+ outputs: [],
+ needs: [{
+ name: "video",
+ capability: mediaOperationsCapabilities.renderStill,
+ returns: blobTypes.blob,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.planAudio.name,
+ inputs: [
+ { name: "composition", type: compositionTypes.composition },
+ { name: "timeline", type: timelineTypes.timeline },
+ ],
+ outputs: [{ name: "plan", type: mediaOperationsTypes.audioProgramPlan }],
+ needs: [],
+ },
+ {
+ name: mediaOperationsProducers.renderAudio.name,
+ inputs: [{ name: "plan", type: mediaOperationsTypes.audioProgramPlan }],
+ outputs: [],
+ needs: [{
+ name: "audio",
+ capability: mediaOperationsCapabilities.renderAudio,
+ returns: mediaTypes.timelineAudio,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.renderAudioRange.name,
+ inputs: [{ name: "plan", type: mediaOperationsTypes.audioProgramPlan }, { name: "range", type: mediaTypes.frameRange }],
+ outputs: [],
+ needs: [{
+ name: "audio",
+ capability: mediaOperationsCapabilities.renderAudio,
+ returns: mediaTypes.timelineAudio,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.mux.name,
+ inputs: [
+ { name: "visual", type: mediaTypes.timelineVisual },
+ { name: "audio", type: mediaTypes.timelineAudio },
+ ],
+ outputs: [],
+ needs: [{
+ name: "media",
+ capability: mediaOperationsCapabilities.mux,
+ returns: mediaTypes.muxed,
+ }],
+ },
+ {
+ name: mediaOperationsProducers.projectMuxed.name,
+ inputs: [{ name: "media", type: mediaTypes.muxed }],
+ outputs: [{ name: "video", type: blobTypes.blob }],
+ needs: [],
+ },
+ ],
+};
diff --git a/packages/media-operations/src/operations.ts b/packages/media-operations/src/operations.ts
new file mode 100644
index 00000000..7842b7b6
--- /dev/null
+++ b/packages/media-operations/src/operations.ts
@@ -0,0 +1,247 @@
+import { verifyMediaInspection } from "@hypit/hypit/media";
+import type { MediaAudioStream, MediaInspection, MediaVideoStream } from "@hypit/hypit/media";
+import { canonicalize } from "@hypit/hypit/protocol";
+import type { BlobRef } from "@hypit/hypit/protocol";
+
+import type {
+ AudioExtractionRequest,
+ FrameExtractionRequest,
+ MediaAudioSelector,
+ MediaTransformProgram,
+ MediaVideoSelector,
+ StillVideoLayout,
+ StillVideoRequest,
+ StillVideoSegment,
+} from "./types.js";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function finiteNonNegative(value: unknown, subject: string): number {
+ assert(typeof value === "number" && Number.isFinite(value) && value >= 0,
+ `${subject} must be a finite non-negative number`);
+ return value;
+}
+
+function streamIndex(value: unknown, subject: string): number {
+ assert(Number.isSafeInteger(value) && (value as number) >= 0, `${subject} must be a non-negative integer`);
+ return value as number;
+}
+
+function object(value: unknown, subject: string): Record<string, unknown> {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);
+ return value as Record<string, unknown>;
+}
+
+export function verifyMediaTransformProgram(value: unknown): asserts value is MediaTransformProgram {
+ const program = object(value, "MediaTransformProgram");
+ assert(Array.isArray(program.operations) && program.operations.length > 0,
+ "MediaTransformProgram requires at least one operation");
+ for (const [index, raw] of program.operations.entries()) {
+ const operation = object(raw, `MediaTransformProgram operation ${index}`);
+ if (operation.kind === "trim") {
+ const start = operation.startSec;
+ const end = operation.endSec;
+ const tail = operation.tailSec;
+ assert(start !== undefined || end !== undefined || tail !== undefined,
+ `MediaTransformProgram trim ${index} must specify startSec, endSec or tailSec`);
+ if (start !== undefined) finiteNonNegative(start, `MediaTransformProgram trim ${index}.startSec`);
+ if (end !== undefined) finiteNonNegative(end, `MediaTransformProgram trim ${index}.endSec`);
+ if (tail !== undefined) finiteNonNegative(tail, `MediaTransformProgram trim ${index}.tailSec`);
+ assert(end === undefined || tail === undefined,
+ `MediaTransformProgram trim ${index} cannot specify both endSec and tailSec`);
+ if (start !== undefined && end !== undefined) {
+ assert((end as number) > (start as number),
+ `MediaTransformProgram trim ${index}.endSec must be after startSec`);
+ }
+ continue;
+ }
+ assert(operation.kind === "retime", `MediaTransformProgram operation ${index} kind is invalid`);
+ assert(typeof operation.rate === "number" && Number.isFinite(operation.rate) && operation.rate > 0
+ && operation.rate <= 100,
+ `MediaTransformProgram retime ${index}.rate must be in (0, 100]`);
+ }
+}
+
+export function sealMediaTransformProgram(value: MediaTransformProgram): MediaTransformProgram {
+ const result = canonicalize(value) as unknown as MediaTransformProgram;
+ verifyMediaTransformProgram(result);
+ return result;
+}
+
+function verifyVideoSelector(value: unknown, subject: string): asserts value is MediaVideoSelector {
+ const selector = object(value, subject);
+ assert(selector.mode === "primary-moving" || selector.mode === "stream-index", `${subject}.mode is invalid`);
+ if (selector.mode === "stream-index") streamIndex(selector.streamIndex, `${subject}.streamIndex`);
+}
+
+function verifyAudioSelector(value: unknown, subject: string, allowNone: boolean): asserts value is MediaAudioSelector {
+ const selector = object(value, subject);
+ assert(selector.mode === "default" || selector.mode === "stream-index" || (allowNone && selector.mode === "none"),
+ `${subject}.mode is invalid`);
+ if (selector.mode === "stream-index") streamIndex(selector.streamIndex, `${subject}.streamIndex`);
+}
+
+export function verifyAudioExtractionRequest(value: unknown): asserts value is AudioExtractionRequest {
+ const request = object(value, "AudioExtractionRequest");
+ verifyAudioSelector(request.audio, "AudioExtractionRequest.audio", false);
+ const output = object(request.output, "AudioExtractionRequest.output");
+ assert(output.container === "wav" && output.codec === "pcm_s16le"
+ && output.sampleRate === 48_000 && output.channels === 2,
+ "AudioExtractionRequest output profile is unsupported");
+}
+
+export function sealAudioExtractionRequest(value: AudioExtractionRequest): AudioExtractionRequest {
+ const result = canonicalize(value) as unknown as AudioExtractionRequest;
+ verifyAudioExtractionRequest(result);
+ return result;
+}
+
+export function verifyFrameExtractionRequest(value: unknown): asserts value is FrameExtractionRequest {
+ const request = object(value, "FrameExtractionRequest");
+ verifyVideoSelector(request.video, "FrameExtractionRequest.video");
+ const at = object(request.at, "FrameExtractionRequest.at");
+ assert(at.kind === "first" || at.kind === "last" || at.kind === "frame" || at.kind === "time",
+ "FrameExtractionRequest.at.kind is invalid");
+ if (at.kind === "frame") streamIndex(at.index, "FrameExtractionRequest.at.index");
+ if (at.kind === "time") finiteNonNegative(at.seconds, "FrameExtractionRequest.at.seconds");
+ const output = object(request.output, "FrameExtractionRequest.output");
+ assert(output.format === "png", "FrameExtractionRequest output format is unsupported");
+}
+
+export function sealFrameExtractionRequest(value: FrameExtractionRequest): FrameExtractionRequest {
+ const result = canonicalize(value) as unknown as FrameExtractionRequest;
+ verifyFrameExtractionRequest(result);
+ return result;
+}
+
+export function verifyStillVideoLayout(value: unknown): asserts value is StillVideoLayout {
+ const layout = object(value, "StillVideoLayout");
+ assert(Array.isArray(layout.weights) && layout.weights.length > 0, "StillVideoLayout needs at least one weight");
+ for (const weight of layout.weights) {
+ assert(typeof weight === "number" && Number.isFinite(weight) && weight > 0, "StillVideoLayout weights must be positive");
+ }
+}
+
+export function sealStillVideoLayout(value: StillVideoLayout): StillVideoLayout {
+ const result = canonicalize(value) as unknown as StillVideoLayout;
+ verifyStillVideoLayout(result);
+ return result;
+}
+
+/**
+ * Spread `frameCount` whole frames over the weights. Every picture first holds one frame; the rest
+ * are shared by weight, each picture taking the floor of its share and the frames left over going
+ * one each to the largest fractional remainders, earlier picture first on a tie. Deterministic, and
+ * refused when there are fewer frames than pictures.
+ */
+export function planStillVideoSegments(frameCount: number, weights: readonly number[]): readonly StillVideoSegment[] {
+ assert(Number.isSafeInteger(frameCount) && frameCount > 0, "Still video frame count must be positive");
+ assert(weights.length > 0, "Still video needs at least one picture");
+ assert(frameCount >= weights.length, `Still video has ${frameCount} frames for ${weights.length} pictures; each picture needs at least one`);
+ const total = weights.reduce((sum, weight) => sum + weight, 0);
+ const shared = frameCount - weights.length;
+ const exact = weights.map((weight) => shared * weight / total);
+ const floors = exact.map((share) => 1 + Math.floor(share));
+ let remaining = frameCount - floors.reduce((sum, item) => sum + item, 0);
+ const order = exact.map((share, index) => ({ index, fraction: share - Math.floor(share) }))
+ .sort((left, right) => right.fraction - left.fraction || left.index - right.index);
+ for (const { index } of order) {
+ if (remaining === 0) break;
+ floors[index] = floors[index]! + 1;
+ remaining -= 1;
+ }
+ const segments: StillVideoSegment[] = [];
+ let start = 0;
+ for (const frames of floors) {
+ segments.push({ startFrame: start, endFrameExclusive: start + frames });
+ start += frames;
+ }
+ return segments;
+}
+
+export function verifyStillVideoRequest(value: unknown): asserts value is StillVideoRequest {
+ const request = object(value, "StillVideoRequest");
+ const frameRate = object(request.frameRate, "StillVideoRequest.frameRate");
+ assert(Number.isSafeInteger(frameRate.numerator) && (frameRate.numerator as number) > 0
+ && Number.isSafeInteger(frameRate.denominator) && (frameRate.denominator as number) > 0,
+ "StillVideoRequest.frameRate must be a positive rational");
+ assert(Number.isSafeInteger(request.frameCount) && (request.frameCount as number) > 0,
+ "StillVideoRequest.frameCount must be positive");
+ const output = object(request.output, "StillVideoRequest.output");
+ assert(output.container === "mp4" && output.codec === "h264" && output.pixelFormat === "yuv420p",
+ "StillVideoRequest output profile is unsupported");
+ assert(Array.isArray(request.segments) && request.segments.length > 0, "StillVideoRequest needs at least one segment");
+ let cursor = 0;
+ for (const item of request.segments) {
+ const segment = object(item, "StillVideoRequest.segments[]");
+ assert(segment.startFrame === cursor, "StillVideoRequest segments must be contiguous from frame 0");
+ assert(Number.isSafeInteger(segment.endFrameExclusive) && (segment.endFrameExclusive as number) > cursor,
+ "StillVideoRequest segments must each hold at least one frame");
+ cursor = segment.endFrameExclusive as number;
+ if (segment.source !== undefined) {
+ const source = object(segment.source, "StillVideoRequest.segments[].source");
+ assert(source.kind === "blob" && typeof source.mediaType === "string" && source.mediaType.startsWith("image/"),
+ "StillVideoRequest segment source must be an image Artifact");
+ }
+ }
+ assert(cursor === request.frameCount, "StillVideoRequest segments must cover exactly frameCount frames");
+}
+
+export function sealStillVideoRequest(value: StillVideoRequest): StillVideoRequest {
+ const result = canonicalize(value) as unknown as StillVideoRequest;
+ verifyStillVideoRequest(result);
+ return result;
+}
+
+/** Attach one picture to the first segment that has none; the pictures arrive in authored order. */
+export function bindStillVideoSource(request: StillVideoRequest, source: BlobRef): StillVideoRequest {
+ verifyStillVideoRequest(request);
+ assert(source.kind === "blob" && source.mediaType.startsWith("image/"), "Still video source must be an image Artifact");
+ const index = request.segments.findIndex((segment) => segment.source === undefined);
+ assert(index >= 0, "Still video already has a picture for every segment");
+ return sealStillVideoRequest({
+ ...request,
+ segments: request.segments.map((segment, at) => at === index ? { ...segment, source } : segment),
+ });
+}
+
+function uniqueDefault<T extends { readonly disposition: { readonly default: boolean }; readonly index: number }>(
+ values: readonly T[],
+ subject: string,
+): T {
+ assert(values.length > 0, `${subject} has no eligible stream`);
+ const defaults = values.filter((item) => item.disposition.default);
+ if (defaults.length === 1) return defaults[0]!;
+ assert(defaults.length === 0, `${subject} has multiple default streams: ${defaults.map((item) => item.index).join(", ")}`);
+ assert(values.length === 1, `${subject} is ambiguous: ${values.map((item) => item.index).join(", ")}`);
+ return values[0]!;
+}
+
+export function selectVideoStream(inspection: MediaInspection, selector: MediaVideoSelector): MediaVideoStream {
+ verifyMediaInspection(inspection);
+ verifyVideoSelector(selector, "Video selector");
+ const moving = inspection.streams.filter((item): item is MediaVideoStream =>
+ item.kind === "video" && item.role === "moving" && !item.disposition.attachedPicture
+ && item.timingStatus === "admissible" && item.startPts !== undefined && item.endPts !== undefined);
+ if (selector.mode === "primary-moving") return uniqueDefault(moving, "Primary moving video");
+ const result = moving.find((item) => item.index === selector.streamIndex);
+ assert(result !== undefined, `Video stream ${selector.streamIndex} is not an eligible moving stream`);
+ return result;
+}
+
+export function selectAudioStream(
+ inspection: MediaInspection,
+ selector: Exclude<MediaAudioSelector, { readonly mode: "none" }>,
+): MediaAudioStream {
+ verifyMediaInspection(inspection);
+ verifyAudioSelector(selector, "Audio selector", false);
+ const audio = inspection.streams.filter((item): item is MediaAudioStream =>
+ item.kind === "audio" && item.timingStatus === "admissible"
+ && item.startPts !== undefined && item.endPts !== undefined);
+ if (selector.mode === "default") return uniqueDefault(audio, "Default audio");
+ const result = audio.find((item) => item.index === selector.streamIndex);
+ assert(result !== undefined, `Audio stream ${selector.streamIndex} is not eligible`);
+ return result;
+}
diff --git a/packages/media-operations/src/selection.ts b/packages/media-operations/src/selection.ts
new file mode 100644
index 00000000..bdd51714
--- /dev/null
+++ b/packages/media-operations/src/selection.ts
@@ -0,0 +1,108 @@
+import { sealMediaStreamSelection, verifyMediaInspection } from "@hypit/hypit/media";
+import type { MediaAudioStream, MediaInspection, MediaStreamSelection, MediaVideoStream } from "@hypit/hypit/media";
+import { canonicalize } from "@hypit/hypit/protocol";
+
+import type { MediaSelectionRequest } from "./types.js";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+export function sealMediaSelectionRequest(value: MediaSelectionRequest): MediaSelectionRequest {
+ return canonicalize(value) as unknown as MediaSelectionRequest;
+}
+
+export function verifyMediaSelectionRequest(value: unknown): asserts value is MediaSelectionRequest {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "MediaSelectionRequest must be an object");
+ const item = value as MediaSelectionRequest;
+ assert(item.video?.mode === "primary-moving" || item.video?.mode === "stream-index" || item.video?.mode === "none",
+ "MediaSelectionRequest video mode is invalid");
+ assert(item.audio?.mode === "default" || item.audio?.mode === "stream-index" || item.audio?.mode === "none",
+ "MediaSelectionRequest audio mode is invalid");
+ if (item.video.mode === "stream-index") {
+ assert(Number.isSafeInteger(item.video.streamIndex) && item.video.streamIndex >= 0,
+ "MediaSelectionRequest video stream index is invalid");
+ }
+ if (item.audio.mode === "stream-index") {
+ assert(Number.isSafeInteger(item.audio.streamIndex) && item.audio.streamIndex >= 0,
+ "MediaSelectionRequest audio stream index is invalid");
+ }
+ assert(item.spanAuthority === "video" || item.spanAuthority === "audio", "MediaSelectionRequest authority is invalid");
+ assert(item.spanAuthority === "video" ? item.video.mode !== "none" : item.audio.mode !== "none",
+ "MediaSelectionRequest authority stream is disabled");
+ assert(Number.isSafeInteger(item.frameRate?.numerator) && item.frameRate.numerator > 0
+ && Number.isSafeInteger(item.frameRate?.denominator) && item.frameRate.denominator > 0,
+ "MediaSelectionRequest frame rate is invalid");
+}
+
+function selectUniqueDefault<T extends { readonly disposition: { readonly default: boolean }; readonly index: number }>(
+ values: readonly T[],
+ subject: string,
+): T {
+ assert(values.length > 0, `${subject} has no eligible stream`);
+ const defaults = values.filter((item) => item.disposition.default);
+ if (defaults.length === 1) return defaults[0]!;
+ if (defaults.length > 1) throw new Error(`${subject} has multiple default streams: ${defaults.map((item) => item.index).join(", ")}`);
+ if (values.length === 1) return values[0]!;
+ throw new Error(`${subject} is ambiguous: ${values.map((item) => item.index).join(", ")}`);
+}
+
+function selectedVideo(inspection: MediaInspection, request: MediaSelectionRequest): MediaVideoStream | undefined {
+ if (request.video.mode === "none") return undefined;
+ const videoRequest = request.video;
+ const moving = inspection.streams.filter((stream): stream is MediaVideoStream =>
+ stream.kind === "video"
+ && stream.role === "moving"
+ && stream.disposition.attachedPicture === false
+ && stream.timingStatus === "admissible"
+ && stream.startPts !== undefined
+ && stream.endPts !== undefined);
+ if (videoRequest.mode === "primary-moving") return selectUniqueDefault(moving, "Primary moving video");
+ const selected = inspection.streams.find((stream): stream is MediaVideoStream =>
+ stream.kind === "video" && stream.index === videoRequest.streamIndex);
+ assert(selected !== undefined, `Video stream ${videoRequest.streamIndex} does not exist`);
+ assert(selected.role === "moving" && selected.disposition.attachedPicture === false,
+ `Video stream ${videoRequest.streamIndex} is not a moving visual stream`);
+ assert(selected.timingStatus === "admissible", `Video stream ${videoRequest.streamIndex} timing is not admissible`);
+ assert(selected.startPts !== undefined && selected.endPts !== undefined,
+ `Video stream ${videoRequest.streamIndex} has no presentation interval`);
+ return selected;
+}
+
+function selectedAudio(inspection: MediaInspection, request: MediaSelectionRequest): MediaAudioStream | undefined {
+ if (request.audio.mode === "none") return undefined;
+ const audioRequest = request.audio;
+ const audio = inspection.streams.filter((stream): stream is MediaAudioStream =>
+ stream.kind === "audio" && stream.timingStatus === "admissible"
+ && stream.startPts !== undefined && stream.endPts !== undefined);
+ if (audioRequest.mode === "default") return selectUniqueDefault(audio, "Default audio");
+ const selected = audio.find((stream) => stream.index === audioRequest.streamIndex);
+ assert(selected !== undefined, `Audio stream ${audioRequest.streamIndex} does not exist or has no presentation interval`);
+ return selected;
+}
+
+/**
+ * Selects container streams only. Choosing an audio stream never claims that it is narrated speech;
+ * Narrative alignment remains a separate semantic Producer with an explicit Narrative edge.
+ */
+export function selectMediaStreams(
+ inspection: MediaInspection,
+ request: MediaSelectionRequest,
+): MediaStreamSelection {
+ verifyMediaInspection(inspection);
+ verifyMediaSelectionRequest(request);
+ const video = selectedVideo(inspection, request);
+ const audio = selectedAudio(inspection, request);
+ assert(video !== undefined || audio !== undefined, "Media stream selection is empty");
+ const policy = request.video.mode === "stream-index" || request.audio.mode === "stream-index"
+ ? "explicit-streams@1" as const
+ : video !== undefined && audio !== undefined
+ ? "primary-moving-default-audio@1" as const
+ : video !== undefined ? "primary-moving@1" as const : "default-audio@1" as const;
+ return sealMediaStreamSelection({
+ ...(video === undefined ? {} : { videoStreamIndex: video.index }),
+ ...(audio === undefined ? {} : { audioStreamIndex: audio.index }),
+ spanAuthority: request.spanAuthority,
+ policy,
+ });
+}
diff --git a/packages/media-operations/src/surface.ts b/packages/media-operations/src/surface.ts
new file mode 100644
index 00000000..b0ab1ccd
--- /dev/null
+++ b/packages/media-operations/src/surface.ts
@@ -0,0 +1,369 @@
+import { assertEmptyElement as empty, localName, textAttribute as text, type StructuredElement, type StructuredSurfaceHandler, type SurfaceResolvedReference, type MarkupAttributeValue } from "@hypit/hypit/markup";
+import { sameType, type CanonicalValue } from "@hypit/hypit/protocol";
+import { blobTypes } from "@hypit/hypit/blob";
+import { mediaTypes } from "@hypit/hypit/media";
+import { temporalTypes } from "@hypit/hypit/temporal";
+import { parseTemporalDuration } from "@hypit/hypit/temporal/markup";
+import { timelineTypes, type Clock } from "@hypit/hypit/timeline";
+import { recipeType, type Recipe } from "@hypit/hypit/recipe";
+
+import {
+ extractAudioFragment,
+ extractFrameFragment,
+ createStillVideoFragment,
+ synchronizedMediaFragment,
+ transformMediaFragment,
+} from "./fragment.js";
+import { mediaOperationsTypes } from "./manifest.js";
+import {
+ sealAudioExtractionRequest,
+ sealFrameExtractionRequest,
+ sealMediaTransformProgram,
+} from "./operations.js";
+import { sealMediaSelectionRequest } from "./selection.js";
+import type {
+ AudioExtractionRequest,
+ FrameExtractionRequest,
+ MediaSelectionRequest,
+ MediaTransformOperation,
+} from "./types.js";
+
+function ref(
+ raw: MarkupAttributeValue | undefined,
+ label: string,
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
+ const value = resolve(raw.path);
+ if (value === undefined || !sameType(value.type, blobTypes.blob)) throw new Error(`${label} must resolve to Blob.`);
+ return value;
+}
+
+function typedReference(
+ raw: MarkupAttributeValue | undefined,
+ label: string,
+ expected: SurfaceResolvedReference["type"],
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
+ const value = resolve(raw.path);
+ if (value === undefined || !sameType(value.type, expected)) throw new Error(`${label} has the wrong Type.`);
+ return value;
+}
+
+function authoredRecipe(
+ raw: MarkupAttributeValue | undefined,
+ label: string,
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): Recipe {
+ const value = typedReference(raw, label, recipeType, resolve);
+ if (value.record?.value.kind !== "inline") throw new Error(`${label} must be an authored SVS Recipe.`);
+ const recipe = value.record.value.value as unknown as Recipe;
+ const unknown = Object.keys(recipe.properties).filter((name) => !["audio", "span-authority", "video"].includes(name));
+ if (unknown.length > 0) throw new Error(`${label} only accepts video, audio and span-authority; found ${unknown.join(", ")}.`);
+ return recipe;
+}
+
+function recipeText(recipe: Recipe, name: string, label: string): string {
+ const value: CanonicalValue | undefined = recipe.properties[name];
+ if (typeof value !== "string" || value.trim().length === 0) {
+ throw new Error(`${label} requires string property ${name}.`);
+ }
+ return value;
+}
+
+function stream(value: string, kind: "video"): MediaSelectionRequest["video"];
+function stream(value: string, kind: "audio"): MediaSelectionRequest["audio"];
+function stream(value: string, kind: "video" | "audio"): MediaSelectionRequest["video"] | MediaSelectionRequest["audio"] {
+ if (kind === "video" && value === "primary-moving") return { mode: "primary-moving" };
+ if (kind === "audio" && value === "default") return { mode: "default" };
+ if (value === "none") return { mode: "none" };
+ const match = /^stream:(\d+)$/u.exec(value);
+ if (match === null) throw new Error(`${kind} must be ${kind === "video" ? "primary-moving" : "default"}, none or stream:<index>.`);
+ return { mode: "stream-index", streamIndex: Number(match[1]) };
+}
+
+function exactAttributes(element: StructuredElement, required: readonly string[], optional: readonly string[] = []): void {
+ const expected = new Set([...required, ...optional]);
+ const unknown = Object.keys(element.attributes).filter((name) => !expected.has(name));
+ if (unknown.length > 0 || required.some((name) => element.attributes[name] === undefined)) {
+ throw new Error(`${element.name} requires ${required.join(", ")}`
+ + (optional.length === 0 ? "" : `; optional: ${optional.join(", ")}`));
+ }
+}
+
+function seconds(value: string, subject: string, allowZero = true): number {
+ const match = /^(\d+(?:\.\d+)?)(?:s)?$/u.exec(value.trim());
+ if (match === null) throw new Error(`${subject} must be seconds such as 0.25s or 2`);
+ const result = Number(match[1]);
+ if (!Number.isFinite(result) || result < 0 || (!allowZero && result === 0)) {
+ throw new Error(`${subject} must be ${allowZero ? "non-negative" : "positive"}`);
+ }
+ return result;
+}
+
+function audioSelector(value: string): AudioExtractionRequest["audio"] {
+ if (value === "default") return { mode: "default" };
+ const match = /^stream:(\d+)$/u.exec(value);
+ if (match === null) throw new Error("ExtractAudio.audio must be default or stream:<index>.");
+ return { mode: "stream-index", streamIndex: Number(match[1]) };
+}
+
+function videoSelector(value: string): FrameExtractionRequest["video"] {
+ if (value === "primary-moving") return { mode: "primary-moving" };
+ const match = /^stream:(\d+)$/u.exec(value);
+ if (match === null) throw new Error("video must be primary-moving or stream:<index>.");
+ return { mode: "stream-index", streamIndex: Number(match[1]) };
+}
+
+function transformOperations(element: StructuredElement): readonly MediaTransformOperation[] {
+ const result: MediaTransformOperation[] = [];
+ for (const child of element.children) {
+ if (child.kind === "text") {
+ if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Trim and Retime children.`);
+ continue;
+ }
+ const name = localName(child.name);
+ empty(child);
+ if (name === "Trim") {
+ exactAttributes(child, [], ["start", "end", "tail"]);
+ const start = child.attributes.start;
+ const end = child.attributes.end;
+ const tail = child.attributes.tail;
+ if (start === undefined && end === undefined && tail === undefined) {
+ throw new Error(`${child.name} requires at least one of start, end or tail.`);
+ }
+ if (end !== undefined && tail !== undefined) throw new Error(`${child.name} cannot combine end and tail.`);
+ for (const [attribute, value] of [["start", start], ["end", end], ["tail", tail]] as const) {
+ if (value !== undefined && typeof value !== "string") throw new Error(`${child.name}.${attribute} must be text.`);
+ }
+ result.push({
+ kind: "trim",
+ ...(typeof start === "string" ? { startSec: seconds(start, `${child.name}.start`) } : {}),
+ ...(typeof end === "string" ? { endSec: seconds(end, `${child.name}.end`, false) } : {}),
+ ...(typeof tail === "string" ? { tailSec: seconds(tail, `${child.name}.tail`, false) } : {}),
+ });
+ continue;
+ }
+ if (name === "Retime") {
+ exactAttributes(child, ["rate"]);
+ const rate = Number(text(child, "rate"));
+ if (!Number.isFinite(rate) || rate <= 0 || rate > 100) throw new Error(`${child.name}.rate must be in (0, 100].`);
+ result.push({ kind: "retime", rate });
+ continue;
+ }
+ throw new Error(`${element.name} accepts only Trim and Retime children.`);
+ }
+ if (result.length === 0) throw new Error(`${element.name} requires at least one Trim or Retime child.`);
+ return result;
+}
+
+export const decodeSynchronizedMediaSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ const required = ["id", "source", "clock"];
+ const optional = ["audio", "recipe", "span-authority", "video"];
+ exactAttributes(element, required, optional);
+ const hasRecipe = element.attributes.recipe !== undefined;
+ const policyCount = ["video", "audio", "span-authority"].filter((name) => element.attributes[name] !== undefined).length;
+ if ((hasRecipe && policyCount !== 0) || (!hasRecipe && policyCount !== 3)) {
+ throw new Error(`${element.name} requires exactly one of recipe or video/audio/span-authority.`);
+ }
+ if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
+ throw new Error(`${element.name} must be empty.`);
+ }
+ const id = text(element, "id");
+ const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
+ const clock = typedReference(element.attributes.clock, `${element.name}.clock`, timelineTypes.clock, resolveReference);
+ const clockValue = clock.record?.value.kind === "inline"
+ ? clock.record.value.value as unknown as Clock
+ : undefined;
+ if (clockValue === undefined) throw new Error(`${element.name}.clock must be an authored Clock record.`);
+ const normalizationRecipe = hasRecipe ? authoredRecipe(element.attributes.recipe, `${element.name}.recipe`, resolveReference) : undefined;
+ const video = stream(normalizationRecipe === undefined ? text(element, "video") : recipeText(normalizationRecipe, "video", `${element.name}.recipe`), "video");
+ const audio = stream(normalizationRecipe === undefined ? text(element, "audio") : recipeText(normalizationRecipe, "audio", `${element.name}.recipe`), "audio");
+ const spanAuthority = normalizationRecipe === undefined
+ ? text(element, "span-authority")
+ : recipeText(normalizationRecipe, "span-authority", `${element.name}.recipe`);
+ if (spanAuthority !== "video" && spanAuthority !== "audio") throw new Error(`${element.name}.span-authority must be video or audio.`);
+ const request = sealMediaSelectionRequest({
+ video,
+ audio,
+ spanAuthority,
+ frameRate: clockValue.frameRate,
+ });
+ const requestId = `${id}.request`;
+ const domainId = `${id}.domain-spec`;
+ return {
+ records: [
+ { id: requestId, type: mediaOperationsTypes.selectionRequest, value: { kind: "inline", value: request }, range: element.range },
+ { id: domainId, type: mediaTypes.domainSpec, value: { kind: "inline", value: { id } }, range: element.range },
+ ],
+ components: [{
+ id,
+ fragment: synchronizedMediaFragment.id,
+ inputs: { source: source.ref, request: { kind: "record", id: requestId }, domain: { kind: "record", id: domainId } },
+ outputs: { media: `${id}.media`, domain: `${id}.domain`, extent: `${id}.extent` },
+ range: element.range,
+ }],
+ fragments: [synchronizedMediaFragment],
+ };
+};
+
+/** The pictures of a StillVideo in authored order, with their weights: one `source`, or Still children. */
+function stillPictures(
+ element: StructuredElement,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+): readonly { readonly source: SurfaceResolvedReference; readonly weight: number }[] {
+ const children = element.children.filter((child) => child.kind === "element" || child.value.trim().length > 0);
+ if (element.attributes.source !== undefined) {
+ if (children.length > 0) throw new Error(`${element.name} takes either source or Still children, not both.`);
+ return [{ source: ref(element.attributes.source, `${element.name}.source`, resolveReference), weight: 1 }];
+ }
+ const pictures = children.map((child) => {
+ if (child.kind !== "element" || localName(child.name) !== "Still") {
+ throw new Error(`${element.name} accepts only Still children.`);
+ }
+ exactAttributes(child, ["source"], ["weight"]);
+ empty(child);
+ const rawWeight = child.attributes.weight;
+ let weight = 1;
+ if (rawWeight !== undefined) {
+ if (typeof rawWeight !== "string") throw new Error(`${child.name}.weight must be text.`);
+ weight = Number(rawWeight);
+ if (!Number.isFinite(weight) || weight <= 0) throw new Error(`${child.name}.weight must be a positive number.`);
+ }
+ return { source: ref(child.attributes.source, `${child.name}.source`, resolveReference), weight };
+ });
+ if (pictures.length === 0) throw new Error(`${element.name} requires a source or at least one Still child.`);
+ return pictures;
+}
+
+export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ exactAttributes(element, ["id", "duration", "clock"], ["source"]);
+ const id = text(element, "id");
+ const pictures = stillPictures(element, resolveReference);
+ // The author supplies an exact temporal value. Planning later proves that it lands on a whole
+ // frame for the selected Clock; no speech-specific duration or rounding policy is involved.
+ const duration = parseTemporalDuration(text(element, "duration"), `${element.name}.duration`);
+ const durationId = `${id}.duration`;
+ const layoutId = `${id}.layout`;
+ const clock = typedReference(
+ element.attributes.clock,
+ `${element.name}.clock`,
+ timelineTypes.clock,
+ resolveReference,
+ );
+ const fragment = createStillVideoFragment(pictures.length);
+ return {
+ records: [{
+ id: durationId,
+ type: temporalTypes.duration,
+ value: { kind: "inline", value: duration },
+ range: element.range,
+ }, {
+ id: layoutId,
+ type: mediaOperationsTypes.stillVideoLayout,
+ value: { kind: "inline", value: {
+ weights: pictures.map((picture) => picture.weight),
+ } },
+ range: element.range,
+ }],
+ components: [{
+ id,
+ fragment: fragment.id,
+ inputs: {
+ duration: { kind: "record", id: durationId },
+ clock: clock.ref,
+ layout: { kind: "record", id: layoutId },
+ ...Object.fromEntries(pictures.map((picture, index) => [`source-${index}`, picture.source.ref])),
+ },
+ outputs: { video: `${id}.video` },
+ range: element.range,
+ }],
+ fragments: [fragment],
+ };
+};
+
+export const decodeTransformMediaSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ exactAttributes(element, ["id", "source"]);
+ const id = text(element, "id");
+ const source = typedReference(element.attributes.source, `${element.name}.source`, mediaTypes.synchronized, resolveReference);
+ const program = sealMediaTransformProgram({
+ operations: transformOperations(element),
+ });
+ const programId = `${id}.program`;
+ return {
+ records: [
+ { id: programId, type: mediaOperationsTypes.transformProgram, value: { kind: "inline", value: program }, range: element.range },
+ ],
+ components: [{
+ id,
+ fragment: transformMediaFragment.id,
+ inputs: {
+ media: source.ref,
+ program: { kind: "record", id: programId },
+ },
+ outputs: { video: `${id}.video` },
+ range: element.range,
+ }],
+ fragments: [transformMediaFragment],
+ };
+};
+
+export const decodeExtractAudioSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ exactAttributes(element, ["id", "source", "audio"]);
+ empty(element);
+ const id = text(element, "id");
+ const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
+ const request = sealAudioExtractionRequest({
+ audio: audioSelector(text(element, "audio")),
+ output: { container: "wav", codec: "pcm_s16le", sampleRate: 48_000, channels: 2 },
+ });
+ const requestId = `${id}.request`;
+ return {
+ records: [{ id: requestId, type: mediaOperationsTypes.audioExtractionRequest,
+ value: { kind: "inline", value: request }, range: element.range }],
+ components: [{
+ id,
+ fragment: extractAudioFragment.id,
+ inputs: { source: source.ref, request: { kind: "record", id: requestId } },
+ outputs: { audio: `${id}.audio` },
+ range: element.range,
+ }],
+ fragments: [extractAudioFragment],
+ };
+};
+
+function frameSelector(value: string, subject: string): FrameExtractionRequest["at"] {
+ if (value === "first") return { kind: "first" };
+ if (value === "last") return { kind: "last" };
+ const frame = /^frame:(\d+)$/u.exec(value);
+ if (frame !== null) return { kind: "frame", index: Number(frame[1]) };
+ const time = /^time:(.+)$/u.exec(value);
+ if (time !== null) return { kind: "time", seconds: seconds(time[1]!, subject) };
+ throw new Error(`${subject} must be first, last, frame:<index> or time:<seconds>.`);
+}
+
+export const decodeExtractFrameSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ exactAttributes(element, ["id", "source", "video", "at"]);
+ empty(element);
+ const id = text(element, "id");
+ const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
+ const request = sealFrameExtractionRequest({
+ video: videoSelector(text(element, "video")),
+ at: frameSelector(text(element, "at"), `${element.name}.at`),
+ output: { format: "png" },
+ });
+ const requestId = `${id}.request`;
+ return {
+ records: [{ id: requestId, type: mediaOperationsTypes.frameExtractionRequest,
+ value: { kind: "inline", value: request }, range: element.range }],
+ components: [{
+ id,
+ fragment: extractFrameFragment.id,
+ inputs: { source: source.ref, request: { kind: "record", id: requestId } },
+ outputs: { image: `${id}.image` },
+ range: element.range,
+ }],
+ fragments: [extractFrameFragment],
+ };
+};
diff --git a/packages/media-operations/src/types.ts b/packages/media-operations/src/types.ts
new file mode 100644
index 00000000..6f2af8dd
--- /dev/null
+++ b/packages/media-operations/src/types.ts
@@ -0,0 +1,172 @@
+import type { AudioLevelAutomation } from "@hypit/hypit/composition";
+import type { BlobRef } from "@hypit/hypit/protocol";
+import type { MediaFrameRange, MediaInspection, MediaRational, MediaStreamSelection, TimelineVisual, TimelineAudio } from "@hypit/hypit/media";
+
+export type MediaSelectionRequest = {
+ readonly video:
+ | { readonly mode: "primary-moving" }
+ | { readonly mode: "stream-index"; readonly streamIndex: number }
+ | { readonly mode: "none" };
+ readonly audio:
+ | { readonly mode: "default" }
+ | { readonly mode: "stream-index"; readonly streamIndex: number }
+ | { readonly mode: "none" };
+ readonly spanAuthority: "video" | "audio";
+ readonly frameRate: MediaRational;
+};
+
+export type InspectMediaNeed = {
+ readonly source: BlobRef;
+};
+
+export type MediaVideoSelector =
+ | { readonly mode: "primary-moving" }
+ | { readonly mode: "stream-index"; readonly streamIndex: number };
+
+export type MediaAudioSelector =
+ | { readonly mode: "default" }
+ | { readonly mode: "stream-index"; readonly streamIndex: number }
+ | { readonly mode: "none" };
+
+export type MediaTransformOperation =
+ | {
+ readonly kind: "trim";
+ /** Seconds in the operation's current timeline, not necessarily the source timeline. */
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly tailSec?: number;
+ }
+ | {
+ readonly kind: "retime";
+ /** Pitch-preserving tempo ratio; 2 means twice as fast and therefore half as long. */
+ readonly rate: number;
+ };
+
+/** Ordered, deterministic A/V operations over already synchronized media. */
+export type MediaTransformProgram = {
+ readonly operations: readonly MediaTransformOperation[];
+};
+
+export type AudioExtractionRequest = {
+ readonly audio: Exclude<MediaAudioSelector, { readonly mode: "none" }>;
+ readonly output: {
+ readonly container: "wav";
+ readonly codec: "pcm_s16le";
+ readonly sampleRate: 48_000;
+ readonly channels: 2;
+ };
+};
+
+export type FrameExtractionRequest = {
+ readonly video: MediaVideoSelector;
+ readonly at:
+ | { readonly kind: "first" }
+ | { readonly kind: "last" }
+ | { readonly kind: "frame"; readonly index: number }
+ | { readonly kind: "time"; readonly seconds: number };
+ readonly output: { readonly format: "png" };
+};
+
+/** How the still video divides its literal duration among its pictures: one weight per picture, in order. */
+export type StillVideoLayout = {
+ readonly weights: readonly number[];
+};
+
+/** One picture's span of the still video, in frames; `source` is bound one picture at a time. */
+export type StillVideoSegment = {
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+ readonly source?: BlobRef;
+};
+
+/** Exact video-only clip requested from one or more authored still images spread over a literal duration. */
+export type StillVideoRequest = {
+ readonly frameRate: MediaRational;
+ readonly frameCount: number;
+ readonly output: {
+ readonly container: "mp4";
+ readonly codec: "h264";
+ readonly pixelFormat: "yuv420p";
+ };
+ readonly segments: readonly StillVideoSegment[];
+};
+
+export type TransformMediaNeed = {
+ readonly media: import("@hypit/hypit/media").SynchronizedMedia;
+ readonly program: MediaTransformProgram;
+};
+
+export type ExtractAudioNeed = {
+ readonly source: BlobRef;
+ readonly streamIndex: number;
+ readonly output: AudioExtractionRequest["output"];
+};
+
+export type ExtractFrameNeed = {
+ readonly source: BlobRef;
+ readonly streamIndex: number;
+ readonly sourceFrameCount: number;
+ readonly at: FrameExtractionRequest["at"];
+ readonly output: FrameExtractionRequest["output"];
+};
+
+/** Every segment of the request carries its picture by the time the Need is made. */
+export type RenderStillVideoNeed = {
+ readonly request: StillVideoRequest;
+};
+
+export type NormalizeMediaNeed = {
+ readonly source: BlobRef;
+ readonly inspection: MediaInspection;
+ readonly selection: MediaStreamSelection;
+ readonly frameRate: MediaRational;
+ readonly audio: {
+ readonly sampleRate: 48_000;
+ readonly channels: 2;
+ readonly codec: "pcm_s16le";
+ readonly loudness: "preserve";
+ };
+};
+
+export type AudioProgramClip = AudioLevelAutomation & {
+ readonly id: string;
+ readonly artifact: BlobRef;
+ readonly targetStartSample: number;
+ readonly targetEndSampleExclusive: number;
+ readonly sourceSampleFrames: number;
+ readonly sourceStartSample: number;
+ readonly sourceEndSampleExclusive: number;
+ readonly sourceLoop: boolean;
+ readonly sourcePhaseSample: number;
+ /** Tempo ratio; Audio execution preserves pitch. */
+ readonly playbackRate: number;
+ /** Complete owning Clip Window; fades do not restart at source-time piece boundaries. */
+ readonly mixStartSample: number;
+ readonly mixEndSampleExclusive: number;
+ readonly gain: number;
+ readonly fadeInSamples: number;
+ readonly fadeOutSamples: number;
+};
+
+/** Pure media plan. Executing it is always a Provider Need. */
+export type AudioProgramPlan = {
+ readonly frameRate: MediaRational;
+ readonly frameCount: number;
+ readonly sampleRate: 48_000;
+ readonly sampleFrames: number;
+ readonly clips: readonly AudioProgramClip[];
+ readonly mix: {
+ readonly normalize: false;
+ readonly limiter: "none";
+ };
+};
+
+export type RenderAudioNeed = {
+ readonly plan: AudioProgramPlan;
+ readonly range?: MediaFrameRange;
+};
+
+export type MuxMediaNeed = {
+ readonly visual: TimelineVisual;
+ readonly audio: TimelineAudio;
+};
diff --git a/packages/media-operations/test/manifest.test.ts b/packages/media-operations/test/manifest.test.ts
new file mode 100644
index 00000000..851845e0
--- /dev/null
+++ b/packages/media-operations/test/manifest.test.ts
@@ -0,0 +1,223 @@
+import { blobTypes } from "@hypit/blob";
+import { mediaTypes } from "@hypit/media";
+import { timelineTypes } from "@hypit/timeline";
+import { temporalTypes } from "@hypit/temporal";
+import { recipeType } from "@hypit/recipe";
+import { installRunFragmentFacets, RunFragmentRegistry } from "@hypit/run";
+import assert from "node:assert/strict";
+import test from "node:test";
+import { fixtureResource } from "../../../test/fixture-resource.js";
+
+import {
+ mediaOperationsTypes,
+ decodeSynchronizedMediaSurface,
+ decodeExtractAudioSurface,
+ decodeExtractFrameSurface,
+ decodeTransformMediaSurface,
+ decodeStillVideoSurface,
+ createStillVideoFragment,
+ planStillVideoSegments,
+ bindStillVideoSource,
+ extractAudioFragment,
+ extractFrameFragment,
+ synchronizedMediaFragment,
+ transformMediaFragment,
+ stillVideoFragment,
+} from "../src/index.js";
+import { hypitPackage } from "../src/activation.js";
+
+test("the package exposes StillVideo as an ordinary Run Fragment", () => {
+ const fragments = new RunFragmentRegistry();
+ installRunFragmentFacets(hypitPackage.facets, fragments);
+ assert.equal(fragments.resolve("@hypit/media-operations@1", "still-video"), stillVideoFragment);
+});
+
+test("the Normalize Surface makes inspection and normalization an explicit author graph branch", async () => {
+ const range = { source: "normalize.svml", start: 0, end: 1 };
+ const result = await decodeSynchronizedMediaSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element",
+ name: "pipeline:Normalize",
+ attributes: {
+ id: "motion",
+ source: { kind: "reference", path: "generated" },
+ clock: { kind: "reference", path: "clock" },
+ recipe: { kind: "reference", path: "policy" },
+ },
+ children: [],
+ range,
+ },
+ resolveReference: (path) => path === "generated"
+ ? { path, ref: { kind: "record", id: path }, type: blobTypes.blob }
+ : path === "clock"
+ ? { path, ref: { kind: "record", id: path }, type: timelineTypes.clock,
+ record: { value: { kind: "inline", value: { frameRate: { numerator: 30_000, denominator: 1_001 } } } } as never }
+ : path === "policy"
+ ? { path, ref: { kind: "record", id: path }, type: recipeType,
+ record: { value: { kind: "inline", value: { path, properties: { video: "primary-moving", audio: "none", "span-authority": "video" } } } } as never }
+ : undefined,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ });
+ assert.equal(result.fragments[0]?.id, synchronizedMediaFragment.id);
+ assert.deepEqual(result.components[0]?.outputs, {
+ media: "motion.media", domain: "motion.domain", extent: "motion.extent",
+ });
+ const request = result.records[0]?.value;
+ assert.ok(request?.kind === "inline");
+ assert.deepEqual(request.value, {
+ video: { mode: "primary-moving" },
+ audio: { mode: "none" },
+ spanAuthority: "video",
+ frameRate: { numerator: 30_000, denominator: 1_001 },
+ });
+});
+
+test("media operations are ordinary graph branches over Blob", async () => {
+ const range = { source: "operations.svml", start: 0, end: 1 };
+ const resolveReference = (path: string) => path === "generated.media"
+ ? { path, ref: { kind: "record" as const, id: path }, type: mediaTypes.synchronized }
+ : path === "generated.video"
+ ? { path, ref: { kind: "record" as const, id: path }, type: blobTypes.blob }
+ : undefined;
+ const context = { sourceName: range.source, resolveReference,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); } };
+ const transform = await decodeTransformMediaSurface({
+ ...context,
+ element: {
+ kind: "element", name: "media:Transform", range,
+ attributes: {
+ id: "prepared", source: { kind: "reference", path: "generated.media" },
+ },
+ children: [
+ { kind: "element", name: "media:Trim", attributes: { tail: "0.25s" }, children: [], range },
+ { kind: "element", name: "media:Retime", attributes: { rate: "1.05" }, children: [], range },
+ ],
+ },
+ });
+ assert.equal(transform.fragments[0]?.id, transformMediaFragment.id);
+ assert.deepEqual(transform.components[0]?.outputs, { video: "prepared.video" });
+ const program = transform.records.find((record) => record.id === "prepared.program");
+ assert.ok(program?.value.kind === "inline");
+ assert.deepEqual(program.value.value, {
+ operations: [
+ { kind: "trim", tailSec: 0.25 },
+ { kind: "retime", rate: 1.05 },
+ ],
+ });
+
+ const audio = await decodeExtractAudioSurface({
+ ...context,
+ element: {
+ kind: "element", name: "media:ExtractAudio", range,
+ attributes: { id: "voice-reference", source: { kind: "reference", path: "generated.video" }, audio: "default" },
+ children: [],
+ },
+ });
+ assert.equal(audio.fragments[0]?.id, extractAudioFragment.id);
+ assert.deepEqual(audio.components[0]?.outputs, { audio: "voice-reference.audio" });
+
+ const frame = await decodeExtractFrameSurface({
+ ...context,
+ element: {
+ kind: "element", name: "media:ExtractFrame", range,
+ attributes: { id: "continuity", source: { kind: "reference", path: "generated.video" },
+ video: "primary-moving", at: "last" },
+ children: [],
+ },
+ });
+ assert.equal(frame.fragments[0]?.id, extractFrameFragment.id);
+ assert.deepEqual(frame.components[0]?.outputs, { image: "continuity.image" });
+});
+
+test("StillVideo stops at an ordinary MP4 branch before Normalize", async () => {
+ const range = { source: "still.svml", start: 0, end: 1 };
+ const output = await decodeStillVideoSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element",
+ name: "media:StillVideo",
+ attributes: {
+ id: "opening-still",
+ source: { kind: "reference", path: "opening-head" },
+ duration: "6s",
+ clock: { kind: "reference", path: "clock" },
+ },
+ children: [],
+ range,
+ },
+ resolveReference: (path) => path === "opening-head"
+ ? { path, ref: { kind: "record", id: path }, type: blobTypes.blob }
+ : path === "clock"
+ ? { path, ref: { kind: "record", id: path }, type: timelineTypes.clock }
+ : undefined,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ });
+ assert.equal(output.fragments[0]?.id, stillVideoFragment.id);
+ assert.deepEqual(output.components[0]?.outputs, { video: "opening-still.video" });
+ assert.deepEqual(output.records[0], {
+ id: "opening-still.duration", type: temporalTypes.duration,
+ value: { kind: "inline", value: { unit: "seconds", numerator: 6, denominator: 1 } }, range,
+ }, "the literal duration is the author's exact TemporalDuration Record");
+ assert.deepEqual(output.components[0]?.inputs.duration, { kind: "record", id: "opening-still.duration" });
+ assert.deepEqual(output.components[0]?.inputs["source-0"], { kind: "record", id: "opening-head" });
+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1] } }, "one picture, one whole share");
+ assert.equal(stillVideoFragment.exports[0]?.type.name, blobTypes.blob.name);
+});
+
+test("StillVideo spreads several pictures over one literal duration by weight, in authored order", async () => {
+ const range = { source: "still.svml", start: 0, end: 1 };
+ const resolveReference = (path: string) => path === "clock"
+ ? { path, ref: { kind: "record" as const, id: path }, type: timelineTypes.clock }
+ : { path, ref: { kind: "record" as const, id: path }, type: blobTypes.blob };
+ const child = (source: string, weight?: string) => ({
+ kind: "element" as const, name: "media:Still",
+ attributes: { source: { kind: "reference" as const, path: source }, ...(weight === undefined ? {} : { weight }) },
+ children: [], range,
+ });
+ const output = await decodeStillVideoSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element", name: "media:StillVideo",
+ attributes: { id: "kitchen", duration: "6s", clock: { kind: "reference", path: "clock" } },
+ children: [child("counter"), child("basil", "2"), child("board")],
+ range,
+ },
+ resolveReference,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ });
+ assert.equal(output.fragments[0]?.id, createStillVideoFragment(3).id);
+ assert.notEqual(output.fragments[0]?.id, stillVideoFragment.id, "three pictures are a different shape from one");
+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1, 2, 1] } });
+ assert.deepEqual(
+ ["source-0", "source-1", "source-2"].map((name) => output.components[0]?.inputs[name]),
+ [{ kind: "record", id: "counter" }, { kind: "record", id: "basil" }, { kind: "record", id: "board" }],
+ );
+ await assert.rejects(async () => await decodeStillVideoSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element", name: "media:StillVideo",
+ attributes: { id: "both", source: { kind: "reference", path: "counter" }, duration: "6s", clock: { kind: "reference", path: "clock" } },
+ children: [child("basil")],
+ range,
+ },
+ resolveReference,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ }), /either source or Still children/u);
+});
+
+test("still video frames are whole and every picture holds at least one", () => {
+ assert.deepEqual(planStillVideoSegments(180, [1, 2, 1]).map((item) => item.endFrameExclusive - item.startFrame), [45, 90, 45]);
+ assert.deepEqual(planStillVideoSegments(10, [1, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [4, 3, 3],
+ "the leftover frame goes to the earliest of equal remainders");
+ assert.deepEqual(planStillVideoSegments(10, [1, 1.5, 1]).map((item) => item.endFrameExclusive - item.startFrame), [3, 4, 3]);
+ assert.deepEqual(planStillVideoSegments(3, [100, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [1, 1, 1]);
+ assert.throws(() => planStillVideoSegments(2, [1, 1, 1]), /each picture needs at least one/u);
+ const bound = bindStillVideoSource(
+ { frameRate: { numerator: 30, denominator: 1 }, frameCount: 2, output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: [{ startFrame: 0, endFrameExclusive: 1 }, { startFrame: 1, endFrameExclusive: 2 }] },
+ { kind: "blob", resource: fixtureResource("still:first"), size: 1, mediaType: "image/png" },
+ );
+ assert.equal(bound.segments[0]?.source?.mediaType, "image/png");
+ assert.equal(bound.segments[1]?.source, undefined, "pictures bind in order, one per step");
+});
diff --git a/packages/media-operations/test/mux.test.ts b/packages/media-operations/test/mux.test.ts
new file mode 100644
index 00000000..eca5a5a3
--- /dev/null
+++ b/packages/media-operations/test/mux.test.ts
@@ -0,0 +1,36 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { sealTimelineVisual, sealTimelineAudio } from "@hypit/media";
+import { fixtureResource } from "../../../test/fixture-resource.js";
+import { mediaOperationsComponent } from "../src/component.js";
+import { mediaOperationsProducers } from "../src/manifest.js";
+
+const mux = mediaOperationsComponent.producers.find((item) => item.producer.name === mediaOperationsProducers.mux.name)!;
+
+const cases = [
+ { frameRate: { numerator: 30, denominator: 1 }, frameCount: 149, sampleFrames: 238400 },
+ { frameRate: { numerator: 24000, denominator: 1001 }, frameCount: 149, sampleFrames: 298298 },
+ { frameRate: { numerator: 30000, denominator: 1001 }, frameCount: 149, sampleFrames: 238638 },
+ { frameRate: { numerator: 30000, denominator: 1001 }, frameCount: 1, sampleFrames: 1602 },
+ { frameRate: { numerator: 60000, denominator: 1001 }, frameCount: 1, sampleFrames: 801 },
+];
+
+for (const { frameRate, frameCount, sampleFrames } of cases) {
+ test(`mux accepts the nearest PCM sample boundary for ${frameCount} frames at ${frameRate.numerator}/${frameRate.denominator}`, async () => {
+ const visual = sealTimelineVisual({ frameRate, frameCount, canvas: { width: 540, height: 960 },
+ artifact: { kind: "blob", resource: fixtureResource("mux:visual"), size: 1, mediaType: "video/mp4" } });
+ const run = (samples: number) => {
+ const audio = sealTimelineAudio({ sampleFrames: samples,
+ artifact: { kind: "blob", resource: fixtureResource("mux:audio"), size: 1, mediaType: "audio/wav" } });
+ return mux.handler({ inputs: {
+ visual: { value: { kind: "inline", value: visual } },
+ audio: { value: { kind: "inline", value: audio } },
+ } } as never);
+ };
+ const result = await run(sampleFrames);
+ assert.deepEqual(Object.keys(result.needs), ["media"]);
+ for (const incorrect of [sampleFrames - 1, sampleFrames + 1]) {
+ await assert.rejects(async () => run(incorrect), /different presentation durations/u);
+ }
+ });
+}
diff --git a/packages/media-operations/test/presentation.test.ts b/packages/media-operations/test/presentation.test.ts
new file mode 100644
index 00000000..e677c2f4
--- /dev/null
+++ b/packages/media-operations/test/presentation.test.ts
@@ -0,0 +1,14 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { mediaOperationsComponent } from "../src/component.js";
+import { mediaOperationsProducers } from "../src/manifest.js";
+
+test("audio range request describes its own range without listing every clip", () => {
+ const facet = mediaOperationsComponent.plannedNeeds.find((item) => item.producer.name === mediaOperationsProducers.renderAudioRange.name)!;
+ const plan = { frameRate: { numerator: 30, denominator: 1 }, frameCount: 600, sampleRate: 48000,
+ clips: Array.from({ length: 35 }, () => ({ source: "not a CLI output" })) };
+ assert.deepEqual(facet.present!({ constraints: {}, pendingInputs: [] }), { fields: {}, references: {} });
+ assert.deepEqual(facet.present!({ constraints: { plan, range: { startFrame: 30, endFrameExclusive: 91 } }, pendingInputs: [] }), {
+ fields: { startFrame: [30], endFrameExclusive: [91], frameRate: ["30/1"], sampleRate: [48000] }, references: {},
+ });
+});
diff --git a/packages/media-operations/test/selection.test.ts b/packages/media-operations/test/selection.test.ts
new file mode 100644
index 00000000..2a5af49d
--- /dev/null
+++ b/packages/media-operations/test/selection.test.ts
@@ -0,0 +1,133 @@
+import { sealMediaInspection, verifyMediaStreamSelection } from "@hypit/media";
+import type { MediaInspection, MediaVideoStream } from "@hypit/media";
+import assert from "node:assert/strict";
+import test from "node:test";
+import { fixtureResource } from "../../../test/fixture-resource.js";
+
+
+import {
+ sealMediaSelectionRequest,
+ selectMediaStreams,
+} from "../src/index.js";
+
+const source = {
+ kind: "blob" as const,
+ resource: fixtureResource("grok-container"),
+ size: 123,
+ mediaType: "video/mp4",
+};
+const timeBase = { numerator: 1, denominator: 12_288 } as const;
+const timestamp = (ticks: string) => ({ ticks, timeBase });
+
+function grokInspection(): MediaInspection {
+ return sealMediaInspection({
+ container: { formatNames: ["mov", "mp4"] },
+ streams: [
+ {
+ kind: "video",
+ index: 0,
+ codecType: "video",
+ codecName: "h264",
+ disposition: { default: true, attachedPicture: false },
+ timingStatus: "admissible",
+ timeBase,
+ startPts: timestamp("0"),
+ endPts: timestamp("74240"),
+ decodedUnitCount: 145,
+ role: "moving",
+ width: 464,
+ height: 688,
+ sampleAspectRatio: { numerator: 1, denominator: 1 },
+ rotationDegrees: 0,
+ averageFrameRate: { numerator: 24, denominator: 1 },
+ nominalFrameRate: { numerator: 24, denominator: 1 },
+ },
+ {
+ kind: "audio",
+ index: 1,
+ codecType: "audio",
+ codecName: "aac",
+ disposition: { default: true, attachedPicture: false },
+ timingStatus: "admissible",
+ timeBase: { numerator: 1, denominator: 48_000 },
+ startPts: { ticks: "0", timeBase: { numerator: 1, denominator: 48_000 } },
+ endPts: { ticks: "288000", timeBase: { numerator: 1, denominator: 48_000 } },
+ decodedUnitCount: 283,
+ sampleRate: 48_000,
+ channels: 2,
+ channelLayout: "stereo",
+ decodedSampleFrames: 289_792,
+ },
+ {
+ kind: "video",
+ index: 2,
+ codecType: "video",
+ codecName: "mjpeg",
+ disposition: { default: false, attachedPicture: true },
+ timingStatus: "admissible",
+ timeBase: { numerator: 1, denominator: 90_000 },
+ startPts: { ticks: "0", timeBase: { numerator: 1, denominator: 90_000 } },
+ endPts: { ticks: "543750", timeBase: { numerator: 1, denominator: 90_000 } },
+ decodedUnitCount: 1,
+ role: "attached-picture",
+ width: 464,
+ height: 688,
+ sampleAspectRatio: { numerator: 1, denominator: 1 },
+ rotationDegrees: 0,
+ },
+ ],
+ });
+}
+
+test("primary stream selection excludes Grok's MJPEG attached picture and observes AAC without speech semantics", () => {
+ const inspection = grokInspection();
+ const request = sealMediaSelectionRequest({
+ video: { mode: "primary-moving" },
+ audio: { mode: "default" },
+ spanAuthority: "video",
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ const selection = selectMediaStreams(inspection, request);
+ verifyMediaStreamSelection(selection);
+ assert.equal(selection.videoStreamIndex, 0);
+ assert.equal(selection.audioStreamIndex, 1);
+ assert.equal(selection.spanAuthority, "video");
+ assert.equal("basisResourceId" in selection, false);
+ assert.equal("narrativeDigest" in selection, false);
+});
+
+test("multiple moving streams without one default fail closed instead of guessing by index or size", () => {
+ const inspection = grokInspection();
+ const primary = inspection.streams.find((stream): stream is MediaVideoStream => stream.kind === "video" && stream.index === 0)!;
+ const ambiguous = sealMediaInspection({
+ ...inspection,
+ streams: [
+ { ...primary, disposition: { default: false, attachedPicture: false } },
+ { ...primary, index: 3, width: 1920, height: 1080, disposition: { default: false, attachedPicture: false } },
+ ],
+ });
+ const request = sealMediaSelectionRequest({
+ video: { mode: "primary-moving" },
+ audio: { mode: "none" },
+ spanAuthority: "video",
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ assert.throws(() => selectMediaStreams(ambiguous, request), /ambiguous/u);
+ const explicit = sealMediaSelectionRequest({
+ video: { mode: "stream-index", streamIndex: 3 },
+ audio: { mode: "none" },
+ spanAuthority: "video",
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ assert.equal(selectMediaStreams(ambiguous, explicit).videoStreamIndex, 3);
+});
+
+test("an attached picture cannot be forced into the moving-video normalization path", () => {
+ const request = sealMediaSelectionRequest({
+ video: { mode: "stream-index", streamIndex: 2 },
+ audio: { mode: "none" },
+ spanAuthority: "video",
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ assert.throws(() => selectMediaStreams(grokInspection(), request), /not a moving visual stream/u);
+});
diff --git a/packages/media-operations/test/still-video.test.ts b/packages/media-operations/test/still-video.test.ts
new file mode 100644
index 00000000..bf7aee86
--- /dev/null
+++ b/packages/media-operations/test/still-video.test.ts
@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { mediaOperationsComponent } from "../src/component.js";
+import { mediaOperationsProducers } from "../src/manifest.js";
+
+const planStill = mediaOperationsComponent.producers.find(
+ (item) => item.producer.name === mediaOperationsProducers.planStill.name,
+)!;
+
+function plan(duration: unknown, frameRate = { numerator: 30, denominator: 1 }) {
+ return planStill.handler({ inputs: {
+ duration: { value: { kind: "inline", value: duration } },
+ clock: { value: { kind: "inline", value: { frameRate } } },
+ layout: { value: { kind: "inline", value: { weights: [1] } } },
+ } } as never);
+}
+
+test("StillVideo converts an exact TemporalDuration to whole frames without rounding", async () => {
+ const result = await plan({ unit: "seconds", numerator: 3, denominator: 2 });
+ const request = (result.outputs as { request?: { kind: "inline"; value: unknown } }).request;
+ assert.equal(request?.kind, "inline");
+ assert.equal((request?.value as { frameCount: number }).frameCount, 45);
+
+ await assert.rejects(
+ async () => plan({ unit: "milliseconds", value: 1 }),
+ /must resolve to a positive whole frame count/u,
+ );
+});
diff --git a/packages/provider-image-opencv-local/README.md b/packages/provider-image-opencv-local/README.md
index 9e1a9904..99f6be76 100644
--- a/packages/provider-image-opencv-local/README.md
+++ b/packages/provider-image-opencv-local/README.md
@@ -1,14 +1,14 @@
# `@hypit/provider-image-opencv-local`
-Local OpenCV/NumPy realization of the single `@hypit/raster` capability.
+Local OpenCV/NumPy realization of the two deterministic capabilities owned by
+`@hypit/image-operations`: `transform-image` and `compose-image`.
-Image Transform and Image Compose lower their different author meanings to one closed RasterRequest.
-One Handler stages its declared Resource inputs, and one Python interpreter shares decoding, fit,
-interpolation, alpha and encoding primitives across both variants. It runs in a bounded child process
-and returns one new image Resource. Temporary paths, OpenCV details and diagnostics stay inside the
-Endpoint and never enter the image Product.
+Transform and Compose remain separate Needs. One Provider implements both while its private Python
+interpreter shares decoding, fit, interpolation, alpha and encoding primitives. Each Need runs in a
+bounded child process and returns one new image Resource. Temporary paths, OpenCV details and
+diagnostics stay inside the Endpoint and never enter the author language.
-The Distribution ships the frozen deployment source from `services/image-opencv`. With the Runtime
+The Provider ships its frozen deployment source in `runtime/`. With the Runtime
Adapter's declared configuration, `programs up` installs it only when the machine Program Home has no
healthy environment; the Endpoint, program probe and doctor all resolve the shared `.venv`
interpreter. An explicit `pythonExecutable` selects an operator-managed compatible environment and
diff --git a/packages/provider-image-opencv-local/package.json b/packages/provider-image-opencv-local/package.json
index 305e88ec..5ef6f542 100644
--- a/packages/provider-image-opencv-local/package.json
+++ b/packages/provider-image-opencv-local/package.json
@@ -1,6 +1,6 @@
{
"name": "@hypit/provider-image-opencv-local",
- "version": "0.0.0-dev",
+ "version": "0.1.0",
"license": "SEE LICENSE IN LICENSE",
"private": true,
"type": "module",
@@ -10,18 +10,18 @@
"hypit": {
"activation": "./src/activation.ts"
},
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/endpoint-kit": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/raster": "workspace:*",
- "@hypit/image-opencv-runtime": "workspace:*",
- "@hypit/package-loader-node": "workspace:*",
- "@hypit/runtime-kit": "workspace:*",
- "@hypit/runtime-host-node": "workspace:*"
+ "files": ["dist", "runtime", "README.md"],
+ "peerDependencies": {
+ "@hypit/hypit": "workspace:^",
+ "@hypit/image-operations": "workspace:^",
+ "@hypit/runtime-local": "workspace:^"
},
"devDependencies": {
- "@hypit/driver-node": "workspace:*",
- "@hypit/image-transform": "workspace:*"
+ "@hypit/blob": "workspace:*",
+ "@hypit/executor": "workspace:*",
+ "@hypit/hypit": "workspace:*",
+ "@hypit/image-operations": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/runtime-local": "workspace:*"
}
}
diff --git a/services/image-opencv/.python-version b/packages/provider-image-opencv-local/runtime/.python-version
similarity index 100%
rename from services/image-opencv/.python-version
rename to packages/provider-image-opencv-local/runtime/.python-version
diff --git a/services/image-opencv/README.md b/packages/provider-image-opencv-local/runtime/README.md
similarity index 65%
rename from services/image-opencv/README.md
rename to packages/provider-image-opencv-local/runtime/README.md
index dd812ded..eac8d1f1 100644
--- a/services/image-opencv/README.md
+++ b/packages/provider-image-opencv-local/runtime/README.md
@@ -1,7 +1,7 @@
-# SVML local OpenCV runtime
+# Local OpenCV Provider runtime
-This is the locked Python environment used by `@hypit/provider-image-opencv-local`. It is deployment
-state, not an author-importable SVML package, a queue, a service daemon or part of Core.
+This is the locked Python material owned and shipped by `@hypit/provider-image-opencv-local`. It is
+not a second npm package, author Surface, queue, service daemon or part of Core.
For an ordinary installed Distribution, select the Provider and let `hypit runtime up` create and
reuse this environment in the machine Program Home. The commands below are only for a contributor
@@ -9,8 +9,8 @@ or operator deliberately managing a custom interpreter:
```bash
uv python install 3.13
-uv sync --project services/image-opencv --frozen
-uv run --project services/image-opencv --frozen --no-sync python -c \
+uv sync --project packages/provider-image-opencv-local/runtime --frozen
+uv run --project packages/provider-image-opencv-local/runtime --frozen --no-sync python -c \
'import cv2, numpy; print(cv2.__version__, numpy.__version__)'
```
@@ -21,7 +21,7 @@ Then configure the Provider with the environment's interpreter:
"use": "@hypit/provider-image-opencv-local",
"instance": "image.opencv.local",
"config": {
- "pythonExecutable": "./services/image-opencv/.venv/bin/python",
+ "pythonExecutable": "./packages/provider-image-opencv-local/runtime/.venv/bin/python",
"defaultConcurrency": 2
}
}
diff --git a/services/image-opencv/pyproject.toml b/packages/provider-image-opencv-local/runtime/pyproject.toml
similarity index 67%
rename from services/image-opencv/pyproject.toml
rename to packages/provider-image-opencv-local/runtime/pyproject.toml
index 1a719e37..a9701d32 100644
--- a/services/image-opencv/pyproject.toml
+++ b/packages/provider-image-opencv-local/runtime/pyproject.toml
@@ -1,7 +1,7 @@
[project]
-name = "hypit-image-opencv-runtime"
+name = "hypit-provider-image-opencv-local-runtime"
version = "0.1.0"
-description = "Locked Python runtime for the SVML local OpenCV image Provider"
+description = "Locked Python material owned by the Hypit local OpenCV image Provider"
requires-python = ">=3.13,<3.14"
license = { text = "Hypit Open Source License (Apache-2.0 with additional conditions); see LICENSE at the repository root" }
dependencies = [
diff --git a/services/image-opencv/uv.lock b/packages/provider-image-opencv-local/runtime/uv.lock
similarity index 99%
rename from services/image-opencv/uv.lock
rename to packages/provider-image-opencv-local/runtime/uv.lock
index 8bb7eaab..54e9fbf9 100644
--- a/services/image-opencv/uv.lock
+++ b/packages/provider-image-opencv-local/runtime/uv.lock
@@ -41,7 +41,7 @@ wheels = [
]
[[package]]
-name = "hypit-image-opencv-runtime"
+name = "hypit-provider-image-opencv-local-runtime"
version = "0.1.0"
source = { virtual = "." }
dependencies = [
diff --git a/packages/provider-image-opencv-local/src/activation.ts b/packages/provider-image-opencv-local/src/activation.ts
index 4e5c7250..6b34754a 100644
--- a/packages/provider-image-opencv-local/src/activation.ts
+++ b/packages/provider-image-opencv-local/src/activation.ts
@@ -4,10 +4,10 @@ import {
runtimeConfigObject,
runtimeConfigPositiveInteger,
runtimeConfigString,
-} from "@hypit/runtime-kit";
+} from "@hypit/runtime-local/extension";
import {
diagnoseRuntimeExecutable,
-} from "@hypit/runtime-host-node";
+} from "@hypit/runtime-local/node";
import { resolveLocalOpenCvDeployment } from "./deployment.js";
import { createLocalOpenCvImageProvider } from "./provider.js";
@@ -54,8 +54,8 @@ const localOpenCvRuntimeAdapter = createRuntimeEndpointAdapterFacet({
});
export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- hostFacets: [localOpenCvRuntimeAdapter],
+ format: "hypit.package@1" as const,
+ facets: [localOpenCvRuntimeAdapter],
};
export default hypitPackage;
diff --git a/packages/provider-image-opencv-local/src/deployment.ts b/packages/provider-image-opencv-local/src/deployment.ts
index e21fa6b9..9b9d7b88 100644
--- a/packages/provider-image-opencv-local/src/deployment.ts
+++ b/packages/provider-image-opencv-local/src/deployment.ts
@@ -1,21 +1,15 @@
import { existsSync } from "node:fs";
import { join } from "node:path";
+import { fileURLToPath } from "node:url";
-import { resolveNodePackageResource } from "@hypit/package-loader-node";
-import type { ManagedProgramCommand } from "@hypit/runtime-kit";
+import type { ManagedProgramCommand } from "@hypit/runtime-local/extension";
import {
pythonEnvironmentExecutable,
resolveRuntimeExecutable,
-} from "@hypit/runtime-host-node";
+} from "@hypit/runtime-local/node";
-/**
- * The locked Python project is its own package asset. Repository workspaces and
- * installed npm distributions therefore resolve the same immutable input.
- */
-export const localOpenCvManagedProject = join(
- resolveNodePackageResource("@hypit/image-opencv-runtime", "pyproject.toml", { from: import.meta.url }),
- "..",
-);
+/** The locked Python project is immutable material owned and shipped by this Provider. */
+export const localOpenCvManagedProject = fileURLToPath(new URL("../runtime", import.meta.url));
export type LocalOpenCvDeployment = {
readonly pythonExecutable: string;
diff --git a/packages/provider-image-opencv-local/src/program.ts b/packages/provider-image-opencv-local/src/program.ts
index ccae54a1..89c2940a 100644
--- a/packages/provider-image-opencv-local/src/program.ts
+++ b/packages/provider-image-opencv-local/src/program.ts
@@ -1,10 +1,10 @@
import { execFile } from "node:child_process";
-import type { ManagedProgram, ManagedProgramState } from "@hypit/runtime-kit";
+import type { ManagedProgram, ManagedProgramState } from "@hypit/runtime-local/extension";
import type { LocalOpenCvDeployment } from "./deployment.js";
-/** Kept equal to `services/image-opencv/pyproject.toml` by a test in this package. */
+/** Kept equal to this Provider's bundled `runtime/pyproject.toml` by a package test. */
const REQUIRED_MAJOR = { cv2: 4, numpy: 2 } as const;
const PROBE_PROGRAM =
diff --git a/packages/provider-image-opencv-local/src/provider.ts b/packages/provider-image-opencv-local/src/provider.ts
index 6e2344dc..5aa227e4 100644
--- a/packages/provider-image-opencv-local/src/provider.ts
+++ b/packages/provider-image-opencv-local/src/provider.ts
@@ -4,18 +4,18 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
-import { artifactTypes } from "@hypit/artifact";
-import { defineEndpointPackage } from "@hypit/endpoint-kit";
-import type { EndpointFulfillment } from "@hypit/endpoint-kit";
-import { canonicalize } from "@hypit/protocol";
-import type { CanonicalValue } from "@hypit/protocol";
+import { blobTypes } from "@hypit/hypit/blob";
+import { defineEndpoint } from "@hypit/hypit/endpoint";
+import type { EndpointFulfillment, EndpointInvocationContext } from "@hypit/hypit/endpoint";
+import type { CanonicalValue } from "@hypit/hypit/protocol";
import {
- assertRasterRequest,
- rasterCapabilities,
- rasterOutputMediaType,
- rasterSources,
-} from "@hypit/raster";
-import type { RasterRequest } from "@hypit/raster";
+ assertImageComposeRequest,
+ assertImageTransformRequest,
+ imageComposeSources,
+ imageOperationsCapabilities,
+ imageTransformOutputMediaType,
+} from "@hypit/image-operations";
+import type { ImageComposeRequest, ImageTransformRequest } from "@hypit/image-operations";
export const localOpenCvImageProviderModuleRef = {
name: "@hypit/provider-image-opencv-local",
@@ -41,10 +41,17 @@ function positiveInteger(value: number, subject: string): number {
return value;
}
-function request(value: CanonicalValue): RasterRequest {
- assert(value !== null && typeof value === "object" && !Array.isArray(value), "Raster request must be an object");
- const item = value as unknown as RasterRequest;
- assertRasterRequest(item);
+function transformRequest(value: CanonicalValue): ImageTransformRequest {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "Image transform request must be an object");
+ const item = value as unknown as ImageTransformRequest;
+ assertImageTransformRequest(item);
+ return item;
+}
+
+function composeRequest(value: CanonicalValue): ImageComposeRequest {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "Image compose request must be an object");
+ const item = value as unknown as ImageComposeRequest;
+ assertImageComposeRequest(item);
return item;
}
@@ -93,63 +100,82 @@ export function createLocalOpenCvImageProvider(config: CreateLocalOpenCvImagePro
const maxInputBytes = positiveInteger(config.maxInputBytes ?? 128 * 1024 * 1024, "maxInputBytes");
const maxOutputBytes = positiveInteger(config.maxOutputBytes ?? 256 * 1024 * 1024, "maxOutputBytes");
const script = fileURLToPath(new URL("../runtime/raster_execute.py", import.meta.url));
- return defineEndpointPackage({
- module: localOpenCvImageProviderModuleRef,
- facet: "raster",
+
+ const execute = async (
+ kind: "transform" | "compose",
+ need: ImageTransformRequest | ImageComposeRequest,
+ context: EndpointInvocationContext,
+ ): Promise<EndpointFulfillment> => {
+ const sources = kind === "transform"
+ ? [(need as ImageTransformRequest).source]
+ : imageComposeSources(need as ImageComposeRequest);
+ const uniqueSources = [...new Map(sources.map((source) => [source.resource, source])).values()];
+ const totalInputBytes = uniqueSources.reduce((sum, source) => sum + source.size, 0);
+ assert(totalInputBytes <= maxInputBytes, "Image operation inputs exceed their configured byte limit");
+ const work = await mkdtemp(join(tmpdir(), "hypit-image-opencv-"));
+ try {
+ const paths = new Map<string, string>();
+ for (const [index, source] of uniqueSources.entries()) {
+ const bytes = await context.resources.get(source.resource);
+ assert(bytes !== undefined, `Image source Artifact ${source.resource} is unavailable`);
+ assert(bytes.byteLength === source.size, "Image source size differs from its BlobRef");
+ const path = join(work, `source-${String(index + 1).padStart(4, "0")}.bin`);
+ await writeFile(path, bytes);
+ paths.set(source.resource, path);
+ }
+ const runtimeRequest = kind === "transform"
+ ? {
+ kind,
+ source: paths.get((need as ImageTransformRequest).source.resource),
+ operations: (need as ImageTransformRequest).operations,
+ }
+ : {
+ kind,
+ canvas: (need as ImageComposeRequest).canvas,
+ background: (need as ImageComposeRequest).background,
+ layers: (need as ImageComposeRequest).layers.map((layer) => ({
+ source: paths.get(layer.source.resource), frame: layer.frame, fit: layer.fit,
+ interpolation: layer.interpolation, opacity: layer.opacity,
+ })),
+ };
+ const program = join(work, "request.json");
+ const output = join(work, "output.bin");
+ await writeFile(program, JSON.stringify(runtimeRequest), "utf8");
+ await runProcess({
+ executable: pythonExecutable,
+ args: [script, program, output],
+ timeoutMs: processTimeoutMs,
+ maxStderrBytes: 256 * 1024,
+ });
+ const info = await stat(output);
+ assert(info.isFile() && info.size > 0 && info.size <= maxOutputBytes,
+ "OpenCV image execution produced an invalid output size");
+ const mediaType = kind === "transform"
+ ? imageTransformOutputMediaType(need as ImageTransformRequest)
+ : "image/png";
+ return { value: await context.resources.put(await readFile(output), mediaType) };
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+ };
+
+ return defineEndpoint({
instance: config.instance ?? "image.opencv.local",
pool: config.pool ?? config.instance ?? "image.opencv.local",
pricing: { kind: "local" },
defaultConcurrency: config.defaultConcurrency ?? 1,
capabilities: [{
lifecycle: "immediate" as const,
- capability: rasterCapabilities.execute,
- returns: artifactTypes.blob,
- handler: async (context): Promise<EndpointFulfillment> => {
- const need = request(context.need.constraints);
- const sources = [...new Map(rasterSources(need).map((source) => [source.resource, source])).values()];
- const totalInputBytes = sources.reduce((sum, source) => sum + source.size, 0);
- assert(totalInputBytes <= maxInputBytes, "Raster inputs exceed their configured byte limit");
- const work = await mkdtemp(join(tmpdir(), "hypit-raster-opencv-"));
- try {
- const paths = new Map<string, string>();
- for (const [index, source] of sources.entries()) {
- const bytes = await context.resources.get(source.resource);
- assert(bytes !== undefined, `Raster source Artifact ${source.resource} is unavailable`);
- assert(bytes.byteLength === source.size, "Raster source size differs from its BlobRef");
- const path = join(work, `source-${String(index + 1).padStart(4, "0")}.bin`);
- await writeFile(path, bytes);
- paths.set(source.resource, path);
- }
- const runtimeRequest = need.kind === "transform"
- ? { kind: "transform", source: paths.get(need.source.resource), operations: need.operations }
- : {
- kind: "compose", canvas: need.canvas, background: need.background,
- layers: need.layers.map((layer) => ({
- source: paths.get(layer.source.resource), frame: layer.frame, fit: layer.fit,
- interpolation: layer.interpolation, opacity: layer.opacity,
- })),
- };
- const program = join(work, "request.json");
- const output = join(work, "output.bin");
- await writeFile(program, JSON.stringify(runtimeRequest), "utf8");
- await runProcess({
- executable: pythonExecutable,
- args: [script, program, output],
- timeoutMs: processTimeoutMs,
- maxStderrBytes: 256 * 1024,
- });
- const info = await stat(output);
- assert(info.isFile() && info.size > 0 && info.size <= maxOutputBytes,
- "OpenCV raster execution produced an invalid output size");
- const mediaType = rasterOutputMediaType(need);
- const artifact = await context.resources.put(await readFile(output), mediaType);
- return {
- value: artifact,
- };
- } finally {
- await rm(work, { recursive: true, force: true }).catch(() => {});
- }
- },
+ capability: imageOperationsCapabilities.transform,
+ returns: blobTypes.blob,
+ handler: async (context): Promise<EndpointFulfillment> =>
+ await execute("transform", transformRequest(context.need.constraints), context),
+ }, {
+ lifecycle: "immediate" as const,
+ capability: imageOperationsCapabilities.compose,
+ returns: blobTypes.blob,
+ handler: async (context): Promise<EndpointFulfillment> =>
+ await execute("compose", composeRequest(context.need.constraints), context),
}],
});
}
diff --git a/packages/provider-image-opencv-local/test/provider.test.ts b/packages/provider-image-opencv-local/test/provider.test.ts
index 7d55fc2d..976f61be 100644
--- a/packages/provider-image-opencv-local/test/provider.test.ts
+++ b/packages/provider-image-opencv-local/test/provider.test.ts
@@ -3,34 +3,45 @@ import { spawnSync } from "node:child_process";
import { join, resolve } from "node:path";
import test from "node:test";
-import { artifactTypes } from "@hypit/artifact";
-import { EndpointRegistry, MemoryResourceStore } from "@hypit/driver-node";
-import {
- gptImageDenoiseV1,
- sealImageTransformProgram,
-} from "@hypit/image-transform";
-import type { ImageTransformProgram } from "@hypit/image-transform";
+import { blobTypes } from "@hypit/blob";
+import { EndpointRegistry, MemoryResourceStore } from "@hypit/executor";
+import { sealImageTransformProgram } from "@hypit/image-operations";
+import type { ImageTransformProgram } from "@hypit/image-operations";
import {
createLocalOpenCvImageProvider,
localOpenCvImageProviderModuleRef,
} from "@hypit/provider-image-opencv-local";
import { canonicalize } from "@hypit/protocol";
import type { BlobRef, Need } from "@hypit/protocol";
-import { rasterCapabilities } from "@hypit/raster";
+import { imageOperationsCapabilities } from "@hypit/image-operations";
import { resolveLocalOpenCvDeployment } from "../src/deployment.js";
import { localOpenCvProgram } from "../src/program.js";
-function need(source: BlobRef, program: ImageTransformProgram = gptImageDenoiseV1): Need {
+const denoiseProgram = sealImageTransformProgram({
+ operations: [{
+ kind: "denoise",
+ method: "nlm-ycrcb",
+ lumaStrength: 1,
+ chromaStrength: 3,
+ templateWindow: 7,
+ searchWindow: 21,
+ saturationRecovery: 1,
+ }, {
+ kind: "encode",
+ format: "png",
+ }],
+});
+
+function need(source: BlobRef, program: ImageTransformProgram = denoiseProgram): Need {
const constraints = canonicalize({
- kind: "transform",
source,
operations: program.operations,
});
return {
id: "need:image-transform",
- capability: rasterCapabilities.execute,
- returns: artifactTypes.blob,
+ capability: imageOperationsCapabilities.transform,
+ returns: blobTypes.blob,
constraints,
result: "record:image-transform",
};
@@ -38,7 +49,6 @@ function need(source: BlobRef, program: ImageTransformProgram = gptImageDenoiseV
function composeNeed(source: BlobRef): Need {
const constraints = canonicalize({
- kind: "compose",
canvas: {
widthPx: 3, heightPx: 2,
},
@@ -50,7 +60,7 @@ function composeNeed(source: BlobRef): Need {
}],
});
return {
- id: "need:image-compose", capability: rasterCapabilities.execute, returns: artifactTypes.blob,
+ id: "need:image-compose", capability: imageOperationsCapabilities.compose, returns: blobTypes.blob,
constraints, result: "record:image-compose",
};
}
@@ -59,8 +69,12 @@ test("the OpenCV package is one replaceable Endpoint with no second queue", asyn
const provider = createLocalOpenCvImageProvider({ defaultConcurrency: 3 });
assert.equal(provider.instance.id, "image.opencv.local");
assert.deepEqual(provider.offers, [{
- capability: rasterCapabilities.execute,
- returns: artifactTypes.blob,
+ capability: imageOperationsCapabilities.transform,
+ returns: blobTypes.blob,
+ endpoint: "image.opencv.local",
+ }, {
+ capability: imageOperationsCapabilities.compose,
+ returns: blobTypes.blob,
endpoint: "image.opencv.local",
}]);
});
@@ -94,7 +108,7 @@ const liveEnabled = process.env.HYPIT_OPENCV_TESTS === "1";
const openCvPython = process.env.HYPIT_OPENCV_PYTHON ?? "python3";
const hasOpenCv = spawnSync(openCvPython, ["-c", "import cv2, numpy"], { stdio: "ignore", windowsHide: true }).status === 0;
-test("the local Provider returns only a new image BlobArtifact", {
+test("the local Provider returns only a new image Blob", {
skip: !liveEnabled || !hasOpenCv,
}, async () => {
const resources = new MemoryResourceStore();
@@ -110,7 +124,7 @@ test("the local Provider returns only a new image BlobArtifact", {
height: 64,
fit: "stretch",
interpolation: "nearest",
- }, ...gptImageDenoiseV1.operations],
+ }, ...denoiseProgram.operations],
});
const request = need(source, program);
const registry = new EndpointRegistry();
diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.json
deleted file mode 100644
index 7256fd9f..00000000
--- a/packages/provider-media-local/package.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "@hypit/provider-media-local",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": {
- ".": "./src/index.ts"
- },
- "hypit": {
- "activation": "./src/activation.ts"
- },
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/endpoint-kit": "workspace:*",
- "@hypit/media": "workspace:*",
- "@hypit/media-execution": "workspace:*",
- "@hypit/media-pipeline": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/runtime": "workspace:*",
- "@hypit/runtime-kit": "workspace:*",
- "@hypit/runtime-host-node": "workspace:*",
- "@hypit/speech-evidence": "workspace:*"
- },
- "devDependencies": {
- "@hypit/driver-node": "workspace:*"
- }
-}
diff --git a/packages/provider-media-local/src/index.ts b/packages/provider-media-local/src/index.ts
deleted file mode 100644
index 00013723..00000000
--- a/packages/provider-media-local/src/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export { createLocalMediaProvider, localMediaProviderModuleRef } from "./provider.js";
-export type { CreateLocalMediaProviderOptions } from "./provider.js";
diff --git a/packages/raster/README.md b/packages/raster/README.md
deleted file mode 100644
index 1177a039..00000000
--- a/packages/raster/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# @hypit/raster
-
-The closed deterministic raster-execution waist. It owns one `execute-raster` capability and a
-finite `RasterRequest` union. Author packages such as Image Transform and Image Compose lower their
-different author meanings into this contract; Providers implement the pixel engine once.
-
-Raster contains no author Surface, queue, filesystem, OpenCV or remote placement policy.
diff --git a/packages/raster/package.json b/packages/raster/package.json
deleted file mode 100644
index a99dd88e..00000000
--- a/packages/raster/package.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "name": "@hypit/raster",
- "version": "0.0.0-dev",
- "license": "SEE LICENSE IN LICENSE",
- "private": true,
- "type": "module",
- "exports": { ".": "./src/index.ts" },
- "hypit": { "activation": "./src/activation.ts" },
- "dependencies": {
- "@hypit/artifact": "workspace:*",
- "@hypit/protocol": "workspace:*",
- "@hypit/spatial": "workspace:*"
- }
-}
diff --git a/packages/raster/src/activation.ts b/packages/raster/src/activation.ts
deleted file mode 100644
index 26372cf3..00000000
--- a/packages/raster/src/activation.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { rasterManifest } from "./index.js";
-
-export const hypitPackage = {
- format: "hypit.node-package@1" as const,
- modules: [{ manifest: rasterManifest }],
-};
-export default hypitPackage;
diff --git a/packages/raster/src/index.ts b/packages/raster/src/index.ts
deleted file mode 100644
index 6161ba0f..00000000
--- a/packages/raster/src/index.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-export { rasterCapabilities, rasterDependency, rasterManifest, rasterModuleRef } from "./manifest.js";
-export { rasterTransformOperationSchema } from "./schema.js";
-export {
- assertRasterRequest, assertRasterTransformOperations, rasterComposeRequest,
- rasterOutputMediaType, rasterSources, rasterTransformRequest,
-} from "./program.js";
-export type * from "./types.js";
diff --git a/packages/raster/src/manifest.ts b/packages/raster/src/manifest.ts
deleted file mode 100644
index 33a09e23..00000000
--- a/packages/raster/src/manifest.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-import { artifactDependency, artifactTypes } from "@hypit/artifact";
-import type { CapabilityRef, ModuleManifest } from "@hypit/protocol";
-
-export const rasterModuleRef = { name: "@hypit/raster", version: "1" } as const;
-export const rasterCapabilities = {
- execute: { module: rasterModuleRef, name: "execute-raster" },
-} satisfies Record<string, CapabilityRef>;
-export const rasterManifest: ModuleManifest = {
- format: "hypit.module@1", name: rasterModuleRef.name, version: rasterModuleRef.version,
- dependencies: [artifactDependency], types: [],
- capabilities: [{ name: rasterCapabilities.execute.name, returns: artifactTypes.blob }],
- producers: [],
-};
-export const rasterDependency = { module: rasterModuleRef } as const;
diff --git a/packages/raster/src/types.ts b/packages/raster/src/types.ts
deleted file mode 100644
index 70dfa434..00000000
--- a/packages/raster/src/types.ts
+++ /dev/null
@@ -1,61 +0,0 @@
-import type { BlobRef } from "@hypit/protocol";
-import type { Canvas, SpatialFrame } from "@hypit/spatial";
-
-export type RasterInterpolation = "nearest" | "linear" | "cubic" | "area" | "lanczos";
-export type RasterFit = "contain" | "cover" | "stretch";
-
-export type RasterCropOperation = {
- readonly kind: "crop"; readonly unit: "fraction" | "pixel";
- readonly x: number; readonly y: number; readonly width: number; readonly height: number;
-};
-export type RasterResizeOperation = {
- readonly kind: "resize"; readonly width: number; readonly height: number;
- readonly fit: RasterFit; readonly interpolation: RasterInterpolation; readonly background?: string;
-};
-export type RasterRotateOperation = { readonly kind: "rotate"; readonly degrees: 90 | 180 | 270 };
-export type RasterFlipOperation = { readonly kind: "flip"; readonly axis: "horizontal" | "vertical" | "both" };
-export type RasterDenoiseOperation = {
- readonly kind: "denoise"; readonly method: "nlm-ycrcb";
- readonly lumaStrength: number; readonly chromaStrength: number;
- readonly templateWindow: number; readonly searchWindow: number; readonly saturationRecovery: number;
-};
-export type RasterColorOperation = {
- readonly kind: "color"; readonly exposureStops: number; readonly contrast: number;
- readonly saturation: number; readonly temperature: number; readonly tint: number; readonly gamma: number;
-};
-export type RasterSharpenOperation = {
- readonly kind: "sharpen"; readonly amount: number; readonly radius: number; readonly threshold: number;
-};
-export type RasterBlurOperation = { readonly kind: "blur"; readonly sigma: number };
-export type RasterAlphaOperation = {
- readonly kind: "alpha"; readonly mode: "preserve" | "flatten"; readonly background?: string;
-};
-export type RasterEncodeOperation = {
- readonly kind: "encode"; readonly format: "png" | "jpeg" | "webp";
- readonly quality?: number; readonly background?: string;
-};
-export type RasterTransformOperation =
- | RasterCropOperation | RasterResizeOperation | RasterRotateOperation | RasterFlipOperation
- | RasterDenoiseOperation | RasterColorOperation | RasterSharpenOperation | RasterBlurOperation
- | RasterAlphaOperation | RasterEncodeOperation;
-
-export type RasterLayer = {
- readonly source: BlobRef;
- readonly frame: SpatialFrame;
- readonly fit: RasterFit;
- readonly interpolation: RasterInterpolation;
- readonly opacity: number;
-};
-
-export type RasterTransformRequest = {
- readonly kind: "transform";
- readonly source: BlobRef;
- readonly operations: readonly RasterTransformOperation[];
-};
-export type RasterComposeRequest = {
- readonly kind: "compose";
- readonly canvas: Canvas;
- readonly background: string;
- readonly layers: readonly RasterLayer[];
-};
-export type RasterRequest = RasterTransformRequest | RasterComposeRequest;
diff --git a/packages/raster/test/raster.test.ts b/packages/raster/test/raster.test.ts
deleted file mode 100644
index 3c1c4644..00000000
--- a/packages/raster/test/raster.test.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-import { fixtureResource } from "../../../test/fixture-resource.js";
-
-import {
- assertRasterRequest,
- rasterCapabilities,
- rasterComposeRequest,
- rasterOutputMediaType,
- rasterSources,
- rasterTransformRequest,
-} from "@hypit/raster";
-
-const image = { kind: "blob" as const, resource: fixtureResource("raster-image"), size: 12, mediaType: "image/png" };
-
-test("one exact Raster capability carries both closed deterministic request variants", () => {
- const transform = rasterTransformRequest(image, [{ kind: "encode", format: "webp", quality: 90 }]);
- const compose = rasterComposeRequest({
- canvas: {
- widthPx: 100, heightPx: 200,
- },
- background: "#00000000",
- layers: [{
- source: image,
- frame: { xPx: 0, yPx: 0, widthPx: 100, heightPx: 200 },
- fit: "contain", interpolation: "lanczos", opacity: 1,
- }],
- });
- assert.deepEqual(rasterCapabilities.execute.name, "execute-raster");
- assert.equal(transform.kind, "transform");
- assert.equal(compose.kind, "compose");
- assert.equal(rasterOutputMediaType(transform), "image/webp");
- assert.equal(rasterOutputMediaType(compose), "image/png");
- assert.deepEqual(rasterSources(transform), [image]);
- assert.deepEqual(rasterSources(compose), [image]);
-});
-
-test("Raster refuses unknown execution meaning rather than letting a Provider guess", () => {
- assert.throws(() => assertRasterRequest({
- kind: "magic", source: image,
- } as never), /kind is invalid/u);
-});
diff --git a/services/image-opencv/package.json b/services/image-opencv/package.json
deleted file mode 100644
index 49feda65..00000000
--- a/services/image-opencv/package.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "name": "@hypit/image-opencv-runtime",
- "version": "0.1.0",
- "license": "SEE LICENSE IN LICENSE",
- "type": "module",
- "exports": {
- "./pyproject.toml": "./pyproject.toml"
- },
- "files": [
- "pyproject.toml",
- "uv.lock",
- "README.md"
- ]
-}