Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat: add embeddable HTML document output (#387)
Sharky Wang398ce9c6d14b
diff --git a/package.json b/package.jsonindex b141c370..23c4f27a 100644--- a/package.json+++ b/package.json@@ -258,6 +258,7 @@"@hypit/gpt-image": "workspace:*","@hypit/gpt-image-kits": "workspace:*","@hypit/grok-imagine": "workspace:*",+ "@hypit/html-document": "workspace:*","@hypit/html-program": "workspace:*","@hypit/image-operations": "workspace:*","@hypit/interview-emoji-reveal": "workspace:*",diff --git a/packages/html-document/README.md b/packages/html-document/README.mdnew file mode 100644index 00000000..1bb6de6c--- /dev/null+++ b/packages/html-document/README.md@@ -0,0 +1,71 @@+# @hypit/html-document++Build a browser-playable document from a Composition and Timeline:++```xml+<import as="page" from="@hypit/html-document@1"/>+<page:Document id="preview" composition={main.composition} timeline={animation.timeline}/>+```++Select the document in the Run:++```xml+<?svml using="@hypit/markup/run@1"?>+<svrun version="1">+ <author source="./main.svml"/>+ <target output="preview.document"/>+</svrun>+```++The `HtmlDocument` result contains a compiled `HtmlProgram` and its audio plan.+The Result repository preserves referenced fonts, images, video and audio using+ordinary resource bindings. Upstream material requirements are handled by the Run.++Hosts turn the result into HTML by supplying material URLs:++```ts+import { materializeHtmlDocument } from "@hypit/html-document";++const html = materializeHtmlDocument(document, artifact => materialUrl(artifact), {+ autoplay: true,+ loop: true,+ muted: true,+ responsive: true,+});+```++The URL resolver can return relative paths, HTTP URLs or data URLs. It must+provide every resource referenced by the document. Autoplay, looping, muting+and responsive canvas sizing default to `true`.++The generated page displays the composition inside the iframe's viewport:++```html+<iframe title="Preview" src="./preview.html"></iframe>+```++Hosts can also assign the generated HTML to `iframe.srcdoc`. The host page+provides any playback controls and uses the document's JavaScript API.++The page exposes `window.__hypitDocument` with `ready`, `play()`, `pause()`,+`seekFrame(frame)`, `setMuted(value)`, and read-only `frame` and `playing` values.+`ready` resolves when the initial frame and its materials are prepared.+Frame indices are integers in `[0, document.program.frameCount)`.+Audible playback follows the browser's user-activation policy.++An embedding parent can send playback commands through `postMessage`:++```ts+iframe.contentWindow.postMessage({+ type: "hypit.html-document.command@1",+ action: "seek",+ frame: 90,+}, "*");+```++Actions are `play`, `pause`, `seek` with `frame`, `mute` with `muted`, and `state`.+The document accepts commands from its embedding parent and serializes them.+It sends `hypit.html-document.state@1` messages containing `frame`, `frameCount`,+`frameRate` and `playing`, at up to ten updates per second during playback and+immediately after state changes. Command errors use `hypit.html-document.error@1`.+The parent checks that `event.source === iframe.contentWindow` before using a message.diff --git a/packages/html-document/package.json b/packages/html-document/package.jsonnew file mode 100644index 00000000..b3e2319d--- /dev/null+++ b/packages/html-document/package.json@@ -0,0 +1,21 @@+{+ "name": "@hypit/html-document",+ "version": "0.1.0",+ "license": "SEE LICENSE IN LICENSE",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "files": ["dist", "src", "README.md"],+ "hypit": { "activation": "./src/activation.ts" },+ "dependencies": {+ "@hypit/composition": "workspace:^0.1.0",+ "@hypit/html-program": "workspace:^0.1.0",+ "@hypit/media-operations": "workspace:^0.2.0",+ "@hypit/temporal": "workspace:^0.1.0",+ "@hypit/timeline": "workspace:^0.1.0",+ "linkedom": "0.18.13"+ },+ "peerDependencies": { "@hypit/hypit": "workspace:^0.3.1" }+}diff --git a/packages/html-document/src/activation.ts b/packages/html-document/src/activation.tsnew file mode 100644index 00000000..9b3a249d--- /dev/null+++ b/packages/html-document/src/activation.ts@@ -0,0 +1,17 @@+import { createMarkupSurfaceFacet } from "@hypit/hypit/markup";+import { createProducerPackageFacet } from "@hypit/hypit/producer";+import { createAdmissionPackageFacet } from "@hypit/hypit/admission";+import { htmlDocumentManifest, htmlDocumentModuleRef, htmlDocumentMarkupSurfaces } from "./manifest.js";+import { htmlDocumentComponent } from "./component.js";+import { decodeHtmlDocumentSurface } from "./surface.js";++export const hypitPackage = {+ format: "hypit.package@1" as const,+ modules: [{ manifest: htmlDocumentManifest }],+ facets: [+ createMarkupSurfaceFacet({ module: htmlDocumentModuleRef, declaration: htmlDocumentMarkupSurfaces[0], handler: decodeHtmlDocumentSurface }),+ createProducerPackageFacet(htmlDocumentComponent),+ createAdmissionPackageFacet(htmlDocumentComponent),+ ],+};+export default hypitPackage;diff --git a/packages/html-document/src/component.ts b/packages/html-document/src/component.tsnew file mode 100644index 00000000..79e91633--- /dev/null+++ b/packages/html-document/src/component.ts@@ -0,0 +1,19 @@+import type { Composition } from "@hypit/composition";+import type { Timeline } from "@hypit/timeline";+import type { ProducerPackage } from "@hypit/hypit/producer";+import type { AdmissionPackage } from "@hypit/hypit/admission";+import { canonicalize } from "@hypit/hypit/protocol";+import { compileHtmlDocument, assertHtmlDocument } from "./document.js";+import { htmlDocumentProducers, htmlDocumentTypes } from "./manifest.js";++export const htmlDocumentComponent = {+ producers: [{ producer: htmlDocumentProducers.compile, handler: ({ inputs }) => {+ const composition = inputs.composition!.value, timeline = inputs.timeline!.value;+ if (composition.kind !== "inline" || timeline.kind !== "inline") throw new Error("HTML document requires inline Composition and Timeline");+ return { outputs: { document: { kind: "inline", value: canonicalize(compileHtmlDocument(composition.value as unknown as Composition, timeline.value as unknown as Timeline)) } }, needs: {} };+ } }],+ validators: [{ type: htmlDocumentTypes.document, handler({ value }) {+ if (value.kind !== "inline") throw new Error("HTML document must be inline");+ assertHtmlDocument(value.value);+ } }],+} satisfies ProducerPackage & AdmissionPackage;diff --git a/packages/html-document/src/document.ts b/packages/html-document/src/document.tsnew file mode 100644index 00000000..123a67fd--- /dev/null+++ b/packages/html-document/src/document.ts@@ -0,0 +1,170 @@+import { parseHTML } from "linkedom";+import type { Composition } from "@hypit/composition";+import { assertHtmlProgram, compileHtmlProgram, materializeHtmlProgram } from "@hypit/html-program";+import type { HtmlProgram, ResourceUrlResolver } from "@hypit/html-program";+import { compileAudioProgramPlan, verifyAudioProgramPlan } from "@hypit/media-operations";+import type { AudioProgramPlan } from "@hypit/media-operations";+import type { Timeline } from "@hypit/timeline";+import type { BlobRef } from "@hypit/hypit/protocol";+import { injectRuntimeShim } from "@hypit/html-program/playback";++export type HtmlDocument = {+ readonly format: "hypit.html-document@1";+ readonly program: HtmlProgram;+ readonly audio: AudioProgramPlan;+ readonly bindings: readonly { readonly id: string; readonly artifact: BlobRef }[];+};+export type HtmlDocumentPlaybackOptions = {+ readonly autoplay?: boolean;+ readonly loop?: boolean;+ readonly muted?: boolean;+ readonly responsive?: boolean;+};++export function compileHtmlDocument(composition: Composition, timeline: Timeline): HtmlDocument {+ const program = compileHtmlProgram(composition, timeline), audio = compileAudioProgramPlan(composition, timeline);+ const resources = new Map([...program.artifacts.map(entry => entry.artifact), ...audio.clips.map(clip => clip.artifact)].map(artifact => [artifact.resource, artifact]));+ return { format: "hypit.html-document@1", program, audio, bindings: [...resources].map(([id, artifact]) => ({ id, artifact })) };+}++// Result storage rebinds BlobRefs. Keep template-local identities independent of those addresses.+function boundProgram(document: HtmlDocument): HtmlProgram {+ if (!Array.isArray(document.bindings)) throw new Error("HTML document material bindings are required");+ const aliases = new Map(document.bindings.map(binding => [binding.artifact?.resource, binding.id]));+ const ids = new Set(document.bindings.map(binding => binding.id));+ if (ids.size !== document.bindings.length) throw new Error("HTML document repeats a material identity");+ const bind = (artifact: BlobRef): BlobRef => {+ const id = aliases.get(artifact.resource);+ if (id === undefined) throw new Error("HTML document material has no binding");+ return { ...artifact, resource: id as BlobRef["resource"] };+ };+ return { ...document.program,+ artifacts: document.program.artifacts.map(entry => ({ ...entry, artifact: bind(entry.artifact) })),+ surfaces: document.program.surfaces.map(surface => ({ ...surface, artifact: bind(surface.artifact) })),+ };+}++export function assertHtmlDocument(value: unknown): asserts value is HtmlDocument {+ if (value === null || typeof value !== "object" || (value as HtmlDocument).format !== "hypit.html-document@1") throw new Error("Unsupported HTML document format");+ const { program, audio } = value as HtmlDocument;+ assertHtmlProgram(boundProgram(value as HtmlDocument));+ verifyAudioProgramPlan(audio);+ if (program.frameCount !== audio.frameCount || program.frameRate.numerator !== audio.frameRate.numerator || program.frameRate.denominator !== audio.frameRate.denominator) throw new Error("HTML picture and sound must use the same frame clock");+}++const attribute = (value: string) => value.replaceAll("&", "&").replaceAll('"', """).replaceAll("<", "<").replaceAll(">", ">");++/** Materialize picture and sound using the host's URLs and shared frame playback. */+function renderDocumentProgram(program: HtmlProgram, audio: AudioProgramPlan, resolve: ResourceUrlResolver): { html: string; preview: string } {+ assertHtmlProgram(program);+ verifyAudioProgramPlan(audio);+ const html = materializeHtmlProgram(program, resolve);+ const clips = audio.clips.map(clip => `<audio class="hypit-studio-audio" preload="auto"+ src="${attribute(resolve(clip.artifact))}"+ data-start="${clip.targetStartSample / 48_000}"+ data-duration="${(clip.targetEndSampleExclusive - clip.targetStartSample) / 48_000}"+ data-media-start="${clip.sourceStartSample / 48_000}"+ data-media-end="${clip.sourceEndSampleExclusive / 48_000}"+ data-loop="${clip.sourceLoop}" data-phase="${clip.sourcePhaseSample / 48_000}"+ data-playback-rate="${clip.playbackRate}" data-gain="${clip.gain}"+ data-level-automation="${encodeURIComponent(JSON.stringify({ gainEnvelope: clip.gainEnvelope, audibility: clip.audibility, fadeInSamples: clip.fadeInSamples, fadeOutSamples: clip.fadeOutSamples, mixStartSample: clip.mixStartSample, mixEndSampleExclusive: clip.mixEndSampleExclusive }))}"></audio>`).join("");+ return { html, preview: injectRuntimeShim(html, clips) };+}++/** Produce a browser page. Hosts resolve material to HTTP, relative or embedded URLs. */+export function materializeHtmlDocument(document: HtmlDocument, resolve: ResourceUrlResolver, options: HtmlDocumentPlaybackOptions = {}): string {+ assertHtmlDocument(document);+ const program = boundProgram(document), audio = document.audio;+ const materials = new Map(document.bindings.map(binding => [binding.id, binding.artifact]));+ const rendered = renderDocumentProgram(program, audio, artifact => resolve(materials.get(artifact.resource) ?? artifact)).preview;+ const { document: page } = parseHTML(rendered);+ const scripts = Array.from(page.querySelectorAll("script")).map(script => {+ const source = script.textContent ?? "";+ script.remove();+ return source;+ });+ const html = page.toString();+ const settings = JSON.stringify({ autoplay: options.autoplay ?? true, loop: options.loop ?? true, muted: options.muted ?? true, responsive: options.responsive ?? true });+ const driver = `<script>+(() => {+ const options = ${settings}, count = ${program.frameCount}, fps = ${program.frameRate.numerator}/${program.frameRate.denominator};+ let frame = 0, playing = false, request, generation = 0, started, lastReport = 0;+ function report(force = false) {+ if (window.parent === window) return;+ const now = performance.now();+ if (!force && now - lastReport < 100) return;+ lastReport = now;+ window.parent.postMessage({type:'hypit.html-document.state@1',frame,frameCount:count,frameRate:fps,playing}, '*');+ }+ let readyResolve, readyReject;+ const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject; });+ function pause() {+ playing = false; generation++; cancelAnimationFrame(request);+ return ready.then(async () => { await window.__hypitSeekFrame(frame); report(true); });+ }+ async function play() {+ await ready;+ if (playing) return;+ playing = true; started = performance.now() - frame * 1000 / fps;+ report(true);+ const current = ++generation;+ async function tick(now) {+ if (!playing || generation !== current) return;+ const elapsed = Math.floor((now - started) * fps / 1000);+ frame = options.loop ? elapsed % count : Math.min(elapsed, count - 1);+ try { await window.__hypitPlayFrame(frame); }+ catch (error) { playing = false; window.dispatchEvent(new CustomEvent('hypit-playback-error', {detail: error})); return; }+ if (!playing || generation !== current) return;+ report();+ if (!options.loop && elapsed >= count - 1) { await pause(); return; }+ request = requestAnimationFrame(tick);+ }+ request = requestAnimationFrame(tick);+ }+ window.__hypitDocument = {+ ready, play, pause,+ get frame() { return frame; }, get playing() { return playing; },+ async seekFrame(value) {+ if (!Number.isSafeInteger(value) || value < 0 || value >= count) throw new RangeError('Frame outside HTML document');+ await pause(); frame = value; await window.__hypitSeekFrame(frame); report(true);+ },+ async setMuted(value) { await ready; await window.__hypitSetMuted(value); }+ };+ let commands = Promise.resolve();+ window.addEventListener('message', event => {+ if (event.source !== window.parent || window.parent === window || event.data?.type !== 'hypit.html-document.command@1') return;+ const {action,frame:target,muted} = event.data;+ commands = commands.then(async () => {+ await ready;+ if (action === 'play') await play();+ else if (action === 'pause') await pause();+ else if (action === 'seek') await window.__hypitDocument.seekFrame(target);+ else if (action === 'mute') await window.__hypitDocument.setMuted(muted);+ else if (action === 'state') report(true);+ }).catch(error => window.parent.postMessage({type:'hypit.html-document.error@1',message:String(error)}, '*'));+ });+ window.addEventListener('load', async () => {+ try {+ await document.fonts.ready;+ await new Promise(requestAnimationFrame);+ ${scripts.join("\n")}+ await window.__hypitFrameProgram.prepare();+ await window.__hypitFrameReady;+ await window.__hypitSetMuted(options.muted);+ if (options.responsive) {+ const root = document.querySelector('[data-hypit-program-root]');+ function resize() {+ const scale = Math.min(innerWidth/${program.canvas.width}, innerHeight/${program.canvas.height});+ Object.assign(root.style, {position:'absolute', transformOrigin:'top left', transform:'scale('+scale+')', left:((innerWidth-${program.canvas.width}*scale)/2)+'px', top:((innerHeight-${program.canvas.height}*scale)/2)+'px'});+ }+ new ResizeObserver(resize).observe(document.documentElement); resize();+ }+ readyResolve();+ report(true);+ if (options.autoplay) await play();+ } catch (error) { readyReject(error); }+ }, {once:true});+})();+</script>`;+ return html.replace("</body>", driver + "</body>");+}diff --git a/packages/html-document/src/fragment.ts b/packages/html-document/src/fragment.tsnew file mode 100644index 00000000..a7a28967--- /dev/null+++ b/packages/html-document/src/fragment.ts@@ -0,0 +1,14 @@+import { sealGraphFragment } from "@hypit/hypit/author";+import { compositionTypes } from "@hypit/composition";+import { timelineTypes } from "@hypit/timeline";+import { htmlDocumentProducers, htmlDocumentTypes } from "./manifest.js";++export const htmlDocumentFragment = sealGraphFragment({+ inputs: [{ name: "composition", type: compositionTypes.composition }, { name: "timeline", type: timelineTypes.timeline }],+ operations: [{+ id: "compile-document", producer: htmlDocumentProducers.compile,+ inputs: { composition: { kind: "fragment-input", name: "composition" }, timeline: { kind: "fragment-input", name: "timeline" } },+ result: { kind: "output", name: "document" },+ }],+ exports: [{ name: "document", type: htmlDocumentTypes.document, root: { kind: "fragment-operation", operation: "compile-document" } }],+});diff --git a/packages/html-document/src/index.ts b/packages/html-document/src/index.tsnew file mode 100644index 00000000..69bf2684--- /dev/null+++ b/packages/html-document/src/index.ts@@ -0,0 +1,6 @@+export { compileHtmlDocument, assertHtmlDocument, materializeHtmlDocument } from "./document.js";+export type { HtmlDocument, HtmlDocumentPlaybackOptions } from "./document.js";+export { htmlDocumentFragment } from "./fragment.js";+export { htmlDocumentComponent } from "./component.js";+export { decodeHtmlDocumentSurface } from "./surface.js";+export { htmlDocumentManifest, htmlDocumentModuleRef, htmlDocumentMarkupSurfaces, htmlDocumentTypes, htmlDocumentProducers, htmlDocumentSchema } from "./manifest.js";diff --git a/packages/html-document/src/manifest.ts b/packages/html-document/src/manifest.tsnew file mode 100644index 00000000..fdd1669d--- /dev/null+++ b/packages/html-document/src/manifest.ts@@ -0,0 +1,50 @@+import { compositionDependency, compositionTypes } from "@hypit/composition";+import { htmlProgramModuleRef, htmlProgramSchema } from "@hypit/html-program";+import { audioProgramPlanSchema, mediaOperationsModuleRef } from "@hypit/media-operations";+import { timelineDependency, timelineTypes } from "@hypit/timeline";+import type { ModuleManifest, ValueSchema } from "@hypit/hypit/protocol";++export const htmlDocumentModuleRef = { name: "@hypit/html-document", version: "1" } as const;+export const htmlDocumentTypes = { document: { module: htmlDocumentModuleRef, name: "HtmlDocument" } } as const;+export const htmlDocumentProducers = { compile: { module: htmlDocumentModuleRef, name: "compile-document" } } as const;+export const htmlDocumentSchema: ValueSchema = {+ kind: "object",+ fields: {+ format: { schema: { kind: "literal", value: "hypit.html-document@1" } },+ program: { schema: htmlProgramSchema },+ audio: { schema: audioProgramPlanSchema },+ bindings: { schema: { kind: "array", items: { kind: "object", fields: {+ id: { schema: { kind: "string", minLength: 1 } },+ artifact: { schema: { kind: "object", fields: {+ kind: { schema: { kind: "literal", value: "blob" } },+ resource: { schema: { kind: "string", minLength: 1 } },+ size: { schema: { kind: "number", integer: true, minimum: 0 } },+ mediaType: { schema: { kind: "string", minLength: 1 } },+ } } },+ } } } },+ },+};+export const htmlDocumentMarkupSurfaces = [{+ name: "document", tag: "Document", mode: "structured",+ outputs: [htmlDocumentTypes.document],+ vocabulary: {+ summary: "Produces a playable HTML document with its picture, sound and material references.",+ attributes: [+ { name: "id", kind: "identifier", required: true, summary: "Names the document output." },+ { name: "composition", kind: "reference", required: true, accepts: [compositionTypes.composition], summary: "Selects the picture and sound." },+ { name: "timeline", kind: "reference", required: true, accepts: [timelineTypes.timeline], summary: "Selects the frame clock and duration." },+ ],+ ports: [{ name: "document", type: htmlDocumentTypes.document, summary: "The document, addressed as <id>.document." }],+ example: '<page:Document id="preview" composition={main.composition} timeline={animation.timeline}/>',+ },+}] as const;+export const htmlDocumentManifest: ModuleManifest = {+ format: "hypit.module@1", ...htmlDocumentModuleRef,+ dependencies: [compositionDependency, timelineDependency, { module: htmlProgramModuleRef }, { module: mediaOperationsModuleRef }],+ types: [{ name: "HtmlDocument" }], capabilities: [],+ producers: [{+ name: htmlDocumentProducers.compile.name,+ inputs: [{ name: "composition", type: compositionTypes.composition }, { name: "timeline", type: timelineTypes.timeline }],+ outputs: [{ name: "document", type: htmlDocumentTypes.document }], needs: [],+ }],+};diff --git a/packages/html-document/src/surface.ts b/packages/html-document/src/surface.tsnew file mode 100644index 00000000..a9a41bf4--- /dev/null+++ b/packages/html-document/src/surface.ts@@ -0,0 +1,25 @@+import { compositionTypes } from "@hypit/composition";+import { resolveTemporalContext } from "@hypit/temporal/markup";+import { sameType } from "@hypit/hypit/protocol";+import type { StructuredSurfaceHandler } from "@hypit/hypit/markup";+import { htmlDocumentFragment } from "./fragment.js";++export const decodeHtmlDocumentSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {+ if (Object.keys(element.attributes).some(name => !["id", "composition", "timeline"].includes(name))) {+ throw new Error(`${element.name} contains unsupported attributes`);+ }+ if (element.children.some(child => child.kind === "element" || child.value.trim().length > 0)) {+ throw new Error(`${element.name} must be empty`);+ }+ const id = element.attributes.id;+ if (typeof id !== "string" || id.length === 0) throw new Error(`${element.name}.id is required`);+ const reference = element.attributes.composition;+ if (typeof reference !== "object" || reference.kind !== "reference") throw new Error(`${element.name}.composition must be a reference`);+ const composition = resolveReference(reference.path);+ if (composition === undefined || !sameType(composition.type, compositionTypes.composition)) throw new Error(`${element.name}.composition must reference Composition`);+ const context = resolveTemporalContext({ element, resolveReference });+ return {+ records: [], fragments: [htmlDocumentFragment],+ components: [{ id, fragment: htmlDocumentFragment.id, inputs: { composition: composition.ref, timeline: context.timeline.ref }, outputs: { document: `${id}.document` }, range: element.range }],+ };+};diff --git a/packages/html-program/README.md b/packages/html-program/README.mdindex 856da6e8..724de26e 100644--- a/packages/html-program/README.md+++ b/packages/html-program/README.md@@ -68,3 +68,10 @@ fonts, images and frame listeners before pixels are accepted.There is no raw-HTML reverse compiler. A snapshot or Provider receives the typed`HtmlProgram` produced by compilation; generated HTML is not treated as a secondsource language.++## Browser playback++Hosts import `injectRuntimeShim` from `@hypit/html-program/playback` to add+frame playback to materialized HTML. The script provides `__hypitSeekFrame`,+`__hypitPlayFrame` and `__hypitSetMuted`; the host supplies playback controls+and serves the referenced materials.diff --git a/packages/html-program/package.json b/packages/html-program/package.jsonindex e224940f..b043f0d2 100644--- a/packages/html-program/package.json+++ b/packages/html-program/package.json@@ -5,7 +5,8 @@"private": true,"type": "module","exports": {- ".": "./src/index.ts"+ ".": "./src/index.ts",+ "./playback": "./src/playback.ts"},"hypit": {"activation": "./src/activation.ts"diff --git a/packages/html-program/src/playback.ts b/packages/html-program/src/playback.tsnew file mode 100644index 00000000..4031f13c--- /dev/null+++ b/packages/html-program/src/playback.ts@@ -0,0 +1,229 @@+import { audioEnvelopeGainAt } from "@hypit/composition";++/**+ * Shared frame driver for a compiled HTML renderer document.+ *+ * HTML renderer remains the owner of layout and animation. The host supplies+ * an absolute Program frame, places every timed Present/material at that+ * instant, and waits until a discrete media seek is actually decoded.+ */+function shim(): string {+ return String.raw`+<script>+(function () {+ var muted = false;+ var root = document.querySelector('[data-hypit-program-root]');+ var numerator = Number(root && root.getAttribute('data-hypit-frame-numerator')) || 30;+ var denominator = Number(root && root.getAttribute('data-hypit-frame-denominator')) || 1;+ var fps = numerator / denominator;+ var frameSeconds = 1 / fps;+ var currentSeconds = 0;+ var playing = false;+ var seekRevision = 0;+ if (root) {+ root.style.width = (root.getAttribute('data-hypit-width') || '0') + 'px';+ root.style.height = (root.getAttribute('data-hypit-height') || '0') + 'px';+ }++ var visualClips = [];+ for (var clip of document.querySelectorAll('.hypit-visual-present')) {+ visualClips.push({+ element: clip,+ startFrame: Number(clip.getAttribute('data-hypit-present-start-frame')),+ endFrame: Number(clip.getAttribute('data-hypit-present-end-frame'))+ });+ }+ // Sampling runs are separate video elements with their own absolute spans.+ // Addressing only their containing Present makes every run paint at once.+ var media = [];+ for (var element of document.querySelectorAll('.hypit-visual-present video')) {+ var present = element.closest('.hypit-visual-present');+ var ownStart = element.getAttribute('data-start');+ var ownDuration = element.getAttribute('data-duration');+ media.push({+ element: element,+ startFrame: Number(element.getAttribute('data-hypit-start-frame')),+ endFrame: Number(element.getAttribute('data-hypit-end-frame')),+ sourceFrame: element.getAttribute('data-hypit-source-frame').split('/').map(BigInt),+ sourceRate: element.getAttribute('data-hypit-source-rate').split('/').map(BigInt),+ sourceFps: element.getAttribute('data-hypit-source-fps').split('/').map(Number)+ });+ }+ var audioContext;+ var programAudio = [];+ for (var element of document.querySelectorAll('.hypit-studio-audio')) {+ programAudio.push({+ element: element,+ start: parseFloat(element.getAttribute('data-start') || '0') || 0,+ duration: parseFloat(element.getAttribute('data-duration') || '0') || 0,+ mediaStart: parseFloat(element.getAttribute('data-media-start') || '0') || 0,+ mediaEnd: parseFloat(element.getAttribute('data-media-end') || '0') || 0,+ loop: element.getAttribute('data-loop') === 'true',+ phase: parseFloat(element.getAttribute('data-phase') || '0') || 0,+ rate: parseFloat(element.getAttribute('data-playback-rate') || '1') || 1,+ gain: Number(element.getAttribute('data-gain') ?? '1'),+ levelAutomation: JSON.parse(decodeURIComponent(element.getAttribute('data-level-automation') || '%7B%7D'))+ });+ }++ function scheduleEnvelope(param, points, sample, now) {+ param.cancelScheduledValues(now);+ param.setValueAtTime(audioEnvelopeGainAt(points, sample), now);+ for (var point of points || []) {+ if (point.sample > sample) param.linearRampToValueAtTime(point.gain, now + (point.sample - sample) / 48000);+ }+ }++ // Each factor has its own GainNode so intersecting ramps multiply exactly.+ // These are the same absolute-sample envelopes consumed by the render mixer.+ function placeAudioGain(record, seconds) {+ if (!record.nodes) {+ var source = audioContext.createMediaElementSource(record.element);+ record.nodes = Array.from({ length: 5 }, function () { return audioContext.createGain(); });+ var previous = source;+ for (var node of record.nodes) { previous.connect(node); previous = node; }+ previous.connect(audioContext.destination);+ record.element.volume = 1;+ }+ var sample = seconds * 48000;+ var now = audioContext.currentTime;+ var p = record.levelAutomation;+ var start = p.mixStartSample ?? record.start * 48000;+ var end = p.mixEndSampleExclusive ?? (record.start + record.duration) * 48000;+ record.nodes[0].gain.setValueAtTime(record.gain, now);+ scheduleEnvelope(record.nodes[1].gain, p.gainEnvelope, sample, now);+ scheduleEnvelope(record.nodes[2].gain, p.fadeInSamples > 0+ ? [{ sample: start, gain: 0 }, { sample: start + p.fadeInSamples, gain: 1 }] : undefined, sample, now);+ scheduleEnvelope(record.nodes[3].gain, p.fadeOutSamples > 0+ ? [{ sample: end - p.fadeOutSamples, gain: 1 }, { sample: end, gain: 0 }] : undefined, sample, now);+ var gate = record.nodes[4].gain;+ gate.cancelScheduledValues(now);+ gate.setValueAtTime(p.audibility === undefined || p.audibility.some(function (span) {+ return sample >= span.startSample && sample < span.endSampleExclusive;+ }) ? 1 : 0, now);+ for (var span of p.audibility || []) {+ if (span.startSample > sample) gate.setValueAtTime(1, now + (span.startSample - sample) / 48000);+ if (span.endSampleExclusive > sample) gate.setValueAtTime(0, now + (span.endSampleExclusive - sample) / 48000);+ }+ }++ function seekDecoded(element, target) {+ if (Math.abs(element.currentTime - target) < 0.0005 && element.readyState >= 2) return Promise.resolve();+ return new Promise(function (resolve) {+ var done = function () {+ element.removeEventListener('seeked', done);+ element.removeEventListener('error', done);+ resolve();+ };+ element.addEventListener('seeked', done, { once: true });+ element.addEventListener('error', done, { once: true });+ try { element.currentTime = target; }+ catch (error) { done(); }+ });+ }++ function placeHTML(frame) {+ if (!window.__hypitFrameProgram) return Promise.reject(new Error('HTML Program page ABI is missing'));+ return window.__hypitFrameProgram.applyFrame(frame);+ }++ function apply(seconds, scrubbing) {+ currentSeconds = Math.max(0, seconds);+ var revision = ++seekRevision;+ var waits = [];+ var programFrame = Math.max(0, Math.round(currentSeconds * fps));+ for (var record of media) {+ var inside = programFrame >= record.startFrame && programFrame < record.endFrame;+ var element = record.element;+ element.style.visibility = inside ? 'visible' : 'hidden';+ // Normalized picture Artifacts are deliberately silent. Program sound+ // comes from the exact AudioTrack below rather than from a video sidecar.+ element.muted = true;+ if (!inside) { element.pause(); continue; }+ // A hold remains one compact target interval. Its exact zero source rate+ // cannot be assigned to HTMLMediaElement.playbackRate, so keep the native+ // decoder paused at the authored source frame instead.+ if (record.sourceRate[0] === 0n) {+ var heldFrame = Number(record.sourceFrame[0] / record.sourceFrame[1]);+ var heldTarget = (heldFrame + 0.5) * record.sourceFps[1] / record.sourceFps[0];+ if (Number.isFinite(element.duration) && element.duration > 0) {+ heldTarget = Math.min(heldTarget, Math.max(0, element.duration - 0.001));+ }+ element.pause();+ waits.push(seekDecoded(element, heldTarget));+ continue;+ }+ var a = record.sourceFrame, r = record.sourceRate;+ var sourceFrame = Number((a[0] * r[1] + BigInt(programFrame - record.startFrame) * r[0] * a[1]) / (a[1] * r[1]));+ var target = (sourceFrame + 0.5) * record.sourceFps[1] / record.sourceFps[0];+ if (Number.isFinite(element.duration) && element.duration > 0) {+ target = Math.min(target, Math.max(0, element.duration - 0.001));+ }+ if (scrubbing || record.sourceRate[0] <= 0n || programFrame === record.endFrame - 1) {+ element.pause();+ waits.push(seekDecoded(element, target));+ } else {+ // During continuous playback the decoder owns its clock. A frame event+ // only corrects visible drift; making every refresh a media seek is what+ // previously froze the preview between frames.+ if (Math.abs(element.currentTime - target) > 0.08) element.currentTime = target;+ element.playbackRate = Number(record.sourceRate[0]) / Number(record.sourceRate[1]) * fps * record.sourceFps[1] / record.sourceFps[0];+ if (element.paused) {+ var started = element.play();+ if (started) started.catch(function () {});+ }+ }+ }+ for (var record of programAudio) {+ var local = currentSeconds - record.start;+ var inside = local >= 0 && local < record.duration;+ var element = record.element;+ element.muted = muted || scrubbing;+ if (!inside) { element.pause(); continue; }+ var interval = Math.max(0, record.mediaEnd - record.mediaStart);+ var advanced = (local + frameSeconds / 2) * record.rate;+ var target = record.mediaStart + advanced;+ if (record.loop && interval > 0) target = record.mediaStart + ((record.phase + advanced) % interval);+ else target = Math.min(target, Math.max(record.mediaStart, record.mediaEnd - 0.000001));+ if (scrubbing || local >= record.duration - frameSeconds - 0.000001) {+ element.pause();+ waits.push(seekDecoded(element, target));+ } else {+ if (Math.abs(element.currentTime - target) > 0.08) element.currentTime = target;+ element.playbackRate = record.rate;+ placeAudioGain(record, currentSeconds + frameSeconds / 2);+ if (element.paused) {+ var started = element.play();+ if (started) started.catch(function () {});+ }+ }+ }+ waits.push(placeHTML(programFrame));+ return Promise.all(waits).then(function () { return revision === seekRevision; });+ }++ window.__hypitSetMuted = function (value) {+ muted = !!value;+ return apply(currentSeconds, !playing);+ };+ window.__hypitSeekFrame = function (frame) {+ playing = false;+ return apply(frame / fps, true);+ };+ window.__hypitPlayFrame = function (frame) {+ playing = true;+ if (programAudio.length && !audioContext) audioContext = new AudioContext();+ if (audioContext && audioContext.state === 'suspended') audioContext.resume();+ return apply(frame / fps, false);+ };+ window.__hypitFrameReady = apply(0, true);+ window.addEventListener('load', function () { window.__hypitFrameReady = apply(currentSeconds, true); });+})();+</script>`;+}++export function injectRuntimeShim(html: string, audio = ""): string {+ const script = audio + shim().replace("(function () {", `(function () {\n${audioEnvelopeGainAt.toString()}`);+ const at = html.lastIndexOf("</body>");+ return at < 0 ? html + script : html.slice(0, at) + script + html.slice(at);+}diff --git a/packages/studio/src/preview/runtime-shim.ts b/packages/studio/src/preview/runtime-shim.tsindex 12107a80..370a73e3 100644--- a/packages/studio/src/preview/runtime-shim.ts+++ b/packages/studio/src/preview/runtime-shim.ts@@ -1,229 +1 @@-import { audioEnvelopeGainAt } from "@hypit/composition";--/**- * Studio's frame driver for a compiled HTML renderer document.- *- * HTML renderer remains the owner of layout and animation. Studio only supplies- * an absolute Program frame, places every timed Present/material at that- * instant, and waits until a discrete media seek is actually decoded.- */-function shim(): string {- return String.raw`-<script>-(function () {- var muted = false;- var root = document.querySelector('[data-hypit-program-root]');- var numerator = Number(root && root.getAttribute('data-hypit-frame-numerator')) || 30;- var denominator = Number(root && root.getAttribute('data-hypit-frame-denominator')) || 1;- var fps = numerator / denominator;- var frameSeconds = 1 / fps;- var currentSeconds = 0;- var playing = false;- var seekRevision = 0;- if (root) {- root.style.width = (root.getAttribute('data-hypit-width') || '0') + 'px';- root.style.height = (root.getAttribute('data-hypit-height') || '0') + 'px';- }-- var visualClips = [];- for (var clip of document.querySelectorAll('.hypit-visual-present')) {- visualClips.push({- element: clip,- startFrame: Number(clip.getAttribute('data-hypit-present-start-frame')),- endFrame: Number(clip.getAttribute('data-hypit-present-end-frame'))- });- }- // Sampling runs are separate video elements with their own absolute spans.- // Addressing only their containing Present makes every run paint at once.- var media = [];- for (var element of document.querySelectorAll('.hypit-visual-present video')) {- var present = element.closest('.hypit-visual-present');- var ownStart = element.getAttribute('data-start');- var ownDuration = element.getAttribute('data-duration');- media.push({- element: element,- startFrame: Number(element.getAttribute('data-hypit-start-frame')),- endFrame: Number(element.getAttribute('data-hypit-end-frame')),- sourceFrame: element.getAttribute('data-hypit-source-frame').split('/').map(BigInt),- sourceRate: element.getAttribute('data-hypit-source-rate').split('/').map(BigInt),- sourceFps: element.getAttribute('data-hypit-source-fps').split('/').map(Number)- });- }- var audioContext;- var programAudio = [];- for (var element of document.querySelectorAll('.hypit-studio-audio')) {- programAudio.push({- element: element,- start: parseFloat(element.getAttribute('data-start') || '0') || 0,- duration: parseFloat(element.getAttribute('data-duration') || '0') || 0,- mediaStart: parseFloat(element.getAttribute('data-media-start') || '0') || 0,- mediaEnd: parseFloat(element.getAttribute('data-media-end') || '0') || 0,- loop: element.getAttribute('data-loop') === 'true',- phase: parseFloat(element.getAttribute('data-phase') || '0') || 0,- rate: parseFloat(element.getAttribute('data-playback-rate') || '1') || 1,- gain: Number(element.getAttribute('data-gain') ?? '1'),- levelAutomation: JSON.parse(decodeURIComponent(element.getAttribute('data-level-automation') || '%7B%7D'))- });- }-- function scheduleEnvelope(param, points, sample, now) {- param.cancelScheduledValues(now);- param.setValueAtTime(audioEnvelopeGainAt(points, sample), now);- for (var point of points || []) {- if (point.sample > sample) param.linearRampToValueAtTime(point.gain, now + (point.sample - sample) / 48000);- }- }-- // Each factor has its own GainNode so intersecting ramps multiply exactly.- // These are the same absolute-sample envelopes consumed by the render mixer.- function placeAudioGain(record, seconds) {- if (!record.nodes) {- var source = audioContext.createMediaElementSource(record.element);- record.nodes = Array.from({ length: 5 }, function () { return audioContext.createGain(); });- var previous = source;- for (var node of record.nodes) { previous.connect(node); previous = node; }- previous.connect(audioContext.destination);- record.element.volume = 1;- }- var sample = seconds * 48000;- var now = audioContext.currentTime;- var p = record.levelAutomation;- var start = p.mixStartSample ?? record.start * 48000;- var end = p.mixEndSampleExclusive ?? (record.start + record.duration) * 48000;- record.nodes[0].gain.setValueAtTime(record.gain, now);- scheduleEnvelope(record.nodes[1].gain, p.gainEnvelope, sample, now);- scheduleEnvelope(record.nodes[2].gain, p.fadeInSamples > 0- ? [{ sample: start, gain: 0 }, { sample: start + p.fadeInSamples, gain: 1 }] : undefined, sample, now);- scheduleEnvelope(record.nodes[3].gain, p.fadeOutSamples > 0- ? [{ sample: end - p.fadeOutSamples, gain: 1 }, { sample: end, gain: 0 }] : undefined, sample, now);- var gate = record.nodes[4].gain;- gate.cancelScheduledValues(now);- gate.setValueAtTime(p.audibility === undefined || p.audibility.some(function (span) {- return sample >= span.startSample && sample < span.endSampleExclusive;- }) ? 1 : 0, now);- for (var span of p.audibility || []) {- if (span.startSample > sample) gate.setValueAtTime(1, now + (span.startSample - sample) / 48000);- if (span.endSampleExclusive > sample) gate.setValueAtTime(0, now + (span.endSampleExclusive - sample) / 48000);- }- }-- function seekDecoded(element, target) {- if (Math.abs(element.currentTime - target) < 0.0005 && element.readyState >= 2) return Promise.resolve();- return new Promise(function (resolve) {- var done = function () {- element.removeEventListener('seeked', done);- element.removeEventListener('error', done);- resolve();- };- element.addEventListener('seeked', done, { once: true });- element.addEventListener('error', done, { once: true });- try { element.currentTime = target; }- catch (error) { done(); }- });- }-- function placeHTML(frame) {- if (!window.__hypitFrameProgram) return Promise.reject(new Error('HTML Program page ABI is missing'));- return window.__hypitFrameProgram.applyFrame(frame);- }-- function apply(seconds, scrubbing) {- currentSeconds = Math.max(0, seconds);- var revision = ++seekRevision;- var waits = [];- var programFrame = Math.max(0, Math.round(currentSeconds * fps));- for (var record of media) {- var inside = programFrame >= record.startFrame && programFrame < record.endFrame;- var element = record.element;- element.style.visibility = inside ? 'visible' : 'hidden';- // Normalized picture Artifacts are deliberately silent. Program sound- // comes from the exact AudioTrack below rather than from a video sidecar.- element.muted = true;- if (!inside) { element.pause(); continue; }- // A hold remains one compact target interval. Its exact zero source rate- // cannot be assigned to HTMLMediaElement.playbackRate, so keep the native- // decoder paused at the authored source frame instead.- if (record.sourceRate[0] === 0n) {- var heldFrame = Number(record.sourceFrame[0] / record.sourceFrame[1]);- var heldTarget = (heldFrame + 0.5) * record.sourceFps[1] / record.sourceFps[0];- if (Number.isFinite(element.duration) && element.duration > 0) {- heldTarget = Math.min(heldTarget, Math.max(0, element.duration - 0.001));- }- element.pause();- waits.push(seekDecoded(element, heldTarget));- continue;- }- var a = record.sourceFrame, r = record.sourceRate;- var sourceFrame = Number((a[0] * r[1] + BigInt(programFrame - record.startFrame) * r[0] * a[1]) / (a[1] * r[1]));- var target = (sourceFrame + 0.5) * record.sourceFps[1] / record.sourceFps[0];- if (Number.isFinite(element.duration) && element.duration > 0) {- target = Math.min(target, Math.max(0, element.duration - 0.001));- }- if (scrubbing || record.sourceRate[0] <= 0n || programFrame === record.endFrame - 1) {- element.pause();- waits.push(seekDecoded(element, target));- } else {- // During continuous playback the decoder owns its clock. A frame event- // only corrects visible drift; making every refresh a media seek is what- // previously froze the preview between frames.- if (Math.abs(element.currentTime - target) > 0.08) element.currentTime = target;- element.playbackRate = Number(record.sourceRate[0]) / Number(record.sourceRate[1]) * fps * record.sourceFps[1] / record.sourceFps[0];- if (element.paused) {- var started = element.play();- if (started) started.catch(function () {});- }- }- }- for (var record of programAudio) {- var local = currentSeconds - record.start;- var inside = local >= 0 && local < record.duration;- var element = record.element;- element.muted = muted || scrubbing;- if (!inside) { element.pause(); continue; }- var interval = Math.max(0, record.mediaEnd - record.mediaStart);- var advanced = (local + frameSeconds / 2) * record.rate;- var target = record.mediaStart + advanced;- if (record.loop && interval > 0) target = record.mediaStart + ((record.phase + advanced) % interval);- else target = Math.min(target, Math.max(record.mediaStart, record.mediaEnd - 0.000001));- if (scrubbing || local >= record.duration - frameSeconds - 0.000001) {- element.pause();- waits.push(seekDecoded(element, target));- } else {- if (Math.abs(element.currentTime - target) > 0.08) element.currentTime = target;- element.playbackRate = record.rate;- placeAudioGain(record, currentSeconds + frameSeconds / 2);- if (element.paused) {- var started = element.play();- if (started) started.catch(function () {});- }- }- }- waits.push(placeHTML(programFrame));- return Promise.all(waits).then(function () { return revision === seekRevision; });- }-- window.__hypitSetMuted = function (value) {- muted = !!value;- return apply(currentSeconds, !playing);- };- window.__hypitSeekFrame = function (frame) {- playing = false;- return apply(frame / fps, true);- };- window.__hypitPlayFrame = function (frame) {- playing = true;- if (programAudio.length && !audioContext) audioContext = new AudioContext();- if (audioContext && audioContext.state === 'suspended') audioContext.resume();- return apply(frame / fps, false);- };- window.__hypitFrameReady = apply(0, true);- window.addEventListener('load', function () { window.__hypitFrameReady = apply(currentSeconds, true); });-})();-</script>`;-}--export function injectRuntimeShim(html: string, audio = ""): string {- const script = audio + shim().replace("(function () {", `(function () {\n${audioEnvelopeGainAt.toString()}`);- const at = html.lastIndexOf("</body>");- return at < 0 ? html + script : html.slice(0, at) + script + html.slice(at);-}+export { injectRuntimeShim } from "@hypit/html-program/playback";diff --git a/packages/video/package.json b/packages/video/package.jsonindex 268ffc80..4d5affef 100644--- a/packages/video/package.json+++ b/packages/video/package.json@@ -27,6 +27,7 @@"@hypit/gpt-image": "workspace:^0.2.0","@hypit/gpt-image-kits": "workspace:^0.1.1","@hypit/grok-imagine": "workspace:^0.2.0",+ "@hypit/html-document": "workspace:^0.1.0","@hypit/html-program": "workspace:^0.1.0","@hypit/html-video": "workspace:^0.2.0","@hypit/image-operations": "workspace:^0.2.0",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex f7206653..8abdd0f4 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -120,6 +120,9 @@ importers:'@hypit/grok-imagine':specifier: workspace:*version: link:packages/grok-imagine+ '@hypit/html-document':+ specifier: workspace:*+ version: link:packages/html-document'@hypit/html-program':specifier: workspace:*version: link:packages/html-program@@ -1269,6 +1272,30 @@ importers:specifier: workspace:*version: link:../text+ packages/html-document:+ dependencies:+ '@hypit/composition':+ specifier: workspace:^0.1.0+ version: link:../composition+ '@hypit/html-program':+ specifier: workspace:^0.1.0+ version: link:../html-program+ '@hypit/hypit':+ specifier: workspace:^0.3.1+ version: link:../..+ '@hypit/media-operations':+ specifier: workspace:^0.2.0+ version: link:../media-operations+ '@hypit/temporal':+ specifier: workspace:^0.1.0+ version: link:../temporal+ '@hypit/timeline':+ specifier: workspace:^0.1.0+ version: link:../timeline+ linkedom:+ specifier: 0.18.13+ version: 0.18.13+packages/html-program:dependencies:'@hypit/composition':@@ -2733,6 +2760,9 @@ importers:'@hypit/grok-imagine':specifier: workspace:^0.2.0version: link:../grok-imagine+ '@hypit/html-document':+ specifier: workspace:^0.1.0+ version: link:../html-document'@hypit/html-program':specifier: workspace:^0.1.0version: link:../html-program