Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(player): play video files with type="video/..." (#4673)
Miguel Ángel326bd0c9a198
diff --git a/docs/packages/player.mdx b/docs/packages/player.mdxindex a94a587f87..add5ce3d89 100644--- a/docs/packages/player.mdx+++ b/docs/packages/player.mdx@@ -47,7 +47,8 @@ Set `autoplay muted` only when playback should start without a user gesture.| Attribute | Type | Default | Description || --------------- | ------- | ------- | --------------------------------------------------------- |-| `src` | string | — | URL or relative path to composition HTML |+| `src` | string | — | URL or relative path to composition HTML, or to a video file with `type` |+| `type` | string | — | A `video/...` type (e.g. `video/mp4`) plays `src` as a video file || `srcdoc` | string | — | Composition HTML already available as a string || `width` | number | 1920 | Native composition width used for aspect ratio || `height` | number | 1080 | Native composition height used for aspect ratio |@@ -68,6 +69,10 @@ Player also accepts `shader-capture-scale` and `shader-loading` for previewingprojects that use shader transitions. These are preview controls, not compositionauthoring attributes.+With a `video/...` `type`, `src` plays in a `<video>` behind the same playback API and+events. `scenes`, `setRuntimeData()`, `setColorGrading()`, `iframeElement` and `stopMedia()`+are composition-only, and `srcdoc` wins over a video `src`, as it does in an iframe.+## JavaScript APIThe main API follows familiar media-player behavior:@@ -113,7 +118,8 @@ player.addEventListener("timeupdate", (event) => {| `ratechange` | — | Playback rate changed || `volumechange` | — | Volume or muted state changed || `scenes` | `{ scenes }` | The runtime reported its scene list |-| `error` | `{ message }` | Load or runtime error |+| `error` | `{ message }` | Load or runtime error; a video adds its `MediaError` `code` |+| `playbackerror` | `{ source, error }` | The browser blocked playback, for example unmuted autoplay |## Advanced: iframe accessdiff --git a/packages/player/README.md b/packages/player/README.mdindex 96659e39fb..a38e92605b 100644--- a/packages/player/README.md+++ b/packages/player/README.md@@ -38,6 +38,19 @@ import "@hyperframes/player";// Vue: <hyperframes-player :src="url" controls />```+### Video files++Set `type` to a video type and the player plays `src` in a `<video playsinline>` instead of loading it as a composition. The same API and events apply: `play()`, `pause()`, `seek()`, `currentTime`, `duration`, `ready`, `timeupdate`, `play`, `pause`, `ended`, `durationchange` and `resize`, with the video's own size as `compositionWidth`/`compositionHeight`. A video that fails to load fires `error` at once with `{ message, code }`, where `code` is the `MediaError` code. A `play()` the browser blocks (for example, unmuted autoplay) fires `playbackerror` with `{ source: "video" }` and leaves the player paused. `poster`, `controls`, `loop`, `muted`, `volume`, `playback-rate` and `autoplay` work as they do for a composition. `scenes`, `setRuntimeData()`, `setColorGrading()`, `iframeElement` and `stopMedia()` are composition-only: a video has no scenes or runtime, its iframe stays blank, and `stopMedia()` does not stop the video. `srcdoc` wins over a video `src`, as it does in an iframe.++```html+<hyperframes-player+ type="video/mp4"+ src="./render.mp4"+ poster="./poster.jpg"+ controls+></hyperframes-player>+```+### Poster imageShow a static image before playback starts:@@ -54,7 +67,8 @@ Show a static image before playback starts:| Attribute | Type | Default | Description || ----------------------- | ------------------------------- | ------------- | --------------------------------------------------------------------------- |-| `src` | string | — | URL to the composition HTML file |+| `src` | string | — | URL to the composition HTML file, or to a video file with `type` |+| `type` | string | — | A `video/...` type (e.g. `video/mp4`) plays `src` as a video file || `audio-src` | string | — | Audio URL for parent-frame playback (mobile) || `width` | number | 1920 | Composition width in pixels (aspect ratio) || `height` | number | 1080 | Composition height in pixels (aspect ratio) |@@ -236,7 +250,8 @@ function StudioPreview({ src }: { src: string }) {| `pause` | — | Playback paused || `timeupdate` | `{ currentTime }` | Playback position changed (~10 fps) || `ended` | — | Reached the end (when not looping) |-| `error` | `{ message }` | Composition failed to load |+| `error` | `{ message }` (video mode: `{ message, code }`) | Composition or video failed to load |+| `playbackerror` | `{ source, error }` | The browser blocked playback || `shadertransitionstate` | `{ compositionId, state }` | Shader transition cache/capture progress |```jsdiff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.tsindex 6fc57214df..481a1cb689 100644--- a/packages/player/src/hyperframes-player.test.ts+++ b/packages/player/src/hyperframes-player.test.ts@@ -2991,6 +2991,637 @@ describe("HyperframesPlayer composition dimension attributes", () => {});});+describe("HyperframesPlayer video mode", () => {+ type VideoPlayer = HTMLElement & {+ play: () => void;+ pause: () => void;+ seek: (t: number) => void;+ currentTime: number;+ duration: number;+ paused: boolean;+ ready: boolean;+ muted: boolean;+ volume: number;+ playbackRate: number;+ iframeElement: HTMLIFrameElement;+ _onIframeLoad: () => void;+ };++ const FILM = "https://cdn.example.com/film.mp4";+ let player: VideoPlayer;+ let playSpy: MockInstance<HTMLMediaElement["play"]>;+ let frames: FrameRequestCallback[];++ function createPlayer(attrs: Record<string, string>): VideoPlayer {+ const el = document.createElement("hyperframes-player") as VideoPlayer;+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);+ document.body.appendChild(el);+ return el;+ }++ function videoOf(el: VideoPlayer): HTMLVideoElement {+ const video = el.shadowRoot?.querySelector("video");+ if (!video) throw new Error("no <video> in the player");+ return video;+ }++ function setMedia(video: HTMLMediaElement, props: Record<string, unknown>) {+ for (const [name, value] of Object.entries(props)) {+ Object.defineProperty(video, name, { configurable: true, writable: true, value });+ }+ }++ function loadMetadata(video: HTMLVideoElement, duration = 6, width = 1080, height = 1920) {+ setMedia(video, { duration, videoWidth: width, videoHeight: height });+ video.dispatchEvent(new Event("durationchange"));+ video.dispatchEvent(new Event("loadedmetadata"));+ }++ function flushFrame() {+ const frame = frames.shift();+ if (!frame) throw new Error("no animation frame queued");+ frame(performance.now());+ }++ beforeEach(async () => {+ await import("./hyperframes-player.js");+ // Like a browser, play() and pause() flip `paused` at once; their events come later, if at all.+ playSpy = vi+ .spyOn(HTMLMediaElement.prototype, "play")+ .mockImplementation(function (this: HTMLMediaElement) {+ setMedia(this, { paused: false });+ return Promise.resolve();+ });+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(+ function (this: HTMLMediaElement) {+ setMedia(this, { paused: true });+ },+ );+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);+ frames = [];+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {+ frames.push(callback);+ return frames.length;+ });+ player = createPlayer({ type: "video/mp4", src: FILM });+ });++ afterEach(() => {+ document.body.innerHTML = "";+ vi.restoreAllMocks();+ });++ it("plays a video file in a <video playsinline>, not in the composition iframe", () => {+ const video = videoOf(player);++ expect(video.getAttribute("src")).toBe(FILM);+ expect(video.playsInline).toBe(true);+ expect(player.iframeElement.src).not.toContain("film.mp4");+ expect(player.iframeElement.hidden).toBe(true);+ });++ it("keeps a composition player free of any <video>", () => {+ const composition = createPlayer({ src: "https://cdn.example.com/index.html" });++ expect(composition.shadowRoot?.querySelector("video")).toBeNull();+ expect(composition.iframeElement.src).toContain("index.html");+ });++ it("fires ready with the video's duration and size", () => {+ const readies: unknown[] = [];+ player.addEventListener("ready", (event) => readies.push((event as CustomEvent).detail));++ loadMetadata(videoOf(player));++ expect(readies).toEqual([{ duration: 6, compositionWidth: 1080, compositionHeight: 1920 }]);+ expect(player.ready).toBe(true);+ });++ it("fires the video's first resize with the player already ready", () => {+ const seen: string[] = [];+ player.addEventListener("resize", () =>+ seen.push(`resize ready=${player.ready} d=${player.duration}`),+ );++ loadMetadata(videoOf(player));++ expect(seen).toEqual(["resize ready=true d=6"]);+ });++ it("stays ready when the hidden iframe finishes loading", () => {+ loadMetadata(videoOf(player));++ player._onIframeLoad();++ expect(player.ready).toBe(true);+ expect(player.duration).toBe(6);+ });++ it("drives the video with play, pause and seek", () => {+ const video = videoOf(player);+ loadMetadata(video);++ player.play();+ expect(playSpy).toHaveBeenCalledTimes(1);+ expect(player.paused).toBe(false);++ player.seek(2.5);+ expect(video.currentTime).toBe(2.5);+ expect(player.currentTime).toBe(2.5);+ expect(player.paused).toBe(true);+ expect(HTMLMediaElement.prototype.pause).toHaveBeenCalled();+ });++ it("reports time while playing and fires ended at the end", () => {+ const video = videoOf(player);+ loadMetadata(video);+ const times: number[] = [];+ let ended = 0;+ player.addEventListener("timeupdate", (event) =>+ times.push((event as CustomEvent<{ currentTime: number }>).detail.currentTime),+ );+ player.addEventListener("ended", () => ended++);++ player.play();+ setMedia(video, { currentTime: 6 });+ flushFrame();++ expect(times).toEqual([6]);+ expect(ended).toBe(1);+ expect(player.paused).toBe(true);+ });++ it("restarts at the end with loop instead of firing ended", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.setAttribute("loop", "");+ let ended = 0;+ player.addEventListener("ended", () => ended++);++ player.play();+ setMedia(video, { currentTime: 6 });+ flushFrame();++ expect(ended).toBe(0);+ expect(video.currentTime).toBe(0);+ expect(playSpy).toHaveBeenCalledTimes(2);+ expect(player.paused).toBe(false);+ });++ it("fires error at once with the video's error code", () => {+ const video = videoOf(player);+ const errors: unknown[] = [];+ player.addEventListener("error", (event) =>+ errors.push((event as unknown as CustomEvent).detail),+ );++ setMedia(video, { error: { code: 4, message: "Format not supported" } });+ video.dispatchEvent(new Event("error"));++ expect(errors).toEqual([{ message: "Format not supported", code: 4 }]);+ });++ it("fires durationchange and resize when the video changes after ready", () => {+ const video = videoOf(player);+ loadMetadata(video);+ const seen: string[] = [];+ player.addEventListener("durationchange", (event) =>+ seen.push(`duration ${(event as CustomEvent<{ duration: number }>).detail.duration}`),+ );+ player.addEventListener("resize", () => seen.push("resize"));++ setMedia(video, { duration: 8, videoWidth: 1920, videoHeight: 1080 });+ video.dispatchEvent(new Event("durationchange"));+ video.dispatchEvent(new Event("resize"));++ expect(seen).toEqual(["duration 8", "resize"]);+ });++ it("mirrors muted and volume onto the video", () => {+ const video = videoOf(player);++ player.muted = true;+ player.volume = 0.25;++ expect(video.muted).toBe(true);+ expect(video.volume).toBe(0.25);+ });++ it("plays on ready with autoplay", () => {+ const autoplay = createPlayer({ type: "video/mp4", src: FILM, autoplay: "", muted: "" });++ loadMetadata(videoOf(autoplay));++ expect(playSpy).toHaveBeenCalledTimes(1);+ expect(autoplay.paused).toBe(false);+ });++ it("reports a blocked play and lands paused", async () => {+ loadMetadata(videoOf(player));+ playSpy.mockRejectedValueOnce(new DOMException("blocked", "NotAllowedError"));+ const errors: unknown[] = [];+ player.addEventListener("playbackerror", (event) =>+ errors.push((event as CustomEvent<{ source: string }>).detail.source),+ );++ player.play();+ await Promise.resolve();+ await Promise.resolve();++ expect(errors).toEqual(["video"]);+ expect(player.paused).toBe(true);+ });++ it("ignores a play interrupted by pause", async () => {+ loadMetadata(videoOf(player));+ playSpy.mockRejectedValueOnce(new DOMException("interrupted", "AbortError"));+ const errors: unknown[] = [];+ player.addEventListener("playbackerror", (event) => errors.push(event));++ player.play();+ await Promise.resolve();+ await Promise.resolve();++ expect(errors).toEqual([]);+ });++ it("stops the composition probe when a composition player switches to video", () => {+ vi.useFakeTimers();+ try {+ const switching = createPlayer({ src: "https://cdn.example.com/index.html" });+ const errors: unknown[] = [];+ switching.addEventListener("error", (event) => errors.push(event));+ switching._onIframeLoad();++ switching.setAttribute("type", "video/mp4");+ vi.advanceTimersByTime(10_000);++ expect(errors).toEqual([]);+ } finally {+ vi.useRealTimers();+ }+ });++ it("keeps the playback rate across a new src", () => {+ player.playbackRate = 2;++ player.setAttribute("src", "https://cdn.example.com/other.mp4");++ expect(videoOf(player).defaultPlaybackRate).toBe(2);+ expect(videoOf(player).playbackRate).toBe(2);+ });++ it("leaves a playing video alone when a shader or sandbox option changes", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.seek(3);++ player.setAttribute("shader-capture-scale", "2");+ player.setAttribute("sandbox-origin", "");++ expect(player.currentTime).toBe(3);+ expect(player.ready).toBe(true);+ });++ it("follows a pause the page did not ask for, but not the one at the end", () => {+ const video = videoOf(player);+ loadMetadata(video);+ const pauses: Event[] = [];+ player.addEventListener("pause", (event) => pauses.push(event));++ player.play();+ setMedia(video, { paused: true, ended: true });+ video.dispatchEvent(new Event("pause"));+ expect(player.paused).toBe(false);++ setMedia(video, { ended: false });+ video.dispatchEvent(new Event("pause"));+ expect(player.paused).toBe(true);+ expect(pauses).toHaveLength(1);+ });++ it("ignores a pause event that lands after the player played again", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.play();+ player.seek(2);+ player.play();+ player.pause();+ player.play();+ const pauses: Event[] = [];+ player.addEventListener("pause", (event) => pauses.push(event));++ video.dispatchEvent(new Event("pause"));++ expect(player.paused).toBe(false);+ expect(pauses).toEqual([]);+ });++ it("follows a play the page did not ask for, but not a stale one", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.play();+ player.pause();+ video.dispatchEvent(new Event("play"));+ expect(player.paused).toBe(true);++ const plays: Event[] = [];+ player.addEventListener("play", (event) => plays.push(event));+ setMedia(video, { paused: false });+ video.dispatchEvent(new Event("play"));++ expect(player.paused).toBe(false);+ expect(plays).toHaveLength(1);+ });++ it("reads the paused frame from the video, not the last clock sample", () => {+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });+ const video = videoOf(withControls);+ loadMetadata(video);+ withControls.play();+ setMedia(video, { currentTime: 2.37 });++ withControls.pause();++ expect(withControls.currentTime).toBe(2.37);+ expect(withControls.shadowRoot?.textContent).toContain("0:02 / 0:06");+ });++ it("switches from a playing video to a composition paused", () => {+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });+ loadMetadata(videoOf(withControls));+ withControls.play();++ withControls.removeAttribute("type");++ expect(withControls.paused).toBe(true);+ expect(withControls.shadowRoot?.querySelector('[aria-label="Play"]')).not.toBeNull();+ });++ it("reports a load failure after play as error only", async () => {+ const video = videoOf(player);+ let reject: (error: unknown) => void = () => {};+ playSpy.mockImplementationOnce(() => new Promise((_resolve, fail) => (reject = fail)));+ const seen: string[] = [];+ for (const type of ["pause", "error", "playbackerror"]) {+ player.addEventListener(type, () => seen.push(type));+ }+ loadMetadata(video);+ player.play();++ video.dispatchEvent(new Event("error"));+ reject(new DOMException("no supported source", "NotSupportedError"));+ await Promise.resolve();+ await Promise.resolve();++ expect(seen).toEqual(["pause", "error"]);+ });++ it("reports a blocked play once when the page already paused", async () => {+ loadMetadata(videoOf(player));+ playSpy.mockRejectedValueOnce(new DOMException("blocked", "NotAllowedError"));+ const seen: string[] = [];+ for (const type of ["pause", "playbackerror"]) {+ player.addEventListener(type, () => seen.push(type));+ }++ player.play();+ player.pause();+ await Promise.resolve();+ await Promise.resolve();++ expect(seen).toEqual(["pause", "playbackerror"]);+ });++ it("ignores a blocked play that lands after the video was removed", async () => {+ let reject: (error: unknown) => void = () => {};+ playSpy.mockImplementationOnce(() => new Promise((_resolve, fail) => (reject = fail)));+ loadMetadata(videoOf(player));+ player.play();+ const seen: Event[] = [];+ player.addEventListener("playbackerror", (event) => seen.push(event));++ player.removeAttribute("type");+ reject(new DOMException("blocked", "NotAllowedError"));+ await Promise.resolve();+ await Promise.resolve();++ expect(seen).toEqual([]);+ });++ it("ignores a composition's late runtime message once it plays a video", () => {+ window.dispatchEvent(+ new MessageEvent("message", {+ source: player.iframeElement.contentWindow,+ data: { source: "hf-preview", type: "timeline", durationInFrames: 120, scenes: [] },+ }),+ );++ expect(player.ready).toBe(false);+ expect(player.duration).toBe(0);+ });++ it("gives srcdoc precedence over a video src", () => {+ loadMetadata(videoOf(player));+ player.play();++ player.setAttribute("srcdoc", "<p>composition</p>");++ expect(player.shadowRoot?.querySelector("video")).toBeNull();+ expect(player.iframeElement.hidden).toBe(false);+ expect(player.iframeElement.getAttribute("srcdoc")).toContain("composition");+ });++ it("plays the video src once srcdoc is removed", () => {+ const both = createPlayer({ type: "video/mp4", src: FILM, srcdoc: "<p>composition</p>" });+ expect(both.shadowRoot?.querySelector("video")).toBeNull();++ both.removeAttribute("srcdoc");+ loadMetadata(videoOf(both));+ both.play();++ expect(videoOf(both).getAttribute("src")).toBe(FILM);+ expect(playSpy).toHaveBeenCalledTimes(1);+ expect(both.paused).toBe(false);+ });++ it("reads the video type without regard to case or padding", () => {+ const upper = createPlayer({ type: " Video/MP4 ", src: FILM });++ expect(videoOf(upper).getAttribute("src")).toBe(FILM);+ });++ it("fires timeupdate with the exact frame before pause", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.play();+ setMedia(video, { currentTime: 2.37 });+ const seen: string[] = [];+ player.addEventListener("timeupdate", (event) =>+ seen.push(`timeupdate ${(event as CustomEvent<{ currentTime: number }>).detail.currentTime}`),+ );+ player.addEventListener("pause", () => seen.push("pause"));++ player.pause();++ expect(seen).toEqual(["timeupdate 2.37", "pause"]);+ });++ it("starts paused when srcdoc replaces a playing video", () => {+ loadMetadata(videoOf(player));+ player.play();++ player.setAttribute("srcdoc", "<p>composition</p>");++ expect(player.paused).toBe(true);+ });++ it("autoplays the video once srcdoc is removed from a playing composition", () => {+ const both = createPlayer({+ type: "video/mp4",+ src: FILM,+ srcdoc: "<p>composition</p>",+ autoplay: "",+ muted: "",+ });+ both.play();++ both.removeAttribute("srcdoc");+ loadMetadata(videoOf(both));++ expect(playSpy).toHaveBeenCalledTimes(1);+ expect(both.paused).toBe(false);+ });++ it("keeps a srcdoc composition when type changes under it", () => {+ const both = createPlayer({ type: "video/mp4", src: FILM, srcdoc: "<p>composition</p>" });+ window.dispatchEvent(+ new MessageEvent("message", {+ source: both.iframeElement.contentWindow,+ data: { source: "hf-preview", type: "timeline", durationInFrames: 120, scenes: [] },+ }),+ );+ expect(both.ready).toBe(true);++ both.removeAttribute("type");++ expect(both.ready).toBe(true);+ });++ it("shows a new src as paused at the start in the controls", () => {+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });+ const video = videoOf(withControls);+ loadMetadata(video);+ withControls.play();+ withControls.seek(3);+ withControls.play();+ expect(withControls.shadowRoot?.textContent).toContain("0:03 / 0:06");++ // As in a browser, the new source has no duration until its metadata loads.+ setMedia(video, { duration: Number.NaN });+ withControls.setAttribute("src", "https://cdn.example.com/other.mp4");++ const playButton = withControls.shadowRoot?.querySelector('[aria-label="Play"]');+ expect(playButton).not.toBeNull();+ expect(withControls.shadowRoot?.textContent).toContain("0:00 / 0:00");+ expect(withControls.duration).toBe(0);+ });++ it("stops playing when the video fails", () => {+ const video = videoOf(player);+ loadMetadata(video);+ player.play();++ video.dispatchEvent(new Event("error"));++ expect(player.paused).toBe(true);+ });++ it("removes the video when the player leaves the page and brings it back on return", () => {+ player.remove();+ expect(player.shadowRoot?.querySelector("video")).toBeNull();+ expect(player.iframeElement.hidden).toBe(false);++ document.body.appendChild(player);+ expect(videoOf(player).getAttribute("src")).toBe(FILM);+ });++ it("goes back to a composition when the video type is removed", () => {+ player.removeAttribute("type");++ expect(player.shadowRoot?.querySelector("video")).toBeNull();+ expect(player.iframeElement.hidden).toBe(false);+ expect(player.iframeElement.src).toContain("film.mp4");+ });+});++// Video mode must leave a player without a video type exactly as it was.+describe("HyperframesPlayer composition behaviour outside video mode", () => {+ type CompositionPlayer = HTMLElement & {+ play: () => void;+ pause: () => void;+ currentTime: number;+ paused: boolean;+ iframeElement: HTMLIFrameElement;+ _currentTime: number;+ _directTimelineAdapter: unknown;+ };+ const COMPOSITION = "composition.html";++ function createPlayer(attrs: Record<string, string>): CompositionPlayer {+ const el = document.createElement("hyperframes-player") as CompositionPlayer;+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);+ document.body.appendChild(el);+ return el;+ }++ beforeEach(async () => {+ await import("./hyperframes-player.js");+ });++ afterEach(() => {+ document.body.innerHTML = "";+ vi.restoreAllMocks();+ });++ it("does not reset paused when a composition src changes", () => {+ const player = createPlayer({ src: COMPOSITION });+ player.play();++ player.setAttribute("src", "other-composition.html");++ expect(player.paused).toBe(false);+ });++ it("keeps the last clock sample when a direct timeline pauses", () => {+ const player = createPlayer({ src: COMPOSITION });+ player._directTimelineAdapter = {+ duration: () => 6,+ time: () => 2.37,+ seek: () => {},+ play: () => {},+ pause: () => {},+ };+ player._currentTime = 1;+ const updates: Event[] = [];+ player.addEventListener("timeupdate", (event) => updates.push(event));++ player.pause();++ expect(player.currentTime).toBe(1);+ expect(updates).toEqual([]);+ });++ it("does not reload src when srcdoc is removed", () => {+ const player = createPlayer({ src: COMPOSITION, srcdoc: "<p>composition</p>" });+ player.iframeElement.setAttribute("src", "about:blank#kept");++ player.removeAttribute("srcdoc");++ expect(player.iframeElement.getAttribute("src")).toBe("about:blank#kept");+ });+});+describe("HyperframesPlayer click-to-play", () => {type ClickPlayer = HTMLElement & {play: () => void;diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.tsindex 078126e9c3..c0b0158ad3 100644--- a/packages/player/src/hyperframes-player.ts+++ b/packages/player/src/hyperframes-player.ts@@ -19,6 +19,7 @@ import { createShaderLoader } from "./shader-loader-element.js";import { ShaderLoaderState } from "./shader-loader-state.js";import { PLAYER_STYLES } from "./styles.js";import { type DirectTimelineAdapter } from "./timeline-adapters.js";+import { createVideoSource, isVideoType, type VideoSource } from "./video-source.js";import { runtimeProtocolMetadata } from "@hyperframes/core/runtime/protocol";import {FIRST_FRAME_READINESS_SCOPE,@@ -96,6 +97,7 @@ class HyperframesPlayer extends HTMLElement {"poster","playback-rate","audio-src",+ "type",SANDBOX_ORIGIN_ATTR,RUNTIME_SRC_ATTR,SHADER_CAPTURE_SCALE_ATTR,@@ -145,6 +147,7 @@ class HyperframesPlayer extends HTMLElement {private _runtimeDataRequestId = 0;private _pendingRuntimeData = new Map<string, PendingRuntimeDataDelivery>();private _afterUpdate: Array<() => void> | null = null;+ private _videoSource: VideoSource | null = null;constructor() {super();@@ -216,8 +219,7 @@ class HyperframesPlayer extends HTMLElement {if (this.hasAttribute("audio-src")) this._media.setupFromUrl(this.getAttribute("audio-src")!);if (this.hasAttribute("srcdoc"))this.iframe.srcdoc = prepareSrcdocForElement(this, this.getAttribute("srcdoc")!);- if (this.hasAttribute("src"))- this.iframe.src = prepareSrcForElement(this, this.getAttribute("src")!);+ if (this.hasAttribute("src")) this._loadSrc(this.getAttribute("src")!);// Host-environment audio lock: when the embedding host (e.g. Claude// desktop) drops the `audio-locked` attribute, attributeChangedCallback@@ -229,6 +231,7 @@ class HyperframesPlayer extends HTMLElement {disconnectedCallback() {this._connected = false;+ this._teardownVideo();this._sendControl("pause");this._stopIframeMedia();this.resizeObserver.disconnect();@@ -255,19 +258,27 @@ class HyperframesPlayer extends HTMLElement {// message fire before connectedCallback installs the parent message listener. Initial// attributes are applied below by connectedCallback; only live changes navigate here.if (!this.isConnected) break;- if (val) {- // A different composition: like a <video> given a new src, it does not inherit a queued play.- this._pendingPlay = false;- this._abandonComposition("Composition navigated before runtime data was applied");- this.iframe.src = prepareSrcForElement(this, val);- }+ if (val) this._navigateSrc(val);+ break;+ case "type": {+ const src = this.getAttribute("src");+ if (!this.isConnected || src === null || !!this._videoSource === this._wantsVideo()) break;+ this._navigateSrc(src);break;+ }case "srcdoc":if (!this.isConnected) break;this._pendingPlay = false;this._abandonComposition("Composition navigated before runtime data was applied");- if (val !== null) this.iframe.srcdoc = prepareSrcdocForElement(this, val);- else this.iframe.removeAttribute("srcdoc");+ if (val !== null) {+ this._pauseForVideoSwitch();+ this._teardownVideo();+ this.iframe.srcdoc = prepareSrcdocForElement(this, val);+ } else {+ this.iframe.removeAttribute("srcdoc");+ const src = this.getAttribute("src");+ if (src !== null && this._wantsVideo()) this._navigateSrc(src);+ }break;case SANDBOX_ORIGIN_ATTR:this._applySandboxOriginPolicy(this.isConnected && oldVal !== val);@@ -310,6 +321,7 @@ class HyperframesPlayer extends HTMLElement {case "volume": {const v = Math.max(0, Math.min(1, parseFloat(val || "1")));this._volume = v;+ this._syncVideoAudio();this._media.updateVolume(v);this._sendControl("set-volume", { volume: v });this.controlsApi?.updateVolume(v);@@ -345,6 +357,8 @@ class HyperframesPlayer extends HTMLElement {}private _reloadForSandboxOriginPolicy(): void {+ // A video does not play in the iframe, so neither reload applies to it.+ if (this._videoSource) return;this._abandonComposition("Sandbox policy changed before runtime data was applied");const srcdoc = this.getAttribute("srcdoc");if (srcdoc !== null) {@@ -352,7 +366,8 @@ class HyperframesPlayer extends HTMLElement {return;}const src = this.getAttribute("src");- this.iframe.src = src === null ? "about:blank" : prepareSrcForElement(this, src);+ if (src === null) this.iframe.src = "about:blank";+ else this._loadSrc(src);}/**@@ -419,6 +434,7 @@ class HyperframesPlayer extends HTMLElement {// after the user stopped playback.this._pendingPlay = false;if (!this._tryDirectTimelinePause()) this._sendControl("pause");+ else this._showPausedVideoTime();this._directTimelineClock.stop();this._stopParentTickClock();if (this._media.audioOwner === "parent") this._media.pauseAll();@@ -632,6 +648,7 @@ class HyperframesPlayer extends HTMLElement {return;}this._media.updateMuted(val !== null);+ this._syncVideoAudio();this._setIframeMediaMuted(val !== null);this._sendControl("set-muted", { muted: val !== null });this.controlsApi?.updateMuted(val !== null);@@ -868,6 +885,7 @@ class HyperframesPlayer extends HTMLElement {}private _reloadShaderOptions(): void {+ if (this._videoSource) return;// This navigates the frame, so readiness has to fall with it. Leaving// `_runtimeBridgeReady` true lets a delivery post into a document that is being// replaced, where it can only end in a delivery timeout rather than the immediate,@@ -877,12 +895,120 @@ class HyperframesPlayer extends HTMLElement {this.iframe.srcdoc = prepareSrcdocForElement(this, this.getAttribute("srcdoc") || "");return;}- if (this.hasAttribute("src")) {- this.iframe.src = prepareSrcForElement(this, this.getAttribute("src") || "");+ if (this.hasAttribute("src")) this._loadSrc(this.getAttribute("src") || "");+ }++ // A different source inherits no queued play.+ private _navigateSrc(src: string): void {+ this._pendingPlay = false;+ this._pauseForVideoSwitch();+ this._abandonComposition("Composition navigated before runtime data was applied");+ this._loadSrc(src);+ }++ /** A `type="video/..."` src plays in a `<video>`, unless `srcdoc` (which wins, as in an iframe)+ * shows a composition. */+ private _wantsVideo(): boolean {+ return isVideoType(this.getAttribute("type")) && !this.hasAttribute("srcdoc");+ }++ // Like a <video> given a new src, a switch into or out of a video starts paused.+ private _pauseForVideoSwitch(): void {+ if (!this._videoSource && !this._wantsVideo()) return;+ this._paused = true;+ this.controlsApi?.updatePlaying(false);+ }++ private _loadSrc(src: string): void {+ if (this._wantsVideo()) {+ this._loadVideo(src);+ return;}+ this._teardownVideo();+ this.iframe.src = prepareSrcForElement(this, src);+ }++ private _loadVideo(src: string): void {+ if (!this._videoSource) {+ this._videoSource = createVideoSource({+ onMetadata: (video) => this._applyThenEmit(() => this._onVideoReady(video)),+ onDurationChange: (video) => this._applyTimelineDuration(video.duration),+ onResize: (video) => this._applyVideoSize(video),+ // Follow a play or pause the page did not ask for, but not a queued one it since reversed.+ onPlay: (video) => {+ if (!video.paused && this._paused) this.play();+ },+ onPause: (video) => {+ if (video.paused && !video.ended && !this._paused) this.pause();+ },+ onError: (message, code) => {+ if (!this._paused) this.pause();+ this._emit(new CustomEvent("error", { detail: { message, code } }));+ },+ onPlayRejected: (error) => this._onVideoPlayRejected(error),+ });+ this.container.appendChild(this._videoSource.video);+ this.iframe.hidden = true;+ // Unloads a composition this player was showing before it switched to video.+ this.iframe.src = "about:blank";+ }+ this.probe.stop();+ const { video, adapter } = this._videoSource;+ this._directTimelineAdapter = adapter;+ this._currentTime = 0;+ this._duration = 0;+ this.controlsApi?.updateTime(0, 0);+ this._syncVideoAudio();+ adapter.timeScale?.(this.playbackRate);+ video.src = src;+ }++ private _teardownVideo(): void {+ if (!this._videoSource) return;+ this._videoSource.destroy();+ this._videoSource = null;+ this.iframe.hidden = false;+ }++ private _syncVideoAudio(): void {+ if (!this._videoSource) return;+ this._videoSource.video.muted = this.muted;+ this._videoSource.video.volume = this._volume;+ }++ private _onVideoReady(video: HTMLVideoElement): void {+ this._applyTimelineDuration(video.duration);+ this._applyVideoSize(video);+ this._ready = true;+ this._dispatchReady();+ // A video has no composition assets to wait on; this settles at once.+ this._waitForAssetsReady(null);+ this._afterEvents(() => {+ if (this.hasAttribute("autoplay") && this._paused) this.play();+ });+ }++ private _applyTimelineDuration(duration: number): void {+ if (!Number.isFinite(duration) || duration <= 0) return;+ this._setDuration(duration);+ this.controlsApi?.updateTime(this._currentTime, duration);+ }++ private _applyVideoSize(video: HTMLVideoElement): void {+ if (video.videoWidth > 0 && video.videoHeight > 0) {+ this._setCompositionSize(video.videoWidth, video.videoHeight);+ }+ }++ // Only a blocked play is a playbackerror: a failed load already fired `error`.+ private _onVideoPlayRejected(error: unknown): void {+ if (!(error instanceof DOMException && error.name === "NotAllowedError")) return;+ if (!this._paused) this.pause();+ this._emit(new CustomEvent("playbackerror", { detail: { source: "video", error } }));}private _trySyncSeek(timeInSeconds: number): boolean {+ if (this._videoSource) return false;try {const win = this.iframe.contentWindow as| (Window & { __player?: { seek?: (t: number) => void } })@@ -897,17 +1023,13 @@ class HyperframesPlayer extends HTMLElement {}private _withDirectTimeline(fn: (tl: DirectTimelineAdapter) => void): boolean {- const resolved = this.probe.resolveDirectTimelineAdapter();+ const resolved = this._videoSource ? null : this.probe.resolveDirectTimelineAdapter();const tl = resolved || this._directTimelineAdapter;if (!tl) return false;try {fn(tl);if (resolved && resolved !== this._directTimelineAdapter) {- const duration = resolved.duration();- if (Number.isFinite(duration) && duration > 0) {- this._setDuration(duration);- this.controlsApi?.updateTime(this._currentTime, duration);- }+ this._applyTimelineDuration(resolved.duration());}this._directTimelineAdapter = tl;return true;@@ -933,6 +1055,14 @@ class HyperframesPlayer extends HTMLElement {return this._withDirectTimeline((tl) => void tl.pause());}+ // The clock samples every ~100 ms; like a <video>, a pause reports the exact frame.+ private _showPausedVideoTime(): void {+ if (!this._videoSource) return;+ this._currentTime = this._videoSource.video.currentTime;+ this.controlsApi?.updateTime(this._currentTime, this._duration);+ this._emit(new CustomEvent("timeupdate", { detail: { currentTime: this._currentTime } }));+ }+/*** Widget-frame RAF loop that sends "tick" postMessages to the composition* iframe on every frame. Used for the runtime bridge path so that animation@@ -961,6 +1091,8 @@ class HyperframesPlayer extends HTMLElement {}private _onMessage(e: MessageEvent) {+ // The iframe window outlives its documents: a late composition message must not reach a video.+ if (this._videoSource) return;this._applyThenEmit(() => this._handleRuntimeMessage(e));}@@ -1281,7 +1413,8 @@ class HyperframesPlayer extends HTMLElement {}private _onIframeLoad() {- if (!this._connected) return;+ // In video mode the iframe only ever loads about:blank; its load must not reset the video.+ if (!this._connected || this._videoSource) return;// The runtime posts its timeline at DOMContentLoaded, before `load`, and every// host-initiated navigation clears `_ready` first. So a ready player already holds this// document's handshake (an opaque origin reads as null); a paused runtime never posts it again.diff --git a/packages/player/src/styles.ts b/packages/player/src/styles.tsindex 7ac23afa81..4f69773187 100644--- a/packages/player/src/styles.ts+++ b/packages/player/src/styles.ts@@ -30,6 +30,15 @@ export const PLAYER_STYLES = /* css */ `pointer-events: auto;}+ .hfp-video {+ position: absolute;+ inset: 0;+ width: 100%;+ height: 100%;+ object-fit: contain;+ pointer-events: none;+ }+.hfp-poster {position: absolute;inset: 0;diff --git a/packages/player/src/video-source.ts b/packages/player/src/video-source.tsnew file mode 100644index 0000000000..0d39ce844c--- /dev/null+++ b/packages/player/src/video-source.ts@@ -0,0 +1,76 @@+// Video mode: a `<video playsinline>` behind the direct-timeline adapter a same-origin+// `__timelines` composition uses, so play, seek, loop and `ended` share one code path.++import type { DirectTimelineAdapter } from "./timeline-adapters.js";++export function isVideoType(type: string | null): boolean {+ return type?.trim().toLowerCase().startsWith("video/") ?? false;+}++export interface VideoSourceCallbacks {+ onMetadata: (video: HTMLVideoElement) => void;+ onDurationChange: (video: HTMLVideoElement) => void;+ onResize: (video: HTMLVideoElement) => void;+ onPlay: (video: HTMLVideoElement) => void;+ onPause: (video: HTMLVideoElement) => void;+ onError: (message: string, code: number | null) => void;+ onPlayRejected: (error: unknown) => void;+}++export interface VideoSource {+ video: HTMLVideoElement;+ adapter: DirectTimelineAdapter;+ destroy: () => void;+}++export function createVideoSource(callbacks: VideoSourceCallbacks): VideoSource {+ const video = document.createElement("video");+ video.className = "hfp-video";+ video.playsInline = true;+ video.preload = "metadata";++ const listening = new AbortController();+ const { signal } = listening;+ video.addEventListener("loadedmetadata", () => callbacks.onMetadata(video), { signal });+ video.addEventListener("durationchange", () => callbacks.onDurationChange(video), { signal });+ video.addEventListener("resize", () => callbacks.onResize(video), { signal });+ video.addEventListener("play", () => callbacks.onPlay(video), { signal });+ video.addEventListener("pause", () => callbacks.onPause(video), { signal });+ video.addEventListener(+ "error",+ () =>+ callbacks.onError(video.error?.message || "Video failed to load", video.error?.code ?? null),+ { signal },+ );++ const adapter: DirectTimelineAdapter = {+ duration: () => video.duration,+ time: () => video.currentTime,+ seek: (timeInSeconds) => {+ video.currentTime = timeInSeconds;+ },+ // A rejection that lands after destroy() belongs to a source that is gone.+ play: () =>+ video.play().catch((error: unknown) => {+ if (!signal.aborted) callbacks.onPlayRejected(error);+ }),+ pause: () => video.pause(),+ // The default rate survives a new src; the current rate alone would reset to it.+ timeScale: (rate) => {+ video.defaultPlaybackRate = rate;+ video.playbackRate = rate;+ },+ };++ return {+ video,+ adapter,+ destroy: () => {+ listening.abort();+ video.pause();+ video.removeAttribute("src");+ video.load();+ video.remove();+ },+ };+}