Skip to content

Write HTML. Render video. Built for agents.

feat(player): play video files with type="video/..." (#4673)

Miguel Ángel326bd0c9a198
diff --git a/docs/packages/player.mdx b/docs/packages/player.mdx
index a94a587f87..add5ce3d89 100644
--- a/docs/packages/player.mdx
+++ b/docs/packages/player.mdx
@@ -47,7 +47,8 @@ Set `autoplay muted` only when playback should start without a user gesture.
| Attribute | Type | Default | Description |
| --------------- | ------- | ------- | --------------------------------------------------------- |
-| `src` | string | — | URL or relative path to composition HTML |
+| `src` | string | — | URL or relative path to composition HTML, or to a video file with `type` |
+| `type` | string | — | A `video/...` type (e.g. `video/mp4`) plays `src` as a video file |
| `srcdoc` | string | — | Composition HTML already available as a string |
| `width` | number | 1920 | Native composition width used for aspect ratio |
| `height` | number | 1080 | Native composition height used for aspect ratio |
@@ -68,6 +69,10 @@ Player also accepts `shader-capture-scale` and `shader-loading` for previewing
projects that use shader transitions. These are preview controls, not composition
authoring attributes.
+With a `video/...` `type`, `src` plays in a `<video>` behind the same playback API and
+events. `scenes`, `setRuntimeData()`, `setColorGrading()`, `iframeElement` and `stopMedia()`
+are composition-only, and `srcdoc` wins over a video `src`, as it does in an iframe.
+
## JavaScript API
The main API follows familiar media-player behavior:
@@ -113,7 +118,8 @@ player.addEventListener("timeupdate", (event) => {
| `ratechange` | — | Playback rate changed |
| `volumechange` | — | Volume or muted state changed |
| `scenes` | `{ scenes }` | The runtime reported its scene list |
-| `error` | `{ message }` | Load or runtime error |
+| `error` | `{ message }` | Load or runtime error; a video adds its `MediaError` `code` |
+| `playbackerror` | `{ source, error }` | The browser blocked playback, for example unmuted autoplay |
## Advanced: iframe access
diff --git a/packages/player/README.md b/packages/player/README.md
index 96659e39fb..a38e92605b 100644
--- a/packages/player/README.md
+++ b/packages/player/README.md
@@ -38,6 +38,19 @@ import "@hyperframes/player";
// Vue: <hyperframes-player :src="url" controls />
```
+### Video files
+
+Set `type` to a video type and the player plays `src` in a `<video playsinline>` instead of loading it as a composition. The same API and events apply: `play()`, `pause()`, `seek()`, `currentTime`, `duration`, `ready`, `timeupdate`, `play`, `pause`, `ended`, `durationchange` and `resize`, with the video's own size as `compositionWidth`/`compositionHeight`. A video that fails to load fires `error` at once with `{ message, code }`, where `code` is the `MediaError` code. A `play()` the browser blocks (for example, unmuted autoplay) fires `playbackerror` with `{ source: "video" }` and leaves the player paused. `poster`, `controls`, `loop`, `muted`, `volume`, `playback-rate` and `autoplay` work as they do for a composition. `scenes`, `setRuntimeData()`, `setColorGrading()`, `iframeElement` and `stopMedia()` are composition-only: a video has no scenes or runtime, its iframe stays blank, and `stopMedia()` does not stop the video. `srcdoc` wins over a video `src`, as it does in an iframe.
+
+```html
+<hyperframes-player
+ type="video/mp4"
+ src="./render.mp4"
+ poster="./poster.jpg"
+ controls
+></hyperframes-player>
+```
+
### Poster image
Show a static image before playback starts:
@@ -54,7 +67,8 @@ Show a static image before playback starts:
| Attribute | Type | Default | Description |
| ----------------------- | ------------------------------- | ------------- | --------------------------------------------------------------------------- |
-| `src` | string | — | URL to the composition HTML file |
+| `src` | string | — | URL to the composition HTML file, or to a video file with `type` |
+| `type` | string | — | A `video/...` type (e.g. `video/mp4`) plays `src` as a video file |
| `audio-src` | string | — | Audio URL for parent-frame playback (mobile) |
| `width` | number | 1920 | Composition width in pixels (aspect ratio) |
| `height` | number | 1080 | Composition height in pixels (aspect ratio) |
@@ -236,7 +250,8 @@ function StudioPreview({ src }: { src: string }) {
| `pause` | — | Playback paused |
| `timeupdate` | `{ currentTime }` | Playback position changed (~10 fps) |
| `ended` | — | Reached the end (when not looping) |
-| `error` | `{ message }` | Composition failed to load |
+| `error` | `{ message }` (video mode: `{ message, code }`) | Composition or video failed to load |
+| `playbackerror` | `{ source, error }` | The browser blocked playback |
| `shadertransitionstate` | `{ compositionId, state }` | Shader transition cache/capture progress |
```js
diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.ts
index 6fc57214df..481a1cb689 100644
--- a/packages/player/src/hyperframes-player.test.ts
+++ b/packages/player/src/hyperframes-player.test.ts
@@ -2991,6 +2991,637 @@ describe("HyperframesPlayer composition dimension attributes", () => {
});
});
+describe("HyperframesPlayer video mode", () => {
+ type VideoPlayer = HTMLElement & {
+ play: () => void;
+ pause: () => void;
+ seek: (t: number) => void;
+ currentTime: number;
+ duration: number;
+ paused: boolean;
+ ready: boolean;
+ muted: boolean;
+ volume: number;
+ playbackRate: number;
+ iframeElement: HTMLIFrameElement;
+ _onIframeLoad: () => void;
+ };
+
+ const FILM = "https://cdn.example.com/film.mp4";
+ let player: VideoPlayer;
+ let playSpy: MockInstance<HTMLMediaElement["play"]>;
+ let frames: FrameRequestCallback[];
+
+ function createPlayer(attrs: Record<string, string>): VideoPlayer {
+ const el = document.createElement("hyperframes-player") as VideoPlayer;
+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);
+ document.body.appendChild(el);
+ return el;
+ }
+
+ function videoOf(el: VideoPlayer): HTMLVideoElement {
+ const video = el.shadowRoot?.querySelector("video");
+ if (!video) throw new Error("no <video> in the player");
+ return video;
+ }
+
+ function setMedia(video: HTMLMediaElement, props: Record<string, unknown>) {
+ for (const [name, value] of Object.entries(props)) {
+ Object.defineProperty(video, name, { configurable: true, writable: true, value });
+ }
+ }
+
+ function loadMetadata(video: HTMLVideoElement, duration = 6, width = 1080, height = 1920) {
+ setMedia(video, { duration, videoWidth: width, videoHeight: height });
+ video.dispatchEvent(new Event("durationchange"));
+ video.dispatchEvent(new Event("loadedmetadata"));
+ }
+
+ function flushFrame() {
+ const frame = frames.shift();
+ if (!frame) throw new Error("no animation frame queued");
+ frame(performance.now());
+ }
+
+ beforeEach(async () => {
+ await import("./hyperframes-player.js");
+ // Like a browser, play() and pause() flip `paused` at once; their events come later, if at all.
+ playSpy = vi
+ .spyOn(HTMLMediaElement.prototype, "play")
+ .mockImplementation(function (this: HTMLMediaElement) {
+ setMedia(this, { paused: false });
+ return Promise.resolve();
+ });
+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(
+ function (this: HTMLMediaElement) {
+ setMedia(this, { paused: true });
+ },
+ );
+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);
+ frames = [];
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
+ frames.push(callback);
+ return frames.length;
+ });
+ player = createPlayer({ type: "video/mp4", src: FILM });
+ });
+
+ afterEach(() => {
+ document.body.innerHTML = "";
+ vi.restoreAllMocks();
+ });
+
+ it("plays a video file in a <video playsinline>, not in the composition iframe", () => {
+ const video = videoOf(player);
+
+ expect(video.getAttribute("src")).toBe(FILM);
+ expect(video.playsInline).toBe(true);
+ expect(player.iframeElement.src).not.toContain("film.mp4");
+ expect(player.iframeElement.hidden).toBe(true);
+ });
+
+ it("keeps a composition player free of any <video>", () => {
+ const composition = createPlayer({ src: "https://cdn.example.com/index.html" });
+
+ expect(composition.shadowRoot?.querySelector("video")).toBeNull();
+ expect(composition.iframeElement.src).toContain("index.html");
+ });
+
+ it("fires ready with the video's duration and size", () => {
+ const readies: unknown[] = [];
+ player.addEventListener("ready", (event) => readies.push((event as CustomEvent).detail));
+
+ loadMetadata(videoOf(player));
+
+ expect(readies).toEqual([{ duration: 6, compositionWidth: 1080, compositionHeight: 1920 }]);
+ expect(player.ready).toBe(true);
+ });
+
+ it("fires the video's first resize with the player already ready", () => {
+ const seen: string[] = [];
+ player.addEventListener("resize", () =>
+ seen.push(`resize ready=${player.ready} d=${player.duration}`),
+ );
+
+ loadMetadata(videoOf(player));
+
+ expect(seen).toEqual(["resize ready=true d=6"]);
+ });
+
+ it("stays ready when the hidden iframe finishes loading", () => {
+ loadMetadata(videoOf(player));
+
+ player._onIframeLoad();
+
+ expect(player.ready).toBe(true);
+ expect(player.duration).toBe(6);
+ });
+
+ it("drives the video with play, pause and seek", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+
+ player.play();
+ expect(playSpy).toHaveBeenCalledTimes(1);
+ expect(player.paused).toBe(false);
+
+ player.seek(2.5);
+ expect(video.currentTime).toBe(2.5);
+ expect(player.currentTime).toBe(2.5);
+ expect(player.paused).toBe(true);
+ expect(HTMLMediaElement.prototype.pause).toHaveBeenCalled();
+ });
+
+ it("reports time while playing and fires ended at the end", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ const times: number[] = [];
+ let ended = 0;
+ player.addEventListener("timeupdate", (event) =>
+ times.push((event as CustomEvent<{ currentTime: number }>).detail.currentTime),
+ );
+ player.addEventListener("ended", () => ended++);
+
+ player.play();
+ setMedia(video, { currentTime: 6 });
+ flushFrame();
+
+ expect(times).toEqual([6]);
+ expect(ended).toBe(1);
+ expect(player.paused).toBe(true);
+ });
+
+ it("restarts at the end with loop instead of firing ended", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.setAttribute("loop", "");
+ let ended = 0;
+ player.addEventListener("ended", () => ended++);
+
+ player.play();
+ setMedia(video, { currentTime: 6 });
+ flushFrame();
+
+ expect(ended).toBe(0);
+ expect(video.currentTime).toBe(0);
+ expect(playSpy).toHaveBeenCalledTimes(2);
+ expect(player.paused).toBe(false);
+ });
+
+ it("fires error at once with the video's error code", () => {
+ const video = videoOf(player);
+ const errors: unknown[] = [];
+ player.addEventListener("error", (event) =>
+ errors.push((event as unknown as CustomEvent).detail),
+ );
+
+ setMedia(video, { error: { code: 4, message: "Format not supported" } });
+ video.dispatchEvent(new Event("error"));
+
+ expect(errors).toEqual([{ message: "Format not supported", code: 4 }]);
+ });
+
+ it("fires durationchange and resize when the video changes after ready", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ const seen: string[] = [];
+ player.addEventListener("durationchange", (event) =>
+ seen.push(`duration ${(event as CustomEvent<{ duration: number }>).detail.duration}`),
+ );
+ player.addEventListener("resize", () => seen.push("resize"));
+
+ setMedia(video, { duration: 8, videoWidth: 1920, videoHeight: 1080 });
+ video.dispatchEvent(new Event("durationchange"));
+ video.dispatchEvent(new Event("resize"));
+
+ expect(seen).toEqual(["duration 8", "resize"]);
+ });
+
+ it("mirrors muted and volume onto the video", () => {
+ const video = videoOf(player);
+
+ player.muted = true;
+ player.volume = 0.25;
+
+ expect(video.muted).toBe(true);
+ expect(video.volume).toBe(0.25);
+ });
+
+ it("plays on ready with autoplay", () => {
+ const autoplay = createPlayer({ type: "video/mp4", src: FILM, autoplay: "", muted: "" });
+
+ loadMetadata(videoOf(autoplay));
+
+ expect(playSpy).toHaveBeenCalledTimes(1);
+ expect(autoplay.paused).toBe(false);
+ });
+
+ it("reports a blocked play and lands paused", async () => {
+ loadMetadata(videoOf(player));
+ playSpy.mockRejectedValueOnce(new DOMException("blocked", "NotAllowedError"));
+ const errors: unknown[] = [];
+ player.addEventListener("playbackerror", (event) =>
+ errors.push((event as CustomEvent<{ source: string }>).detail.source),
+ );
+
+ player.play();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(errors).toEqual(["video"]);
+ expect(player.paused).toBe(true);
+ });
+
+ it("ignores a play interrupted by pause", async () => {
+ loadMetadata(videoOf(player));
+ playSpy.mockRejectedValueOnce(new DOMException("interrupted", "AbortError"));
+ const errors: unknown[] = [];
+ player.addEventListener("playbackerror", (event) => errors.push(event));
+
+ player.play();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(errors).toEqual([]);
+ });
+
+ it("stops the composition probe when a composition player switches to video", () => {
+ vi.useFakeTimers();
+ try {
+ const switching = createPlayer({ src: "https://cdn.example.com/index.html" });
+ const errors: unknown[] = [];
+ switching.addEventListener("error", (event) => errors.push(event));
+ switching._onIframeLoad();
+
+ switching.setAttribute("type", "video/mp4");
+ vi.advanceTimersByTime(10_000);
+
+ expect(errors).toEqual([]);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("keeps the playback rate across a new src", () => {
+ player.playbackRate = 2;
+
+ player.setAttribute("src", "https://cdn.example.com/other.mp4");
+
+ expect(videoOf(player).defaultPlaybackRate).toBe(2);
+ expect(videoOf(player).playbackRate).toBe(2);
+ });
+
+ it("leaves a playing video alone when a shader or sandbox option changes", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.seek(3);
+
+ player.setAttribute("shader-capture-scale", "2");
+ player.setAttribute("sandbox-origin", "");
+
+ expect(player.currentTime).toBe(3);
+ expect(player.ready).toBe(true);
+ });
+
+ it("follows a pause the page did not ask for, but not the one at the end", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ const pauses: Event[] = [];
+ player.addEventListener("pause", (event) => pauses.push(event));
+
+ player.play();
+ setMedia(video, { paused: true, ended: true });
+ video.dispatchEvent(new Event("pause"));
+ expect(player.paused).toBe(false);
+
+ setMedia(video, { ended: false });
+ video.dispatchEvent(new Event("pause"));
+ expect(player.paused).toBe(true);
+ expect(pauses).toHaveLength(1);
+ });
+
+ it("ignores a pause event that lands after the player played again", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.play();
+ player.seek(2);
+ player.play();
+ player.pause();
+ player.play();
+ const pauses: Event[] = [];
+ player.addEventListener("pause", (event) => pauses.push(event));
+
+ video.dispatchEvent(new Event("pause"));
+
+ expect(player.paused).toBe(false);
+ expect(pauses).toEqual([]);
+ });
+
+ it("follows a play the page did not ask for, but not a stale one", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.play();
+ player.pause();
+ video.dispatchEvent(new Event("play"));
+ expect(player.paused).toBe(true);
+
+ const plays: Event[] = [];
+ player.addEventListener("play", (event) => plays.push(event));
+ setMedia(video, { paused: false });
+ video.dispatchEvent(new Event("play"));
+
+ expect(player.paused).toBe(false);
+ expect(plays).toHaveLength(1);
+ });
+
+ it("reads the paused frame from the video, not the last clock sample", () => {
+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });
+ const video = videoOf(withControls);
+ loadMetadata(video);
+ withControls.play();
+ setMedia(video, { currentTime: 2.37 });
+
+ withControls.pause();
+
+ expect(withControls.currentTime).toBe(2.37);
+ expect(withControls.shadowRoot?.textContent).toContain("0:02 / 0:06");
+ });
+
+ it("switches from a playing video to a composition paused", () => {
+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });
+ loadMetadata(videoOf(withControls));
+ withControls.play();
+
+ withControls.removeAttribute("type");
+
+ expect(withControls.paused).toBe(true);
+ expect(withControls.shadowRoot?.querySelector('[aria-label="Play"]')).not.toBeNull();
+ });
+
+ it("reports a load failure after play as error only", async () => {
+ const video = videoOf(player);
+ let reject: (error: unknown) => void = () => {};
+ playSpy.mockImplementationOnce(() => new Promise((_resolve, fail) => (reject = fail)));
+ const seen: string[] = [];
+ for (const type of ["pause", "error", "playbackerror"]) {
+ player.addEventListener(type, () => seen.push(type));
+ }
+ loadMetadata(video);
+ player.play();
+
+ video.dispatchEvent(new Event("error"));
+ reject(new DOMException("no supported source", "NotSupportedError"));
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(seen).toEqual(["pause", "error"]);
+ });
+
+ it("reports a blocked play once when the page already paused", async () => {
+ loadMetadata(videoOf(player));
+ playSpy.mockRejectedValueOnce(new DOMException("blocked", "NotAllowedError"));
+ const seen: string[] = [];
+ for (const type of ["pause", "playbackerror"]) {
+ player.addEventListener(type, () => seen.push(type));
+ }
+
+ player.play();
+ player.pause();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(seen).toEqual(["pause", "playbackerror"]);
+ });
+
+ it("ignores a blocked play that lands after the video was removed", async () => {
+ let reject: (error: unknown) => void = () => {};
+ playSpy.mockImplementationOnce(() => new Promise((_resolve, fail) => (reject = fail)));
+ loadMetadata(videoOf(player));
+ player.play();
+ const seen: Event[] = [];
+ player.addEventListener("playbackerror", (event) => seen.push(event));
+
+ player.removeAttribute("type");
+ reject(new DOMException("blocked", "NotAllowedError"));
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(seen).toEqual([]);
+ });
+
+ it("ignores a composition's late runtime message once it plays a video", () => {
+ window.dispatchEvent(
+ new MessageEvent("message", {
+ source: player.iframeElement.contentWindow,
+ data: { source: "hf-preview", type: "timeline", durationInFrames: 120, scenes: [] },
+ }),
+ );
+
+ expect(player.ready).toBe(false);
+ expect(player.duration).toBe(0);
+ });
+
+ it("gives srcdoc precedence over a video src", () => {
+ loadMetadata(videoOf(player));
+ player.play();
+
+ player.setAttribute("srcdoc", "<p>composition</p>");
+
+ expect(player.shadowRoot?.querySelector("video")).toBeNull();
+ expect(player.iframeElement.hidden).toBe(false);
+ expect(player.iframeElement.getAttribute("srcdoc")).toContain("composition");
+ });
+
+ it("plays the video src once srcdoc is removed", () => {
+ const both = createPlayer({ type: "video/mp4", src: FILM, srcdoc: "<p>composition</p>" });
+ expect(both.shadowRoot?.querySelector("video")).toBeNull();
+
+ both.removeAttribute("srcdoc");
+ loadMetadata(videoOf(both));
+ both.play();
+
+ expect(videoOf(both).getAttribute("src")).toBe(FILM);
+ expect(playSpy).toHaveBeenCalledTimes(1);
+ expect(both.paused).toBe(false);
+ });
+
+ it("reads the video type without regard to case or padding", () => {
+ const upper = createPlayer({ type: " Video/MP4 ", src: FILM });
+
+ expect(videoOf(upper).getAttribute("src")).toBe(FILM);
+ });
+
+ it("fires timeupdate with the exact frame before pause", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.play();
+ setMedia(video, { currentTime: 2.37 });
+ const seen: string[] = [];
+ player.addEventListener("timeupdate", (event) =>
+ seen.push(`timeupdate ${(event as CustomEvent<{ currentTime: number }>).detail.currentTime}`),
+ );
+ player.addEventListener("pause", () => seen.push("pause"));
+
+ player.pause();
+
+ expect(seen).toEqual(["timeupdate 2.37", "pause"]);
+ });
+
+ it("starts paused when srcdoc replaces a playing video", () => {
+ loadMetadata(videoOf(player));
+ player.play();
+
+ player.setAttribute("srcdoc", "<p>composition</p>");
+
+ expect(player.paused).toBe(true);
+ });
+
+ it("autoplays the video once srcdoc is removed from a playing composition", () => {
+ const both = createPlayer({
+ type: "video/mp4",
+ src: FILM,
+ srcdoc: "<p>composition</p>",
+ autoplay: "",
+ muted: "",
+ });
+ both.play();
+
+ both.removeAttribute("srcdoc");
+ loadMetadata(videoOf(both));
+
+ expect(playSpy).toHaveBeenCalledTimes(1);
+ expect(both.paused).toBe(false);
+ });
+
+ it("keeps a srcdoc composition when type changes under it", () => {
+ const both = createPlayer({ type: "video/mp4", src: FILM, srcdoc: "<p>composition</p>" });
+ window.dispatchEvent(
+ new MessageEvent("message", {
+ source: both.iframeElement.contentWindow,
+ data: { source: "hf-preview", type: "timeline", durationInFrames: 120, scenes: [] },
+ }),
+ );
+ expect(both.ready).toBe(true);
+
+ both.removeAttribute("type");
+
+ expect(both.ready).toBe(true);
+ });
+
+ it("shows a new src as paused at the start in the controls", () => {
+ const withControls = createPlayer({ type: "video/mp4", src: FILM, controls: "" });
+ const video = videoOf(withControls);
+ loadMetadata(video);
+ withControls.play();
+ withControls.seek(3);
+ withControls.play();
+ expect(withControls.shadowRoot?.textContent).toContain("0:03 / 0:06");
+
+ // As in a browser, the new source has no duration until its metadata loads.
+ setMedia(video, { duration: Number.NaN });
+ withControls.setAttribute("src", "https://cdn.example.com/other.mp4");
+
+ const playButton = withControls.shadowRoot?.querySelector('[aria-label="Play"]');
+ expect(playButton).not.toBeNull();
+ expect(withControls.shadowRoot?.textContent).toContain("0:00 / 0:00");
+ expect(withControls.duration).toBe(0);
+ });
+
+ it("stops playing when the video fails", () => {
+ const video = videoOf(player);
+ loadMetadata(video);
+ player.play();
+
+ video.dispatchEvent(new Event("error"));
+
+ expect(player.paused).toBe(true);
+ });
+
+ it("removes the video when the player leaves the page and brings it back on return", () => {
+ player.remove();
+ expect(player.shadowRoot?.querySelector("video")).toBeNull();
+ expect(player.iframeElement.hidden).toBe(false);
+
+ document.body.appendChild(player);
+ expect(videoOf(player).getAttribute("src")).toBe(FILM);
+ });
+
+ it("goes back to a composition when the video type is removed", () => {
+ player.removeAttribute("type");
+
+ expect(player.shadowRoot?.querySelector("video")).toBeNull();
+ expect(player.iframeElement.hidden).toBe(false);
+ expect(player.iframeElement.src).toContain("film.mp4");
+ });
+});
+
+// Video mode must leave a player without a video type exactly as it was.
+describe("HyperframesPlayer composition behaviour outside video mode", () => {
+ type CompositionPlayer = HTMLElement & {
+ play: () => void;
+ pause: () => void;
+ currentTime: number;
+ paused: boolean;
+ iframeElement: HTMLIFrameElement;
+ _currentTime: number;
+ _directTimelineAdapter: unknown;
+ };
+ const COMPOSITION = "composition.html";
+
+ function createPlayer(attrs: Record<string, string>): CompositionPlayer {
+ const el = document.createElement("hyperframes-player") as CompositionPlayer;
+ for (const [name, value] of Object.entries(attrs)) el.setAttribute(name, value);
+ document.body.appendChild(el);
+ return el;
+ }
+
+ beforeEach(async () => {
+ await import("./hyperframes-player.js");
+ });
+
+ afterEach(() => {
+ document.body.innerHTML = "";
+ vi.restoreAllMocks();
+ });
+
+ it("does not reset paused when a composition src changes", () => {
+ const player = createPlayer({ src: COMPOSITION });
+ player.play();
+
+ player.setAttribute("src", "other-composition.html");
+
+ expect(player.paused).toBe(false);
+ });
+
+ it("keeps the last clock sample when a direct timeline pauses", () => {
+ const player = createPlayer({ src: COMPOSITION });
+ player._directTimelineAdapter = {
+ duration: () => 6,
+ time: () => 2.37,
+ seek: () => {},
+ play: () => {},
+ pause: () => {},
+ };
+ player._currentTime = 1;
+ const updates: Event[] = [];
+ player.addEventListener("timeupdate", (event) => updates.push(event));
+
+ player.pause();
+
+ expect(player.currentTime).toBe(1);
+ expect(updates).toEqual([]);
+ });
+
+ it("does not reload src when srcdoc is removed", () => {
+ const player = createPlayer({ src: COMPOSITION, srcdoc: "<p>composition</p>" });
+ player.iframeElement.setAttribute("src", "about:blank#kept");
+
+ player.removeAttribute("srcdoc");
+
+ expect(player.iframeElement.getAttribute("src")).toBe("about:blank#kept");
+ });
+});
+
describe("HyperframesPlayer click-to-play", () => {
type ClickPlayer = HTMLElement & {
play: () => void;
diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.ts
index 078126e9c3..c0b0158ad3 100644
--- a/packages/player/src/hyperframes-player.ts
+++ b/packages/player/src/hyperframes-player.ts
@@ -19,6 +19,7 @@ import { createShaderLoader } from "./shader-loader-element.js";
import { ShaderLoaderState } from "./shader-loader-state.js";
import { PLAYER_STYLES } from "./styles.js";
import { type DirectTimelineAdapter } from "./timeline-adapters.js";
+import { createVideoSource, isVideoType, type VideoSource } from "./video-source.js";
import { runtimeProtocolMetadata } from "@hyperframes/core/runtime/protocol";
import {
FIRST_FRAME_READINESS_SCOPE,
@@ -96,6 +97,7 @@ class HyperframesPlayer extends HTMLElement {
"poster",
"playback-rate",
"audio-src",
+ "type",
SANDBOX_ORIGIN_ATTR,
RUNTIME_SRC_ATTR,
SHADER_CAPTURE_SCALE_ATTR,
@@ -145,6 +147,7 @@ class HyperframesPlayer extends HTMLElement {
private _runtimeDataRequestId = 0;
private _pendingRuntimeData = new Map<string, PendingRuntimeDataDelivery>();
private _afterUpdate: Array<() => void> | null = null;
+ private _videoSource: VideoSource | null = null;
constructor() {
super();
@@ -216,8 +219,7 @@ class HyperframesPlayer extends HTMLElement {
if (this.hasAttribute("audio-src")) this._media.setupFromUrl(this.getAttribute("audio-src")!);
if (this.hasAttribute("srcdoc"))
this.iframe.srcdoc = prepareSrcdocForElement(this, this.getAttribute("srcdoc")!);
- if (this.hasAttribute("src"))
- this.iframe.src = prepareSrcForElement(this, this.getAttribute("src")!);
+ if (this.hasAttribute("src")) this._loadSrc(this.getAttribute("src")!);
// Host-environment audio lock: when the embedding host (e.g. Claude
// desktop) drops the `audio-locked` attribute, attributeChangedCallback
@@ -229,6 +231,7 @@ class HyperframesPlayer extends HTMLElement {
disconnectedCallback() {
this._connected = false;
+ this._teardownVideo();
this._sendControl("pause");
this._stopIframeMedia();
this.resizeObserver.disconnect();
@@ -255,19 +258,27 @@ class HyperframesPlayer extends HTMLElement {
// message fire before connectedCallback installs the parent message listener. Initial
// attributes are applied below by connectedCallback; only live changes navigate here.
if (!this.isConnected) break;
- if (val) {
- // A different composition: like a <video> given a new src, it does not inherit a queued play.
- this._pendingPlay = false;
- this._abandonComposition("Composition navigated before runtime data was applied");
- this.iframe.src = prepareSrcForElement(this, val);
- }
+ if (val) this._navigateSrc(val);
+ break;
+ case "type": {
+ const src = this.getAttribute("src");
+ if (!this.isConnected || src === null || !!this._videoSource === this._wantsVideo()) break;
+ this._navigateSrc(src);
break;
+ }
case "srcdoc":
if (!this.isConnected) break;
this._pendingPlay = false;
this._abandonComposition("Composition navigated before runtime data was applied");
- if (val !== null) this.iframe.srcdoc = prepareSrcdocForElement(this, val);
- else this.iframe.removeAttribute("srcdoc");
+ if (val !== null) {
+ this._pauseForVideoSwitch();
+ this._teardownVideo();
+ this.iframe.srcdoc = prepareSrcdocForElement(this, val);
+ } else {
+ this.iframe.removeAttribute("srcdoc");
+ const src = this.getAttribute("src");
+ if (src !== null && this._wantsVideo()) this._navigateSrc(src);
+ }
break;
case SANDBOX_ORIGIN_ATTR:
this._applySandboxOriginPolicy(this.isConnected && oldVal !== val);
@@ -310,6 +321,7 @@ class HyperframesPlayer extends HTMLElement {
case "volume": {
const v = Math.max(0, Math.min(1, parseFloat(val || "1")));
this._volume = v;
+ this._syncVideoAudio();
this._media.updateVolume(v);
this._sendControl("set-volume", { volume: v });
this.controlsApi?.updateVolume(v);
@@ -345,6 +357,8 @@ class HyperframesPlayer extends HTMLElement {
}
private _reloadForSandboxOriginPolicy(): void {
+ // A video does not play in the iframe, so neither reload applies to it.
+ if (this._videoSource) return;
this._abandonComposition("Sandbox policy changed before runtime data was applied");
const srcdoc = this.getAttribute("srcdoc");
if (srcdoc !== null) {
@@ -352,7 +366,8 @@ class HyperframesPlayer extends HTMLElement {
return;
}
const src = this.getAttribute("src");
- this.iframe.src = src === null ? "about:blank" : prepareSrcForElement(this, src);
+ if (src === null) this.iframe.src = "about:blank";
+ else this._loadSrc(src);
}
/**
@@ -419,6 +434,7 @@ class HyperframesPlayer extends HTMLElement {
// after the user stopped playback.
this._pendingPlay = false;
if (!this._tryDirectTimelinePause()) this._sendControl("pause");
+ else this._showPausedVideoTime();
this._directTimelineClock.stop();
this._stopParentTickClock();
if (this._media.audioOwner === "parent") this._media.pauseAll();
@@ -632,6 +648,7 @@ class HyperframesPlayer extends HTMLElement {
return;
}
this._media.updateMuted(val !== null);
+ this._syncVideoAudio();
this._setIframeMediaMuted(val !== null);
this._sendControl("set-muted", { muted: val !== null });
this.controlsApi?.updateMuted(val !== null);
@@ -868,6 +885,7 @@ class HyperframesPlayer extends HTMLElement {
}
private _reloadShaderOptions(): void {
+ if (this._videoSource) return;
// This navigates the frame, so readiness has to fall with it. Leaving
// `_runtimeBridgeReady` true lets a delivery post into a document that is being
// replaced, where it can only end in a delivery timeout rather than the immediate,
@@ -877,12 +895,120 @@ class HyperframesPlayer extends HTMLElement {
this.iframe.srcdoc = prepareSrcdocForElement(this, this.getAttribute("srcdoc") || "");
return;
}
- if (this.hasAttribute("src")) {
- this.iframe.src = prepareSrcForElement(this, this.getAttribute("src") || "");
+ if (this.hasAttribute("src")) this._loadSrc(this.getAttribute("src") || "");
+ }
+
+ // A different source inherits no queued play.
+ private _navigateSrc(src: string): void {
+ this._pendingPlay = false;
+ this._pauseForVideoSwitch();
+ this._abandonComposition("Composition navigated before runtime data was applied");
+ this._loadSrc(src);
+ }
+
+ /** A `type="video/..."` src plays in a `<video>`, unless `srcdoc` (which wins, as in an iframe)
+ * shows a composition. */
+ private _wantsVideo(): boolean {
+ return isVideoType(this.getAttribute("type")) && !this.hasAttribute("srcdoc");
+ }
+
+ // Like a <video> given a new src, a switch into or out of a video starts paused.
+ private _pauseForVideoSwitch(): void {
+ if (!this._videoSource && !this._wantsVideo()) return;
+ this._paused = true;
+ this.controlsApi?.updatePlaying(false);
+ }
+
+ private _loadSrc(src: string): void {
+ if (this._wantsVideo()) {
+ this._loadVideo(src);
+ return;
}
+ this._teardownVideo();
+ this.iframe.src = prepareSrcForElement(this, src);
+ }
+
+ private _loadVideo(src: string): void {
+ if (!this._videoSource) {
+ this._videoSource = createVideoSource({
+ onMetadata: (video) => this._applyThenEmit(() => this._onVideoReady(video)),
+ onDurationChange: (video) => this._applyTimelineDuration(video.duration),
+ onResize: (video) => this._applyVideoSize(video),
+ // Follow a play or pause the page did not ask for, but not a queued one it since reversed.
+ onPlay: (video) => {
+ if (!video.paused && this._paused) this.play();
+ },
+ onPause: (video) => {
+ if (video.paused && !video.ended && !this._paused) this.pause();
+ },
+ onError: (message, code) => {
+ if (!this._paused) this.pause();
+ this._emit(new CustomEvent("error", { detail: { message, code } }));
+ },
+ onPlayRejected: (error) => this._onVideoPlayRejected(error),
+ });
+ this.container.appendChild(this._videoSource.video);
+ this.iframe.hidden = true;
+ // Unloads a composition this player was showing before it switched to video.
+ this.iframe.src = "about:blank";
+ }
+ this.probe.stop();
+ const { video, adapter } = this._videoSource;
+ this._directTimelineAdapter = adapter;
+ this._currentTime = 0;
+ this._duration = 0;
+ this.controlsApi?.updateTime(0, 0);
+ this._syncVideoAudio();
+ adapter.timeScale?.(this.playbackRate);
+ video.src = src;
+ }
+
+ private _teardownVideo(): void {
+ if (!this._videoSource) return;
+ this._videoSource.destroy();
+ this._videoSource = null;
+ this.iframe.hidden = false;
+ }
+
+ private _syncVideoAudio(): void {
+ if (!this._videoSource) return;
+ this._videoSource.video.muted = this.muted;
+ this._videoSource.video.volume = this._volume;
+ }
+
+ private _onVideoReady(video: HTMLVideoElement): void {
+ this._applyTimelineDuration(video.duration);
+ this._applyVideoSize(video);
+ this._ready = true;
+ this._dispatchReady();
+ // A video has no composition assets to wait on; this settles at once.
+ this._waitForAssetsReady(null);
+ this._afterEvents(() => {
+ if (this.hasAttribute("autoplay") && this._paused) this.play();
+ });
+ }
+
+ private _applyTimelineDuration(duration: number): void {
+ if (!Number.isFinite(duration) || duration <= 0) return;
+ this._setDuration(duration);
+ this.controlsApi?.updateTime(this._currentTime, duration);
+ }
+
+ private _applyVideoSize(video: HTMLVideoElement): void {
+ if (video.videoWidth > 0 && video.videoHeight > 0) {
+ this._setCompositionSize(video.videoWidth, video.videoHeight);
+ }
+ }
+
+ // Only a blocked play is a playbackerror: a failed load already fired `error`.
+ private _onVideoPlayRejected(error: unknown): void {
+ if (!(error instanceof DOMException && error.name === "NotAllowedError")) return;
+ if (!this._paused) this.pause();
+ this._emit(new CustomEvent("playbackerror", { detail: { source: "video", error } }));
}
private _trySyncSeek(timeInSeconds: number): boolean {
+ if (this._videoSource) return false;
try {
const win = this.iframe.contentWindow as
| (Window & { __player?: { seek?: (t: number) => void } })
@@ -897,17 +1023,13 @@ class HyperframesPlayer extends HTMLElement {
}
private _withDirectTimeline(fn: (tl: DirectTimelineAdapter) => void): boolean {
- const resolved = this.probe.resolveDirectTimelineAdapter();
+ const resolved = this._videoSource ? null : this.probe.resolveDirectTimelineAdapter();
const tl = resolved || this._directTimelineAdapter;
if (!tl) return false;
try {
fn(tl);
if (resolved && resolved !== this._directTimelineAdapter) {
- const duration = resolved.duration();
- if (Number.isFinite(duration) && duration > 0) {
- this._setDuration(duration);
- this.controlsApi?.updateTime(this._currentTime, duration);
- }
+ this._applyTimelineDuration(resolved.duration());
}
this._directTimelineAdapter = tl;
return true;
@@ -933,6 +1055,14 @@ class HyperframesPlayer extends HTMLElement {
return this._withDirectTimeline((tl) => void tl.pause());
}
+ // The clock samples every ~100 ms; like a <video>, a pause reports the exact frame.
+ private _showPausedVideoTime(): void {
+ if (!this._videoSource) return;
+ this._currentTime = this._videoSource.video.currentTime;
+ this.controlsApi?.updateTime(this._currentTime, this._duration);
+ this._emit(new CustomEvent("timeupdate", { detail: { currentTime: this._currentTime } }));
+ }
+
/**
* Widget-frame RAF loop that sends "tick" postMessages to the composition
* iframe on every frame. Used for the runtime bridge path so that animation
@@ -961,6 +1091,8 @@ class HyperframesPlayer extends HTMLElement {
}
private _onMessage(e: MessageEvent) {
+ // The iframe window outlives its documents: a late composition message must not reach a video.
+ if (this._videoSource) return;
this._applyThenEmit(() => this._handleRuntimeMessage(e));
}
@@ -1281,7 +1413,8 @@ class HyperframesPlayer extends HTMLElement {
}
private _onIframeLoad() {
- if (!this._connected) return;
+ // In video mode the iframe only ever loads about:blank; its load must not reset the video.
+ if (!this._connected || this._videoSource) return;
// The runtime posts its timeline at DOMContentLoaded, before `load`, and every
// host-initiated navigation clears `_ready` first. So a ready player already holds this
// document's handshake (an opaque origin reads as null); a paused runtime never posts it again.
diff --git a/packages/player/src/styles.ts b/packages/player/src/styles.ts
index 7ac23afa81..4f69773187 100644
--- a/packages/player/src/styles.ts
+++ b/packages/player/src/styles.ts
@@ -30,6 +30,15 @@ export const PLAYER_STYLES = /* css */ `
pointer-events: auto;
}
+ .hfp-video {
+ position: absolute;
+ inset: 0;
+ width: 100%;
+ height: 100%;
+ object-fit: contain;
+ pointer-events: none;
+ }
+
.hfp-poster {
position: absolute;
inset: 0;
diff --git a/packages/player/src/video-source.ts b/packages/player/src/video-source.ts
new file mode 100644
index 0000000000..0d39ce844c
--- /dev/null
+++ b/packages/player/src/video-source.ts
@@ -0,0 +1,76 @@
+// Video mode: a `<video playsinline>` behind the direct-timeline adapter a same-origin
+// `__timelines` composition uses, so play, seek, loop and `ended` share one code path.
+
+import type { DirectTimelineAdapter } from "./timeline-adapters.js";
+
+export function isVideoType(type: string | null): boolean {
+ return type?.trim().toLowerCase().startsWith("video/") ?? false;
+}
+
+export interface VideoSourceCallbacks {
+ onMetadata: (video: HTMLVideoElement) => void;
+ onDurationChange: (video: HTMLVideoElement) => void;
+ onResize: (video: HTMLVideoElement) => void;
+ onPlay: (video: HTMLVideoElement) => void;
+ onPause: (video: HTMLVideoElement) => void;
+ onError: (message: string, code: number | null) => void;
+ onPlayRejected: (error: unknown) => void;
+}
+
+export interface VideoSource {
+ video: HTMLVideoElement;
+ adapter: DirectTimelineAdapter;
+ destroy: () => void;
+}
+
+export function createVideoSource(callbacks: VideoSourceCallbacks): VideoSource {
+ const video = document.createElement("video");
+ video.className = "hfp-video";
+ video.playsInline = true;
+ video.preload = "metadata";
+
+ const listening = new AbortController();
+ const { signal } = listening;
+ video.addEventListener("loadedmetadata", () => callbacks.onMetadata(video), { signal });
+ video.addEventListener("durationchange", () => callbacks.onDurationChange(video), { signal });
+ video.addEventListener("resize", () => callbacks.onResize(video), { signal });
+ video.addEventListener("play", () => callbacks.onPlay(video), { signal });
+ video.addEventListener("pause", () => callbacks.onPause(video), { signal });
+ video.addEventListener(
+ "error",
+ () =>
+ callbacks.onError(video.error?.message || "Video failed to load", video.error?.code ?? null),
+ { signal },
+ );
+
+ const adapter: DirectTimelineAdapter = {
+ duration: () => video.duration,
+ time: () => video.currentTime,
+ seek: (timeInSeconds) => {
+ video.currentTime = timeInSeconds;
+ },
+ // A rejection that lands after destroy() belongs to a source that is gone.
+ play: () =>
+ video.play().catch((error: unknown) => {
+ if (!signal.aborted) callbacks.onPlayRejected(error);
+ }),
+ pause: () => video.pause(),
+ // The default rate survives a new src; the current rate alone would reset to it.
+ timeScale: (rate) => {
+ video.defaultPlaybackRate = rate;
+ video.playbackRate = rate;
+ },
+ };
+
+ return {
+ video,
+ adapter,
+ destroy: () => {
+ listening.abort();
+ video.pause();
+ video.removeAttribute("src");
+ video.load();
+ video.remove();
+ },
+ };
+}