Skip to content

Automate browser based workflows with AI

fix(SKY-8938): finalize webm recordings so they have end timestamp (#5551)

Shuchang Zhengd1582e0013cf
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index bffe99ad6..60b102d8e 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -15,6 +15,7 @@ from skyvern.webeye.browser_factory import BrowserContextFactory
from skyvern.webeye.browser_manager import BrowserManager
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.real_browser_state import RealBrowserState
+from skyvern.webeye.video_utils import finalize_webm
LOG = structlog.get_logger()
@@ -292,8 +293,9 @@ class RealBrowserManager(BrowserManager):
for i, video_artifact in enumerate(browser_state.browser_artifacts.video_artifacts):
path = video_artifact.video_path
if path and os.path.exists(path=path):
- with open(path, "rb") as f:
- browser_state.browser_artifacts.video_artifacts[i].video_data = f.read()
+ # Remux via ffmpeg so the WebM container has a valid Duration + Cues,
+ # even when browser_context.close() was killed mid-finalization.
+ browser_state.browser_artifacts.video_artifacts[i].video_data = await finalize_webm(path)
else:
LOG.debug(
"Video path not found",
diff --git a/skyvern/webeye/video_utils.py b/skyvern/webeye/video_utils.py
new file mode 100644
index 000000000..ac6be6ec8
--- /dev/null
+++ b/skyvern/webeye/video_utils.py
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import tempfile
+
+import structlog
+
+LOG = structlog.get_logger()
+
+FFMPEG_BINARY = "ffmpeg"
+FFMPEG_REMUX_TIMEOUT_SECONDS = 30
+
+
+async def finalize_webm(src_path: str) -> bytes:
+ """Return the bytes of a WebM recording with a valid end timestamp.
+
+ Browser recordings can end up with an unfinalized Matroska container when
+ ``browser_context.close()`` is killed mid-shutdown (close timeout, OOM,
+ pod eviction). The file is left with an "unknown size" Segment, no
+ ``Duration`` element, and no ``Cues`` index — so players show no end
+ timestamp and can't seek. Remuxing via ffmpeg stream-copy rewrites the
+ container with a backfilled Segment size, a Duration element, and Cues
+ at the front for HTTP streaming. VP8 clusters are copied unmodified.
+
+ Falls back to the raw bytes if ffmpeg is unavailable, times out, or
+ errors so recording upload is never regressed relative to today.
+ """
+ if not os.path.exists(src_path):
+ raise FileNotFoundError(src_path)
+
+ if shutil.which(FFMPEG_BINARY) is None:
+ LOG.warning("ffmpeg binary not found on PATH, returning raw webm bytes", src=src_path)
+ with open(src_path, "rb") as f:
+ return f.read()
+
+ with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as dst_tmp:
+ dst_path = dst_tmp.name
+
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ FFMPEG_BINARY,
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-i",
+ src_path,
+ "-c",
+ "copy",
+ "-cues_to_front",
+ "1",
+ "-reserve_index_space",
+ "200k",
+ dst_path,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ try:
+ _, stderr = await asyncio.wait_for(proc.communicate(), timeout=FFMPEG_REMUX_TIMEOUT_SECONDS)
+ except asyncio.TimeoutError:
+ proc.kill()
+ await proc.wait()
+ LOG.warning("ffmpeg remux timed out, returning raw webm bytes", src=src_path)
+ with open(src_path, "rb") as f:
+ return f.read()
+
+ if proc.returncode != 0 or not os.path.exists(dst_path) or os.path.getsize(dst_path) == 0:
+ LOG.warning(
+ "ffmpeg remux failed, returning raw webm bytes",
+ src=src_path,
+ returncode=proc.returncode,
+ stderr=stderr.decode(errors="replace")[:500] if stderr else "",
+ )
+ with open(src_path, "rb") as f:
+ return f.read()
+
+ with open(dst_path, "rb") as f:
+ return f.read()
+ finally:
+ if os.path.exists(dst_path):
+ try:
+ os.unlink(dst_path)
+ except OSError:
+ LOG.debug("failed to cleanup ffmpeg temp output", path=dst_path, exc_info=True)
diff --git a/tests/unit/test_video_utils.py b/tests/unit/test_video_utils.py
new file mode 100644
index 000000000..de9fe45d3
--- /dev/null
+++ b/tests/unit/test_video_utils.py
@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import asyncio
+import struct
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from skyvern.webeye import video_utils
+from skyvern.webeye.video_utils import finalize_webm
+
+
+def _write_unfinalized_webm(path: str) -> bytes:
+ """Write a WebM whose Segment has the "unknown size" VINT and no Duration/Cues.
+
+ This mirrors the shape of recordings produced when Chromium's muxer is
+ killed before finalizing the container.
+ """
+ ebml_header = bytes.fromhex(
+ "1a45dfa3" # EBML
+ "9f" # size = 31
+ "4286"
+ "81"
+ "01" # EBMLVersion = 1
+ "42f7"
+ "81"
+ "01" # EBMLReadVersion = 1
+ "42f2"
+ "81"
+ "04" # EBMLMaxIDLength = 4
+ "42f3"
+ "81"
+ "08" # EBMLMaxSizeLength = 8
+ "4282"
+ "84"
+ "7765626d" # DocType = "webm"
+ "4287"
+ "81"
+ "02" # DocTypeVersion = 2
+ "4285"
+ "81"
+ "02" # DocTypeReadVersion = 2
+ )
+ segment_header = bytes.fromhex("18538067") + bytes.fromhex("01ffffffffffffff") # "unknown size"
+ # One tiny SimpleBlock-bearing Cluster so ffmpeg sees something.
+ cluster_body = bytes.fromhex("e78100") # Timestamp = 0
+ cluster = bytes.fromhex("1f43b675") + struct.pack(">B", 0x80 | len(cluster_body)) + cluster_body
+ data = ebml_header + segment_header + cluster
+ with open(path, "wb") as f:
+ f.write(data)
+ return data
+
+
+async def test_finalize_webm_missing_file_raises(tmp_path) -> None:
+ missing = str(tmp_path / "nope.webm")
+ with pytest.raises(FileNotFoundError):
+ await finalize_webm(missing)
+
+
+async def test_finalize_webm_without_ffmpeg_returns_raw(tmp_path) -> None:
+ src = str(tmp_path / "raw.webm")
+ expected = _write_unfinalized_webm(src)
+
+ with patch.object(video_utils, "shutil") as mock_shutil:
+ mock_shutil.which.return_value = None
+ result = await finalize_webm(src)
+
+ assert result == expected
+
+
+async def test_finalize_webm_invokes_ffmpeg_with_cues_at_front(tmp_path) -> None:
+ src = str(tmp_path / "src.webm")
+ _write_unfinalized_webm(src)
+
+ captured_args: list[str] = []
+
+ async def fake_exec(*args, **kwargs):
+ captured_args.extend(args)
+ # ffmpeg "writes" an output file, then we simulate a successful run
+ output_path = args[-1]
+ with open(output_path, "wb") as f:
+ f.write(b"REMUXED_CONTENT")
+ proc = AsyncMock()
+ proc.returncode = 0
+ proc.communicate = AsyncMock(return_value=(b"", b""))
+ proc.wait = AsyncMock(return_value=0)
+ return proc
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffmpeg"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ result = await finalize_webm(src)
+
+ assert result == b"REMUXED_CONTENT"
+ assert "-c" in captured_args and "copy" in captured_args
+ assert "-cues_to_front" in captured_args
+ # cues_to_front must be followed by "1"
+ assert captured_args[captured_args.index("-cues_to_front") + 1] == "1"
+ assert "-reserve_index_space" in captured_args
+ assert captured_args[0] == video_utils.FFMPEG_BINARY
+ assert captured_args[-2] != src # output path is the temp file, not input
+ assert src in captured_args
+
+
+async def test_finalize_webm_falls_back_on_ffmpeg_error(tmp_path) -> None:
+ src = str(tmp_path / "src.webm")
+ expected = _write_unfinalized_webm(src)
+
+ async def fake_exec(*args, **kwargs):
+ proc = AsyncMock()
+ proc.returncode = 1
+ proc.communicate = AsyncMock(return_value=(b"", b"boom"))
+ proc.wait = AsyncMock(return_value=1)
+ return proc
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffmpeg"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ result = await finalize_webm(src)
+
+ assert result == expected
+
+
+async def test_finalize_webm_falls_back_on_timeout(tmp_path) -> None:
+ src = str(tmp_path / "src.webm")
+ expected = _write_unfinalized_webm(src)
+
+ async def fake_exec(*args, **kwargs):
+ proc = AsyncMock()
+ proc.communicate = AsyncMock(side_effect=asyncio.TimeoutError())
+ proc.kill = lambda: None
+ proc.wait = AsyncMock(return_value=0)
+ return proc
+
+ async def immediate_timeout(coro, timeout):
+ coro.close()
+ raise asyncio.TimeoutError()
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffmpeg"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ with patch.object(video_utils.asyncio, "wait_for", side_effect=immediate_timeout):
+ result = await finalize_webm(src)
+
+ assert result == expected
+
+
+async def test_finalize_webm_end_to_end_sets_duration(tmp_path) -> None:
+ import shutil
+ import subprocess
+
+ if shutil.which("ffmpeg") is None:
+ pytest.skip("ffmpeg not installed")
+
+ src = str(tmp_path / "real.webm")
+ subprocess.check_call(
+ [
+ "ffmpeg",
+ "-y",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "color=black:size=320x180:rate=25:duration=2",
+ "-c:v",
+ "libvpx",
+ "-f",
+ "webm",
+ src,
+ ]
+ )
+
+ output = await finalize_webm(src)
+ # Duration element tag is 0x4489 — must be present after remux.
+ assert b"\x44\x89" in output[:4096]