Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Cut live recording draft latency: instant placeholders, max-wait debounce, deterministic nav/wait blocks (backend) (#6553)
Cindy Lidbe733bf7e17
diff --git a/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2 b/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2deleted file mode 100644index 772b67ef0..000000000--- a/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2+++ /dev/null@@ -1,13 +0,0 @@-Given a "go to URL" action, come up with a block label suitable for a browser agent. It should include a short version of a url, if the url exists.--MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.--Reply in JSON format with the following keys:-{- "block_label": str // A label for the block. Lower case. Underscores are permitted.-}--go to URL action:-```-{{ action }}-```diff --git a/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2 b/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2deleted file mode 100644index 7e5101609..000000000--- a/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2+++ /dev/null@@ -1,15 +0,0 @@-Given a wait action, come up with a block label. It should include the duration in seconds, and a short version of a url, if the url exists.--If the duration for the wait action is in milliseconds, so be sure to convert to seconds.--MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.--Reply in JSON format with the following keys:-{- "block_label": str // A label for the block. Lower case. Underscores are permitted.-}--Wait action:-```-{{ action }}-```diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex cddcf1fbd..20ad42a23 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -1,6 +1,7 @@import asyncioimport dataclassesimport enum+import timeimport typing as timport structlog@@ -13,9 +14,15 @@ from skyvern.client.types.workflow_definition_yaml_blocks_item import ()from skyvern.forge.sdk.routes.streaming.channels import exfiltration as streaming_exfiltrationfrom skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEventSource-from skyvern.services.browser_recording.service import Processor+from skyvern.services.browser_recording.service import (+ Processor,+ deterministic_input_text_parameter_key,+ normalize_recording_block_label,+)from skyvern.services.browser_recording.types import (Action,+ ActionBlockable,+ ActionInputText,ActionKind,ExfiltratedCdpEvent,ExfiltratedConsoleEvent,@@ -29,7 +36,8 @@ from skyvern.services.browser_recording.types import (LOG = structlog.get_logger(__name__)-INTERPRETATION_DEBOUNCE_SECONDS = 1.5+INTERPRETATION_DEBOUNCE_SECONDS = 0.4+INTERPRETATION_MAX_WAIT_SECONDS = 2.0SIGNIFICANT_CONSOLE_EVENT_TYPES = {"blur","change",@@ -161,6 +169,67 @@ def _draft_step_from_block(return None+def _action_display_text(action: Action) -> str:+ for text in action.target.texts or []:+ cleaned = " ".join(text.split())+ if cleaned:+ return cleaned[:80]++ return (action.target.tag_name or "element").lower()+++_PLACEHOLDER_VERBS: dict[ActionKind, str] = {+ ActionKind.CLICK: "Click",+ ActionKind.HOVER: "Hover over",+ ActionKind.INPUT_TEXT: "Fill",+}+++def _placeholder_step_from_action(+ *,+ browser_session_id: str,+ action_index: int,+ action: ActionBlockable,+) -> RecordingDraftStep:+ """+ A draft step built purely from exfiltrated event data, shown to the user+ immediately while LLM enrichment runs in the background.+ """+ step_id = f"{browser_session_id}-recording-step-{action_index}"+ text = _action_display_text(action)+ verb = _PLACEHOLDER_VERBS[action.kind]+ title = f"{verb} '{text}'"++ parameters: list[dict[str, t.Any]] = []+ parameter_keys: list[str] = []+ if isinstance(action, ActionInputText):+ parameter_key = deterministic_input_text_parameter_key(action)+ navigation_goal = f"{verb} '{text}' with {{{{ {parameter_key} }}}}."+ parameters = [{"key": parameter_key}]+ parameter_keys = [parameter_key]+ else:+ navigation_goal = f"{verb} '{text}'."++ return RecordingDraftStep(+ step_id=step_id,+ action_kind=action.kind,+ block_type="action",+ label=normalize_recording_block_label(f"{action.kind.value}_{text}", fallback=action.kind.value),+ title=title,+ navigation_goal=navigation_goal,+ status=RecordingDraftStepStatus.INTERPRETING,+ editable_fields=[+ RecordingDraftStepEditableField.LABEL,+ RecordingDraftStepEditableField.TITLE,+ RecordingDraftStepEditableField.NAVIGATION_GOAL,+ ],+ parameters=parameters,+ parameter_keys=parameter_keys,+ timestamp_start=action.timestamp_start,+ timestamp_end=action.timestamp_end,+ )++class RecordingInterpretationSession:def __init__(self,@@ -170,12 +239,14 @@ class RecordingInterpretationSession:workflow_permanent_id: str,on_update: OnRecordingInterpretationUpdate,debounce_seconds: float = INTERPRETATION_DEBOUNCE_SECONDS,+ max_wait_seconds: float = INTERPRETATION_MAX_WAIT_SECONDS,) -> None:self.browser_session_id = browser_session_idself.organization_id = organization_idself.workflow_permanent_id = workflow_permanent_idself.on_update = on_updateself.debounce_seconds = debounce_seconds+ self.max_wait_seconds = max_wait_secondsself.events: list[ExfiltratedEvent] = []self.steps: list[RecordingDraftStep] = []self.emitted_action_count = 0@@ -183,6 +254,8 @@ class RecordingInterpretationSession:self.pending = Falseself.finalized = Falseself._debounce_task: asyncio.Task[None] | None = None+ self._pending_since: float | None = None+ self._enrichment_tasks: set[asyncio.Task[None]] = set()self._interpret_lock = asyncio.Lock()self._action_machines: list[sm.StateMachine] = [sm.Click(),@@ -204,36 +277,53 @@ class RecordingInterpretationSession:if not any(event_should_trigger_interpretation(event) for event in reified_events):return+ if self._pending_since is None:+ self._pending_since = time.monotonic()+self.pending = Trueself.finalized = Falseself._emit_update()self._schedule_interpretation()def cancel(self) -> None:- if self._debounce_task and not self._debounce_task.done():- self._debounce_task.cancel()- self._debounce_task = None+ self._cancel_debounce()+ for task in list(self._enrichment_tasks):+ task.cancel()+ self._enrichment_tasks.clear()async def flush(self) -> list[RecordingDraftStep]:- self.cancel()+ self._cancel_debounce()await self._interpret(finalized=True)return self.steps+ def _cancel_debounce(self) -> None:+ if self._debounce_task and not self._debounce_task.done():+ self._debounce_task.cancel()+ self._debounce_task = None+def _schedule_interpretation(self) -> None:- self.cancel()- self._debounce_task = asyncio.create_task(self._debounced_interpret())+ self._cancel_debounce()++ delay = self.debounce_seconds+ if self._pending_since is not None:+ deadline = self._pending_since + self.max_wait_seconds+ delay = min(delay, max(0.0, deadline - time.monotonic()))- async def _debounced_interpret(self) -> None:+ self._debounce_task = asyncio.create_task(self._debounced_interpret(delay))++ async def _debounced_interpret(self, delay: float) -> None:+ # No finally-cleanup of self._debounce_task here: by the time a+ # cancelled task unwinds, the reference may already point at a newer+ # task, and clearing it would orphan that one.try:- await asyncio.sleep(self.debounce_seconds)+ await asyncio.sleep(delay)await self._interpret(finalized=False)except asyncio.CancelledError:return- finally:- self._debounce_task = Noneasync def _interpret(self, *, finalized: bool) -> None:async with self._interpret_lock:+ self._pending_since = Noneprocessor = Processor(self.browser_session_id,self.organization_id,@@ -249,53 +339,132 @@ class RecordingInterpretationSession:)self._processed_event_count = len(self.events)- actions = self._all_actions- new_actions = actions[self.emitted_action_count :]-- if not new_actions:- self.pending = False- self.finalized = finalized- self._emit_update()- return-- try:- blocks = await processor.actions_to_blocks(new_actions)- except Exception:- self.pending = False- self.finalized = finalized- self._emit_update()- LOG.exception(- "Failed to interpret live browser recording actions",- browser_session_id=self.browser_session_id,- organization_id=self.organization_id,- workflow_permanent_id=self.workflow_permanent_id,- )- return-- if len(blocks) < len(new_actions):- LOG.warning(- "Live interpretation produced fewer blocks than actions",- browser_session_id=self.browser_session_id,- action_count=len(new_actions),- block_count=len(blocks),- )+ new_actions = self._all_actions[self.emitted_action_count :]- for offset, (action, block) in enumerate(zip(new_actions, blocks, strict=False)):+ for offset, action in enumerate(new_actions):action_index = self.emitted_action_count + offset- draft_step = _draft_step_from_block(- browser_session_id=self.browser_session_id,- action_index=action_index,- action=action,- block=block,- )+ draft_step = await self._step_from_action(processor, action_index, action)if draft_step:+ draft_step.label = self._unique_step_label(draft_step.label, draft_step)self.steps.append(draft_step)- self.emitted_action_count += len(blocks)+ self.emitted_action_count += len(new_actions)self.pending = False- self.finalized = finalized+ self.finalized = finalized and not self._enrichment_tasks+ self._emit_update()++ if not finalized:+ return++ # Hold the finalized signal until in-flight LLM enrichment lands, so+ # commit paths never capture placeholder metadata.+ await self._drain_enrichment()+ if not self.finalized:+ self.finalized = Trueself._emit_update()+ async def _step_from_action(+ self,+ processor: Processor,+ action_index: int,+ action: Action,+ ) -> RecordingDraftStep | None:+ """+ Build a draft step for the action immediately. Deterministic kinds+ (goto_url, wait) come back final; agent-action kinds come back as+ placeholders with LLM enrichment scheduled in the background.+ """+ if action.kind in (ActionKind.CLICK, ActionKind.HOVER, ActionKind.INPUT_TEXT):+ blockable = t.cast(ActionBlockable, action)+ step = _placeholder_step_from_action(+ browser_session_id=self.browser_session_id,+ action_index=action_index,+ action=blockable,+ )+ self._schedule_enrichment(processor, action_index, step, blockable)+ return step++ if action.kind in (ActionKind.URL_CHANGE, ActionKind.WAIT):+ block: OutputBlock+ if action.kind == ActionKind.URL_CHANGE:+ block = await processor.create_url_block(action)+ else:+ block = await processor.create_wait_block(action)++ return _draft_step_from_block(+ browser_session_id=self.browser_session_id,+ action_index=action_index,+ action=action,+ block=block,+ )++ LOG.warning(+ "Unknown action kind in live interpretation",+ action_kind=action.kind,+ browser_session_id=self.browser_session_id,+ )+ return None++ def _schedule_enrichment(+ self,+ processor: Processor,+ action_index: int,+ step: RecordingDraftStep,+ action: ActionBlockable,+ ) -> None:+ task = asyncio.create_task(self._enrich_step(processor, action_index, step, action))+ self._enrichment_tasks.add(task)+ task.add_done_callback(self._enrichment_tasks.discard)++ async def _enrich_step(+ self,+ processor: Processor,+ action_index: int,+ step: RecordingDraftStep,+ action: ActionBlockable,+ ) -> None:+ enriched: RecordingDraftStep | None = None++ try:+ block = await processor.create_action_block(action)+ enriched = _draft_step_from_block(+ browser_session_id=self.browser_session_id,+ action_index=action_index,+ action=action,+ block=block,+ )+ except Exception:+ LOG.exception(+ "Failed to enrich recording draft step; keeping deterministic placeholder",+ browser_session_id=self.browser_session_id,+ organization_id=self.organization_id,+ step_id=step.step_id,+ )++ if enriched:+ step.label = self._unique_step_label(enriched.label, step)+ step.title = enriched.title+ step.navigation_goal = enriched.navigation_goal+ step.parameters = enriched.parameters+ step.parameter_keys = enriched.parameter_keys++ step.status = RecordingDraftStepStatus.READY+ self._emit_update()++ async def _drain_enrichment(self) -> None:+ if self._enrichment_tasks:+ await asyncio.gather(*list(self._enrichment_tasks), return_exceptions=True)++ def _unique_step_label(self, label: str, step: RecordingDraftStep) -> str:+ existing = {s.label for s in self.steps if s.step_id != step.step_id}+ if label not in existing:+ return label++ suffix = 2+ while f"{label}_{suffix}" in existing:+ suffix += 1+ return f"{label}_{suffix}"+def _emit_update(self) -> None:self.session_revision += 1update = RecordingInterpretationUpdate(diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex cc4c396e0..26456b74f 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -5,6 +5,7 @@ import pathlibimport reimport typing as timport zlib+from urllib.parse import urlparseimport structlog@@ -20,6 +21,7 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.services.browser_recording.types import (Action,ActionBlockable,+ ActionInputText,ActionKind,ActionUrlChange,ActionWait,@@ -93,6 +95,31 @@ def _is_duplicate_action(candidate: Action, existing_actions: list[Action]) -> b)+def deterministic_goto_url_label(url: str) -> str:+ host = ""+ try:+ host = urlparse(url).netloc+ except ValueError:+ pass++ return normalize_recording_block_label(f"goto_{host}" if host else None, fallback="goto_url")+++def deterministic_wait_seconds(duration_ms: int) -> int:+ return int(max(duration_ms / 1000.0, ActionWait.MIN_DURATION_THRESHOLD_MS / 1000.0))+++def deterministic_input_text_parameter_key(action: ActionInputText) -> str:+ target = action.target+ for candidate in (target.id, *(target.texts or []), target.sky_id):+ if not candidate:+ continue+ key = normalize_recording_block_label(str(candidate), fallback="")+ if key:+ return key.lower()+ return "input_value"++def normalize_recording_block_label(label: str | None, *, fallback: str) -> str:candidate = (label or "").strip()candidate = re.sub(r"\W+", "_", candidate)@@ -475,59 +502,21 @@ class Processor:return blockasync def create_url_block(self, action: ActionUrlChange) -> WorkflowDefinitionYamlBlocksItem_GotoUrl:- """- Create a YAML goto URL block from an `ActionUrlChange`.- """-- prompt_name = "recording-go-to-url-block-prompt"-- metadata_prompt = prompt_engine.load_prompt(- prompt_name,- action=action,- )-- metadata_response = await app.LLM_API_HANDLER(- prompt=metadata_prompt,- prompt_name=prompt_name,- organization_id=self.organization_id,- )-- block_label: str = metadata_response.get("block_label", None) or "goto_url"-- block = WorkflowDefinitionYamlBlocksItem_GotoUrl(- label=block_label,+ """Build a goto-url block with a host-derived label and the action URL."""+ return WorkflowDefinitionYamlBlocksItem_GotoUrl(+ label=deterministic_goto_url_label(action.url),url=action.url,)- return block-async def create_wait_block(self, action: ActionWait) -> WorkflowDefinitionYamlBlocksItem_Wait:- """- Create a YAML wait block from an `ActionWait`.- """-- prompt_name = "recording-wait-block-prompt"-- metadata_prompt = prompt_engine.load_prompt(- prompt_name,- action=action,- )+ """Build a wait block with a duration-derived label and wait seconds."""+ wait_sec = deterministic_wait_seconds(action.duration_ms)- metadata_response = await app.LLM_API_HANDLER(- prompt=metadata_prompt,- prompt_name=prompt_name,- organization_id=self.organization_id,+ return WorkflowDefinitionYamlBlocksItem_Wait(+ label=f"wait_{wait_sec}s",+ wait_sec=wait_sec,)- block_label: str = metadata_response.get("block_label", None) or "wait"-- block = WorkflowDefinitionYamlBlocksItem_Wait(- label=block_label,- wait_sec=int(max(action.duration_ms / 1000.0, ActionWait.MIN_DURATION_THRESHOLD_MS / 1000.0)),- )-- return block-async def process(self,compressed_chunks: list[str],diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex 571a12589..a5db3d150 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -2,13 +2,35 @@Just an example unit test for now. Will expand later."""+import asyncio+import timeimport typing as t-from skyvern.services.browser_recording.service import Processor, summarize_exfiltrated_recording_events+import pytest++from skyvern.forge import app+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEvent as StreamingExfiltratedEvent+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (+ ExfiltratedEventSource as StreamingExfiltratedEventSource,+)+from skyvern.services.browser_recording.interpretation import RecordingInterpretationSession+from skyvern.services.browser_recording.service import (+ Processor,+ deterministic_input_text_parameter_key,+ summarize_exfiltrated_recording_events,+)from skyvern.services.browser_recording.types import (+ ActionInputText,+ ActionKind,+ ActionTarget,+ ActionUrlChange,+ ActionWait,ExfiltratedCdpEvent,ExfiltratedConsoleEvent,ExfiltratedEventCdpParams,+ Mouse,+ RecordingDraftStepStatus,+ RecordingInterpretationUpdate,)ORG_ID = "org_123"@@ -143,6 +165,276 @@ def test_hover() -> None:assert len(actions) == 1+def _empty_action_target() -> ActionTarget:+ return ActionTarget(mouse=Mouse(xp=None, yp=None))+++def make_streaming_console_click(timestamp_ms: float) -> StreamingExfiltratedEvent:+ return StreamingExfiltratedEvent(+ event_name="user_interaction",+ params={+ "type": "click",+ "target": {"id": "button-1", "skyId": "sky-123", "tagName": "BUTTON", "text": ["Click me"]},+ "timestamp": timestamp_ms,+ "url": "https://example.com",+ "activeElement": {"tagName": "BUTTON"},+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},+ "mousePosition": {"xp": 0.5, "yp": 0.5},+ },+ source=StreamingExfiltratedEventSource.CONSOLE,+ timestamp=timestamp_ms / 1000.0,+ )+++def make_streaming_nav_event(url: str, timestamp: float) -> StreamingExfiltratedEvent:+ return StreamingExfiltratedEvent(+ event_name="nav:frame_started_navigating",+ params={"url": url},+ source=StreamingExfiltratedEventSource.CDP,+ timestamp=timestamp,+ )+++def make_streaming_console_input(+ *,+ timestamp_ms: float,+ input_value: str,+ target_id: str = "email",+ target_text: str = "Email",+) -> list[StreamingExfiltratedEvent]:+ target = {+ "id": target_id,+ "skyId": "sky-email",+ "tagName": "INPUT",+ "text": [target_text],+ "value": input_value,+ }+ common = {+ "target": target,+ "timestamp": timestamp_ms,+ "url": "https://example.com",+ "activeElement": {"tagName": "INPUT"},+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},+ "mousePosition": {"xp": 0.5, "yp": 0.5},+ }++ return [+ StreamingExfiltratedEvent(+ event_name="user_interaction",+ params={"type": "focus", **common},+ source=StreamingExfiltratedEventSource.CONSOLE,+ timestamp=timestamp_ms / 1000.0,+ ),+ StreamingExfiltratedEvent(+ event_name="user_interaction",+ params={"type": "keydown", "key": "a", **common},+ source=StreamingExfiltratedEventSource.CONSOLE,+ timestamp=(timestamp_ms + 1) / 1000.0,+ ),+ StreamingExfiltratedEvent(+ event_name="user_interaction",+ params={"type": "blur", **common},+ source=StreamingExfiltratedEventSource.CONSOLE,+ timestamp=(timestamp_ms + 2) / 1000.0,+ ),+ ]+++def test_create_url_block_is_deterministic() -> None:+ action = ActionUrlChange(+ kind=ActionKind.URL_CHANGE,+ target=_empty_action_target(),+ timestamp_start=1000.0,+ timestamp_end=1000.0,+ url="https://example.com/products?page=2",+ )++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ block = asyncio.run(processor.create_url_block(action))++ assert block.label == "goto_example_com"+ assert block.url == "https://example.com/products?page=2"+++def test_create_wait_block_is_deterministic() -> None:+ action = ActionWait(+ kind=ActionKind.WAIT,+ target=_empty_action_target(),+ timestamp_start=1000.0,+ timestamp_end=8000.0,+ url="https://example.com",+ duration_ms=7000,+ )++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ block = asyncio.run(processor.create_wait_block(action))++ assert block.label == "wait_7s"+ assert block.wait_sec == 7+++def test_create_wait_block_floors_at_minimum_duration() -> None:+ action = ActionWait(+ kind=ActionKind.WAIT,+ target=_empty_action_target(),+ timestamp_start=1000.0,+ timestamp_end=2000.0,+ url="https://example.com",+ duration_ms=1000,+ )++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ block = asyncio.run(processor.create_wait_block(action))++ assert block.wait_sec == ActionWait.MIN_DURATION_THRESHOLD_MS // 1000+++def test_input_text_parameter_key_is_derived_from_target_metadata() -> None:+ action = ActionInputText(+ kind=ActionKind.INPUT_TEXT,+ target=ActionTarget(+ id="customer_email",+ sky_id="sky-email",+ tag_name="INPUT",+ texts=["Email"],+ mouse=Mouse(xp=0.5, yp=0.5),+ ),+ timestamp_start=1000.0,+ timestamp_end=1001.0,+ url="https://example.com",+ input_value="secret123",+ )++ assert deterministic_input_text_parameter_key(action) == "customer_email"+++async def test_input_text_placeholder_parameterizes_value_on_enrichment_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )++ session.ingest_events(make_streaming_console_input(timestamp_ms=1000.0, input_value="secret123"))+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].status == RecordingDraftStepStatus.READY+ assert "secret123" not in (steps[0].navigation_goal or "")+ assert steps[0].navigation_goal == "Fill 'Email' with {{ email }}."+ assert steps[0].parameter_keys == ["email"]+ assert steps[0].parameters == [{"key": "email"}]+++async def test_live_interpretation_emits_placeholder_then_enriched(monkeypatch: pytest.MonkeyPatch) -> None:+ release_llm = asyncio.Event()++ async def fake_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ await release_llm.wait()+ return {+ "block_label": "click_submit",+ "title": "Click Submit",+ "prompt": "Click the submit button.",+ }++ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)++ updates: list[RecordingInterpretationUpdate] = []+ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=updates.append,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )++ session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])+ await asyncio.sleep(0.05)++ # the placeholder draft is visible before the LLM responds+ updates_with_steps = [update for update in updates if update.steps]+ assert updates_with_steps+ placeholder = updates_with_steps[-1].steps[0]+ assert placeholder.status == RecordingDraftStepStatus.INTERPRETING+ assert placeholder.title == "Click 'Click me'"+ assert placeholder.navigation_goal == "Click 'Click me'."++ release_llm.set()+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].status == RecordingDraftStepStatus.READY+ assert steps[0].title == "Click Submit"+ assert steps[0].navigation_goal == "Click the submit button."+ assert updates[-1].finalized is True+++async def test_live_interpretation_enrichment_failure_keeps_placeholder(monkeypatch: pytest.MonkeyPatch) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )++ session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].status == RecordingDraftStepStatus.READY+ assert steps[0].title == "Click 'Click me'"+++async def test_live_interpretation_max_wait_fires_during_continuous_events() -> None:+ updates: list[RecordingInterpretationUpdate] = []+ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=updates.append,+ debounce_seconds=0.05,+ max_wait_seconds=0.1,+ )++ # Significant events arrive faster than the quiet debounce, so a pure+ # trailing debounce would defer interpretation until the stream stops.+ start = time.monotonic()+ sequence = 0+ while time.monotonic() - start < 0.35:+ session.ingest_events([make_streaming_nav_event(url=f"https://example.com/{sequence}", timestamp=time.time())])+ sequence += 1+ await asyncio.sleep(0.02)++ try:+ assert any(update.steps for update in updates)+ finally:+ session.cancel()+ # let the cancelled debounce task unwind before the loop closes+ await asyncio.sleep(0.01)++def test_summarize_exfiltrated_recording_events_mixed() -> None:target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])click = make_click_event(target=target, timestamp=1000.0)diff --git a/tests/unit/test_browser_recording_interpretation.py b/tests/unit/test_browser_recording_interpretation.pyindex 3fc548d01..10d4d9699 100644--- a/tests/unit/test_browser_recording_interpretation.py+++ b/tests/unit/test_browser_recording_interpretation.py@@ -17,7 +17,10 @@ from skyvern.services.browser_recording.interpretation import (from skyvern.services.browser_recording.service import Processorfrom skyvern.services.browser_recording.types import (ActionKind,+ ActionTarget,+ ActionWait,ExfiltratedConsoleEvent,+ Mouse,RecordingDraftStep,)@@ -220,7 +223,7 @@ async def test_recording_interpretation_session_flush_cancels_debounce(monkeypatdebounce_started = asyncio.Event()allow_debounce_finish = asyncio.Event()- async def fake_debounced_interpret(self: RecordingInterpretationSession) -> None:+ async def fake_debounced_interpret(self: RecordingInterpretationSession, delay: float) -> None:debounce_started.set()await allow_debounce_finish.wait()@@ -252,11 +255,24 @@ async def test_recording_interpretation_session_flush_cancels_debounce(monkeypat@pytest.mark.asyncio-async def test_recording_interpretation_session_advances_only_matched_blocks(monkeypatch: pytest.MonkeyPatch) -> None:- wait_block = WorkflowDefinitionYamlBlocksItem_Wait(label="wait", wait_sec=5)+async def test_recording_interpretation_session_advances_past_unhandled_actions(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ wait_action = ActionWait(+ kind=ActionKind.WAIT,+ target=ActionTarget(mouse=Mouse(xp=None, yp=None)),+ timestamp_start=1000.0,+ timestamp_end=8000.0,+ url="https://example.com",+ duration_ms=7000,+ )+ unhandled_action = MagicMock()+ unhandled_action.kind = "unsupported"+processor = MagicMock()- processor.events_to_actions.return_value = [MagicMock(), MagicMock()]- processor.actions_to_blocks = AsyncMock(return_value=[wait_block])+ processor.create_wait_block = AsyncMock(+ return_value=WorkflowDefinitionYamlBlocksItem_Wait(label="wait_7s", wait_sec=7),+ )monkeypatch.setattr("skyvern.services.browser_recording.interpretation.Processor",lambda *args, **kwargs: processor,@@ -270,9 +286,9 @@ async def test_recording_interpretation_session_advances_only_matched_blocks(mon)session.events = [MagicMock(), MagicMock()]session._processed_event_count = len(session.events)- session._all_actions = processor.events_to_actions.return_value+ session._all_actions = [wait_action, unhandled_action]await session._interpret(finalized=False)- assert session.emitted_action_count == 1+ assert session.emitted_action_count == 2assert len(session.steps) == 1