Skip to content

Automate browser based workflows with AI

Cut live recording draft latency: instant placeholders, max-wait debounce, deterministic nav/wait blocks (backend) (#6553)

Cindy Lidbe733bf7e17
diff --git a/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2 b/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2
deleted file mode 100644
index 772b67ef0..000000000
--- a/skyvern/forge/prompts/skyvern/recording-go-to-url-block-prompt.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-Given a "go to URL" action, come up with a block label suitable for a browser agent. It should include a short version of a url, if the url exists.
-
-MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
-
-Reply in JSON format with the following keys:
-{
- "block_label": str // A label for the block. Lower case. Underscores are permitted.
-}
-
-go to URL action:
-```
-{{ action }}
-```
diff --git a/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2 b/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2
deleted file mode 100644
index 7e5101609..000000000
--- a/skyvern/forge/prompts/skyvern/recording-wait-block-prompt.j2
+++ /dev/null
@@ -1,15 +0,0 @@
-Given a wait action, come up with a block label. It should include the duration in seconds, and a short version of a url, if the url exists.
-
-If the duration for the wait action is in milliseconds, so be sure to convert to seconds.
-
-MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
-
-Reply in JSON format with the following keys:
-{
- "block_label": str // A label for the block. Lower case. Underscores are permitted.
-}
-
-Wait action:
-```
-{{ action }}
-```
diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.py
index cddcf1fbd..20ad42a23 100644
--- a/skyvern/services/browser_recording/interpretation.py
+++ b/skyvern/services/browser_recording/interpretation.py
@@ -1,6 +1,7 @@
import asyncio
import dataclasses
import enum
+import time
import typing as t
import structlog
@@ -13,9 +14,15 @@ from skyvern.client.types.workflow_definition_yaml_blocks_item import (
)
from skyvern.forge.sdk.routes.streaming.channels import exfiltration as streaming_exfiltration
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEventSource
-from skyvern.services.browser_recording.service import Processor
+from skyvern.services.browser_recording.service import (
+ Processor,
+ deterministic_input_text_parameter_key,
+ normalize_recording_block_label,
+)
from skyvern.services.browser_recording.types import (
Action,
+ ActionBlockable,
+ ActionInputText,
ActionKind,
ExfiltratedCdpEvent,
ExfiltratedConsoleEvent,
@@ -29,7 +36,8 @@ from skyvern.services.browser_recording.types import (
LOG = structlog.get_logger(__name__)
-INTERPRETATION_DEBOUNCE_SECONDS = 1.5
+INTERPRETATION_DEBOUNCE_SECONDS = 0.4
+INTERPRETATION_MAX_WAIT_SECONDS = 2.0
SIGNIFICANT_CONSOLE_EVENT_TYPES = {
"blur",
"change",
@@ -161,6 +169,67 @@ def _draft_step_from_block(
return None
+def _action_display_text(action: Action) -> str:
+ for text in action.target.texts or []:
+ cleaned = " ".join(text.split())
+ if cleaned:
+ return cleaned[:80]
+
+ return (action.target.tag_name or "element").lower()
+
+
+_PLACEHOLDER_VERBS: dict[ActionKind, str] = {
+ ActionKind.CLICK: "Click",
+ ActionKind.HOVER: "Hover over",
+ ActionKind.INPUT_TEXT: "Fill",
+}
+
+
+def _placeholder_step_from_action(
+ *,
+ browser_session_id: str,
+ action_index: int,
+ action: ActionBlockable,
+) -> RecordingDraftStep:
+ """
+ A draft step built purely from exfiltrated event data, shown to the user
+ immediately while LLM enrichment runs in the background.
+ """
+ step_id = f"{browser_session_id}-recording-step-{action_index}"
+ text = _action_display_text(action)
+ verb = _PLACEHOLDER_VERBS[action.kind]
+ title = f"{verb} '{text}'"
+
+ parameters: list[dict[str, t.Any]] = []
+ parameter_keys: list[str] = []
+ if isinstance(action, ActionInputText):
+ parameter_key = deterministic_input_text_parameter_key(action)
+ navigation_goal = f"{verb} '{text}' with {{{{ {parameter_key} }}}}."
+ parameters = [{"key": parameter_key}]
+ parameter_keys = [parameter_key]
+ else:
+ navigation_goal = f"{verb} '{text}'."
+
+ return RecordingDraftStep(
+ step_id=step_id,
+ action_kind=action.kind,
+ block_type="action",
+ label=normalize_recording_block_label(f"{action.kind.value}_{text}", fallback=action.kind.value),
+ title=title,
+ navigation_goal=navigation_goal,
+ status=RecordingDraftStepStatus.INTERPRETING,
+ editable_fields=[
+ RecordingDraftStepEditableField.LABEL,
+ RecordingDraftStepEditableField.TITLE,
+ RecordingDraftStepEditableField.NAVIGATION_GOAL,
+ ],
+ parameters=parameters,
+ parameter_keys=parameter_keys,
+ timestamp_start=action.timestamp_start,
+ timestamp_end=action.timestamp_end,
+ )
+
+
class RecordingInterpretationSession:
def __init__(
self,
@@ -170,12 +239,14 @@ class RecordingInterpretationSession:
workflow_permanent_id: str,
on_update: OnRecordingInterpretationUpdate,
debounce_seconds: float = INTERPRETATION_DEBOUNCE_SECONDS,
+ max_wait_seconds: float = INTERPRETATION_MAX_WAIT_SECONDS,
) -> None:
self.browser_session_id = browser_session_id
self.organization_id = organization_id
self.workflow_permanent_id = workflow_permanent_id
self.on_update = on_update
self.debounce_seconds = debounce_seconds
+ self.max_wait_seconds = max_wait_seconds
self.events: list[ExfiltratedEvent] = []
self.steps: list[RecordingDraftStep] = []
self.emitted_action_count = 0
@@ -183,6 +254,8 @@ class RecordingInterpretationSession:
self.pending = False
self.finalized = False
self._debounce_task: asyncio.Task[None] | None = None
+ self._pending_since: float | None = None
+ self._enrichment_tasks: set[asyncio.Task[None]] = set()
self._interpret_lock = asyncio.Lock()
self._action_machines: list[sm.StateMachine] = [
sm.Click(),
@@ -204,36 +277,53 @@ class RecordingInterpretationSession:
if not any(event_should_trigger_interpretation(event) for event in reified_events):
return
+ if self._pending_since is None:
+ self._pending_since = time.monotonic()
+
self.pending = True
self.finalized = False
self._emit_update()
self._schedule_interpretation()
def cancel(self) -> None:
- if self._debounce_task and not self._debounce_task.done():
- self._debounce_task.cancel()
- self._debounce_task = None
+ self._cancel_debounce()
+ for task in list(self._enrichment_tasks):
+ task.cancel()
+ self._enrichment_tasks.clear()
async def flush(self) -> list[RecordingDraftStep]:
- self.cancel()
+ self._cancel_debounce()
await self._interpret(finalized=True)
return self.steps
+ def _cancel_debounce(self) -> None:
+ if self._debounce_task and not self._debounce_task.done():
+ self._debounce_task.cancel()
+ self._debounce_task = None
+
def _schedule_interpretation(self) -> None:
- self.cancel()
- self._debounce_task = asyncio.create_task(self._debounced_interpret())
+ self._cancel_debounce()
+
+ delay = self.debounce_seconds
+ if self._pending_since is not None:
+ deadline = self._pending_since + self.max_wait_seconds
+ delay = min(delay, max(0.0, deadline - time.monotonic()))
- async def _debounced_interpret(self) -> None:
+ self._debounce_task = asyncio.create_task(self._debounced_interpret(delay))
+
+ async def _debounced_interpret(self, delay: float) -> None:
+ # No finally-cleanup of self._debounce_task here: by the time a
+ # cancelled task unwinds, the reference may already point at a newer
+ # task, and clearing it would orphan that one.
try:
- await asyncio.sleep(self.debounce_seconds)
+ await asyncio.sleep(delay)
await self._interpret(finalized=False)
except asyncio.CancelledError:
return
- finally:
- self._debounce_task = None
async def _interpret(self, *, finalized: bool) -> None:
async with self._interpret_lock:
+ self._pending_since = None
processor = Processor(
self.browser_session_id,
self.organization_id,
@@ -249,53 +339,132 @@ class RecordingInterpretationSession:
)
self._processed_event_count = len(self.events)
- actions = self._all_actions
- new_actions = actions[self.emitted_action_count :]
-
- if not new_actions:
- self.pending = False
- self.finalized = finalized
- self._emit_update()
- return
-
- try:
- blocks = await processor.actions_to_blocks(new_actions)
- except Exception:
- self.pending = False
- self.finalized = finalized
- self._emit_update()
- LOG.exception(
- "Failed to interpret live browser recording actions",
- browser_session_id=self.browser_session_id,
- organization_id=self.organization_id,
- workflow_permanent_id=self.workflow_permanent_id,
- )
- return
-
- if len(blocks) < len(new_actions):
- LOG.warning(
- "Live interpretation produced fewer blocks than actions",
- browser_session_id=self.browser_session_id,
- action_count=len(new_actions),
- block_count=len(blocks),
- )
+ new_actions = self._all_actions[self.emitted_action_count :]
- for offset, (action, block) in enumerate(zip(new_actions, blocks, strict=False)):
+ for offset, action in enumerate(new_actions):
action_index = self.emitted_action_count + offset
- draft_step = _draft_step_from_block(
- browser_session_id=self.browser_session_id,
- action_index=action_index,
- action=action,
- block=block,
- )
+ draft_step = await self._step_from_action(processor, action_index, action)
if draft_step:
+ draft_step.label = self._unique_step_label(draft_step.label, draft_step)
self.steps.append(draft_step)
- self.emitted_action_count += len(blocks)
+ self.emitted_action_count += len(new_actions)
self.pending = False
- self.finalized = finalized
+ self.finalized = finalized and not self._enrichment_tasks
+ self._emit_update()
+
+ if not finalized:
+ return
+
+ # Hold the finalized signal until in-flight LLM enrichment lands, so
+ # commit paths never capture placeholder metadata.
+ await self._drain_enrichment()
+ if not self.finalized:
+ self.finalized = True
self._emit_update()
+ async def _step_from_action(
+ self,
+ processor: Processor,
+ action_index: int,
+ action: Action,
+ ) -> RecordingDraftStep | None:
+ """
+ Build a draft step for the action immediately. Deterministic kinds
+ (goto_url, wait) come back final; agent-action kinds come back as
+ placeholders with LLM enrichment scheduled in the background.
+ """
+ if action.kind in (ActionKind.CLICK, ActionKind.HOVER, ActionKind.INPUT_TEXT):
+ blockable = t.cast(ActionBlockable, action)
+ step = _placeholder_step_from_action(
+ browser_session_id=self.browser_session_id,
+ action_index=action_index,
+ action=blockable,
+ )
+ self._schedule_enrichment(processor, action_index, step, blockable)
+ return step
+
+ if action.kind in (ActionKind.URL_CHANGE, ActionKind.WAIT):
+ block: OutputBlock
+ if action.kind == ActionKind.URL_CHANGE:
+ block = await processor.create_url_block(action)
+ else:
+ block = await processor.create_wait_block(action)
+
+ return _draft_step_from_block(
+ browser_session_id=self.browser_session_id,
+ action_index=action_index,
+ action=action,
+ block=block,
+ )
+
+ LOG.warning(
+ "Unknown action kind in live interpretation",
+ action_kind=action.kind,
+ browser_session_id=self.browser_session_id,
+ )
+ return None
+
+ def _schedule_enrichment(
+ self,
+ processor: Processor,
+ action_index: int,
+ step: RecordingDraftStep,
+ action: ActionBlockable,
+ ) -> None:
+ task = asyncio.create_task(self._enrich_step(processor, action_index, step, action))
+ self._enrichment_tasks.add(task)
+ task.add_done_callback(self._enrichment_tasks.discard)
+
+ async def _enrich_step(
+ self,
+ processor: Processor,
+ action_index: int,
+ step: RecordingDraftStep,
+ action: ActionBlockable,
+ ) -> None:
+ enriched: RecordingDraftStep | None = None
+
+ try:
+ block = await processor.create_action_block(action)
+ enriched = _draft_step_from_block(
+ browser_session_id=self.browser_session_id,
+ action_index=action_index,
+ action=action,
+ block=block,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to enrich recording draft step; keeping deterministic placeholder",
+ browser_session_id=self.browser_session_id,
+ organization_id=self.organization_id,
+ step_id=step.step_id,
+ )
+
+ if enriched:
+ step.label = self._unique_step_label(enriched.label, step)
+ step.title = enriched.title
+ step.navigation_goal = enriched.navigation_goal
+ step.parameters = enriched.parameters
+ step.parameter_keys = enriched.parameter_keys
+
+ step.status = RecordingDraftStepStatus.READY
+ self._emit_update()
+
+ async def _drain_enrichment(self) -> None:
+ if self._enrichment_tasks:
+ await asyncio.gather(*list(self._enrichment_tasks), return_exceptions=True)
+
+ def _unique_step_label(self, label: str, step: RecordingDraftStep) -> str:
+ existing = {s.label for s in self.steps if s.step_id != step.step_id}
+ if label not in existing:
+ return label
+
+ suffix = 2
+ while f"{label}_{suffix}" in existing:
+ suffix += 1
+ return f"{label}_{suffix}"
+
def _emit_update(self) -> None:
self.session_revision += 1
update = RecordingInterpretationUpdate(
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index cc4c396e0..26456b74f 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -5,6 +5,7 @@ import pathlib
import re
import typing as t
import zlib
+from urllib.parse import urlparse
import structlog
@@ -20,6 +21,7 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.services.browser_recording.types import (
Action,
ActionBlockable,
+ ActionInputText,
ActionKind,
ActionUrlChange,
ActionWait,
@@ -93,6 +95,31 @@ def _is_duplicate_action(candidate: Action, existing_actions: list[Action]) -> b
)
+def deterministic_goto_url_label(url: str) -> str:
+ host = ""
+ try:
+ host = urlparse(url).netloc
+ except ValueError:
+ pass
+
+ return normalize_recording_block_label(f"goto_{host}" if host else None, fallback="goto_url")
+
+
+def deterministic_wait_seconds(duration_ms: int) -> int:
+ return int(max(duration_ms / 1000.0, ActionWait.MIN_DURATION_THRESHOLD_MS / 1000.0))
+
+
+def deterministic_input_text_parameter_key(action: ActionInputText) -> str:
+ target = action.target
+ for candidate in (target.id, *(target.texts or []), target.sky_id):
+ if not candidate:
+ continue
+ key = normalize_recording_block_label(str(candidate), fallback="")
+ if key:
+ return key.lower()
+ return "input_value"
+
+
def normalize_recording_block_label(label: str | None, *, fallback: str) -> str:
candidate = (label or "").strip()
candidate = re.sub(r"\W+", "_", candidate)
@@ -475,59 +502,21 @@ class Processor:
return block
async def create_url_block(self, action: ActionUrlChange) -> WorkflowDefinitionYamlBlocksItem_GotoUrl:
- """
- Create a YAML goto URL block from an `ActionUrlChange`.
- """
-
- prompt_name = "recording-go-to-url-block-prompt"
-
- metadata_prompt = prompt_engine.load_prompt(
- prompt_name,
- action=action,
- )
-
- metadata_response = await app.LLM_API_HANDLER(
- prompt=metadata_prompt,
- prompt_name=prompt_name,
- organization_id=self.organization_id,
- )
-
- block_label: str = metadata_response.get("block_label", None) or "goto_url"
-
- block = WorkflowDefinitionYamlBlocksItem_GotoUrl(
- label=block_label,
+ """Build a goto-url block with a host-derived label and the action URL."""
+ return WorkflowDefinitionYamlBlocksItem_GotoUrl(
+ label=deterministic_goto_url_label(action.url),
url=action.url,
)
- return block
-
async def create_wait_block(self, action: ActionWait) -> WorkflowDefinitionYamlBlocksItem_Wait:
- """
- Create a YAML wait block from an `ActionWait`.
- """
-
- prompt_name = "recording-wait-block-prompt"
-
- metadata_prompt = prompt_engine.load_prompt(
- prompt_name,
- action=action,
- )
+ """Build a wait block with a duration-derived label and wait seconds."""
+ wait_sec = deterministic_wait_seconds(action.duration_ms)
- metadata_response = await app.LLM_API_HANDLER(
- prompt=metadata_prompt,
- prompt_name=prompt_name,
- organization_id=self.organization_id,
+ return WorkflowDefinitionYamlBlocksItem_Wait(
+ label=f"wait_{wait_sec}s",
+ wait_sec=wait_sec,
)
- block_label: str = metadata_response.get("block_label", None) or "wait"
-
- block = WorkflowDefinitionYamlBlocksItem_Wait(
- label=block_label,
- wait_sec=int(max(action.duration_ms / 1000.0, ActionWait.MIN_DURATION_THRESHOLD_MS / 1000.0)),
- )
-
- return block
-
async def process(
self,
compressed_chunks: list[str],
diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.py
index 571a12589..a5db3d150 100644
--- a/tests/unit/services/test_browser_recording.py
+++ b/tests/unit/services/test_browser_recording.py
@@ -2,13 +2,35 @@
Just an example unit test for now. Will expand later.
"""
+import asyncio
+import time
import typing as t
-from skyvern.services.browser_recording.service import Processor, summarize_exfiltrated_recording_events
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEvent as StreamingExfiltratedEvent
+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
+ ExfiltratedEventSource as StreamingExfiltratedEventSource,
+)
+from skyvern.services.browser_recording.interpretation import RecordingInterpretationSession
+from skyvern.services.browser_recording.service import (
+ Processor,
+ deterministic_input_text_parameter_key,
+ summarize_exfiltrated_recording_events,
+)
from skyvern.services.browser_recording.types import (
+ ActionInputText,
+ ActionKind,
+ ActionTarget,
+ ActionUrlChange,
+ ActionWait,
ExfiltratedCdpEvent,
ExfiltratedConsoleEvent,
ExfiltratedEventCdpParams,
+ Mouse,
+ RecordingDraftStepStatus,
+ RecordingInterpretationUpdate,
)
ORG_ID = "org_123"
@@ -143,6 +165,276 @@ def test_hover() -> None:
assert len(actions) == 1
+def _empty_action_target() -> ActionTarget:
+ return ActionTarget(mouse=Mouse(xp=None, yp=None))
+
+
+def make_streaming_console_click(timestamp_ms: float) -> StreamingExfiltratedEvent:
+ return StreamingExfiltratedEvent(
+ event_name="user_interaction",
+ params={
+ "type": "click",
+ "target": {"id": "button-1", "skyId": "sky-123", "tagName": "BUTTON", "text": ["Click me"]},
+ "timestamp": timestamp_ms,
+ "url": "https://example.com",
+ "activeElement": {"tagName": "BUTTON"},
+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},
+ "mousePosition": {"xp": 0.5, "yp": 0.5},
+ },
+ source=StreamingExfiltratedEventSource.CONSOLE,
+ timestamp=timestamp_ms / 1000.0,
+ )
+
+
+def make_streaming_nav_event(url: str, timestamp: float) -> StreamingExfiltratedEvent:
+ return StreamingExfiltratedEvent(
+ event_name="nav:frame_started_navigating",
+ params={"url": url},
+ source=StreamingExfiltratedEventSource.CDP,
+ timestamp=timestamp,
+ )
+
+
+def make_streaming_console_input(
+ *,
+ timestamp_ms: float,
+ input_value: str,
+ target_id: str = "email",
+ target_text: str = "Email",
+) -> list[StreamingExfiltratedEvent]:
+ target = {
+ "id": target_id,
+ "skyId": "sky-email",
+ "tagName": "INPUT",
+ "text": [target_text],
+ "value": input_value,
+ }
+ common = {
+ "target": target,
+ "timestamp": timestamp_ms,
+ "url": "https://example.com",
+ "activeElement": {"tagName": "INPUT"},
+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},
+ "mousePosition": {"xp": 0.5, "yp": 0.5},
+ }
+
+ return [
+ StreamingExfiltratedEvent(
+ event_name="user_interaction",
+ params={"type": "focus", **common},
+ source=StreamingExfiltratedEventSource.CONSOLE,
+ timestamp=timestamp_ms / 1000.0,
+ ),
+ StreamingExfiltratedEvent(
+ event_name="user_interaction",
+ params={"type": "keydown", "key": "a", **common},
+ source=StreamingExfiltratedEventSource.CONSOLE,
+ timestamp=(timestamp_ms + 1) / 1000.0,
+ ),
+ StreamingExfiltratedEvent(
+ event_name="user_interaction",
+ params={"type": "blur", **common},
+ source=StreamingExfiltratedEventSource.CONSOLE,
+ timestamp=(timestamp_ms + 2) / 1000.0,
+ ),
+ ]
+
+
+def test_create_url_block_is_deterministic() -> None:
+ action = ActionUrlChange(
+ kind=ActionKind.URL_CHANGE,
+ target=_empty_action_target(),
+ timestamp_start=1000.0,
+ timestamp_end=1000.0,
+ url="https://example.com/products?page=2",
+ )
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ block = asyncio.run(processor.create_url_block(action))
+
+ assert block.label == "goto_example_com"
+ assert block.url == "https://example.com/products?page=2"
+
+
+def test_create_wait_block_is_deterministic() -> None:
+ action = ActionWait(
+ kind=ActionKind.WAIT,
+ target=_empty_action_target(),
+ timestamp_start=1000.0,
+ timestamp_end=8000.0,
+ url="https://example.com",
+ duration_ms=7000,
+ )
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ block = asyncio.run(processor.create_wait_block(action))
+
+ assert block.label == "wait_7s"
+ assert block.wait_sec == 7
+
+
+def test_create_wait_block_floors_at_minimum_duration() -> None:
+ action = ActionWait(
+ kind=ActionKind.WAIT,
+ target=_empty_action_target(),
+ timestamp_start=1000.0,
+ timestamp_end=2000.0,
+ url="https://example.com",
+ duration_ms=1000,
+ )
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ block = asyncio.run(processor.create_wait_block(action))
+
+ assert block.wait_sec == ActionWait.MIN_DURATION_THRESHOLD_MS // 1000
+
+
+def test_input_text_parameter_key_is_derived_from_target_metadata() -> None:
+ action = ActionInputText(
+ kind=ActionKind.INPUT_TEXT,
+ target=ActionTarget(
+ id="customer_email",
+ sky_id="sky-email",
+ tag_name="INPUT",
+ texts=["Email"],
+ mouse=Mouse(xp=0.5, yp=0.5),
+ ),
+ timestamp_start=1000.0,
+ timestamp_end=1001.0,
+ url="https://example.com",
+ input_value="secret123",
+ )
+
+ assert deterministic_input_text_parameter_key(action) == "customer_email"
+
+
+async def test_input_text_placeholder_parameterizes_value_on_enrichment_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+
+ session.ingest_events(make_streaming_console_input(timestamp_ms=1000.0, input_value="secret123"))
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].status == RecordingDraftStepStatus.READY
+ assert "secret123" not in (steps[0].navigation_goal or "")
+ assert steps[0].navigation_goal == "Fill 'Email' with {{ email }}."
+ assert steps[0].parameter_keys == ["email"]
+ assert steps[0].parameters == [{"key": "email"}]
+
+
+async def test_live_interpretation_emits_placeholder_then_enriched(monkeypatch: pytest.MonkeyPatch) -> None:
+ release_llm = asyncio.Event()
+
+ async def fake_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ await release_llm.wait()
+ return {
+ "block_label": "click_submit",
+ "title": "Click Submit",
+ "prompt": "Click the submit button.",
+ }
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)
+
+ updates: list[RecordingInterpretationUpdate] = []
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=updates.append,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+
+ session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])
+ await asyncio.sleep(0.05)
+
+ # the placeholder draft is visible before the LLM responds
+ updates_with_steps = [update for update in updates if update.steps]
+ assert updates_with_steps
+ placeholder = updates_with_steps[-1].steps[0]
+ assert placeholder.status == RecordingDraftStepStatus.INTERPRETING
+ assert placeholder.title == "Click 'Click me'"
+ assert placeholder.navigation_goal == "Click 'Click me'."
+
+ release_llm.set()
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].status == RecordingDraftStepStatus.READY
+ assert steps[0].title == "Click Submit"
+ assert steps[0].navigation_goal == "Click the submit button."
+ assert updates[-1].finalized is True
+
+
+async def test_live_interpretation_enrichment_failure_keeps_placeholder(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+
+ session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].status == RecordingDraftStepStatus.READY
+ assert steps[0].title == "Click 'Click me'"
+
+
+async def test_live_interpretation_max_wait_fires_during_continuous_events() -> None:
+ updates: list[RecordingInterpretationUpdate] = []
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=updates.append,
+ debounce_seconds=0.05,
+ max_wait_seconds=0.1,
+ )
+
+ # Significant events arrive faster than the quiet debounce, so a pure
+ # trailing debounce would defer interpretation until the stream stops.
+ start = time.monotonic()
+ sequence = 0
+ while time.monotonic() - start < 0.35:
+ session.ingest_events([make_streaming_nav_event(url=f"https://example.com/{sequence}", timestamp=time.time())])
+ sequence += 1
+ await asyncio.sleep(0.02)
+
+ try:
+ assert any(update.steps for update in updates)
+ finally:
+ session.cancel()
+ # let the cancelled debounce task unwind before the loop closes
+ await asyncio.sleep(0.01)
+
+
def test_summarize_exfiltrated_recording_events_mixed() -> None:
target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])
click = make_click_event(target=target, timestamp=1000.0)
diff --git a/tests/unit/test_browser_recording_interpretation.py b/tests/unit/test_browser_recording_interpretation.py
index 3fc548d01..10d4d9699 100644
--- a/tests/unit/test_browser_recording_interpretation.py
+++ b/tests/unit/test_browser_recording_interpretation.py
@@ -17,7 +17,10 @@ from skyvern.services.browser_recording.interpretation import (
from skyvern.services.browser_recording.service import Processor
from skyvern.services.browser_recording.types import (
ActionKind,
+ ActionTarget,
+ ActionWait,
ExfiltratedConsoleEvent,
+ Mouse,
RecordingDraftStep,
)
@@ -220,7 +223,7 @@ async def test_recording_interpretation_session_flush_cancels_debounce(monkeypat
debounce_started = asyncio.Event()
allow_debounce_finish = asyncio.Event()
- async def fake_debounced_interpret(self: RecordingInterpretationSession) -> None:
+ async def fake_debounced_interpret(self: RecordingInterpretationSession, delay: float) -> None:
debounce_started.set()
await allow_debounce_finish.wait()
@@ -252,11 +255,24 @@ async def test_recording_interpretation_session_flush_cancels_debounce(monkeypat
@pytest.mark.asyncio
-async def test_recording_interpretation_session_advances_only_matched_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
- wait_block = WorkflowDefinitionYamlBlocksItem_Wait(label="wait", wait_sec=5)
+async def test_recording_interpretation_session_advances_past_unhandled_actions(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ wait_action = ActionWait(
+ kind=ActionKind.WAIT,
+ target=ActionTarget(mouse=Mouse(xp=None, yp=None)),
+ timestamp_start=1000.0,
+ timestamp_end=8000.0,
+ url="https://example.com",
+ duration_ms=7000,
+ )
+ unhandled_action = MagicMock()
+ unhandled_action.kind = "unsupported"
+
processor = MagicMock()
- processor.events_to_actions.return_value = [MagicMock(), MagicMock()]
- processor.actions_to_blocks = AsyncMock(return_value=[wait_block])
+ processor.create_wait_block = AsyncMock(
+ return_value=WorkflowDefinitionYamlBlocksItem_Wait(label="wait_7s", wait_sec=7),
+ )
monkeypatch.setattr(
"skyvern.services.browser_recording.interpretation.Processor",
lambda *args, **kwargs: processor,
@@ -270,9 +286,9 @@ async def test_recording_interpretation_session_advances_only_matched_blocks(mon
)
session.events = [MagicMock(), MagicMock()]
session._processed_event_count = len(session.events)
- session._all_actions = processor.events_to_actions.return_value
+ session._all_actions = [wait_action, unhandled_action]
await session._interpret(finalized=False)
- assert session.emitted_action_count == 1
+ assert session.emitted_action_count == 2
assert len(session.steps) == 1