Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
perf(record): emit interpretation deltas instead of full snapshots (#6995)
Cindy Lie52e90fd79cc
diff --git a/skyvern/config.py b/skyvern/config.pyindex af76e59d4..a348a3f40 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -295,6 +295,10 @@ class Settings(BaseSettings):# model keeps the click->labeled-draft latency low. Falls back to the default# LLM_API_HANDLER when the key is unset or not registered in this environment.RECORDING_ENRICHMENT_LLM_KEY: str = "GEMINI_3.1_FLASH_LITE"+ # Server-side kill switch for delta interpretation updates. Deltas are also+ # gated per-connection on the client declaring support (begin-exfiltration+ # supports_interpretation_deltas); this only force-disables them everywhere.+ RECORDING_INTERPRETATION_DELTAS_ENABLED: bool = TrueBROWSER_POLICY_FILE: str = "/etc/chromium/policies/managed/policies.json"BROWSER_LOGS_ENABLED: bool = TrueBROWSER_CURSOR_VISUALIZATION: bool = Falsediff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.pyindex 0995f1310..5c146e949 100644--- a/skyvern/forge/sdk/routes/streaming/channels/message.py+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py@@ -133,6 +133,9 @@ class MessageInBeginExfiltration(Message):kind: t.Literal[MessageKind.BEGIN_EXFILTRATION] = MessageKind.BEGIN_EXFILTRATIONworkflow_permanent_id: str | None = Nonelive_interpretation_enabled: bool = False+ # Client capability: true only when the frontend understands delta interpretation+ # updates. Defaults false so older clients keep receiving full snapshots.+ supports_interpretation_deltas: bool = False@dataclasses.dataclass@@ -268,6 +271,8 @@ class MessageOutRecordingInterpretationUpdate(Message):interpretation_session_id: str = ""session_revision: int = 0steps: list[RecordingDraftStep] = dataclasses.field(default_factory=list)+ changed_steps: list[RecordingDraftStep] = dataclasses.field(default_factory=list)+ is_snapshot: bool = Truepending: bool = Falsefinalized: bool = False@@ -326,6 +331,7 @@ def reify_channel_message(data: dict) -> ChannelMessage:return MessageInBeginExfiltration(workflow_permanent_id=workflow_permanent_id if isinstance(workflow_permanent_id, str) else None,live_interpretation_enabled=bool(data.get("live_interpretation_enabled") or False),+ supports_interpretation_deltas=bool(data.get("supports_interpretation_deltas") or False),)case MessageKind.CEDE_CONTROL:return MessageInCedeControl()@@ -660,6 +666,8 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:interpretation_session_id=update.interpretation_session_id,session_revision=update.session_revision,steps=update.steps,+ changed_steps=update.changed_steps,+ is_snapshot=update.is_snapshot,pending=update.pending,finalized=update.finalized,)@@ -679,6 +687,7 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:organization_id=message_channel.organization_id,workflow_permanent_id=message.workflow_permanent_id,on_update=on_interpretation_update,+ deltas_enabled=message.supports_interpretation_deltas,)live_interpretation_browser_session_id = browser_session_iddiff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex 29945aa70..bc8149647 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -246,11 +246,13 @@ class RecordingInterpretationSession:on_update: OnRecordingInterpretationUpdate,debounce_seconds: float = INTERPRETATION_DEBOUNCE_SECONDS,max_wait_seconds: float = INTERPRETATION_MAX_WAIT_SECONDS,+ deltas_enabled: bool = False,) -> None:self.browser_session_id = browser_session_idself.organization_id = organization_idself.workflow_permanent_id = workflow_permanent_idself.on_update = on_update+ self.set_deltas_enabled(deltas_enabled)self.interpretation_session_id = str(uuid.uuid4())self.debounce_seconds = debounce_secondsself.max_wait_seconds = max_wait_seconds@@ -281,6 +283,10 @@ class RecordingInterpretationSession:if isinstance(machine, sm.Wait):machine.reset()+ def set_deltas_enabled(self, enabled: bool) -> None:+ # Deltas require both the client capability and the server kill switch.+ self._deltas_enabled = enabled and settings.RECORDING_INTERPRETATION_DELTAS_ENABLED+def pause_capture(self) -> None:self._capture_paused = Trueself.reset_wait_capture()@@ -288,6 +294,10 @@ class RecordingInterpretationSession:def resume_capture(self) -> None:self._capture_paused = Falseself.reset_wait_capture()+ # Enrichment deltas that landed while paused were dropped client-side, so+ # send an authoritative snapshot to resync on resume.+ if self.session_revision > 0:+ self._emit()def ingest_events(self, events: list[streaming_exfiltration.ExfiltratedEvent]) -> None:if self._capture_paused:@@ -314,7 +324,8 @@ class RecordingInterpretationSession:self.pending = Trueself.finalized = False- self._emit_update()+ # Pending ping: no step changed yet, just signal interpretation is in flight.+ self._emit(changed_steps=[])self._schedule_interpretation()def cancel(self) -> None:@@ -383,27 +394,29 @@ class RecordingInterpretationSession:new_actions = self._all_actions[self.emitted_action_count :]+ new_steps: list[RecordingDraftStep] = []for offset, action in enumerate(new_actions):action_index = self.emitted_action_count + offsetdraft_step = await self._step_from_action(processor, action_index, action)if draft_step:draft_step.label = self._unique_step_label(draft_step.label, draft_step)self.steps.append(draft_step)+ new_steps.append(draft_step)self.emitted_action_count += len(new_actions)self.pending = Falseself.finalized = finalized and not self._enrichment_tasks- self._emit_update()+ self._emit(changed_steps=new_steps)if not finalized:return- # Hold the finalized signal until in-flight LLM enrichment lands, so- # commit paths never capture placeholder metadata.+ # Hold the finalized signal until in-flight LLM enrichment lands, so commit+ # paths never capture placeholder metadata. Emit a final snapshot so the+ # client's full step list is authoritative regardless of any dropped delta.await self._drain_enrichment()- if not self.finalized:- self.finalized = True- self._emit_update()+ self.finalized = True+ self._emit()async def _step_from_action(self,@@ -492,7 +505,7 @@ class RecordingInterpretationSession:step.parameter_keys = enriched.parameter_keysstep.status = RecordingDraftStepStatus.READY- self._emit_update()+ self._emit(changed_steps=[step])async def _drain_enrichment(self) -> None:while self._enrichment_tasks:@@ -529,12 +542,21 @@ class RecordingInterpretationSession:organization_id=self.organization_id,)- def _emit_update(self) -> None:+ def _emit(self, *, changed_steps: list[RecordingDraftStep] | None = None) -> None:+ # Snapshot when changed_steps is None (full steps, is_snapshot=True); delta+ # otherwise (only the changed steps), keeping each update O(1) instead of+ # re-sending the whole growing list. Both advance session_revision so the+ # client's staleness guard orders them.self.session_revision += 1+ # Send a delta only when the client understands them and a changed set was+ # given; otherwise fall back to a full snapshot (legacy-compatible).+ is_delta = self._deltas_enabled and changed_steps is not Noneupdate = RecordingInterpretationUpdate(interpretation_session_id=self.interpretation_session_id,session_revision=self.session_revision,- steps=self.steps,+ steps=[] if is_delta else self.steps,+ changed_steps=changed_steps if is_delta else [],+ is_snapshot=not is_delta,pending=self.pending,finalized=self.finalized,)diff --git a/skyvern/services/browser_recording/session_registry.py b/skyvern/services/browser_recording/session_registry.pyindex f13383953..b1eb8d3cd 100644--- a/skyvern/services/browser_recording/session_registry.py+++ b/skyvern/services/browser_recording/session_registry.py@@ -27,6 +27,7 @@ class RecordingInterpretationSessionRegistry:organization_id: str,workflow_permanent_id: str,on_update: OnRecordingInterpretationUpdate,+ deltas_enabled: bool = False,) -> None:self._prune_expired_sessions()existing = self._sessions.get(browser_session_id)@@ -37,6 +38,7 @@ class RecordingInterpretationSessionRegistry:and not existing.finalized):existing.on_update = on_update+ existing.set_deltas_enabled(deltas_enabled)self._last_seen[browser_session_id] = time.monotonic()existing.emit_snapshot()return@@ -47,6 +49,7 @@ class RecordingInterpretationSessionRegistry:organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,on_update=on_update,+ deltas_enabled=deltas_enabled,)self._last_seen[browser_session_id] = time.monotonic()diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.pyindex d08364752..1e1274108 100644--- a/skyvern/services/browser_recording/types.py+++ b/skyvern/services/browser_recording/types.py@@ -127,7 +127,12 @@ class RecordingDraftStep(BaseModel):class RecordingInterpretationUpdate(BaseModel):interpretation_session_id: strsession_revision: int+ # Authoritative full list when is_snapshot is true. Empty on delta updates.steps: list[RecordingDraftStep] = Field(default_factory=list)+ # Per-step upserts (by step_id) when is_snapshot is false. Keeps each update+ # O(1) instead of re-sending the whole growing steps list.+ changed_steps: list[RecordingDraftStep] = Field(default_factory=list)+ is_snapshot: bool = Truepending: bool = Falsefinalized: bool = Falsediff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.pyindex 8105e8a38..51a38b520 100644--- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py@@ -104,6 +104,31 @@ class TestReifyChannelMessage:assert msg.workflow_permanent_id == "wpid_123"assert msg.live_interpretation_enabled is True+ def test_begin_exfiltration_defaults_no_delta_support(self) -> None:+ # An existing client that predates deltas omits the field; it must default+ # false so the backend keeps sending full snapshots (no breakage).+ msg = reify_channel_message(+ {+ "kind": "begin-exfiltration",+ "workflow_permanent_id": "wpid_123",+ "live_interpretation_enabled": True,+ }+ )+ assert isinstance(msg, MessageInBeginExfiltration)+ assert msg.supports_interpretation_deltas is False++ def test_begin_exfiltration_honors_declared_delta_support(self) -> None:+ msg = reify_channel_message(+ {+ "kind": "begin-exfiltration",+ "workflow_permanent_id": "wpid_123",+ "live_interpretation_enabled": True,+ "supports_interpretation_deltas": True,+ }+ )+ assert isinstance(msg, MessageInBeginExfiltration)+ assert msg.supports_interpretation_deltas is True+def test_recording_rearm_capture(self) -> None:assert isinstance(reify_channel_message({"kind": "recording-rearm-capture"}),diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex 986420a00..bd6c99b7a 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -37,6 +37,7 @@ from skyvern.services.browser_recording.types import (ExfiltratedConsoleEvent,ExfiltratedEventCdpParams,Mouse,+ RecordingDraftStep,RecordingDraftStepStatus,RecordingInterpretationUpdate,)@@ -486,10 +487,14 @@ async def test_live_interpretation_emits_placeholder_then_enriched(monkeypatch:session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])await asyncio.sleep(0.05)- # the placeholder draft is visible before the LLM responds- updates_with_steps = [update for update in updates if update.steps]+ # the placeholder draft is visible before the LLM responds — it arrives as a+ # delta (changed_steps), since only snapshots carry the full steps list.+ def emitted_steps(u: RecordingInterpretationUpdate) -> list[RecordingDraftStep]:+ return u.steps if u.is_snapshot else u.changed_steps++ updates_with_steps = [update for update in updates if emitted_steps(update)]assert updates_with_steps- placeholder = updates_with_steps[-1].steps[0]+ placeholder = emitted_steps(updates_with_steps[-1])[0]assert placeholder.status == RecordingDraftStepStatus.INTERPRETINGassert placeholder.title == "Click 'Click me'"assert placeholder.navigation_goal == "Click 'Click me'."@@ -582,7 +587,8 @@ async def test_live_interpretation_max_wait_fires_during_continuous_events() ->await asyncio.sleep(0.02)try:- assert any(update.steps for update in updates)+ # steps land in changed_steps (delta) or steps (snapshot) depending on emit type.+ assert any(update.steps or update.changed_steps for update in updates)finally:session.cancel()# let the cancelled debounce task unwind before the loop closesdiff --git a/tests/unit/test_browser_recording_interpretation.py b/tests/unit/test_browser_recording_interpretation.pyindex bed7a1cad..a36944c25 100644--- a/tests/unit/test_browser_recording_interpretation.py+++ b/tests/unit/test_browser_recording_interpretation.py@@ -24,6 +24,7 @@ from skyvern.services.browser_recording.types import (Mouse,RecordingDraftStep,RecordingDraftStepStatus,+ RecordingInterpretationUpdate,)ORG_ID = "org_123"@@ -583,3 +584,96 @@ def test_start_session_resumes_after_websocket_disconnect_without_stop() -> Noneassert registry._sessions[PBS_ID] is sessionassert reconnect_updates == [4]+++async def test_emits_deltas_for_steps_and_snapshot_on_finalize(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:+ return {"block_label": "click_submit", "title": "Click Submit", "prompt": "Click submit."}++ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)++ updates: list[RecordingInterpretationUpdate] = []+ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=updates.append,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ deltas_enabled=True,+ )++ session.ingest_events([_click_streaming_event(timestamp=1000.0)])+ await session.flush()++ # Steps arrive as deltas (placeholder + enriched), never re-sending the full list.+ deltas = [u for u in updates if not u.is_snapshot]+ assert any(u.changed_steps for u in deltas)+ assert all(u.steps == [] for u in deltas)++ # Finalize ends with an authoritative snapshot carrying the full list.+ assert updates[-1].is_snapshot is True+ assert updates[-1].finalized is True+ assert len(updates[-1].steps) == 1++ # A delta never smuggles the whole growing list back in.+ assert all(u.is_snapshot or not u.steps for u in updates)+++async def test_no_deltas_when_client_lacks_capability(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:+ return {"block_label": "click", "title": "Click", "prompt": "Click."}++ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)++ updates: list[RecordingInterpretationUpdate] = []+ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=updates.append,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ # deltas_enabled defaults False — a client that didn't opt in gets snapshots.+ )++ session.ingest_events([_click_streaming_event(timestamp=1000.0)])+ await session.flush()++ # Every update is a full snapshot; no changed_steps are ever sent.+ assert all(u.is_snapshot for u in updates)+ assert all(not u.changed_steps for u in updates)+ assert updates[-1].steps # final snapshot still carries the steps+++async def test_resume_capture_emits_resync_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:+ return {"block_label": "click", "title": "Click", "prompt": "Click."}++ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)++ updates: list[RecordingInterpretationUpdate] = []+ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=updates.append,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )++ session.ingest_events([_click_streaming_event(timestamp=1000.0)])+ await asyncio.sleep(0.05)++ session.pause_capture()+ updates.clear()+ session.resume_capture()++ assert len(updates) == 1+ assert updates[0].is_snapshot is True+ session.cancel()