Skip to content

Automate browser based workflows with AI

perf(record): emit interpretation deltas instead of full snapshots (#6995)

Cindy Lie52e90fd79cc
diff --git a/skyvern/config.py b/skyvern/config.py
index af76e59d4..a348a3f40 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -295,6 +295,10 @@ class Settings(BaseSettings):
# model keeps the click->labeled-draft latency low. Falls back to the default
# LLM_API_HANDLER when the key is unset or not registered in this environment.
RECORDING_ENRICHMENT_LLM_KEY: str = "GEMINI_3.1_FLASH_LITE"
+ # Server-side kill switch for delta interpretation updates. Deltas are also
+ # gated per-connection on the client declaring support (begin-exfiltration
+ # supports_interpretation_deltas); this only force-disables them everywhere.
+ RECORDING_INTERPRETATION_DELTAS_ENABLED: bool = True
BROWSER_POLICY_FILE: str = "/etc/chromium/policies/managed/policies.json"
BROWSER_LOGS_ENABLED: bool = True
BROWSER_CURSOR_VISUALIZATION: bool = False
diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.py
index 0995f1310..5c146e949 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/message.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py
@@ -133,6 +133,9 @@ class MessageInBeginExfiltration(Message):
kind: t.Literal[MessageKind.BEGIN_EXFILTRATION] = MessageKind.BEGIN_EXFILTRATION
workflow_permanent_id: str | None = None
live_interpretation_enabled: bool = False
+ # Client capability: true only when the frontend understands delta interpretation
+ # updates. Defaults false so older clients keep receiving full snapshots.
+ supports_interpretation_deltas: bool = False
@dataclasses.dataclass
@@ -268,6 +271,8 @@ class MessageOutRecordingInterpretationUpdate(Message):
interpretation_session_id: str = ""
session_revision: int = 0
steps: list[RecordingDraftStep] = dataclasses.field(default_factory=list)
+ changed_steps: list[RecordingDraftStep] = dataclasses.field(default_factory=list)
+ is_snapshot: bool = True
pending: bool = False
finalized: bool = False
@@ -326,6 +331,7 @@ def reify_channel_message(data: dict) -> ChannelMessage:
return MessageInBeginExfiltration(
workflow_permanent_id=workflow_permanent_id if isinstance(workflow_permanent_id, str) else None,
live_interpretation_enabled=bool(data.get("live_interpretation_enabled") or False),
+ supports_interpretation_deltas=bool(data.get("supports_interpretation_deltas") or False),
)
case MessageKind.CEDE_CONTROL:
return MessageInCedeControl()
@@ -660,6 +666,8 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
interpretation_session_id=update.interpretation_session_id,
session_revision=update.session_revision,
steps=update.steps,
+ changed_steps=update.changed_steps,
+ is_snapshot=update.is_snapshot,
pending=update.pending,
finalized=update.finalized,
)
@@ -679,6 +687,7 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
organization_id=message_channel.organization_id,
workflow_permanent_id=message.workflow_permanent_id,
on_update=on_interpretation_update,
+ deltas_enabled=message.supports_interpretation_deltas,
)
live_interpretation_browser_session_id = browser_session_id
diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.py
index 29945aa70..bc8149647 100644
--- a/skyvern/services/browser_recording/interpretation.py
+++ b/skyvern/services/browser_recording/interpretation.py
@@ -246,11 +246,13 @@ class RecordingInterpretationSession:
on_update: OnRecordingInterpretationUpdate,
debounce_seconds: float = INTERPRETATION_DEBOUNCE_SECONDS,
max_wait_seconds: float = INTERPRETATION_MAX_WAIT_SECONDS,
+ deltas_enabled: bool = False,
) -> None:
self.browser_session_id = browser_session_id
self.organization_id = organization_id
self.workflow_permanent_id = workflow_permanent_id
self.on_update = on_update
+ self.set_deltas_enabled(deltas_enabled)
self.interpretation_session_id = str(uuid.uuid4())
self.debounce_seconds = debounce_seconds
self.max_wait_seconds = max_wait_seconds
@@ -281,6 +283,10 @@ class RecordingInterpretationSession:
if isinstance(machine, sm.Wait):
machine.reset()
+ def set_deltas_enabled(self, enabled: bool) -> None:
+ # Deltas require both the client capability and the server kill switch.
+ self._deltas_enabled = enabled and settings.RECORDING_INTERPRETATION_DELTAS_ENABLED
+
def pause_capture(self) -> None:
self._capture_paused = True
self.reset_wait_capture()
@@ -288,6 +294,10 @@ class RecordingInterpretationSession:
def resume_capture(self) -> None:
self._capture_paused = False
self.reset_wait_capture()
+ # Enrichment deltas that landed while paused were dropped client-side, so
+ # send an authoritative snapshot to resync on resume.
+ if self.session_revision > 0:
+ self._emit()
def ingest_events(self, events: list[streaming_exfiltration.ExfiltratedEvent]) -> None:
if self._capture_paused:
@@ -314,7 +324,8 @@ class RecordingInterpretationSession:
self.pending = True
self.finalized = False
- self._emit_update()
+ # Pending ping: no step changed yet, just signal interpretation is in flight.
+ self._emit(changed_steps=[])
self._schedule_interpretation()
def cancel(self) -> None:
@@ -383,27 +394,29 @@ class RecordingInterpretationSession:
new_actions = self._all_actions[self.emitted_action_count :]
+ new_steps: list[RecordingDraftStep] = []
for offset, action in enumerate(new_actions):
action_index = self.emitted_action_count + offset
draft_step = await self._step_from_action(processor, action_index, action)
if draft_step:
draft_step.label = self._unique_step_label(draft_step.label, draft_step)
self.steps.append(draft_step)
+ new_steps.append(draft_step)
self.emitted_action_count += len(new_actions)
self.pending = False
self.finalized = finalized and not self._enrichment_tasks
- self._emit_update()
+ self._emit(changed_steps=new_steps)
if not finalized:
return
- # Hold the finalized signal until in-flight LLM enrichment lands, so
- # commit paths never capture placeholder metadata.
+ # Hold the finalized signal until in-flight LLM enrichment lands, so commit
+ # paths never capture placeholder metadata. Emit a final snapshot so the
+ # client's full step list is authoritative regardless of any dropped delta.
await self._drain_enrichment()
- if not self.finalized:
- self.finalized = True
- self._emit_update()
+ self.finalized = True
+ self._emit()
async def _step_from_action(
self,
@@ -492,7 +505,7 @@ class RecordingInterpretationSession:
step.parameter_keys = enriched.parameter_keys
step.status = RecordingDraftStepStatus.READY
- self._emit_update()
+ self._emit(changed_steps=[step])
async def _drain_enrichment(self) -> None:
while self._enrichment_tasks:
@@ -529,12 +542,21 @@ class RecordingInterpretationSession:
organization_id=self.organization_id,
)
- def _emit_update(self) -> None:
+ def _emit(self, *, changed_steps: list[RecordingDraftStep] | None = None) -> None:
+ # Snapshot when changed_steps is None (full steps, is_snapshot=True); delta
+ # otherwise (only the changed steps), keeping each update O(1) instead of
+ # re-sending the whole growing list. Both advance session_revision so the
+ # client's staleness guard orders them.
self.session_revision += 1
+ # Send a delta only when the client understands them and a changed set was
+ # given; otherwise fall back to a full snapshot (legacy-compatible).
+ is_delta = self._deltas_enabled and changed_steps is not None
update = RecordingInterpretationUpdate(
interpretation_session_id=self.interpretation_session_id,
session_revision=self.session_revision,
- steps=self.steps,
+ steps=[] if is_delta else self.steps,
+ changed_steps=changed_steps if is_delta else [],
+ is_snapshot=not is_delta,
pending=self.pending,
finalized=self.finalized,
)
diff --git a/skyvern/services/browser_recording/session_registry.py b/skyvern/services/browser_recording/session_registry.py
index f13383953..b1eb8d3cd 100644
--- a/skyvern/services/browser_recording/session_registry.py
+++ b/skyvern/services/browser_recording/session_registry.py
@@ -27,6 +27,7 @@ class RecordingInterpretationSessionRegistry:
organization_id: str,
workflow_permanent_id: str,
on_update: OnRecordingInterpretationUpdate,
+ deltas_enabled: bool = False,
) -> None:
self._prune_expired_sessions()
existing = self._sessions.get(browser_session_id)
@@ -37,6 +38,7 @@ class RecordingInterpretationSessionRegistry:
and not existing.finalized
):
existing.on_update = on_update
+ existing.set_deltas_enabled(deltas_enabled)
self._last_seen[browser_session_id] = time.monotonic()
existing.emit_snapshot()
return
@@ -47,6 +49,7 @@ class RecordingInterpretationSessionRegistry:
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
on_update=on_update,
+ deltas_enabled=deltas_enabled,
)
self._last_seen[browser_session_id] = time.monotonic()
diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.py
index d08364752..1e1274108 100644
--- a/skyvern/services/browser_recording/types.py
+++ b/skyvern/services/browser_recording/types.py
@@ -127,7 +127,12 @@ class RecordingDraftStep(BaseModel):
class RecordingInterpretationUpdate(BaseModel):
interpretation_session_id: str
session_revision: int
+ # Authoritative full list when is_snapshot is true. Empty on delta updates.
steps: list[RecordingDraftStep] = Field(default_factory=list)
+ # Per-step upserts (by step_id) when is_snapshot is false. Keeps each update
+ # O(1) instead of re-sending the whole growing steps list.
+ changed_steps: list[RecordingDraftStep] = Field(default_factory=list)
+ is_snapshot: bool = True
pending: bool = False
finalized: bool = False
diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
index 8105e8a38..51a38b520 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
@@ -104,6 +104,31 @@ class TestReifyChannelMessage:
assert msg.workflow_permanent_id == "wpid_123"
assert msg.live_interpretation_enabled is True
+ def test_begin_exfiltration_defaults_no_delta_support(self) -> None:
+ # An existing client that predates deltas omits the field; it must default
+ # false so the backend keeps sending full snapshots (no breakage).
+ msg = reify_channel_message(
+ {
+ "kind": "begin-exfiltration",
+ "workflow_permanent_id": "wpid_123",
+ "live_interpretation_enabled": True,
+ }
+ )
+ assert isinstance(msg, MessageInBeginExfiltration)
+ assert msg.supports_interpretation_deltas is False
+
+ def test_begin_exfiltration_honors_declared_delta_support(self) -> None:
+ msg = reify_channel_message(
+ {
+ "kind": "begin-exfiltration",
+ "workflow_permanent_id": "wpid_123",
+ "live_interpretation_enabled": True,
+ "supports_interpretation_deltas": True,
+ }
+ )
+ assert isinstance(msg, MessageInBeginExfiltration)
+ assert msg.supports_interpretation_deltas is True
+
def test_recording_rearm_capture(self) -> None:
assert isinstance(
reify_channel_message({"kind": "recording-rearm-capture"}),
diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.py
index 986420a00..bd6c99b7a 100644
--- a/tests/unit/services/test_browser_recording.py
+++ b/tests/unit/services/test_browser_recording.py
@@ -37,6 +37,7 @@ from skyvern.services.browser_recording.types import (
ExfiltratedConsoleEvent,
ExfiltratedEventCdpParams,
Mouse,
+ RecordingDraftStep,
RecordingDraftStepStatus,
RecordingInterpretationUpdate,
)
@@ -486,10 +487,14 @@ async def test_live_interpretation_emits_placeholder_then_enriched(monkeypatch:
session.ingest_events([make_streaming_console_click(timestamp_ms=1000.0)])
await asyncio.sleep(0.05)
- # the placeholder draft is visible before the LLM responds
- updates_with_steps = [update for update in updates if update.steps]
+ # the placeholder draft is visible before the LLM responds — it arrives as a
+ # delta (changed_steps), since only snapshots carry the full steps list.
+ def emitted_steps(u: RecordingInterpretationUpdate) -> list[RecordingDraftStep]:
+ return u.steps if u.is_snapshot else u.changed_steps
+
+ updates_with_steps = [update for update in updates if emitted_steps(update)]
assert updates_with_steps
- placeholder = updates_with_steps[-1].steps[0]
+ placeholder = emitted_steps(updates_with_steps[-1])[0]
assert placeholder.status == RecordingDraftStepStatus.INTERPRETING
assert placeholder.title == "Click 'Click me'"
assert placeholder.navigation_goal == "Click 'Click me'."
@@ -582,7 +587,8 @@ async def test_live_interpretation_max_wait_fires_during_continuous_events() ->
await asyncio.sleep(0.02)
try:
- assert any(update.steps for update in updates)
+ # steps land in changed_steps (delta) or steps (snapshot) depending on emit type.
+ assert any(update.steps or update.changed_steps for update in updates)
finally:
session.cancel()
# let the cancelled debounce task unwind before the loop closes
diff --git a/tests/unit/test_browser_recording_interpretation.py b/tests/unit/test_browser_recording_interpretation.py
index bed7a1cad..a36944c25 100644
--- a/tests/unit/test_browser_recording_interpretation.py
+++ b/tests/unit/test_browser_recording_interpretation.py
@@ -24,6 +24,7 @@ from skyvern.services.browser_recording.types import (
Mouse,
RecordingDraftStep,
RecordingDraftStepStatus,
+ RecordingInterpretationUpdate,
)
ORG_ID = "org_123"
@@ -583,3 +584,96 @@ def test_start_session_resumes_after_websocket_disconnect_without_stop() -> None
assert registry._sessions[PBS_ID] is session
assert reconnect_updates == [4]
+
+
+async def test_emits_deltas_for_steps_and_snapshot_on_finalize(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:
+ return {"block_label": "click_submit", "title": "Click Submit", "prompt": "Click submit."}
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)
+
+ updates: list[RecordingInterpretationUpdate] = []
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=updates.append,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ deltas_enabled=True,
+ )
+
+ session.ingest_events([_click_streaming_event(timestamp=1000.0)])
+ await session.flush()
+
+ # Steps arrive as deltas (placeholder + enriched), never re-sending the full list.
+ deltas = [u for u in updates if not u.is_snapshot]
+ assert any(u.changed_steps for u in deltas)
+ assert all(u.steps == [] for u in deltas)
+
+ # Finalize ends with an authoritative snapshot carrying the full list.
+ assert updates[-1].is_snapshot is True
+ assert updates[-1].finalized is True
+ assert len(updates[-1].steps) == 1
+
+ # A delta never smuggles the whole growing list back in.
+ assert all(u.is_snapshot or not u.steps for u in updates)
+
+
+async def test_no_deltas_when_client_lacks_capability(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:
+ return {"block_label": "click", "title": "Click", "prompt": "Click."}
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)
+
+ updates: list[RecordingInterpretationUpdate] = []
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=updates.append,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ # deltas_enabled defaults False — a client that didn't opt in gets snapshots.
+ )
+
+ session.ingest_events([_click_streaming_event(timestamp=1000.0)])
+ await session.flush()
+
+ # Every update is a full snapshot; no changed_steps are ever sent.
+ assert all(u.is_snapshot for u in updates)
+ assert all(not u.changed_steps for u in updates)
+ assert updates[-1].steps # final snapshot still carries the steps
+
+
+async def test_resume_capture_emits_resync_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fake_llm(*args: object, **kwargs: object) -> dict[str, object]:
+ return {"block_label": "click", "title": "Click", "prompt": "Click."}
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", fake_llm)
+
+ updates: list[RecordingInterpretationUpdate] = []
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=updates.append,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+
+ session.ingest_events([_click_streaming_event(timestamp=1000.0)])
+ await asyncio.sleep(0.05)
+
+ session.pause_capture()
+ updates.clear()
+ session.resume_capture()
+
+ assert len(updates) == 1
+ assert updates[0].is_snapshot is True
+ session.cancel()