Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(record): start a fresh interpretation session per recording (#7002)
Cindy Li3b6cba4f084a
diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.pyindex 971bab269..8d04ab9b1 100644--- a/skyvern/forge/sdk/routes/streaming/channels/message.py+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py@@ -137,6 +137,9 @@ class MessageInBeginExfiltration(Message):# Client capability: true only when the frontend understands delta interpretation# updates. Defaults false so older clients keep receiving full snapshots.supports_interpretation_deltas: bool = False+ # Per-recording id from the client. Distinguishes a reconnect (same id, reuse+ # session) from a new recording (new id, fresh session). None on older clients.+ recording_attempt_id: str | None = None@dataclasses.dataclass@@ -329,10 +332,12 @@ def reify_channel_message(data: dict) -> ChannelMessage:return MessageInAskForClipboardResponse(text=text)case MessageKind.BEGIN_EXFILTRATION:workflow_permanent_id = data.get("workflow_permanent_id")+ recording_attempt_id = data.get("recording_attempt_id")return MessageInBeginExfiltration(workflow_permanent_id=workflow_permanent_id if isinstance(workflow_permanent_id, str) else None,live_interpretation_enabled=bool(data.get("live_interpretation_enabled") or False),supports_interpretation_deltas=bool(data.get("supports_interpretation_deltas") or False),+ recording_attempt_id=recording_attempt_id if isinstance(recording_attempt_id, str) else None,)case MessageKind.CEDE_CONTROL:return MessageInCedeControl()@@ -702,6 +707,7 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:workflow_permanent_id=message.workflow_permanent_id,on_update=on_interpretation_update,deltas_enabled=message.supports_interpretation_deltas,+ recording_attempt_id=message.recording_attempt_id,)live_interpretation_browser_session_id = browser_session_iddiff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex bc8149647..53d08572c 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -247,11 +247,16 @@ class RecordingInterpretationSession:debounce_seconds: float = INTERPRETATION_DEBOUNCE_SECONDS,max_wait_seconds: float = INTERPRETATION_MAX_WAIT_SECONDS,deltas_enabled: bool = False,+ recording_attempt_id: str | None = None,) -> None:self.browser_session_id = browser_session_idself.organization_id = organization_idself.workflow_permanent_id = workflow_permanent_idself.on_update = on_update+ # Identifies the client-side recording attempt. A cached session is+ # reused across reconnects only when this matches; a new recording sends+ # a new id, forcing a fresh session (state machines reset from scratch).+ self.recording_attempt_id = recording_attempt_idself.set_deltas_enabled(deltas_enabled)self.interpretation_session_id = str(uuid.uuid4())self.debounce_seconds = debounce_secondsdiff --git a/skyvern/services/browser_recording/session_registry.py b/skyvern/services/browser_recording/session_registry.pyindex b1eb8d3cd..153679aa4 100644--- a/skyvern/services/browser_recording/session_registry.py+++ b/skyvern/services/browser_recording/session_registry.py@@ -28,14 +28,21 @@ class RecordingInterpretationSessionRegistry:workflow_permanent_id: str,on_update: OnRecordingInterpretationUpdate,deltas_enabled: bool = False,+ recording_attempt_id: str | None = None,) -> None:self._prune_expired_sessions()existing = self._sessions.get(browser_session_id)+ # Reuse the cached session only for a reconnect to the SAME recording. A+ # new recording sends a different recording_attempt_id, so its events must+ # not be appended to the prior session's already-consumed state machines+ # (which would emit zero new draft steps). Clients that omit the id keep+ # the legacy reuse behavior.if (existing is not Noneand existing.workflow_permanent_id == workflow_permanent_idand existing.organization_id == organization_idand not existing.finalized+ and (recording_attempt_id is None or existing.recording_attempt_id == recording_attempt_id)):existing.on_update = on_updateexisting.set_deltas_enabled(deltas_enabled)@@ -50,6 +57,7 @@ class RecordingInterpretationSessionRegistry:workflow_permanent_id=workflow_permanent_id,on_update=on_update,deltas_enabled=deltas_enabled,+ recording_attempt_id=recording_attempt_id,)self._last_seen[browser_session_id] = time.monotonic()diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.pyindex 7df08bb7c..25e6eb24f 100644--- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py@@ -131,6 +131,29 @@ class TestReifyChannelMessage:assert isinstance(msg, MessageInBeginExfiltration)assert msg.supports_interpretation_deltas is True+ def test_begin_exfiltration_defaults_no_recording_attempt_id(self) -> None:+ msg = reify_channel_message(+ {+ "kind": "begin-exfiltration",+ "workflow_permanent_id": "wpid_123",+ "live_interpretation_enabled": True,+ }+ )+ assert isinstance(msg, MessageInBeginExfiltration)+ assert msg.recording_attempt_id is None++ def test_begin_exfiltration_carries_recording_attempt_id(self) -> None:+ msg = reify_channel_message(+ {+ "kind": "begin-exfiltration",+ "workflow_permanent_id": "wpid_123",+ "live_interpretation_enabled": True,+ "recording_attempt_id": "attempt-abc",+ }+ )+ assert isinstance(msg, MessageInBeginExfiltration)+ assert msg.recording_attempt_id == "attempt-abc"+def test_recording_rearm_capture(self) -> None:assert isinstance(reify_channel_message({"kind": "recording-rearm-capture"}),diff --git a/tests/unit/test_browser_recording_interpretation.py b/tests/unit/test_browser_recording_interpretation.pyindex a36944c25..2dc7eaadf 100644--- a/tests/unit/test_browser_recording_interpretation.py+++ b/tests/unit/test_browser_recording_interpretation.py@@ -586,6 +586,72 @@ def test_start_session_resumes_after_websocket_disconnect_without_stop() -> Noneassert reconnect_updates == [4]+def test_start_session_same_recording_attempt_id_reuses_session() -> None:+ from skyvern.services.browser_recording.session_registry import RecordingInterpretationSessionRegistry++ registry = RecordingInterpretationSessionRegistry()+ registry.start_session(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda _: None,+ recording_attempt_id="attempt-1",+ )+ session = registry._sessions[PBS_ID]+ session.session_revision = 5++ registry.start_session(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda _: None,+ recording_attempt_id="attempt-1",+ )++ # Same recording (reconnect) reuses the cached session and its revision.+ assert registry._sessions[PBS_ID] is session+ assert registry._sessions[PBS_ID].session_revision == 5+++def test_start_session_new_recording_attempt_id_forces_fresh_session() -> None:+ from skyvern.services.browser_recording.session_registry import RecordingInterpretationSessionRegistry++ registry = RecordingInterpretationSessionRegistry()+ registry.start_session(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda _: None,+ recording_attempt_id="attempt-1",+ )+ stale = registry._sessions[PBS_ID]+ stale.session_revision = 42+ stale.steps = [+ RecordingDraftStep(+ step_id="step-1",+ action_kind=ActionKind.CLICK,+ block_type="action",+ label="click_submit",+ )+ ]++ registry.start_session(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda _: None,+ recording_attempt_id="attempt-2",+ )++ # A new recording gets a fresh session: not the stale object, revision reset,+ # no carried-over steps.+ fresh = registry._sessions[PBS_ID]+ assert fresh is not stale+ assert fresh.recording_attempt_id == "attempt-2"+ assert fresh.session_revision == 0+ assert fresh.steps == []++@pytest.mark.asyncioasync def test_emits_deltas_for_steps_and_snapshot_on_finalize(monkeypatch: pytest.MonkeyPatch,