Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Gate recorded wait actions on page activity (backend) (#6554)
Cindy Li10519abe0074
diff --git a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.pyindex 7e8da57da..f45e8480f 100644--- a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py+++ b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py@@ -62,6 +62,7 @@ class ExfiltrationChannel(CdpChannel):BINDING_NAME: t.ClassVar[str] = "__skyvern_exfiltrate_event"CONSOLE_DEDUP_TTL_SECONDS: t.ClassVar[float] = 5.0REFRESH_INTERVAL_SECONDS: t.ClassVar[float] = 1.0+ NETWORK_ACTIVITY_THROTTLE_SECONDS: t.ClassVar[float] = 1.0_active_binding_channels: t.ClassVar[weakref.WeakKeyDictionary[Page, "ExfiltrationChannel"]] = (weakref.WeakKeyDictionary())@@ -74,6 +75,9 @@ class ExfiltrationChannel(CdpChannel):self._recent_console_event_fingerprints: dict[str, float] = {}self._pending_event_tasks: set[asyncio.Task[None]] = set()self._refresh_task: asyncio.Task | None = None+ self._network_activity_count = 0+ self._last_network_activity_emit = 0.0+ self._network_activity_flush_task: asyncio.Task[None] | None = Nonesuper().__init__(vnc_channel=vnc_channel)@@ -280,6 +284,32 @@ class ExfiltrationChannel(CdpChannel):exc_info=True,)+ def _handle_network_activity(self) -> None:+ self._network_activity_count += 1++ now = time.monotonic()+ elapsed = now - self._last_network_activity_emit+ if elapsed < self.NETWORK_ACTIVITY_THROTTLE_SECONDS:+ if not self._network_activity_flush_task or self._network_activity_flush_task.done():+ delay = self.NETWORK_ACTIVITY_THROTTLE_SECONDS - elapsed+ self._network_activity_flush_task = asyncio.create_task(self._flush_network_activity_after(delay))+ return++ self._emit_network_activity(now)++ async def _flush_network_activity_after(self, delay: float) -> None:+ await asyncio.sleep(delay)+ self._emit_network_activity(time.monotonic())++ def _emit_network_activity(self, now: float) -> None:+ if self._network_activity_count == 0:+ return++ self._last_network_activity_emit = now+ count = self._network_activity_count+ self._network_activity_count = 0+ self._handle_cdp_event("net:activity", {"count": count})+def _handle_cdp_event(self, event_name: str, params: dict) -> None:LOG.debug(f"{self.class_name} cdp event captured: {event_name}", params=params)@@ -406,6 +436,7 @@ class ExfiltrationChannel(CdpChannel):cdp_session.send("Runtime.enable"),cdp_session.send("DOM.enable"),cdp_session.send("Page.enable"),+ cdp_session.send("Network.enable"),cdp_session.send("Target.setDiscoverTargets", {"discover": True}),]@@ -427,6 +458,9 @@ class ExfiltrationChannel(CdpChannel):"Page.navigatedWithinDocument",lambda params: self._handle_cdp_event("nav:navigated_within_document", params),)+ cdp_session.on("Network.requestWillBeSent", lambda params: self._handle_network_activity())+ cdp_session.on("Network.loadingFinished", lambda params: self._handle_network_activity())+ cdp_session.on("Network.loadingFailed", lambda params: self._handle_network_activity())return self@@ -500,6 +534,12 @@ class ExfiltrationChannel(CdpChannel):await self._refresh_taskself._refresh_task = None+ if self._network_activity_flush_task and not self._network_activity_flush_task.done():+ self._network_activity_flush_task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await self._network_activity_flush_task+ self._network_activity_flush_task = None+pending_event_tasks = list(self._pending_event_tasks)for task in pending_event_tasks:task.cancel()diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex 20ad42a23..0e1c4bc01 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -452,7 +452,7 @@ class RecordingInterpretationSession:self._emit_update()async def _drain_enrichment(self) -> None:- if self._enrichment_tasks:+ while self._enrichment_tasks:await asyncio.gather(*list(self._enrichment_tasks), return_exceptions=True)def _unique_step_label(self, label: str, step: RecordingDraftStep) -> str:diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex 26456b74f..50425851b 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -502,14 +502,24 @@ class Processor:return blockasync def create_url_block(self, action: ActionUrlChange) -> WorkflowDefinitionYamlBlocksItem_GotoUrl:- """Build a goto-url block with a host-derived label and the action URL."""+ """+ Create a YAML goto URL block from an `ActionUrlChange`.++ Fully deterministic: goto blocks carry no LLM-generated metadata, so+ skipping the LLM round-trip makes navigation drafts instant.+ """return WorkflowDefinitionYamlBlocksItem_GotoUrl(label=deterministic_goto_url_label(action.url),url=action.url,)async def create_wait_block(self, action: ActionWait) -> WorkflowDefinitionYamlBlocksItem_Wait:- """Build a wait block with a duration-derived label and wait seconds."""+ """+ Create a YAML wait block from an `ActionWait`.++ Fully deterministic: wait blocks carry no LLM-generated metadata, so+ skipping the LLM round-trip makes wait drafts instant.+ """wait_sec = deterministic_wait_seconds(action.duration_ms)return WorkflowDefinitionYamlBlocksItem_Wait(diff --git a/skyvern/services/browser_recording/state_machines/wait.py b/skyvern/services/browser_recording/state_machines/wait.pyindex b00c3f183..d4bf178af 100644--- a/skyvern/services/browser_recording/state_machines/wait.py+++ b/skyvern/services/browser_recording/state_machines/wait.py@@ -1,3 +1,4 @@+import collectionsimport typing as timport structlog@@ -15,6 +16,11 @@ from .state_machine import StateMachineLOG = structlog.get_logger()+PAGE_ACTIVITY_EVENT_NAME = "net:activity"+PAGE_ACTIVITY_EVENT_NAME_PREFIX = "nav:"++MAX_PAGE_ACTIVITY_TIMESTAMPS = 256+class StateMachineWait(StateMachine):state: t.Literal["void"] = "void"@@ -27,9 +33,20 @@ class StateMachineWait(StateMachine):ActionWait.MIN_DURATION_THRESHOLD_MS,)+ self.page_activity_timestamps_ms: collections.deque[float] = collections.deque(+ maxlen=MAX_PAGE_ACTIVITY_TIMESTAMPS+ )+self.reset()def tick(self, event: ExfiltratedEvent, current_actions: list[Action]) -> ActionWait | None:+ if event.source == "cdp":+ if event.event_name == PAGE_ACTIVITY_EVENT_NAME or event.event_name.startswith(+ PAGE_ACTIVITY_EVENT_NAME_PREFIX+ ):+ self.page_activity_timestamps_ms.append(event.timestamp * 1000.0)+ return None+if event.source != "console":return None@@ -37,34 +54,46 @@ class StateMachineWait(StateMachine):duration_ms = int(event.params.timestamp - self.last_event_timestamp)if duration_ms >= self.threshold_ms:- LOG.debug("~ emitting wait action", duration_ms=duration_ms)-- action_target = ActionTarget(- class_name=None,- id=None,- mouse=Mouse(xp=None, yp=None),- sky_id=None,- tag_name=None,- texts=[],- )-- action = ActionWait(- kind=ActionKind.WAIT.value,- target=action_target,- timestamp_start=self.last_event_timestamp,- timestamp_end=event.params.timestamp,- url=event.params.url,- duration_ms=duration_ms,- )-- self.reset()-- return action+ if not self._page_was_active(self.last_event_timestamp, event.params.timestamp):+ LOG.debug(+ "~ suppressing wait action: no page activity during idle gap",+ duration_ms=duration_ms,+ )+ else:+ LOG.debug("~ emitting wait action", duration_ms=duration_ms)++ action_target = ActionTarget(+ class_name=None,+ id=None,+ mouse=Mouse(xp=None, yp=None),+ sky_id=None,+ tag_name=None,+ texts=[],+ )++ action = ActionWait(+ kind=ActionKind.WAIT.value,+ target=action_target,+ timestamp_start=self.last_event_timestamp,+ timestamp_end=event.params.timestamp,+ url=event.params.url,+ duration_ms=duration_ms,+ )++ self.reset()++ return actionself.last_event_timestamp = event.params.timestampreturn None+ def _page_was_active(self, gap_start_ms: float, gap_end_ms: float) -> bool:+ while self.page_activity_timestamps_ms and self.page_activity_timestamps_ms[0] < gap_start_ms:+ self.page_activity_timestamps_ms.popleft()++ return any(timestamp <= gap_end_ms for timestamp in self.page_activity_timestamps_ms)+def on_action(self, action: Action, current_actions: list[Action]) -> bool:if action.kind == ActionKind.HOVER:return Truediff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.pyindex 6c74fecc0..88b3bda91 100644--- a/skyvern/services/browser_recording/types.py+++ b/skyvern/services/browser_recording/types.py@@ -160,6 +160,9 @@ class ExfiltratedEventCdpParams(BaseModel):# frame_navigated eventsframe: CdpEventFrame | None = None+ # net:activity events+ count: int | None = None+class EventTarget(BaseModel):className: str | None = Nonediff --git a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.pyindex 66daedba4..56f486736 100644--- a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py+++ b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py@@ -249,6 +249,26 @@ class TestExfiltrationChannelEvents:page.on.assert_called_once()assert channel._page_console_captures[page].cdp_session is None+ @pytest.mark.asyncio+ async def test_stop_cancels_pending_network_activity_flush_task(self) -> None:+ events: list[object] = []+ channel, _ = _make_channel(on_event=lambda messages: events.extend(messages))+ channel.NETWORK_ACTIVITY_THROTTLE_SECONDS = 0.05++ channel._handle_network_activity()+ assert len(events) == 1++ channel._handle_network_activity()+ assert len(events) == 1+ assert channel._network_activity_flush_task is not None+ assert not channel._network_activity_flush_task.done()++ await channel.stop()++ assert channel._network_activity_flush_task is None+ await asyncio.sleep(0.1)+ assert len(events) == 1+@pytest.mark.asyncioasync def test_exfiltrate_rearms_existing_page_without_duplicate_listeners(self) -> None:channel, _ = _make_channel()diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex a5db3d150..58d125443 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -13,6 +13,9 @@ from skyvern.forge.sdk.routes.streaming.channels.exfiltration import Exfiltratedfrom skyvern.forge.sdk.routes.streaming.channels.exfiltration import (ExfiltratedEventSource as StreamingExfiltratedEventSource,)+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (+ ExfiltrationChannel,+)from skyvern.services.browser_recording.interpretation import RecordingInterpretationSessionfrom skyvern.services.browser_recording.service import (Processor,@@ -38,6 +41,13 @@ PBS_ID = "pbs_123"WP_ID = "wpid_123"+class DummyVncChannel:+ identity: t.ClassVar[dict[str, t.Any]] = {}+ browser_session: t.ClassVar[None] = None+ x_api_key: t.ClassVar[None] = None+ organization_id: t.ClassVar[str] = ORG_ID++def make_console_event(params: dict[str, t.Any],timestamp: float,@@ -435,6 +445,100 @@ async def test_live_interpretation_max_wait_fires_during_continuous_events() ->await asyncio.sleep(0.01)+async def test_network_activity_trailing_flush_preserves_throttled_activity() -> None:+ events: list[StreamingExfiltratedEvent] = []+ channel = ExfiltrationChannel(+ on_event=lambda messages: events.extend(messages),+ vnc_channel=t.cast(t.Any, DummyVncChannel()),+ )+ channel.NETWORK_ACTIVITY_THROTTLE_SECONDS = 0.01++ channel._handle_network_activity()+ assert len(events) == 1+ assert events[0].params == {"count": 1}++ channel._handle_network_activity()+ assert len(events) == 1++ await asyncio.sleep(0.02)++ assert len(events) == 2+ assert events[1].event_name == "net:activity"+ assert events[1].params == {"count": 1}+++def make_cdp_event(+ event_name: str, timestamp_seconds: float, params: dict[str, t.Any] | None = None+) -> ExfiltratedCdpEvent:+ return ExfiltratedCdpEvent(+ kind="exfiltrated-event",+ event_name=event_name,+ params=ExfiltratedEventCdpParams(**(params or {})),+ source="cdp",+ timestamp=timestamp_seconds,+ )+++def make_focus_event(target: dict[str, t.Any], timestamp: float) -> ExfiltratedConsoleEvent:+ params: dict[str, t.Any] = {+ "type": "focus",+ "target": target,+ "timestamp": timestamp,+ }++ return make_console_event(params=params, timestamp=timestamp)+++def test_wait_suppressed_when_page_idle() -> None:+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])++ events = [+ make_click_event(target=target, timestamp=1000.0),+ make_focus_event(target=target, timestamp=8000.0),+ ]++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ actions = processor.events_to_actions(events)++ assert [action.kind for action in actions] == [ActionKind.CLICK]+++def test_wait_emitted_when_page_showed_network_activity() -> None:+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])++ events = [+ make_click_event(target=target, timestamp=1000.0),+ # page was loading during the idle gap (cdp timestamps are seconds)+ make_cdp_event("net:activity", timestamp_seconds=4.0, params={"count": 12}),+ make_focus_event(target=target, timestamp=8000.0),+ ]++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ actions = processor.events_to_actions(events)++ assert [action.kind for action in actions] == [ActionKind.CLICK, ActionKind.WAIT]+ wait_action = actions[1]+ assert isinstance(wait_action, ActionWait)+ assert wait_action.duration_ms == 7000+++def test_wait_ignores_activity_outside_the_idle_gap() -> None:+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])++ events = [+ # activity happened before the gap even started+ make_cdp_event("net:activity", timestamp_seconds=0.5, params={"count": 3}),+ make_click_event(target=target, timestamp=1000.0),+ make_focus_event(target=target, timestamp=8000.0),+ ]++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ actions = processor.events_to_actions(events)++ assert [action.kind for action in actions] == [ActionKind.CLICK]++def test_summarize_exfiltrated_recording_events_mixed() -> None:target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])click = make_click_event(target=target, timestamp=1000.0)