Skip to content

Automate browser based workflows with AI

Gate recorded wait actions on page activity (backend) (#6554)

Cindy Li10519abe0074
diff --git a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
index 7e8da57da..f45e8480f 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
@@ -62,6 +62,7 @@ class ExfiltrationChannel(CdpChannel):
BINDING_NAME: t.ClassVar[str] = "__skyvern_exfiltrate_event"
CONSOLE_DEDUP_TTL_SECONDS: t.ClassVar[float] = 5.0
REFRESH_INTERVAL_SECONDS: t.ClassVar[float] = 1.0
+ NETWORK_ACTIVITY_THROTTLE_SECONDS: t.ClassVar[float] = 1.0
_active_binding_channels: t.ClassVar[weakref.WeakKeyDictionary[Page, "ExfiltrationChannel"]] = (
weakref.WeakKeyDictionary()
)
@@ -74,6 +75,9 @@ class ExfiltrationChannel(CdpChannel):
self._recent_console_event_fingerprints: dict[str, float] = {}
self._pending_event_tasks: set[asyncio.Task[None]] = set()
self._refresh_task: asyncio.Task | None = None
+ self._network_activity_count = 0
+ self._last_network_activity_emit = 0.0
+ self._network_activity_flush_task: asyncio.Task[None] | None = None
super().__init__(vnc_channel=vnc_channel)
@@ -280,6 +284,32 @@ class ExfiltrationChannel(CdpChannel):
exc_info=True,
)
+ def _handle_network_activity(self) -> None:
+ self._network_activity_count += 1
+
+ now = time.monotonic()
+ elapsed = now - self._last_network_activity_emit
+ if elapsed < self.NETWORK_ACTIVITY_THROTTLE_SECONDS:
+ if not self._network_activity_flush_task or self._network_activity_flush_task.done():
+ delay = self.NETWORK_ACTIVITY_THROTTLE_SECONDS - elapsed
+ self._network_activity_flush_task = asyncio.create_task(self._flush_network_activity_after(delay))
+ return
+
+ self._emit_network_activity(now)
+
+ async def _flush_network_activity_after(self, delay: float) -> None:
+ await asyncio.sleep(delay)
+ self._emit_network_activity(time.monotonic())
+
+ def _emit_network_activity(self, now: float) -> None:
+ if self._network_activity_count == 0:
+ return
+
+ self._last_network_activity_emit = now
+ count = self._network_activity_count
+ self._network_activity_count = 0
+ self._handle_cdp_event("net:activity", {"count": count})
+
def _handle_cdp_event(self, event_name: str, params: dict) -> None:
LOG.debug(f"{self.class_name} cdp event captured: {event_name}", params=params)
@@ -406,6 +436,7 @@ class ExfiltrationChannel(CdpChannel):
cdp_session.send("Runtime.enable"),
cdp_session.send("DOM.enable"),
cdp_session.send("Page.enable"),
+ cdp_session.send("Network.enable"),
cdp_session.send("Target.setDiscoverTargets", {"discover": True}),
]
@@ -427,6 +458,9 @@ class ExfiltrationChannel(CdpChannel):
"Page.navigatedWithinDocument",
lambda params: self._handle_cdp_event("nav:navigated_within_document", params),
)
+ cdp_session.on("Network.requestWillBeSent", lambda params: self._handle_network_activity())
+ cdp_session.on("Network.loadingFinished", lambda params: self._handle_network_activity())
+ cdp_session.on("Network.loadingFailed", lambda params: self._handle_network_activity())
return self
@@ -500,6 +534,12 @@ class ExfiltrationChannel(CdpChannel):
await self._refresh_task
self._refresh_task = None
+ if self._network_activity_flush_task and not self._network_activity_flush_task.done():
+ self._network_activity_flush_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._network_activity_flush_task
+ self._network_activity_flush_task = None
+
pending_event_tasks = list(self._pending_event_tasks)
for task in pending_event_tasks:
task.cancel()
diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.py
index 20ad42a23..0e1c4bc01 100644
--- a/skyvern/services/browser_recording/interpretation.py
+++ b/skyvern/services/browser_recording/interpretation.py
@@ -452,7 +452,7 @@ class RecordingInterpretationSession:
self._emit_update()
async def _drain_enrichment(self) -> None:
- if self._enrichment_tasks:
+ while self._enrichment_tasks:
await asyncio.gather(*list(self._enrichment_tasks), return_exceptions=True)
def _unique_step_label(self, label: str, step: RecordingDraftStep) -> str:
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index 26456b74f..50425851b 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -502,14 +502,24 @@ class Processor:
return block
async def create_url_block(self, action: ActionUrlChange) -> WorkflowDefinitionYamlBlocksItem_GotoUrl:
- """Build a goto-url block with a host-derived label and the action URL."""
+ """
+ Create a YAML goto URL block from an `ActionUrlChange`.
+
+ Fully deterministic: goto blocks carry no LLM-generated metadata, so
+ skipping the LLM round-trip makes navigation drafts instant.
+ """
return WorkflowDefinitionYamlBlocksItem_GotoUrl(
label=deterministic_goto_url_label(action.url),
url=action.url,
)
async def create_wait_block(self, action: ActionWait) -> WorkflowDefinitionYamlBlocksItem_Wait:
- """Build a wait block with a duration-derived label and wait seconds."""
+ """
+ Create a YAML wait block from an `ActionWait`.
+
+ Fully deterministic: wait blocks carry no LLM-generated metadata, so
+ skipping the LLM round-trip makes wait drafts instant.
+ """
wait_sec = deterministic_wait_seconds(action.duration_ms)
return WorkflowDefinitionYamlBlocksItem_Wait(
diff --git a/skyvern/services/browser_recording/state_machines/wait.py b/skyvern/services/browser_recording/state_machines/wait.py
index b00c3f183..d4bf178af 100644
--- a/skyvern/services/browser_recording/state_machines/wait.py
+++ b/skyvern/services/browser_recording/state_machines/wait.py
@@ -1,3 +1,4 @@
+import collections
import typing as t
import structlog
@@ -15,6 +16,11 @@ from .state_machine import StateMachine
LOG = structlog.get_logger()
+PAGE_ACTIVITY_EVENT_NAME = "net:activity"
+PAGE_ACTIVITY_EVENT_NAME_PREFIX = "nav:"
+
+MAX_PAGE_ACTIVITY_TIMESTAMPS = 256
+
class StateMachineWait(StateMachine):
state: t.Literal["void"] = "void"
@@ -27,9 +33,20 @@ class StateMachineWait(StateMachine):
ActionWait.MIN_DURATION_THRESHOLD_MS,
)
+ self.page_activity_timestamps_ms: collections.deque[float] = collections.deque(
+ maxlen=MAX_PAGE_ACTIVITY_TIMESTAMPS
+ )
+
self.reset()
def tick(self, event: ExfiltratedEvent, current_actions: list[Action]) -> ActionWait | None:
+ if event.source == "cdp":
+ if event.event_name == PAGE_ACTIVITY_EVENT_NAME or event.event_name.startswith(
+ PAGE_ACTIVITY_EVENT_NAME_PREFIX
+ ):
+ self.page_activity_timestamps_ms.append(event.timestamp * 1000.0)
+ return None
+
if event.source != "console":
return None
@@ -37,34 +54,46 @@ class StateMachineWait(StateMachine):
duration_ms = int(event.params.timestamp - self.last_event_timestamp)
if duration_ms >= self.threshold_ms:
- LOG.debug("~ emitting wait action", duration_ms=duration_ms)
-
- action_target = ActionTarget(
- class_name=None,
- id=None,
- mouse=Mouse(xp=None, yp=None),
- sky_id=None,
- tag_name=None,
- texts=[],
- )
-
- action = ActionWait(
- kind=ActionKind.WAIT.value,
- target=action_target,
- timestamp_start=self.last_event_timestamp,
- timestamp_end=event.params.timestamp,
- url=event.params.url,
- duration_ms=duration_ms,
- )
-
- self.reset()
-
- return action
+ if not self._page_was_active(self.last_event_timestamp, event.params.timestamp):
+ LOG.debug(
+ "~ suppressing wait action: no page activity during idle gap",
+ duration_ms=duration_ms,
+ )
+ else:
+ LOG.debug("~ emitting wait action", duration_ms=duration_ms)
+
+ action_target = ActionTarget(
+ class_name=None,
+ id=None,
+ mouse=Mouse(xp=None, yp=None),
+ sky_id=None,
+ tag_name=None,
+ texts=[],
+ )
+
+ action = ActionWait(
+ kind=ActionKind.WAIT.value,
+ target=action_target,
+ timestamp_start=self.last_event_timestamp,
+ timestamp_end=event.params.timestamp,
+ url=event.params.url,
+ duration_ms=duration_ms,
+ )
+
+ self.reset()
+
+ return action
self.last_event_timestamp = event.params.timestamp
return None
+ def _page_was_active(self, gap_start_ms: float, gap_end_ms: float) -> bool:
+ while self.page_activity_timestamps_ms and self.page_activity_timestamps_ms[0] < gap_start_ms:
+ self.page_activity_timestamps_ms.popleft()
+
+ return any(timestamp <= gap_end_ms for timestamp in self.page_activity_timestamps_ms)
+
def on_action(self, action: Action, current_actions: list[Action]) -> bool:
if action.kind == ActionKind.HOVER:
return True
diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.py
index 6c74fecc0..88b3bda91 100644
--- a/skyvern/services/browser_recording/types.py
+++ b/skyvern/services/browser_recording/types.py
@@ -160,6 +160,9 @@ class ExfiltratedEventCdpParams(BaseModel):
# frame_navigated events
frame: CdpEventFrame | None = None
+ # net:activity events
+ count: int | None = None
+
class EventTarget(BaseModel):
className: str | None = None
diff --git a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
index 66daedba4..56f486736 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
@@ -249,6 +249,26 @@ class TestExfiltrationChannelEvents:
page.on.assert_called_once()
assert channel._page_console_captures[page].cdp_session is None
+ @pytest.mark.asyncio
+ async def test_stop_cancels_pending_network_activity_flush_task(self) -> None:
+ events: list[object] = []
+ channel, _ = _make_channel(on_event=lambda messages: events.extend(messages))
+ channel.NETWORK_ACTIVITY_THROTTLE_SECONDS = 0.05
+
+ channel._handle_network_activity()
+ assert len(events) == 1
+
+ channel._handle_network_activity()
+ assert len(events) == 1
+ assert channel._network_activity_flush_task is not None
+ assert not channel._network_activity_flush_task.done()
+
+ await channel.stop()
+
+ assert channel._network_activity_flush_task is None
+ await asyncio.sleep(0.1)
+ assert len(events) == 1
+
@pytest.mark.asyncio
async def test_exfiltrate_rearms_existing_page_without_duplicate_listeners(self) -> None:
channel, _ = _make_channel()
diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.py
index a5db3d150..58d125443 100644
--- a/tests/unit/services/test_browser_recording.py
+++ b/tests/unit/services/test_browser_recording.py
@@ -13,6 +13,9 @@ from skyvern.forge.sdk.routes.streaming.channels.exfiltration import Exfiltrated
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
ExfiltratedEventSource as StreamingExfiltratedEventSource,
)
+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
+ ExfiltrationChannel,
+)
from skyvern.services.browser_recording.interpretation import RecordingInterpretationSession
from skyvern.services.browser_recording.service import (
Processor,
@@ -38,6 +41,13 @@ PBS_ID = "pbs_123"
WP_ID = "wpid_123"
+class DummyVncChannel:
+ identity: t.ClassVar[dict[str, t.Any]] = {}
+ browser_session: t.ClassVar[None] = None
+ x_api_key: t.ClassVar[None] = None
+ organization_id: t.ClassVar[str] = ORG_ID
+
+
def make_console_event(
params: dict[str, t.Any],
timestamp: float,
@@ -435,6 +445,100 @@ async def test_live_interpretation_max_wait_fires_during_continuous_events() ->
await asyncio.sleep(0.01)
+async def test_network_activity_trailing_flush_preserves_throttled_activity() -> None:
+ events: list[StreamingExfiltratedEvent] = []
+ channel = ExfiltrationChannel(
+ on_event=lambda messages: events.extend(messages),
+ vnc_channel=t.cast(t.Any, DummyVncChannel()),
+ )
+ channel.NETWORK_ACTIVITY_THROTTLE_SECONDS = 0.01
+
+ channel._handle_network_activity()
+ assert len(events) == 1
+ assert events[0].params == {"count": 1}
+
+ channel._handle_network_activity()
+ assert len(events) == 1
+
+ await asyncio.sleep(0.02)
+
+ assert len(events) == 2
+ assert events[1].event_name == "net:activity"
+ assert events[1].params == {"count": 1}
+
+
+def make_cdp_event(
+ event_name: str, timestamp_seconds: float, params: dict[str, t.Any] | None = None
+) -> ExfiltratedCdpEvent:
+ return ExfiltratedCdpEvent(
+ kind="exfiltrated-event",
+ event_name=event_name,
+ params=ExfiltratedEventCdpParams(**(params or {})),
+ source="cdp",
+ timestamp=timestamp_seconds,
+ )
+
+
+def make_focus_event(target: dict[str, t.Any], timestamp: float) -> ExfiltratedConsoleEvent:
+ params: dict[str, t.Any] = {
+ "type": "focus",
+ "target": target,
+ "timestamp": timestamp,
+ }
+
+ return make_console_event(params=params, timestamp=timestamp)
+
+
+def test_wait_suppressed_when_page_idle() -> None:
+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])
+
+ events = [
+ make_click_event(target=target, timestamp=1000.0),
+ make_focus_event(target=target, timestamp=8000.0),
+ ]
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ actions = processor.events_to_actions(events)
+
+ assert [action.kind for action in actions] == [ActionKind.CLICK]
+
+
+def test_wait_emitted_when_page_showed_network_activity() -> None:
+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])
+
+ events = [
+ make_click_event(target=target, timestamp=1000.0),
+ # page was loading during the idle gap (cdp timestamps are seconds)
+ make_cdp_event("net:activity", timestamp_seconds=4.0, params={"count": 12}),
+ make_focus_event(target=target, timestamp=8000.0),
+ ]
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ actions = processor.events_to_actions(events)
+
+ assert [action.kind for action in actions] == [ActionKind.CLICK, ActionKind.WAIT]
+ wait_action = actions[1]
+ assert isinstance(wait_action, ActionWait)
+ assert wait_action.duration_ms == 7000
+
+
+def test_wait_ignores_activity_outside_the_idle_gap() -> None:
+ target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])
+
+ events = [
+ # activity happened before the gap even started
+ make_cdp_event("net:activity", timestamp_seconds=0.5, params={"count": 3}),
+ make_click_event(target=target, timestamp=1000.0),
+ make_focus_event(target=target, timestamp=8000.0),
+ ]
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ actions = processor.events_to_actions(events)
+
+ assert [action.kind for action in actions] == [ActionKind.CLICK]
+
+
def test_summarize_exfiltrated_recording_events_mixed() -> None:
target = dict(id="button-1", skyId="sky-123", tagName="BUTTON", text=["Click me"])
click = make_click_event(target=target, timestamp=1000.0)