Skip to content

Automate browser based workflows with AI

Let the GCP deploy scripts build as a named service account (SKY-16553) (#8842)

Shuchang Zheng99faea753b11
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index 0888f9e9c..3780968de 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -66,6 +66,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotStreamMessageType,
)
from tests.unit.copilot_test_helpers import FakeCopilotStream
+from tests.unit.scoped_asyncio import ScopedAsyncio
@pytest.mark.parametrize(
@@ -2359,16 +2360,26 @@ async def test_a_failed_screenshot_upload_leaves_the_tool_result_as_it_was(
dictation_upload = asyncio.create_task(asyncio.Event().wait())
app.ARTIFACT_MANAGER.upload_aiotasks_map["wcc_1"].append(dictation_upload)
dictation_waits: list[asyncio.Task[None]] = []
+ save_budgets: list[asyncio.Timeout] = []
+ save_budget_delays: list[float | None] = []
+
+ def save_budget(delay: float | None) -> asyncio.Timeout:
+ save_budget_delays.append(delay)
+ save_budgets.append(asyncio.timeout(delay))
+ return save_budgets[-1]
async def refuse(artifact: Artifact, data: bytes) -> None:
# A dictation request for the same chat starts waiting while this upload is in flight.
dictation_waits.append(asyncio.create_task(app.ARTIFACT_MANAGER.wait_for_upload_aiotasks(["wcc_1"])))
if storage_fault == "stalls":
+ # Expire the save budget only once the upload is in flight; a short real budget let a slow
+ # runner expire it before the upload started.
+ save_budgets[-1].reschedule(asyncio.get_running_loop().time())
await asyncio.Event().wait()
raise RuntimeError("storage unavailable")
monkeypatch.setattr(app.STORAGE, "store_artifact", refuse)
- monkeypatch.setattr(streaming_adapter_module, "_CHAT_SCREENSHOT_SAVE_TIMEOUT_SECONDS", 0.05)
+ monkeypatch.setattr(streaming_adapter_module, "asyncio", ScopedAsyncio(timeout=save_budget))
def tool_results(sent: list[Any]) -> list[dict[str, Any]]:
return [
@@ -2381,10 +2392,16 @@ async def test_a_failed_screenshot_upload_leaves_the_tool_result_as_it_was(
ctx = _test_copilot_context(workflow_copilot_chat_id="wcc_1")
assert _stage_frame(ctx)
+ # Count only the saves made by the drive under test, not by the undisturbed drive above.
+ save_budgets.clear()
+ save_budget_delays.clear()
try:
sent = await _drive(_click_round_trip(), ctx)
await asyncio.sleep(0)
+ # Pins the production save budget, not just the constant's name.
+ assert save_budget_delays == [5]
+ assert [budget.expired() for budget in save_budgets] == [storage_fault == "stalls"]
assert "screenshot" not in [p.type for p in sent]
assert _saved_screenshots(ctx) is None
assert len(tool_results(sent)) == 1
diff --git a/tests/unit/test_workflow_service_managed_browser_profile.py b/tests/unit/test_workflow_service_managed_browser_profile.py
index 3f767ca30..59d91c0ce 100644
--- a/tests/unit/test_workflow_service_managed_browser_profile.py
+++ b/tests/unit/test_workflow_service_managed_browser_profile.py
@@ -3352,7 +3352,8 @@ async def test_forced_retirement_claim_propagates_cancellation(monkeypatch, forc
await asyncio.wait_for(entered.wait(), timeout=5)
if case == "cancelled":
task.cancel()
- done, _ = await asyncio.wait({task}, timeout=1)
+ # A propagation bug leaves the task parked forever; the bound only has to outlast a GC pause (~1s).
+ done, _ = await asyncio.wait({task}, timeout=10)
assert task in done, "retirement claim did not propagate cancellation or timeout"
with pytest.raises(expected) as raised:
await task
diff --git a/tests/unit/webeye/test_screenshot_observability.py b/tests/unit/webeye/test_screenshot_observability.py
index 35de51178..612c4dfbc 100644
--- a/tests/unit/webeye/test_screenshot_observability.py
+++ b/tests/unit/webeye/test_screenshot_observability.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import asyncio
import base64
+import contextlib
from io import BytesIO
from pathlib import Path
from types import SimpleNamespace
@@ -32,6 +33,12 @@ from skyvern.webeye.utils.page import (
_current_viewpoint_screenshot_helper,
_screenshot_observation_fields,
)
+from tests.unit.scoped_asyncio import ScopedAsyncio
+
+# Read before the autouse fixture shrinks it, so a test can run detach under its real budget.
+_PRODUCTION_DETACH_TIMEOUT_SECONDS = page_module.CDP_RESCUE_DETACH_TIMEOUT_SECONDS
+# Long enough that no runner pause expires it, and distinct from the other stages' test budgets.
+_TEST_DETACH_TIMEOUT_SECONDS = 45
def _page() -> MagicMock:
@@ -199,6 +206,19 @@ async def test_detach_failure_is_observable_without_changing_capture_or_cancella
) -> None:
log = MagicMock()
monkeypatch.setattr(page_module, "LOG", log)
+ # Budgets long enough that no runner pause expires them; the test expires the detach budget itself.
+ # Detach gets its own value so a detach bounded by another stage's constant is caught.
+ for name in ("SESSION", "CAPTURE"):
+ monkeypatch.setattr(page_module, f"CDP_RESCUE_{name}_TIMEOUT_SECONDS", 30)
+ monkeypatch.setattr(page_module, "CDP_RESCUE_DETACH_TIMEOUT_SECONDS", _TEST_DETACH_TIMEOUT_SECONDS)
+ wait_budgets: list[tuple[float | None, asyncio.Timeout]] = []
+
+ async def wait_for(awaitable: Any, timeout: float | None) -> Any:
+ async with asyncio.timeout(timeout) as budget:
+ wait_budgets.append((timeout, budget))
+ return await awaitable
+
+ monkeypatch.setattr(page_module, "asyncio", ScopedAsyncio(wait_for=wait_for))
page = _page()
session = page.context.new_cdp_session.return_value
capture_entered = asyncio.Event()
@@ -228,18 +248,23 @@ async def test_detach_failure_is_observable_without_changing_capture_or_cancella
task = asyncio.create_task(_current_viewpoint_screenshot_helper(page))
try:
if cancel_stage == "capture":
- await asyncio.wait_for(capture_entered.wait(), timeout=0.5)
+ await asyncio.wait_for(capture_entered.wait(), timeout=5)
task.cancel()
- await asyncio.wait_for(detach_entered.wait(), timeout=0.5)
+ await asyncio.wait_for(detach_entered.wait(), timeout=5)
if cancel_stage == "detach":
task.cancel()
release_detach.set()
+ detach_delay, detach_budget = wait_budgets[-1]
+ assert detach_delay == _TEST_DETACH_TIMEOUT_SECONDS
+ if failure == "timeout":
+ detach_budget.reschedule(asyncio.get_running_loop().time())
if cancel_stage is not None:
with pytest.raises(asyncio.CancelledError):
- await asyncio.wait_for(task, timeout=0.5)
+ await asyncio.wait_for(task, timeout=5)
else:
- assert (await asyncio.wait_for(task, timeout=0.5)).startswith(page_module._PNG_SIGNATURE)
+ assert (await asyncio.wait_for(task, timeout=5)).startswith(page_module._PNG_SIGNATURE)
assert detach_finished.is_set()
+ assert detach_budget.expired() == (failure == "timeout")
assert not (asyncio.all_tasks() - existing_tasks)
detach_observations = [o for o in _observations(log) if o.get("screenshot.stage") == "detach"]
assert len(detach_observations) == 1
@@ -247,13 +272,35 @@ async def test_detach_failure_is_observable_without_changing_capture_or_cancella
assert observation["screenshot.arm"] == "control"
assert observation["screenshot.primitive"] == "cdp_rescue"
assert observation["screenshot.outcome"] == failure
- assert observation["screenshot.timeout_budget_ms"] == 50
+ assert observation["screenshot.timeout_budget_ms"] == _TEST_DETACH_TIMEOUT_SECONDS * 1000
assert observation["screenshot.elapsed_ms"] >= 0
finally:
+ # Expire any budget still running and bound the drain, so a failed assertion or an unbounded
+ # detach fails fast instead of waiting out (or hanging on) the stalled detach.
release_detach.set()
+ for _, budget in wait_budgets:
+ with contextlib.suppress(RuntimeError): # Only a budget still entered can be expired.
+ budget.reschedule(asyncio.get_running_loop().time())
if not task.done():
task.cancel()
- await asyncio.gather(task, return_exceptions=True)
+ await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=5)
+
+
+async def test_detach_runs_under_its_production_budget(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(page_module, "CDP_RESCUE_DETACH_TIMEOUT_SECONDS", _PRODUCTION_DETACH_TIMEOUT_SECONDS)
+ wait_timeouts: list[float | None] = []
+
+ async def wait_for(awaitable: Any, timeout: float | None) -> Any:
+ wait_timeouts.append(timeout)
+ return await asyncio.wait_for(awaitable, timeout=timeout)
+
+ monkeypatch.setattr(page_module, "asyncio", ScopedAsyncio(wait_for=wait_for))
+ page = _page()
+ assert (await _current_viewpoint_screenshot_helper(page)).startswith(page_module._PNG_SIGNATURE)
+ page.context.new_cdp_session.return_value.detach.assert_awaited_once_with()
+ # Attach, then detach; the literal pins the production detach budget, not just the constant's name.
+ assert wait_timeouts == [page_module.CDP_RESCUE_SESSION_TIMEOUT_SECONDS, 2]
@pytest.mark.asyncio