Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Gate Task V3 auto-observe per run on a PostHog A/B flag (SKY-15239) (#8319)
pedrohsdbae520accde48
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex ce785c68e..e9758684f 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1888,6 +1888,7 @@ class ForgeAgent:starting_url=task.url,downloads_dir=get_download_dir(download_id),organization_id=organization.organization_id,+ task_id=task.task_id,max_action_steps=step_cap,max_turns=max_turns,max_tool_calls=max_tool_calls,diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex b9e93963c..db3036875 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -70,6 +70,7 @@ from skyvern.forge.sdk.services import (from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResultfrom skyvern.forge.sdk.trace import tracedfrom skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVar+from skyvern.forge.taskv3.auto_observe import AutoObserveDecision, auto_observe_from_settingfrom skyvern.schemas.run_enums import RunEngine, RunTypefrom skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestinationfrom skyvern.services.otp_email import EmailOTPSearchError, EmailOTPVerificationContext, build_email_otp_sources@@ -882,6 +883,12 @@ class AgentFunction:) -> RunEngine:return requested_engine+ # OSS resolves auto-observe from the static setting; cloud overrides to bucket a run per task_id.+ async def resolve_task_v3_auto_observe(+ self, *, task_id: str | None, organization_id: str | None+ ) -> AutoObserveDecision:+ return auto_observe_from_setting()+async def record_run_duration(self,run_type: str,diff --git a/skyvern/forge/taskv3/auto_observe.py b/skyvern/forge/taskv3/auto_observe.pynew file mode 100644index 000000000..d16f1a002--- /dev/null+++ b/skyvern/forge/taskv3/auto_observe.py@@ -0,0 +1,23 @@+"""Per-run decision for Task V3 auto-observe. The engine consumes it through the AgentFunction seam,+so an OSS deployment resolves from the static setting while cloud can bucket a run into an A/B."""++from __future__ import annotations++from dataclasses import dataclass+from typing import Literal++from skyvern.config import settings++AutoObserveArm = Literal["treatment", "control", "setting", "default"]+++@dataclass(frozen=True)+class AutoObserveDecision:+ enabled: bool+ arm: AutoObserveArm+++def auto_observe_from_setting() -> AutoObserveDecision:+ if settings.TASK_V3_AUTO_OBSERVE:+ return AutoObserveDecision(enabled=True, arm="setting")+ return AutoObserveDecision(enabled=False, arm="default")diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex 639cad7cd..452cde258 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -29,9 +29,11 @@ from typing import Any, Awaitable, Callableimport structlogfrom skyvern.config import settings+from skyvern.forge import appfrom skyvern.forge.sdk.api.llm.api_handler_factory import VISION_FALLBACK_PROMPT_NAMESfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTaskfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.taskv3.auto_observe import AutoObserveDecisionfrom skyvern.forge.taskv3.loop import (DEFAULT_MAX_SETTLE_DEFERRALS,ActivityRecency,@@ -181,6 +183,7 @@ async def run_task_v3_agent_loop(starting_url: str | None = None,downloads_dir: str | None = None,organization_id: str | None = None,+ task_id: str | None = None,max_turns: int = DEFAULT_MAX_TURNS,max_tool_calls: int = DEFAULT_MAX_TOOL_CALLS,max_action_steps: int | None = None,@@ -291,7 +294,22 @@ async def run_task_v3_agent_loop(verification_blocker=verification_blocker,)tools = browser_tools + (extra_tools or []) + [finish_tool]- auto_observe = settings.TASK_V3_AUTO_OBSERVE and not page_free+ # Resolved through the AgentFunction seam so cloud can bucket the run into an A/B; a page-free+ # run has no observe to append, so it is never resolved and never bands as an arm.+ auto_observe_decision = (+ AutoObserveDecision(enabled=False, arm="default")+ if page_free+ else await app.AGENT_FUNCTION.resolve_task_v3_auto_observe(task_id=task_id, organization_id=organization_id)+ )+ auto_observe = auto_observe_decision.enabled+ if not page_free:+ LOG.info(+ "taskv3 auto-observe resolved",+ task_id=task_id,+ organization_id=organization_id,+ auto_observe=auto_observe,+ auto_observe_arm=auto_observe_decision.arm,+ )base_system_prompt = PAGE_FREE_SYSTEM_PROMPT if page_free else SYSTEM_PROMPT# Keyed on which hooks are present, not completion_probe alone: an extraction blocker-only# case needs the model told it ends the run itself; a wait-only probe has nothing to explain.@@ -353,6 +371,7 @@ async def run_task_v3_agent_loop(tool_seconds=outcome.tool_seconds,action_steps=outcome.action_steps,no_tool_call_turns=outcome.no_tool_call_turns,+ auto_observe_arm=auto_observe_decision.arm,tool_choice_requested=settings.TASK_V3_TOOL_CHOICE_REQUIRED,tool_choice_in_effect=outcome.tool_choice_in_effect,)diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.pyindex 64bbdeb27..4006a8d1d 100644--- a/tests/unit/force_stub_app.py+++ b/tests/unit/force_stub_app.py@@ -45,6 +45,8 @@ def create_forge_stub_app() -> ForgeApp:# Class constant, not a method — _LazyNamespace would auto-mock it into a non-iterable AsyncMock# and break every caller that scans it for close-page phrases.fake_app_module.AGENT_FUNCTION.MAGIC_LINK_CLOSE_SIGNALS = base_agent_function.MAGIC_LINK_CLOSE_SIGNALS+ # Auto-mocked, its truthy MagicMock decision would switch auto-observe on for every engine test.+ fake_app_module.AGENT_FUNCTION.resolve_task_v3_auto_observe = base_agent_function.resolve_task_v3_auto_observefake_app_module.AGENT_FUNCTION.serialize_codeblock_parameters = base_agent_function.serialize_codeblock_parametersfake_app_module.AGENT_FUNCTION.redact_codeblock_parameter_values = (base_agent_function.redact_codeblock_parameter_valuesdiff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex a906884d0..7e7d6e071 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -199,8 +199,9 @@ async def test_execute_task_v3_bills_per_browser_action(monkeypatch: pytest.Monkmonkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None)- # The whole task runs as one loop invocation.+ # The whole task runs as one loop invocation, keyed to this task so per-run gates bucket on it.assert loop_mock.await_count == 1+ assert loop_mock.await_args.kwargs["task_id"] == task.task_idassert step.status == StepStatus.completed# Every reported browser action becomes one action-result with a non-empty results list,