Skip to content

Automate browser based workflows with AI

Gate Task V3 auto-observe per run on a PostHog A/B flag (SKY-15239) (#8319)

pedrohsdbae520accde48
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index ce785c68e..e9758684f 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1888,6 +1888,7 @@ class ForgeAgent:
starting_url=task.url,
downloads_dir=get_download_dir(download_id),
organization_id=organization.organization_id,
+ task_id=task.task_id,
max_action_steps=step_cap,
max_turns=max_turns,
max_tool_calls=max_tool_calls,
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index b9e93963c..db3036875 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -70,6 +70,7 @@ from skyvern.forge.sdk.services import (
from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResult
from skyvern.forge.sdk.trace import traced
from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVar
+from skyvern.forge.taskv3.auto_observe import AutoObserveDecision, auto_observe_from_setting
from skyvern.schemas.run_enums import RunEngine, RunType
from skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestination
from skyvern.services.otp_email import EmailOTPSearchError, EmailOTPVerificationContext, build_email_otp_sources
@@ -882,6 +883,12 @@ class AgentFunction:
) -> RunEngine:
return requested_engine
+ # OSS resolves auto-observe from the static setting; cloud overrides to bucket a run per task_id.
+ async def resolve_task_v3_auto_observe(
+ self, *, task_id: str | None, organization_id: str | None
+ ) -> AutoObserveDecision:
+ return auto_observe_from_setting()
+
async def record_run_duration(
self,
run_type: str,
diff --git a/skyvern/forge/taskv3/auto_observe.py b/skyvern/forge/taskv3/auto_observe.py
new file mode 100644
index 000000000..d16f1a002
--- /dev/null
+++ b/skyvern/forge/taskv3/auto_observe.py
@@ -0,0 +1,23 @@
+"""Per-run decision for Task V3 auto-observe. The engine consumes it through the AgentFunction seam,
+so an OSS deployment resolves from the static setting while cloud can bucket a run into an A/B."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from skyvern.config import settings
+
+AutoObserveArm = Literal["treatment", "control", "setting", "default"]
+
+
+@dataclass(frozen=True)
+class AutoObserveDecision:
+ enabled: bool
+ arm: AutoObserveArm
+
+
+def auto_observe_from_setting() -> AutoObserveDecision:
+ if settings.TASK_V3_AUTO_OBSERVE:
+ return AutoObserveDecision(enabled=True, arm="setting")
+ return AutoObserveDecision(enabled=False, arm="default")
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 639cad7cd..452cde258 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -29,9 +29,11 @@ from typing import Any, Awaitable, Callable
import structlog
from skyvern.config import settings
+from skyvern.forge import app
from skyvern.forge.sdk.api.llm.api_handler_factory import VISION_FALLBACK_PROMPT_NAMES
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.taskv3.auto_observe import AutoObserveDecision
from skyvern.forge.taskv3.loop import (
DEFAULT_MAX_SETTLE_DEFERRALS,
ActivityRecency,
@@ -181,6 +183,7 @@ async def run_task_v3_agent_loop(
starting_url: str | None = None,
downloads_dir: str | None = None,
organization_id: str | None = None,
+ task_id: str | None = None,
max_turns: int = DEFAULT_MAX_TURNS,
max_tool_calls: int = DEFAULT_MAX_TOOL_CALLS,
max_action_steps: int | None = None,
@@ -291,7 +294,22 @@ async def run_task_v3_agent_loop(
verification_blocker=verification_blocker,
)
tools = browser_tools + (extra_tools or []) + [finish_tool]
- auto_observe = settings.TASK_V3_AUTO_OBSERVE and not page_free
+ # Resolved through the AgentFunction seam so cloud can bucket the run into an A/B; a page-free
+ # run has no observe to append, so it is never resolved and never bands as an arm.
+ auto_observe_decision = (
+ AutoObserveDecision(enabled=False, arm="default")
+ if page_free
+ else await app.AGENT_FUNCTION.resolve_task_v3_auto_observe(task_id=task_id, organization_id=organization_id)
+ )
+ auto_observe = auto_observe_decision.enabled
+ if not page_free:
+ LOG.info(
+ "taskv3 auto-observe resolved",
+ task_id=task_id,
+ organization_id=organization_id,
+ auto_observe=auto_observe,
+ auto_observe_arm=auto_observe_decision.arm,
+ )
base_system_prompt = PAGE_FREE_SYSTEM_PROMPT if page_free else SYSTEM_PROMPT
# Keyed on which hooks are present, not completion_probe alone: an extraction blocker-only
# case needs the model told it ends the run itself; a wait-only probe has nothing to explain.
@@ -353,6 +371,7 @@ async def run_task_v3_agent_loop(
tool_seconds=outcome.tool_seconds,
action_steps=outcome.action_steps,
no_tool_call_turns=outcome.no_tool_call_turns,
+ auto_observe_arm=auto_observe_decision.arm,
tool_choice_requested=settings.TASK_V3_TOOL_CHOICE_REQUIRED,
tool_choice_in_effect=outcome.tool_choice_in_effect,
)
diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.py
index 64bbdeb27..4006a8d1d 100644
--- a/tests/unit/force_stub_app.py
+++ b/tests/unit/force_stub_app.py
@@ -45,6 +45,8 @@ def create_forge_stub_app() -> ForgeApp:
# Class constant, not a method — _LazyNamespace would auto-mock it into a non-iterable AsyncMock
# and break every caller that scans it for close-page phrases.
fake_app_module.AGENT_FUNCTION.MAGIC_LINK_CLOSE_SIGNALS = base_agent_function.MAGIC_LINK_CLOSE_SIGNALS
+ # Auto-mocked, its truthy MagicMock decision would switch auto-observe on for every engine test.
+ fake_app_module.AGENT_FUNCTION.resolve_task_v3_auto_observe = base_agent_function.resolve_task_v3_auto_observe
fake_app_module.AGENT_FUNCTION.serialize_codeblock_parameters = base_agent_function.serialize_codeblock_parameters
fake_app_module.AGENT_FUNCTION.redact_codeblock_parameter_values = (
base_agent_function.redact_codeblock_parameter_values
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index a906884d0..7e7d6e071 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -199,8 +199,9 @@ async def test_execute_task_v3_bills_per_browser_action(monkeypatch: pytest.Monk
monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None
)
- # The whole task runs as one loop invocation.
+ # The whole task runs as one loop invocation, keyed to this task so per-run gates bucket on it.
assert loop_mock.await_count == 1
+ assert loop_mock.await_args.kwargs["task_id"] == task.task_id
assert step.status == StepStatus.completed
# Every reported browser action becomes one action-result with a non-empty results list,