Skip to content

Automate browser based workflows with AI

feat(SKY-9121): gate workflow copilot v2 dispatch behind PostHog flag (#5561)

Andrew Neilsond2e8307e552a
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 5739d8495..0657fa0c7 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -937,13 +937,38 @@ async def _new_copilot_chat_post(
return FastAPIEventSourceStream.create(request, stream_handler)
+COPILOT_V2_FLAG_KEY = "ENABLE_WORKFLOW_COPILOT_V2"
+
+
+async def _should_use_copilot_v2(organization: Organization, workflow_permanent_id: str) -> bool:
+ if settings.ENABLE_WORKFLOW_COPILOT_V2:
+ return True
+ try:
+ # distinct_id is the org (not a run id) because this gate is an org-sticky rollout:
+ # copilot chat may not have a stable run at dispatch time, and we want each org to
+ # see the same path across sessions. Contrast with backend.md's default of run-level
+ # ids for per-run randomized experiments.
+ return await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ COPILOT_V2_FLAG_KEY,
+ distinct_id=organization.organization_id,
+ properties={"organization_id": organization.organization_id},
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to evaluate copilot-v2 feature flag; falling back to legacy copilot",
+ organization_id=organization.organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ )
+ return False
+
+
@base_router.post("/workflow/copilot/chat-post", include_in_schema=False)
async def workflow_copilot_chat_post(
request: Request,
chat_request: WorkflowCopilotChatRequest,
organization: Organization = Depends(org_auth_service.get_current_org),
) -> EventSourceResponse:
- if settings.ENABLE_WORKFLOW_COPILOT_V2:
+ if await _should_use_copilot_v2(organization, chat_request.workflow_permanent_id):
return await _new_copilot_chat_post(request, chat_request, organization)
async def stream_handler(stream: EventSourceStream) -> None:
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index a1204a7af..7d3ba35f7 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -23,7 +23,7 @@ import pytest
from skyvern.config import settings
from skyvern.forge import app
-from skyvern.forge.sdk.routes.workflow_copilot import workflow_copilot_chat_post
+from skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_post
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest
@@ -56,6 +56,21 @@ def _install_fake_create(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
return captured
+def _install_mock_provider(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ return_value: bool | None = None,
+ side_effect: BaseException | None = None,
+) -> AsyncMock:
+ mock_provider = AsyncMock()
+ if side_effect is not None:
+ mock_provider.is_feature_enabled_cached.side_effect = side_effect
+ else:
+ mock_provider.is_feature_enabled_cached.return_value = return_value
+ monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", mock_provider)
+ return mock_provider
+
+
@pytest.mark.asyncio
async def test_flag_off_dispatches_to_old_copilot(monkeypatch: pytest.MonkeyPatch) -> None:
"""Flag off -> workflow_copilot_chat_post must use the old-copilot stream handler.
@@ -64,6 +79,9 @@ async def test_flag_off_dispatches_to_old_copilot(monkeypatch: pytest.MonkeyPatc
raise if called, then confirming the old path was used instead.
"""
monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", False)
+ # force_stub_app's _LazyNamespace auto-creates truthy AsyncMocks for any attribute
+ # access, so the provider needs an explicit False stub to keep this test accurate.
+ _install_mock_provider(monkeypatch, return_value=False)
new_copilot_mock = AsyncMock(side_effect=AssertionError("new-copilot path must not run when flag is off"))
monkeypatch.setattr(
@@ -223,3 +241,107 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
restore_mock.assert_awaited_once()
else:
restore_mock.assert_not_awaited()
+
+
+async def test_env_on_short_circuits_posthog(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Env var True must skip the PostHog check entirely and route to v2."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ mock_provider = _install_mock_provider(
+ monkeypatch,
+ side_effect=AssertionError("PostHog must not be consulted when env var is True"),
+ )
+
+ sentinel = object()
+ new_copilot_mock = AsyncMock(return_value=sentinel)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._new_copilot_chat_post",
+ new_copilot_mock,
+ )
+
+ request = MagicMock()
+ request.headers = {}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+
+ assert response is sentinel
+ new_copilot_mock.assert_awaited_once()
+ mock_provider.is_feature_enabled_cached.assert_not_awaited()
+
+
+async def test_env_off_posthog_on_uses_v2(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Env var False + PostHog True -> v2 path, with org_id as distinct_id."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", False)
+ mock_provider = _install_mock_provider(monkeypatch, return_value=True)
+
+ sentinel = object()
+ new_copilot_mock = AsyncMock(return_value=sentinel)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._new_copilot_chat_post",
+ new_copilot_mock,
+ )
+
+ request = MagicMock()
+ request.headers = {}
+ organization = SimpleNamespace(organization_id="org-abc")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+
+ assert response is sentinel
+ new_copilot_mock.assert_awaited_once()
+ mock_provider.is_feature_enabled_cached.assert_awaited_once_with(
+ COPILOT_V2_FLAG_KEY,
+ distinct_id="org-abc",
+ properties={"organization_id": "org-abc"},
+ )
+
+
+async def test_provider_failure_falls_back_to_legacy(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Provider errors (PostHog down, DB hiccup) must not break the endpoint."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", False)
+ _install_mock_provider(monkeypatch, side_effect=RuntimeError("posthog unreachable"))
+
+ new_copilot_mock = AsyncMock(side_effect=AssertionError("v2 path must not run when provider errors"))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._new_copilot_chat_post",
+ new_copilot_mock,
+ )
+
+ captured = _install_fake_create(monkeypatch)
+
+ request = MagicMock()
+ request.headers = {}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+
+ assert response is captured["sentinel"]
+ new_copilot_mock.assert_not_awaited()
+
+
+async def test_env_off_posthog_off_uses_legacy(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Env var False + PostHog False -> legacy stream handler, v2 not reached."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", False)
+ mock_provider = _install_mock_provider(monkeypatch, return_value=False)
+
+ new_copilot_mock = AsyncMock(side_effect=AssertionError("v2 path must not run when both gates are off"))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._new_copilot_chat_post",
+ new_copilot_mock,
+ )
+
+ captured = _install_fake_create(monkeypatch)
+
+ request = MagicMock()
+ request.headers = {}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+
+ assert response is captured["sentinel"]
+ new_copilot_mock.assert_not_awaited()
+ mock_provider.is_feature_enabled_cached.assert_awaited_once()