Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-17491: Inline run reliability below analytics (#8753)
Aaron Perez4a3ca3abccf8
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsxindex 59dbcadbf..2116d5fa1 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx@@ -323,10 +323,10 @@ function WorkflowPage() {</Button></div></div>- <WorkflowReliabilityPanel workflowPermanentId={workflowPermanentId} />{WorkflowAnalyticsPanel ? (<WorkflowAnalyticsPanel workflowPermanentId={workflowPermanentId} />) : null}+ <WorkflowReliabilityPanel workflowPermanentId={workflowPermanentId} /></header><div className="space-y-4"><header>diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsxindex 98bb6afbf..445b6f7a4 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx@@ -8270,7 +8270,6 @@ describe("WorkflowCopilotChat — g2 review gate", () => {});it("keeps Accept withheld until every Turn off for the chat settles, not just the first", async () => {- historyResponse.data.auto_accept = true;await renderChat();await submit("add a step");await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));@@ -8278,6 +8277,34 @@ describe("WorkflowCopilotChat — g2 review gate", () => {streamCalls[0]!.onMessage(proposalResponse("Untested draft."));streamCalls[0]!.resolve();});+ // Arm auto-accept the way a user does, not by seeding the row: an Always accept goes through+ // setAutoAcceptFromWrite and bumps the write counter, which a seeded history read never does.+ cancelPost.mockImplementation((path: string) =>+ path === "/workflow/copilot/apply-proposed-workflow"+ ? Promise.resolve({ data: proposedWorkflowPayload() })+ : Promise.resolve({}),+ );+ historyResponse.data.auto_accept = true;+ await openAcceptMenu();+ await act(async () => {+ clickAlwaysAccept();+ });+ expect(+ await screen.findByRole("button", { name: /Auto-accepting/ }),+ ).toBeTruthy();++ // The next turn stages for review, so the gate is back while auto-accept is still on.+ await submit("add another step");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(2));+ await act(async () => {+ streamCalls[1]!.onMessage(+ proposalResponse("Second untested draft.", { turn_id: "turn-2" }),+ );+ streamCalls[1]!.resolve();+ });+ expect(await screen.findByRole("button", { name: "Review" })).toBeTruthy();+ expect(screen.getByRole("button", { name: /Auto-accepting/ })).toBeTruthy();+const heldDisables: Array<(value: unknown) => void> = [];cancelPost.mockImplementation((path: string) =>path === "/workflow/copilot/disable-auto-accept"@@ -8316,7 +8343,7 @@ describe("WorkflowCopilotChat — g2 review gate", () => {});it("gives the gate its Accept back when Turn off fails", async () => {- historyResponse.data.auto_accept = true;+ // Armed by a real Always accept, so the write counter is in the state a user would leave it.await renderChat();await submit("add a step");await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));@@ -8324,6 +8351,27 @@ describe("WorkflowCopilotChat — g2 review gate", () => {streamCalls[0]!.onMessage(proposalResponse("Untested draft."));streamCalls[0]!.resolve();});+ cancelPost.mockImplementation((path: string) =>+ path === "/workflow/copilot/apply-proposed-workflow"+ ? Promise.resolve({ data: proposedWorkflowPayload() })+ : Promise.resolve({}),+ );+ historyResponse.data.auto_accept = true;+ await openAcceptMenu();+ await act(async () => {+ clickAlwaysAccept();+ });+ await submit("add another step");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(2));+ await act(async () => {+ streamCalls[1]!.onMessage(+ proposalResponse("Second untested draft.", { turn_id: "turn-2" }),+ );+ streamCalls[1]!.resolve();+ });+ expect(await screen.findByRole("button", { name: "Review" })).toBeTruthy();+ expect(screen.getByRole("button", { name: /Auto-accepting/ })).toBeTruthy();+let failTurnOff: (reason: unknown) => void = () => {};cancelPost.mockImplementation((path: string) =>path === "/workflow/copilot/disable-auto-accept"diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsxindex c672eddfc..b7d9c690d 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsx@@ -38,7 +38,7 @@ function WorkflowReliabilityPanel({ workflowPermanentId }: Props) {if (!showsState) {return (- <section className="rounded-md border border-border bg-slate-elevation1/80 px-3 py-2">+ <section><p className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">Reliability</p>@@ -54,7 +54,7 @@ function WorkflowReliabilityPanel({ workflowPermanentId }: Props) {const isAmber = tone === "amber";return (- <section className="rounded-md border border-border bg-slate-elevation1/80 px-3 py-2">+ <section><p className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">Reliability</p>diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex d5327ce07..b29a5f7b0 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -1021,6 +1021,92 @@ async def test_turning_off_auto_accept_mid_turn_keeps_the_pending_review_and_gatassert after_turn.proposed_workflow_metadata.owner_turn_id == "turn-b"+async def test_turning_auto_accept_on_mid_turn_does_not_apply_the_draft_this_turn_staged_for_review(+ sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A mid-turn opt-in only applies to the next turn; this one staged for review."""+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)+ monkeypatch.setattr(app.DATABASE, "workflow_params", repo)+ monkeypatch.setattr(app, "CACHE", None)+ canonical = MagicMock()+ canonical.model_dump.return_value = {"title": "Canonical"}+ canonical_fingerprint = workflow_content_fingerprint({"title": "Canonical"})+ monkeypatch.setattr(+ app.DATABASE,+ "workflows",+ SimpleNamespace(get_workflow_by_permanent_id=AsyncMock(return_value=canonical)),+ )+ organization = SimpleNamespace(organization_id="org-1")+ chat = await repo.create_workflow_copilot_chat(organization_id="org-1", workflow_permanent_id="wpid-1")+ chat_id = chat.workflow_copilot_chat_id+ await repo.publish_workflow_copilot_candidate(+ "org-1",+ chat_id,+ proposal={"title": "Pending review", "_copilot_yaml": "title: Pending review\n"},+ owner_turn_id="turn-a",+ canonical_fingerprint=canonical_fingerprint,+ canonical_title="Pending review",+ disposition="review_untested",+ expected_owner_turn_id=None,+ expected_revision=None,+ )+ # The turn starts with auto-accept off, which is what it told the model.+ turn_start_chat = await repo.get_workflow_copilot_chat_by_id(+ organization_id="org-1", workflow_copilot_chat_id=chat_id+ )+ assert turn_start_chat is not None and turn_start_chat.auto_accept is not True++ # Mid-turn the user clicks Always accept on the gate the previous turn left on screen.+ await repo.update_workflow_copilot_chat("org-1", chat_id, auto_accept=True)++ await repo.publish_workflow_copilot_candidate(+ "org-1",+ chat_id,+ proposal={"title": "Verified fix", "_copilot_yaml": "title: Verified fix\n"},+ owner_turn_id="turn-b",+ canonical_fingerprint=canonical_fingerprint,+ canonical_title="Verified fix",+ disposition="auto_applicable",+ expected_owner_turn_id="turn-a",+ expected_revision=1,+ )+ updated_workflow = MagicMock(title="Verified fix")+ updated_workflow.model_dump.return_value = {"title": "Verified fix"}+ agent_result = AgentResult(+ user_response="Fixed it.",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ response_type="REPLY",+ proposal_disposition="auto_applicable",+ proposal_owner_turn_id="turn-b",+ proposal_revision=1,+ narrative_payload=_narrative_payload(),+ )+ stream = MagicMock(send=AsyncMock(return_value=True))++ await workflow_copilot_route._finalise_normal_turn(+ stream=stream,+ chat=turn_start_chat,+ organization_id="org-1",+ original_workflow=SimpleNamespace(workflow_id="wf-canonical"),+ chat_request=_make_chat_request(),+ agent_result=agent_result,+ )++ frame = stream.send.await_args.args[0]+ assert isinstance(frame, WorkflowCopilotStreamResponseUpdate)+ assert frame.updated_workflow is not None+ assert frame.workflow_applied is False+ after_turn = await workflow_copilot_chat_history(+ credential_recovery_token=None, workflow_copilot_chat_id=chat_id, organization=organization+ )+ # The opt-in stands for the next turn; it just did not reach back into this one.+ assert after_turn.auto_accept is True+ assert after_turn.proposed_workflow_metadata is not None+ assert after_turn.proposed_workflow_metadata.owner_turn_id == "turn-b"++def test_chat_history_serves_a_saved_envelope_back_untouched() -> None:now = datetime.now(UTC)payload = {