Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Surface workflow self-heal reliability (panel + run-page uplink) (#7493)
pedrohsdb409b2577a63c
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.runTags.test.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.runTags.test.tsxindex 434f8ff50..91cb7de20 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.runTags.test.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.runTags.test.tsx@@ -91,6 +91,10 @@ vi.mock("./workflowRun/RunParametersDialog", () => ({RunParametersDialog: () => null,}));+vi.mock("./workflowRun/WorkflowReliabilityPanel", () => ({+ WorkflowReliabilityPanel: () => null,+}));+vi.mock("./hooks/useWorkflowQuery", () => ({useWorkflowQuery: () => ({data: {diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsxindex a1eaad3a2..d24c2d25a 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx@@ -48,6 +48,10 @@ vi.mock("./workflowRun/RunParametersDialog", () => ({RunParametersDialog: () => null,}));+vi.mock("./workflowRun/WorkflowReliabilityPanel", () => ({+ WorkflowReliabilityPanel: () => null,+}));+vi.mock("./hooks/useWorkflowQuery", () => ({useWorkflowQuery: () => ({data: {diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsxindex a252938a4..8f1dffeb1 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx@@ -98,6 +98,7 @@ import {import { useRowSelection } from "@/hooks/useRowSelection";import { RunBulkActionBar } from "@/routes/runs/RunBulkActionBar";import { RunRowContextMenu } from "@/routes/runs/RunRowContextMenu";+import { WorkflowReliabilityPanel } from "./workflowRun/WorkflowReliabilityPanel";function WorkflowPage() {const { workflowPermanentId } = useParams();@@ -307,6 +308,7 @@ function WorkflowPage() {</Button></div></div>+ <WorkflowReliabilityPanel workflowPermanentId={workflowPermanentId} />{WorkflowAnalyticsPanel ? (<WorkflowAnalyticsPanel workflowPermanentId={workflowPermanentId} />) : null}diff --git a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowReliabilityQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowReliabilityQuery.tsnew file mode 100644index 000000000..9b8859bdc--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowReliabilityQuery.ts@@ -0,0 +1,44 @@+import { getClient } from "@/api/AxiosClient";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import {+ getActiveOrgQueryKeyScope,+ getOrgScopedQueryKey,+ useActiveOrgId,+} from "@/store/ActiveOrgContext";+import { useQuery } from "@tanstack/react-query";+import type { WorkflowReliability } from "../types/reliabilityTypes";++type UseWorkflowReliabilityQueryOptions = {+ workflowPermanentId?: string;+ enabled?: boolean;+};++function useWorkflowReliabilityQuery({+ workflowPermanentId,+ enabled = true,+}: UseWorkflowReliabilityQueryOptions) {+ const credentialGetter = useCredentialGetter();+ const activeOrgId = useActiveOrgId();+ const activeOrgQueryKeyScope = getActiveOrgQueryKeyScope(activeOrgId);++ return useQuery<WorkflowReliability>({+ queryKey: getOrgScopedQueryKey(+ ["workflow-reliability", workflowPermanentId],+ activeOrgQueryKeyScope,+ ),+ queryFn: async ({ signal }) => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .get<WorkflowReliability>(+ `/workflows/${workflowPermanentId}/reliability`,+ {+ signal,+ },+ )+ .then((response) => response.data);+ },+ enabled: Boolean(workflowPermanentId) && enabled,+ });+}++export { useWorkflowReliabilityQuery };diff --git a/skyvern-frontend/src/routes/workflows/types/reliabilityTypes.ts b/skyvern-frontend/src/routes/workflows/types/reliabilityTypes.tsnew file mode 100644index 000000000..91556af53--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/types/reliabilityTypes.ts@@ -0,0 +1,15 @@+type WorkflowReliabilityState = "healthy" | "watch" | "action_needed";++type WorkflowReliability = {+ state: WorkflowReliabilityState;+ outcome_risk: boolean;+ scored: boolean;+ window_runs: number;+ healed_runs: number;+ heal_rate: number;+ consecutive_healed_runs: number;+ floor_runs: number;+ outcome_risk_runs: number;+};++export type { WorkflowReliability, WorkflowReliabilityState };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.test.tsxnew file mode 100644index 000000000..0284f4852--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.test.tsx@@ -0,0 +1,131 @@+// @vitest-environment jsdom++import { render, screen } from "@testing-library/react";+import { beforeEach, describe, expect, it, vi } from "vitest";+import { MemoryRouter } from "react-router-dom";++import type { WorkflowReliability } from "../types/reliabilityTypes";++const { mockState } = vi.hoisted(() => ({+ mockState: {+ workflowRun: {+ workflow: { workflow_permanent_id: "wpid_1" },+ },+ reliability: undefined as WorkflowReliability | undefined,+ },+}));++vi.mock("../hooks/useWorkflowRunWithWorkflowQuery", () => ({+ useWorkflowRunWithWorkflowQuery: () => ({+ data: mockState.workflowRun,+ }),+}));++vi.mock("../hooks/useWorkflowReliabilityQuery", () => ({+ useWorkflowReliabilityQuery: () => ({+ data: mockState.reliability,+ }),+}));++import { RunReliabilityUplink } from "./RunReliabilityUplink";++function makeReliability(+ overrides: Partial<WorkflowReliability> = {},+): WorkflowReliability {+ return {+ state: "healthy",+ outcome_risk: false,+ scored: true,+ window_runs: 10,+ healed_runs: 2,+ heal_rate: 0.2,+ consecutive_healed_runs: 1,+ floor_runs: 0,+ outcome_risk_runs: 0,+ ...overrides,+ };+}++describe("RunReliabilityUplink", () => {+ beforeEach(() => {+ mockState.workflowRun = {+ workflow: { workflow_permanent_id: "wpid_1" },+ };+ mockState.reliability = undefined;+ });++ it("renders nothing when reliability is not scored", () => {+ mockState.reliability = makeReliability({+ state: "watch",+ scored: false,+ healed_runs: 3,+ });++ const { container } = render(+ <MemoryRouter>+ <RunReliabilityUplink workflowRunId="wr_1" />+ </MemoryRouter>,+ );++ expect(container.firstChild).toBeNull();+ });++ it("renders nothing when there is no heal or floor activity", () => {+ mockState.reliability = makeReliability({+ state: "action_needed",+ healed_runs: 0,+ floor_runs: 0,+ });++ const { container } = render(+ <MemoryRouter>+ <RunReliabilityUplink workflowRunId="wr_1" />+ </MemoryRouter>,+ );++ expect(container.firstChild).toBeNull();+ });++ it("links with a floor-fallback story when action_needed has no heals", () => {+ mockState.reliability = makeReliability({+ state: "action_needed",+ scored: true,+ healed_runs: 0,+ floor_runs: 3,+ window_runs: 10,+ });++ render(+ <MemoryRouter>+ <RunReliabilityUplink workflowRunId="wr_1" />+ </MemoryRouter>,+ );++ const link = screen.getByRole("link", {+ name: "This workflow fell back to a backup on 3 of the last 10 runs →",+ });+ expect(link).toBeDefined();+ expect(link.getAttribute("href")).toBe("/agents/wpid_1/runs");+ });++ it("links to workflow reliability when scored and not healthy", () => {+ mockState.reliability = makeReliability({+ state: "watch",+ scored: true,+ healed_runs: 4,+ window_runs: 10,+ });++ render(+ <MemoryRouter>+ <RunReliabilityUplink workflowRunId="wr_1" />+ </MemoryRouter>,+ );++ const link = screen.getByRole("link", {+ name: "This workflow self-healed 4 of the last 10 runs →",+ });+ expect(link).toBeDefined();+ expect(link.getAttribute("href")).toBe("/agents/wpid_1/runs");+ });+});diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.tsxnew file mode 100644index 000000000..8fede4b19--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/RunReliabilityUplink.tsx@@ -0,0 +1,46 @@+import { Link } from "react-router-dom";++import { useWorkflowReliabilityQuery } from "../hooks/useWorkflowReliabilityQuery";+import { useWorkflowRunWithWorkflowQuery } from "../hooks/useWorkflowRunWithWorkflowQuery";+import { reliabilityHasActivity } from "./reliabilityStatus";++type Props = {+ workflowRunId?: string;+};++function RunReliabilityUplink({ workflowRunId }: Props) {+ const { data: workflowRun } = useWorkflowRunWithWorkflowQuery({+ workflowRunId,+ });+ const workflowPermanentId = workflowRun?.workflow?.workflow_permanent_id;+ const { data: reliability } = useWorkflowReliabilityQuery({+ workflowPermanentId,+ enabled: Boolean(workflowPermanentId),+ });++ if (+ !workflowPermanentId ||+ !reliability ||+ !reliabilityHasActivity(reliability) ||+ !reliability.scored ||+ reliability.state === "healthy"+ ) {+ return null;+ }++ const story =+ reliability.healed_runs > 0+ ? `This workflow self-healed ${reliability.healed_runs} of the last ${reliability.window_runs} runs`+ : `This workflow fell back to a backup on ${reliability.floor_runs} of the last ${reliability.window_runs} runs`;++ return (+ <Link+ to={`/agents/${workflowPermanentId}/runs`}+ className="inline-flex w-fit items-center rounded-md border border-border bg-slate-elevation2/80 px-2 py-0.5 text-xs text-warning backdrop-blur-sm hover:underline"+ >+ {story} →+ </Link>+ );+}++export { RunReliabilityUplink };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.test.tsxnew file mode 100644index 000000000..fe419565c--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.test.tsx@@ -0,0 +1,119 @@+// @vitest-environment jsdom++import { render, screen } from "@testing-library/react";+import { beforeEach, describe, expect, it, vi } from "vitest";++import type { WorkflowReliability } from "../types/reliabilityTypes";++const { reliabilityState } = vi.hoisted(() => ({+ reliabilityState: {+ reliability: undefined as WorkflowReliability | undefined,+ },+}));++vi.mock("../hooks/useWorkflowReliabilityQuery", () => ({+ useWorkflowReliabilityQuery: () => ({+ data: reliabilityState.reliability,+ }),+}));++import { WorkflowReliabilityPanel } from "./WorkflowReliabilityPanel";++function makeReliability(+ overrides: Partial<WorkflowReliability> = {},+): WorkflowReliability {+ return {+ state: "healthy",+ outcome_risk: false,+ scored: true,+ window_runs: 12,+ healed_runs: 2,+ heal_rate: 2 / 12,+ consecutive_healed_runs: 1,+ floor_runs: 0,+ outcome_risk_runs: 0,+ ...overrides,+ };+}++describe("WorkflowReliabilityPanel", () => {+ beforeEach(() => {+ reliabilityState.reliability = undefined;+ });++ it("renders nothing for an idle window (no heals, no floor fallbacks)", () => {+ reliabilityState.reliability = makeReliability({+ healed_runs: 0,+ floor_runs: 0,+ });++ const { container } = render(+ <WorkflowReliabilityPanel workflowPermanentId="wpid_1" />,+ );++ expect(container.firstChild).toBeNull();+ });++ it("shows an amber floor-fallback story when action_needed has no heals", () => {+ reliabilityState.reliability = makeReliability({+ state: "action_needed",+ scored: true,+ window_runs: 10,+ healed_runs: 0,+ floor_runs: 3,+ heal_rate: 0,+ consecutive_healed_runs: 0,+ });++ render(<WorkflowReliabilityPanel workflowPermanentId="wpid_1" />);++ const pill = screen.getByText("Needs a look");+ expect(pill).toBeDefined();+ expect(pill.className).toContain("text-warning");+ expect(+ screen.getByText("Fell back to a backup on 3 of the last 10 runs"),+ ).toBeDefined();+ });++ it("shows unscored reliability without any state label", () => {+ reliabilityState.reliability = makeReliability({+ state: "watch",+ scored: false,+ window_runs: 3,+ healed_runs: 3,+ heal_rate: 1,+ });++ render(<WorkflowReliabilityPanel workflowPermanentId="wpid_1" />);++ expect(+ screen.getByText(+ "Self-healed in 3 of the last 3 runs - not enough runs to assess yet.",+ ),+ ).toBeDefined();+ expect(screen.queryByText("Watch")).toBeNull();+ expect(screen.queryByText("Needs a look")).toBeNull();+ });++ it("shows an amber state pill for action_needed reliability", () => {+ reliabilityState.reliability = makeReliability({+ state: "action_needed",+ scored: true,+ window_runs: 10,+ healed_runs: 6,+ heal_rate: 0.6,+ consecutive_healed_runs: 3,+ });++ render(<WorkflowReliabilityPanel workflowPermanentId="wpid_1" />);++ const pill = screen.getByText("Needs a look");+ expect(pill).toBeDefined();+ expect(pill.className).toContain("text-warning");+ expect(+ screen.getByText(+ "Self-healed in 6 of the last 10 runs - 3 in a row - 60% rate",+ ),+ ).toBeDefined();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsxnew file mode 100644index 000000000..c672eddfc--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowReliabilityPanel.tsx@@ -0,0 +1,81 @@+import { useWorkflowReliabilityQuery } from "../hooks/useWorkflowReliabilityQuery";+import {+ reliabilityHasActivity,+ reliabilityLabel,+ reliabilityShowsState,+ reliabilityTone,+} from "./reliabilityStatus";++type Props = {+ workflowPermanentId?: string;+};++function formatRate(healRate: number): string {+ return `${Math.round(healRate * 100)}%`;+}++function WorkflowReliabilityPanel({ workflowPermanentId }: Props) {+ const { data: reliability } = useWorkflowReliabilityQuery({+ workflowPermanentId,+ });++ if (!reliability || !reliabilityHasActivity(reliability)) {+ return null;+ }++ const showsState = reliabilityShowsState(reliability);+ const healed = reliability.healed_runs > 0;+ const baseStory = healed+ ? `Self-healed in ${reliability.healed_runs} of the last ${reliability.window_runs} runs`+ : `Fell back to a backup on ${reliability.floor_runs} of the last ${reliability.window_runs} runs`;+ const streakStory =+ healed && reliability.consecutive_healed_runs > 1+ ? ` - ${reliability.consecutive_healed_runs} in a row`+ : "";+ const rateStory = healed+ ? ` - ${formatRate(reliability.heal_rate)} rate`+ : "";++ if (!showsState) {+ return (+ <section className="rounded-md border border-border bg-slate-elevation1/80 px-3 py-2">+ <p className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">+ Reliability+ </p>+ <p className="mt-1 text-sm text-muted-foreground">+ {baseStory} - not enough runs to assess yet.+ </p>+ </section>+ );+ }++ const isHealthy = reliability.state === "healthy";+ const tone = reliabilityTone(reliability.state);+ const isAmber = tone === "amber";++ return (+ <section className="rounded-md border border-border bg-slate-elevation1/80 px-3 py-2">+ <p className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">+ Reliability+ </p>+ <div className="mt-1 flex flex-wrap items-center gap-2">+ {!isHealthy ? (+ <span className="inline-flex items-center rounded border border-border bg-warning/15 px-1.5 py-0.5 text-[10px] font-medium uppercase tracking-wide text-warning">+ {reliabilityLabel(reliability.state)}+ </span>+ ) : null}+ <p+ className={+ isAmber ? "text-sm text-warning" : "text-sm text-muted-foreground"+ }+ >+ {baseStory}+ {streakStory}+ {rateStory}+ </p>+ </div>+ </section>+ );+}++export { WorkflowReliabilityPanel };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsxindex a9ef584b3..d1d90688d 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx@@ -27,6 +27,7 @@ import { useQueryClient } from "@tanstack/react-query";import { useCallback, useEffect, useState } from "react";import { useBrowserStreamingMode } from "@/hooks/useRuntimeConfig";import { RunHealChip } from "./RunHealChip";+import { RunReliabilityUplink } from "./RunReliabilityUplink";export type ActionItem = {block: WorkflowRunBlock;@@ -133,8 +134,9 @@ function WorkflowRunOverview() {return (<AspectRatio ratio={16 / 9}><div className="relative h-full w-full">- <div className="absolute left-2 top-2 z-20">+ <div className="absolute left-2 top-2 z-20 flex flex-col gap-1"><RunHealChip workflowRunId={workflowRunId} />+ <RunReliabilityUplink workflowRunId={workflowRunId} /></div>{shouldShowBrowserStream && (<BrowserStreamdiff --git a/skyvern-frontend/src/routes/workflows/workflowRun/__tests__/reliabilityStatus.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/__tests__/reliabilityStatus.test.tsnew file mode 100644index 000000000..949ac60bb--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/__tests__/reliabilityStatus.test.ts@@ -0,0 +1,66 @@+import { describe, expect, it } from "vitest";++import {+ reliabilityHasActivity,+ reliabilityLabel,+ reliabilityShowsState,+ reliabilityTone,+} from "../reliabilityStatus";+import type { WorkflowReliability } from "../../types/reliabilityTypes";++function makeReliability(+ overrides: Partial<WorkflowReliability> = {},+): WorkflowReliability {+ return {+ state: "healthy",+ outcome_risk: false,+ scored: true,+ window_runs: 12,+ healed_runs: 2,+ heal_rate: 2 / 12,+ consecutive_healed_runs: 1,+ floor_runs: 0,+ outcome_risk_runs: 0,+ ...overrides,+ };+}++describe("reliability status helpers", () => {+ it("maps reliability states to user-facing labels", () => {+ expect(reliabilityLabel("healthy")).toBe("Healthy");+ expect(reliabilityLabel("watch")).toBe("Watch");+ expect(reliabilityLabel("action_needed")).toBe("Needs a look");+ });++ it("keeps reliability tones neutral or amber only", () => {+ expect(reliabilityTone("healthy")).toBe("neutral");+ expect(reliabilityTone("watch")).toBe("amber");+ expect(reliabilityTone("action_needed")).toBe("amber");+ expect(reliabilityTone("healthy")).not.toBe("destructive");+ expect(reliabilityTone("watch")).not.toBe("destructive");+ expect(reliabilityTone("action_needed")).not.toBe("destructive");+ });++ it("reports activity for heals or floor fallbacks, but not an idle window", () => {+ expect(+ reliabilityHasActivity(+ makeReliability({ healed_runs: 0, floor_runs: 0 }),+ ),+ ).toBe(false);+ expect(reliabilityHasActivity(makeReliability({ healed_runs: 1 }))).toBe(+ true,+ );+ expect(+ reliabilityHasActivity(+ makeReliability({ healed_runs: 0, floor_runs: 2 }),+ ),+ ).toBe(true);+ });++ it("shows state only when reliability is scored", () => {+ expect(reliabilityShowsState(makeReliability({ scored: true }))).toBe(true);+ expect(reliabilityShowsState(makeReliability({ scored: false }))).toBe(+ false,+ );+ });+});diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/reliabilityStatus.ts b/skyvern-frontend/src/routes/workflows/workflowRun/reliabilityStatus.tsnew file mode 100644index 000000000..c7f84c736--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/reliabilityStatus.ts@@ -0,0 +1,39 @@+import type {+ WorkflowReliability,+ WorkflowReliabilityState,+} from "../types/reliabilityTypes";++type ReliabilityTone = "neutral" | "amber";++const reliabilityLabels: Record<WorkflowReliabilityState, string> = {+ healthy: "Healthy",+ watch: "Watch",+ action_needed: "Needs a look",+};++function reliabilityHasActivity(reliability: WorkflowReliability): boolean {+ return reliability.healed_runs > 0 || reliability.floor_runs > 0;+}++function reliabilityShowsState(reliability: WorkflowReliability): boolean {+ return reliability.scored;+}++function reliabilityLabel(state: WorkflowReliabilityState): string {+ return reliabilityLabels[state];+}++function reliabilityTone(state: WorkflowReliabilityState): ReliabilityTone {+ if (state === "healthy") {+ return "neutral";+ }+ return "amber";+}++export {+ reliabilityHasActivity,+ reliabilityLabel,+ reliabilityShowsState,+ reliabilityTone,+};+export type { ReliabilityTone };