Skip to content

Automate browser based workflows with AI

15.9 KB
Raw Blame
import { describe, expect, test } from "vitest";

import { ActionTypes, Status, type ActionsApiResponse } from "@/api/types";

import type {
  WorkflowRunBlock,
  WorkflowRunTimelineItem,
} from "../types/workflowRunTypes";
import {
  describeCodeBlockFailure,
  failingCodeLineFromActions,
  failureSupportsScreenshot,
  findRunCodeBlockFailure,
} from "./codeBlockFailure";

type FailedBlock = Pick<
  WorkflowRunBlock,
  "block_type" | "failure_reason" | "error_codes" | "actions" | "output"
>;

function block(overrides: Partial<FailedBlock> = {}): FailedBlock {
  return {
    block_type: "code",
    failure_reason: null,
    error_codes: null,
    actions: null,
    output: null,
    ...overrides,
  };
}

function failedAction(codeLine: number | null): ActionsApiResponse {
  return {
    action_id: `act_${codeLine ?? "none"}`,
    action_type: ActionTypes.NullAction,
    status: Status.Failed,
    task_id: null,
    step_id: null,
    step_order: null,
    action_order: 0,
    confidence_float: null,
    description: null,
    reasoning: null,
    intention: null,
    response: null,
    created_by: null,
    text: null,
    output: codeLine === null ? null : { code_line: codeLine },
  };
}

function timelineBlock(
  overrides: Partial<WorkflowRunBlock>,
): WorkflowRunTimelineItem {
  const failureBlock = {
    ...block(overrides),
    workflow_run_block_id: overrides.workflow_run_block_id ?? "wrb_code",
    continue_on_failure: overrides.continue_on_failure ?? false,
  } as WorkflowRunBlock;
  return {
    type: "block",
    block: failureBlock,
    children: [],
    thought: null,
    created_at: "2026-01-01T00:00:00Z",
    modified_at: "2026-01-01T00:00:00Z",
  };
}

describe("describeCodeBlockFailure", () => {
  test("returns null for blocks that are not code blocks", () => {
    expect(
      describeCodeBlockFailure(
        block({ block_type: "task", failure_reason: "task block failed" }),
      ),
    ).toBeNull();
  });

  test("returns null when there is nothing to classify", () => {
    expect(describeCodeBlockFailure(block())).toBeNull();
  });

  test("treats a sandbox fault as a retry, never a code fix", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["runner_unavailable"],
        failure_reason: "Secure CodeBlock runner is unavailable. Please retry.",
      }),
    );
    expect(failure).toMatchObject({
      kind: "infrastructure",
      code: "runner_unavailable",
      recovery: "retry",
    });
  });

  test("a busy runner is an infrastructure state, not a code defect", () => {
    expect(
      describeCodeBlockFailure(block({ error_codes: ["busy"] })),
    ).toMatchObject({ kind: "infrastructure", recovery: "retry" });
  });

  test("treats an input reassembly memory limit as a retry, not a code fix", () => {
    expect(
      describeCodeBlockFailure(
        block({ error_codes: ["parameter_reassembly_memory_limit_exceeded"] }),
      ),
    ).toMatchObject({
      kind: "infrastructure",
      code: "parameter_reassembly_memory_limit_exceeded",
      recovery: "retry",
      guidance: expect.stringContaining("earlier block pass in less data"),
    });
  });

  test("keeps a code memory limit repairable", () => {
    expect(
      describeCodeBlockFailure(
        block({ error_codes: ["memory_limit_exceeded"] }),
      ),
    ).toMatchObject({
      kind: "limit",
      code: "memory_limit_exceeded",
      recovery: "fix",
      guidance: expect.stringContaining("Process data in smaller batches"),
    });
  });

  test("names the exception and line for a user code error", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["user_code_error"],
        failure_reason:
          "CodeBlock failed with NameError at line 6: name 'min' is not defined.",
      }),
    );
    expect(failure).toMatchObject({
      kind: "user-code",
      title: "The block's code raised NameError",
      line: 6,
      recovery: "fix",
    });
  });

  test("a page failure reads as a browser problem, not a code defect", () => {
    expect(
      describeCodeBlockFailure(
        block({
          error_codes: ["browser_operation_failed"],
          failure_reason:
            "CodeBlock failed because a browser operation failed at line 12: net::ERR_CONNECTION_CLOSED",
        }),
      ),
    ).toMatchObject({
      kind: "browser",
      code: "browser_operation_failed",
      line: 12,
      recovery: "either",
    });
  });

  test("prefers the workflow's declared code over the transport code", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["user_code_error", "no_tables_available"],
        failure_reason: "No tables were available at the requested time.",
      }),
    );
    expect(failure).toMatchObject({
      kind: "declared",
      code: "no_tables_available",
      title: 'The workflow reported "no_tables_available"',
      recovery: "retry",
    });
  });

  test("does not treat an unmapped runner code as workflow-declared", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["future_runner_error"],
        failure_reason: "The code block runner reported a new failure.",
        output: {
          errors: [{ error_code: "future_runner_error" }],
        },
      }),
    );
    expect(failure).toMatchObject({
      kind: "user-code",
      code: "future_runner_error",
      title: "The code block failed",
      recovery: "either",
    });
  });

  test("does not treat two unknown runner codes as workflow-declared", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["future_runner_error", "another_runner_error"],
        failure_reason: "The code block runner reported a new failure.",
      }),
    );
    expect(failure).toMatchObject({
      kind: "user-code",
      code: "future_runner_error",
      title: "The code block failed",
      recovery: "either",
    });
  });

  test("uses the backend error type for an inline declared code", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["no_tables_available"],
        failure_reason: "No tables were available at the requested time.",
        output: {
          errors: [
            {
              error_code: "no_tables_available",
              error_type: "USER_DEFINED_ERROR",
            },
          ],
        },
      }),
    );
    expect(failure).toMatchObject({
      kind: "declared",
      code: "no_tables_available",
      title: 'The workflow reported "no_tables_available"',
      recovery: "retry",
    });
  });

  test("keeps code-specific guidance when the reason names an exception", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["insecure_code_detected"],
        failure_reason:
          "CodeBlock failed with InsecureCodeError at line 4: blocked operation.",
      }),
    );
    expect(failure).toMatchObject({
      title: "The block's code raised InsecureCodeError",
      guidance:
        "Code blocks run in a locked-down sandbox. Rewrite the step using a supported Skyvern helper instead.",
    });
  });

  test("classifies the inline engine's prose when no error code is persisted", () => {
    expect(
      describeCodeBlockFailure(
        block({
          failure_reason:
            "Failed to execute code block. Reason: TimeoutError: code block exceeded 300 seconds",
        }),
      ),
    ).toMatchObject({ kind: "limit", code: "timeout" });
  });

  test("keeps a named user exception ahead of infrastructure wording", () => {
    expect(
      describeCodeBlockFailure(
        block({
          failure_reason:
            "Failed to execute code block. Reason: RuntimeError: browser disconnected while polling",
        }),
      ),
    ).toMatchObject({
      kind: "user-code",
      title: "The block's code raised RuntimeError",
      recovery: "fix",
    });
  });

  test("calls out an ErrorCode that is missing from the mapping", () => {
    const failure = describeCodeBlockFailure(
      block({
        failure_reason:
          "Failed to execute code block. Reason: ErrorCode is not declared in the effective error_code_mapping",
      }),
    );
    expect(failure).toMatchObject({
      kind: "user-code",
      title: "The block raised an undeclared error code",
      recovery: "fix",
    });
    expect(failure?.guidance).toContain("error code mapping");
  });

  test("names the exception behind a redacted inline failure reason", () => {
    expect(
      describeCodeBlockFailure(
        block({
          failure_reason:
            "Failed to execute code block. Reason: SyntaxError: invalid syntax",
        }),
      ),
    ).toMatchObject({
      kind: "user-code",
      title: "The block's code raised SyntaxError",
    });
  });

  test("a machinery failure inside Skyvern is never the block's own raise", () => {
    for (const exception of ["KeyError", "TimeoutError"]) {
      const failure = describeCodeBlockFailure(
        block({
          failure_reason: `CodeBlock failed with ${exception} inside Skyvern while running line 3.`,
        }),
      );
      expect(failure).toMatchObject({
        kind: "infrastructure",
        code: null,
        line: 3,
        recovery: "retry",
      });
      expect(failure?.title).not.toContain("raised");
    }
  });

  test("a user exception whose message mimics the machinery wording stays a user-code failure", () => {
    const failure = describeCodeBlockFailure(
      block({
        failure_reason:
          "Failed to execute code block. Reason: ValueError: failed inside Skyvern while running line 999",
        actions: [failedAction(3)],
      }),
    );
    expect(failure).toMatchObject({ kind: "user-code", code: null, line: 3 });
    expect(failure?.recovery).not.toBe("retry");
  });

  test("falls back to the failing action's line when the reason has none", () => {
    expect(
      describeCodeBlockFailure(
        block({
          failure_reason: "Failed to execute code block.",
          actions: [failedAction(null), failedAction(9)],
        }),
      )?.line,
    ).toBe(9);
  });
});

describe("failingCodeLineFromActions", () => {
  test("takes the first failed action carrying a line", () => {
    expect(failingCodeLineFromActions([failedAction(4), failedAction(7)])).toBe(
      4,
    );
  });

  test("returns null without actions", () => {
    expect(failingCodeLineFromActions(null)).toBeNull();
  });
});

describe("findRunCodeBlockFailure", () => {
  test("skips continued blocks when multiple blocks share the run reason", () => {
    const reason = "Failed to execute code block.";
    const timeline = [
      timelineBlock({
        workflow_run_block_id: "wrb_continued",
        failure_reason: reason,
        error_codes: ["runner_unavailable"],
        continue_on_failure: true,
      }),
      timelineBlock({
        workflow_run_block_id: "wrb_culprit",
        failure_reason: reason,
        error_codes: ["user_code_error"],
      }),
    ];

    expect(findRunCodeBlockFailure(reason, timeline)).toMatchObject({
      code: "user_code_error",
      recovery: "fix",
    });
  });

  test("prefers a body failure over a later finally failure", () => {
    const reason = "Failed to execute code block.";
    const timeline = [
      timelineBlock({
        workflow_run_block_id: "wrb_finally",
        label: "cleanup",
        failure_reason: reason,
        error_codes: ["runner_unavailable"],
      }),
      timelineBlock({
        workflow_run_block_id: "wrb_culprit",
        label: "body",
        failure_reason: reason,
        error_codes: ["user_code_error"],
      }),
    ];

    expect(findRunCodeBlockFailure(reason, timeline, "cleanup")).toMatchObject({
      code: "user_code_error",
      recovery: "fix",
    });
  });

  test("uses the finally failure when it is the only matching culprit", () => {
    const reason = "Code block runner is unavailable.";
    const timeline = [
      timelineBlock({
        workflow_run_block_id: "wrb_finally",
        label: "cleanup",
        failure_reason: reason,
        error_codes: ["runner_unavailable"],
      }),
    ];

    expect(findRunCodeBlockFailure(reason, timeline, "cleanup")).toMatchObject({
      code: "runner_unavailable",
      recovery: "retry",
    });
  });
});

describe("failureSupportsScreenshot", () => {
  // Only faults that are claims about the page have a capture worth offering.
  const kinds = {
    "user-code": true,
    browser: true,
    declared: false,
    infrastructure: false,
    limit: false,
    cancelled: false,
  } as const;

  for (const [kind, supported] of Object.entries(kinds)) {
    test(`${kind} -> ${supported}`, () => {
      expect(
        failureSupportsScreenshot({
          kind: kind as keyof typeof kinds,
          code: null,
          title: "t",
          guidance: "g",
          line: null,
          recovery: "fix",
        }),
      ).toBe(supported);
    });
  }
});

describe("guard denials under unsupported_page_operation", () => {
  // All five guards report the same error code, so the code alone cannot pick the remedy.
  // The regression these guard against: telling someone who exhausted the per-run operation
  // budget, or who navigated somewhere refused, to "use a supported page method" instead.
  const denial = (exceptionClass: string | null): FailedBlock =>
    block({
      error_codes: ["unsupported_page_operation"],
      failure_reason:
        "Browser operation failed while running CodeBlock: denied at line 4",
      output:
        exceptionClass === null
          ? { status: "failed" }
          : { status: "failed", runner_exception_class: exceptionClass },
    });

  test("a quota breach and an egress block get different remedies", () => {
    const quota = describeCodeBlockFailure(
      denial("codeblock.page_operation_broker.PageOperationLimitExceeded"),
    );
    const egress = describeCodeBlockFailure(
      denial("codeblock.page_operation_broker.BlockedEgressError"),
    );

    expect(quota?.title).not.toBe(egress?.title);
    expect(quota?.guidance).not.toBe(egress?.guidance);
    // Neither may keep telling the user the method was the problem.
    expect(quota?.guidance).not.toMatch(/supported page method/i);
    expect(egress?.guidance).not.toMatch(/supported page method/i);
    expect(quota?.guidance).toMatch(/split the work|narrow/i);
    expect(egress?.guidance).toMatch(/url|destination/i);
  });

  test("a handle-table breach is told apart from the operation quota", () => {
    const handles = describeCodeBlockFailure(
      denial("codeblock.page_operation_broker.HandleLimitExceeded"),
    );
    const quota = describeCodeBlockFailure(
      denial("codeblock.page_operation_broker.PageOperationLimitExceeded"),
    );

    expect(handles?.kind).toBe("limit");
    expect(handles?.title).not.toBe(quota?.title);
    expect(handles?.guidance).not.toMatch(/supported page method/i);
    expect(handles?.guidance).toMatch(/browser (elements|objects)|handles/i);
  });

  test("an unrecognised or absent class still falls back to the generic denial", () => {
    for (const value of [
      null,
      "codeblock.page_operation_broker.SomethingNew",
    ]) {
      const failure = describeCodeBlockFailure(denial(value));
      expect(failure?.code).toBe("unsupported_page_operation");
      expect(failure?.title).toBe(
        "The code called an unsupported browser operation",
      );
    }
  });

  test("a declared workflow error still outranks the guard class", () => {
    const failure = describeCodeBlockFailure(
      block({
        error_codes: ["unsupported_page_operation", "no_tables_available"],
        failure_reason: "denied",
        output: {
          status: "failed",
          runner_exception_class:
            "codeblock.page_operation_broker.BlockedEgressError",
          errors: [
            {
              error_type: "USER_DEFINED_ERROR",
              error_code: "no_tables_available",
              reasoning: "denied",
            },
          ],
        },
      }),
    );
    expect(failure?.code).toBe("no_tables_available");
  });
});