Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

12.2 KB
Raw Blame
/**
 * LLM request-failure recovery. The stop-reason split these tests pin is core's four-value
 * protocol (`completed` / `aborted` / `retryable` / `fatal`, upstream #458): a request either
 * retries on the engine's ladder or ends the turn, and a definitive provider rejection never
 * rides the ladder.
 *
 * 1. A truncated stream is retryable: the mock's first attempt ends one event short (no
 *    message_stop), GenerativeModel ends it `retryable` with error_code "malformed", and the
 *    engine reconnects after its backoff base (2s — see `reconnectBackoffMs`; the full
 *    2+4+8+16+30s ladder is pinned by the engine's unit tests, not by a browser test sitting
 *    through real waits). The 2s wait is exactly the countdown floor, so it shows a live
 *    ticker whose seconds go DOWN; clicking "retry now" (立即重试) skips the rest of the wait
 *    and the turn completes normally — no abort. (An HTTP 5xx would not do here: the protocol
 *    SDK retries 5xx internally, so neither the engine nor the Trace ever sees it.)
 * 2. Give-up: a conversation whose transient failures never stop — clicking 放弃 on the
 *    countdown fires the ordinary abort; the engine's abort-during-backoff path ends the turn
 *    and the composer is immediately usable again.
 * 3. A quota rejection (403 insufficient_user_quota) is FATAL: a 4xx other than 408/429 is in
 *    `isFatalProviderRejection`'s allowlist (core/test/llm.test.ts pins exactly this code), so
 *    the turn ends on the first attempt — the provider's own text on screen, no ladder, no
 *    countdown — and the composer is usable again for a fixed key or another model.
 * 4. An authentication failure (401 invalid_api_key) is fatal too, but RECOVERABLE: only the
 *    model reference is fixed at creation — credentials come from the current Project config —
 *    so updating the key and sending again continues the same conversation (no reload).
 */
import { test, expect } from "@playwright/test";
import { provisionAndLogin } from "./auth.mjs";

const BASE = process.env.BASE_URL;
const MOCK = process.env.MOCK_URL;

/** Provision a user with a configured mock model and one session; returns ids for later config updates. */
async function makeSession(page, userId, apiKey = "sk-mock") {
  await provisionAndLogin(page.request, userId, "password123");
  const projects = await (await page.request.get(`${BASE}/api/projects`)).json();
  const projectId = projects.projects[0].projectId;
  await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
    data: {
      defaultModel: { provider: "custom", modelId: "claude-4-8" },
      models: [
        {
          provider: "custom",
          modelId: "claude-4-8",
          apiKey,
          baseUrl: MOCK,
          contextWindow: 200000,
        },
      ],
    },
  });
  const sess = await (
    await page.request.post(`${BASE}/api/projects/${projectId}/agents/default_agent/sessions`, {
      data: { provider: "custom", modelId: "claude-4-8" },
    })
  ).json();
  return { sessionId: sess.session.sessionId, projectId };
}

test("a transient failure retries with a live countdown; 'retry now' skips the wait and the turn completes", async ({
  page,
}) => {
  const { sessionId } = await makeSession(page, "transientuser");

  await page.goto(`${BASE}/chat/${sessionId}`);
  await page.getByPlaceholder(/输入消息/).fill("transient retry test");
  await page.getByRole("button", { name: "发送" }).click();

  // One line for the whole ladder, not one per attempt. The first retry's wait is the
  // backoff base (2s — exactly the countdown floor), so the single retry line advances to
  // its countdown wording right away.
  const ladder = page.getByText(/\[重试\]/);
  await expect(ladder).toHaveCount(1, { timeout: 20000 });
  await expect(ladder).toHaveText(/第 1 次重试/, { timeout: 20000 });

  // A live countdown (whole seconds, ticking down).
  const countdown = page.getByText(/第 1 次重试,\d+ 秒后发起/);
  await expect(countdown).toBeVisible({ timeout: 20000 });
  const readSecs = async () => {
    const txt = await countdown.textContent({ timeout: 500 }).catch(() => null);
    const m = txt === null ? null : /第 1 次重试,(\d+) 秒后发起/.exec(txt);
    return m ? parseInt(m[1], 10) : null;
  };
  const first = await readSecs();
  expect(first).not.toBeNull();
  // Poll until the displayed seconds DECREASE (a live ticker, not a static label).
  let second = first;
  for (let i = 0; i < 12 && second !== null && second >= first; i++) {
    await page.waitForTimeout(300);
    second = await readSecs();
  }
  expect(second).not.toBeNull();
  expect(second).toBeLessThan(first);

  // "Retry now" skips the remaining wait: the retry fires promptly — well before the
  // scheduled 2s would have elapsed (the 1.5s expectation window is the proof: without
  // the skip, the natural timer still had time to go).
  await page.getByRole("button", { name: "立即重试" }).click();
  await expect(page.getByText(/已发起第 1 次重试/)).toBeVisible({
    timeout: 1500,
  });

  // Attempt 2 succeeds: the final answer streams in.
  await expect(page.getByText("Service recovered; the answer is 42.")).toBeVisible({
    timeout: 20000,
  });

  // No abort: the run recovered, the composer stays usable.
  await expect(page.getByText(/已中断/)).toHaveCount(0);
  await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();

  // Trace: the truncated stream recorded as request_end(retryable, error_code "malformed") —
  // the reconnect path — carrying the real failure detail (the Cost center's errors panel
  // reads it from here) and the announced backoff; no abort event.
  const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
  const retries = msgs.messages.filter(
    (m) => m.payload.type === "request_end" && m.payload.status === "retryable",
  );
  expect(retries.length).toBe(1);
  expect(retries[0].payload.error_code).toBe("malformed");
  // AgentHub's own wording for a stream it saw none of ("Streaming response yielded no
  // events"); the point is that the provider's detail is what the Trace keeps.
  expect(retries[0].payload.error_message).toContain("no events");
  expect(retries[0].payload.retry_in_ms).toBe(2000);
  expect(msgs.messages.some((m) => m.payload.type === "abort")).toBe(false);
});

test("'give up' on the countdown aborts the backoff: the turn ends and the composer is usable again", async ({
  page,
}) => {
  const { sessionId } = await makeSession(page, "giveupuser");

  await page.goto(`${BASE}/chat/${sessionId}`);
  await page.getByPlaceholder(/输入消息/).fill("transient giveup test");
  await page.getByRole("button", { name: "发送" }).click();

  // The mock rejects every request: by the 2s wait before retry #4 the countdown (and its
  // inline controls) are on screen.
  await expect(page.getByRole("button", { name: "放弃" })).toBeVisible({ timeout: 20000 });
  await page.getByRole("button", { name: "放弃" }).click();

  // The ordinary abort lands mid-backoff: the engine's abort-during-backoff path ends the
  // turn (abort line, and the retry line flips to the "gave up after attempt N" wording)
  // and the composer is immediately usable again.
  await expect(page.getByText(/已中断/)).toBeVisible({ timeout: 10000 });
  await expect(page.getByText(/次尝试后放弃/)).toBeVisible();
  await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();

  // Trace: the abort says why — the backoff was interrupted, not the user's own stop.
  const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
  const abort = msgs.messages.find((m) => m.payload.type === "abort");
  expect(abort.payload.error_code).toBe("backoff_interrupted");
});

test("a quota-403 is a fatal rejection: one attempt, the provider's text on screen, no ladder", async ({
  page,
}) => {
  const { sessionId } = await makeSession(page, "quotauser");

  await page.goto(`${BASE}/chat/${sessionId}`);
  await page.getByPlaceholder(/输入消息/).fill("quota fatal test");
  await page.getByRole("button", { name: "发送" }).click();

  // Fatal means fatal: the error banner names the provider's own rejection, and the
  // reconnect line never appears — a 4xx is not retried.
  await expect(page.getByText(/\[错误\]:模型请求错误/)).toBeVisible({ timeout: 20000 });
  await expect(page.getByText(/insufficient_user_quota/)).toBeVisible();
  await expect(page.getByText(/\[重试\]/)).toHaveCount(0);
  await expect(page.getByText(/已中断/)).toHaveCount(0);

  // The turn ended; the composer is usable again for a fixed key or another model.
  await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();

  // Trace: one terminal request, classified fatal → "rejected", carrying the provider's text.
  const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
  const rejected = msgs.messages.filter(
    (m) => m.payload.type === "request_end" && m.payload.error_code === "rejected",
  );
  expect(rejected.length).toBe(1);
  expect(rejected[0].payload.status).toBe("fatal");
  expect(rejected[0].payload.error_message).toContain("insufficient_user_quota");
  expect(rejected[0].payload.retry_in_ms).toBeUndefined();
  expect(msgs.messages.some((m) => m.payload.type === "abort")).toBe(false);
});

test("an auth-401 is fatal but recoverable: the banner names the rejected key, and updating it lets the same conversation continue", async ({
  page,
}) => {
  // The mock rejects the provisioned key (`sk-auth-bad`) with a 401 and accepts any other.
  const { sessionId, projectId } = await makeSession(page, "authuser", "sk-auth-bad");

  await page.goto(`${BASE}/chat/${sessionId}`);
  await page.getByPlaceholder(/输入消息/).fill("auth dead test");
  await page.getByRole("button", { name: "发送" }).click();

  // A dead credential is fatal: one attempt, the provider's text (code and message) on
  // screen, no ladder — retrying the same dead key can never help — and the composer stays
  // usable for the fixed key.
  await expect(page.getByText(/\[错误\]:模型请求错误/)).toBeVisible({ timeout: 20000 });
  await expect(page.getByText(/invalid_api_key/)).toBeVisible();
  await expect(page.getByText(/invalid x-api-key/)).toBeVisible();
  await expect(page.getByText(/\[重试\]/)).toHaveCount(0);
  await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();

  // Trace: the credentials failure is the request's own terminal status — fatal, with
  // error_code "auth" — and a fatal turn writes no abort event (there is no interruption to
  // report; the failure is the record).
  const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
  const authEnd = msgs.messages.find(
    (m) => m.payload.type === "request_end" && m.payload.error_code === "auth",
  );
  expect(authEnd).toBeTruthy();
  expect(authEnd.payload.status).toBe("fatal");
  expect(authEnd.payload.error_message).toContain("invalid x-api-key");
  expect(msgs.messages.some((m) => m.payload.type === "abort")).toBe(false);

  // PRIMARY PATH: update the model's key (as the Models page would). The rebuilt runtime
  // re-reads the Project config on the next request, so the SAME conversation continues.
  const put = await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
    data: {
      defaultModel: { provider: "custom", modelId: "claude-4-8" },
      models: [
        {
          provider: "custom",
          modelId: "claude-4-8",
          apiKey: "sk-auth-good",
          baseUrl: MOCK,
          contextWindow: 200000,
        },
      ],
    },
  });
  expect(put.status()).toBe(200);

  await page.getByPlaceholder(/输入消息/).fill("auth dead test after fix");
  await page.getByRole("button", { name: "发送" }).click();
  await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 20000 });

  // Reload AFTER the success: replay still contains the failed attempt, but it is followed
  // by a completed request — the composer stays usable and nothing spins a retry.
  await page.reload();
  await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 15000 });
  await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
  await expect(page.getByText(/\[重试\]/)).toHaveCount(0);
});