Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

test(e2e): prove the task-run log folds a lock-timeout retry

zchpeteref0892fa9b23
diff --git a/frontend/tests/e2e/framework/api-client.ts b/frontend/tests/e2e/framework/api-client.ts
index 3ca8e24a40..8eadd8f530 100644
--- a/frontend/tests/e2e/framework/api-client.ts
+++ b/frontend/tests/e2e/framework/api-client.ts
@@ -788,6 +788,9 @@ export class BytebaseApiClient {
// already defined in the DATA_CLASSIFICATION setting (UpdateProject
// validates it exists).
dataClassificationConfigId?: string;
+ // How many times the driver re-runs the sheet's execution after a
+ // lock-timeout failure, inside the same task run (0 disables retries).
+ executionRetryPolicy?: { maximumRetries?: number };
},
): Promise<void> {
const fields: string[] = [];
@@ -828,6 +831,10 @@ export class BytebaseApiClient {
fields.push("data_classification_config_id");
body.dataClassificationConfigId = settings.dataClassificationConfigId;
}
+ if (settings.executionRetryPolicy !== undefined) {
+ fields.push("execution_retry_policy");
+ body.executionRetryPolicy = settings.executionRetryPolicy;
+ }
if (fields.length === 0) {
throw new Error("updateProjectSettings: no fields specified");
}
diff --git a/frontend/tests/e2e/framework/psql.ts b/frontend/tests/e2e/framework/psql.ts
index fdb94836ca..b387e4893c 100644
--- a/frontend/tests/e2e/framework/psql.ts
+++ b/frontend/tests/e2e/framework/psql.ts
@@ -1,4 +1,4 @@
-import { execFileSync } from "child_process";
+import { type ChildProcess, execFileSync, spawn } from "child_process";
import type { TestEnv } from "./env";
import type { BytebaseApiClient } from "./api-client";
@@ -15,39 +15,26 @@ export async function getInstancePgPort(
return port;
}
+// The sample Postgres listens on a Unix socket in /tmp as role bbsample.
+function psqlArgs(dbName: string, port: string): string[] {
+ return ["-h", "/tmp", "-p", port, "-U", "bbsample", "-d", dbName, "-v", "ON_ERROR_STOP=1"];
+}
+
// Execute SQL via psql over the Unix socket on the sample Postgres instance.
// Used for DDL/DML setup and teardown — Bytebase's query API is read-only.
// Callers that interpolate non-constant values MUST validate them first
// (see masking-exemption's assertSafeSqlIdentifier).
export function execSql(dbName: string, port: string, sql: string): void {
- execFileSync(
- "psql",
- [
- "-h", "/tmp",
- "-p", port,
- "-U", "bbsample",
- "-d", dbName,
- "-v", "ON_ERROR_STOP=1",
- "-c", sql,
- ],
- { stdio: "pipe" },
- );
+ execFileSync("psql", [...psqlArgs(dbName, port), "-c", sql], { stdio: "pipe" });
}
// Execute a larger SQL script through stdin so it does not hit the operating
// system's command-line length limit.
export function execSqlScript(dbName: string, port: string, sql: string): void {
- execFileSync(
- "psql",
- [
- "-h", "/tmp",
- "-p", port,
- "-U", "bbsample",
- "-d", dbName,
- "-v", "ON_ERROR_STOP=1",
- ],
- { input: sql, stdio: ["pipe", "pipe", "pipe"] },
- );
+ execFileSync("psql", psqlArgs(dbName, port), {
+ input: sql,
+ stdio: ["pipe", "pipe", "pipe"],
+ });
}
// Run a read query and return the raw scalar/tuple output (tuples-only,
@@ -56,16 +43,17 @@ export function execSqlScript(dbName: string, port: string, sql: string): void {
// instead of inferring success from the absence of a UI error. Same
// interpolation-safety contract as execSql.
export function querySql(dbName: string, port: string, sql: string): string {
- return execFileSync(
- "psql",
- [
- "-h", "/tmp",
- "-p", port,
- "-U", "bbsample",
- "-d", dbName,
- "-v", "ON_ERROR_STOP=1",
- "-tAc", sql,
- ],
- { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] },
- ).trim();
+ return execFileSync("psql", [...psqlArgs(dbName, port), "-tAc", sql], {
+ encoding: "utf-8",
+ stdio: ["ignore", "pipe", "pipe"],
+ }).trim();
+}
+
+// A psql session that stays open after running `sql`, for a lock a test must
+// hold across steps: the caller ends it by killing the process, and the server
+// rolls back whatever the session left open. Its errors go to the run output.
+export function spawnPsql(dbName: string, port: string, sql: string): ChildProcess {
+ const child = spawn("psql", psqlArgs(dbName, port), { stdio: ["pipe", "ignore", "inherit"] });
+ child.stdin?.write(`${sql}\n`);
+ return child;
}
diff --git a/frontend/tests/e2e/plan-detail/plan-detail-lifecycle.spec.ts b/frontend/tests/e2e/plan-detail/plan-detail-lifecycle.spec.ts
index db443db995..5a0e992336 100644
--- a/frontend/tests/e2e/plan-detail/plan-detail-lifecycle.spec.ts
+++ b/frontend/tests/e2e/plan-detail/plan-detail-lifecycle.spec.ts
@@ -58,7 +58,9 @@ import { PlanDetailPage } from "./plan-detail.page";
import {
seedDraftPlan,
seedReviewPlan,
+ setPermissiveGates,
waitForApprovalStatus,
+ waitForRollout,
} from "./plan-helpers";
import {
createDatabaseChangePlanViaUI,
@@ -153,37 +155,6 @@ async function setApproval(allowSelfApproval: boolean): Promise<void> {
);
}
-async function setPermissive(): Promise<void> {
- await env.api.deletePolicy(env.project, "tag").catch(() => {});
- await env.api.updateProjectSettings(env.project, {
- requireIssueApproval: false,
- requirePlanCheckNoError: false,
- enforceSqlReview: false,
- forceIssueLabels: false,
- });
- // Clear any approval rule a prior describe left in WORKSPACE_APPROVAL —
- // otherwise a leftover rule still forces a pending review (blocking the
- // auto-rollout) even with requireIssueApproval=false. (Each describe must
- // reset its arrival state; the page is shared.)
- await env.api.upsertSetting(
- "WORKSPACE_APPROVAL",
- { workspaceApproval: { rules: [] } },
- "value.workspace_approval",
- );
-}
-
-// Wait (via API) for the backend to auto-create the rollout, so the deploy
-// tests navigate to a page that already shows the frontier Run advance rather
-// than racing the async rollout creation.
-async function waitForRollout(planName: string, timeoutMs = 30_000): Promise<void> {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- if ((await env.api.getPlan(planName)).hasRollout) return;
- await new Promise((r) => setTimeout(r, 1000));
- }
- throw new Error(`rollout was not auto-created for ${planName} in ${timeoutMs}ms`);
-}
-
// Attach a single ERROR-level COLUMN_NO_NULL rule so a nullable column trips it.
async function attachErrorConfig(): Promise<void> {
const id = `e2e-hdr-err-${Date.now()}`;
@@ -733,7 +704,7 @@ test.describe("Running the frontier stage from the header reaches the Deployed s
let planId = "";
test.beforeAll(async () => {
- await setPermissive();
+ await setPermissiveGates(env.api, env.project);
const ts = Date.now();
const seeded = await seedReviewPlan(env, page, {
prefix: "E2E Hdr D1",
@@ -741,7 +712,7 @@ test.describe("Running the frontier stage from the header reaches the Deployed s
runChecks: true,
});
planId = seeded.planId;
- await waitForRollout(seeded.planName);
+ await waitForRollout(env.api, seeded.planName);
});
test("header Run·<stage> runs the frontier, then the slot becomes the Deployed stamp", async () => {
@@ -765,7 +736,7 @@ test.describe("A failed task surfaces Rerun in the header slot (D2)", () => {
let planId = "";
test.beforeAll(async () => {
- await setPermissive();
+ await setPermissiveGates(env.api, env.project);
const ts = Date.now();
// A nonexistent target makes the task fail at execution.
const seeded = await seedReviewPlan(env, page, {
@@ -774,7 +745,7 @@ test.describe("A failed task surfaces Rerun in the header slot (D2)", () => {
runChecks: false,
});
planId = seeded.planId;
- await waitForRollout(seeded.planName);
+ await waitForRollout(env.api, seeded.planName);
});
test("after the task fails, the header advance reads Rerun·<stage>", async () => {
@@ -795,7 +766,7 @@ test.describe("A multi-stage rollout advances the header stage by stage to Deplo
let planId = "";
test.beforeAll(async () => {
- await setPermissive();
+ await setPermissiveGates(env.api, env.project);
const ts = Date.now();
const testDb = await env.api.findDatabaseByShortName("hr_test", env.project);
const prodDb = await env.api.findDatabaseByShortName("hr_prod", env.project);
@@ -812,7 +783,7 @@ test.describe("A multi-stage rollout advances the header stage by stage to Deplo
title: `E2E Hdr D3 ${ts}`,
sql: `ALTER TABLE employee ADD COLUMN IF NOT EXISTS e2e_d3_${ts} TEXT;`,
}));
- await waitForRollout(`${env.project}/plans/${planId}`);
+ await waitForRollout(env.api, `${env.project}/plans/${planId}`);
});
test("the header Run advance walks each stage until the plan is Deployed", async () => {
diff --git a/frontend/tests/e2e/plan-detail/plan-detail-logs.spec.ts b/frontend/tests/e2e/plan-detail/plan-detail-logs.spec.ts
new file mode 100644
index 0000000000..66695f3bde
--- /dev/null
+++ b/frontend/tests/e2e/plan-detail/plan-detail-logs.spec.ts
@@ -0,0 +1,215 @@
+// Plan Detail logs — the task run log viewer inside the deploy section's task
+// card, for a run the driver retried inside itself.
+//
+// A lock held by another session makes the statement time out; the driver
+// retries within the same task run (project Execution retry policy), and the
+// run passes once the lock is released. The user contract (BYT-9993): the log
+// shows one gray "Previous attempts" row above the final attempt's green
+// sections — nothing red, nothing pre-opened — and the failed attempt stays in
+// the record behind it, its error entry intact.
+
+import type { ChildProcess } from "child_process";
+import { expect, type Locator, type Page, test } from "@playwright/test";
+import type { BytebaseApiClient } from "../framework/api-client";
+import { loadTestEnv, type TestEnv } from "../framework/env";
+import { execSqlScript, getInstancePgPort, querySql, spawnPsql } from "../framework/psql";
+import { PlanDetailPage } from "./plan-detail.page";
+import { seedReviewPlan, setPermissiveGates, waitForRollout } from "./plan-helpers";
+
+test.setTimeout(240_000);
+
+// Static identifiers, inlined into SQL below (letters, digits, underscores only).
+const SCHEMA = "e2e_retry_log";
+const TABLE = "t";
+// Counts a run's executions; sequence advances survive the rolled-back attempts.
+const ATTEMPT_SEQUENCE = "attempt";
+const LOCK_TIMEOUT_TEXT = "canceling statement due to lock timeout";
+// Well past the failures a test schedules, so the run cannot fail first.
+const RETRY_BUDGET = 10;
+
+let env: TestEnv & { api: BytebaseApiClient };
+let projectId: string;
+let pgPort: string;
+let lockHolder: ChildProcess | undefined;
+let originalSettings: {
+ requireIssueApproval: boolean;
+ requirePlanCheckNoError: boolean;
+ enforceSqlReview: boolean;
+ forceIssueLabels: boolean;
+ executionRetryPolicy: { maximumRetries?: number };
+};
+
+// The sessions holding the fixture table exclusively (`granted` keeps the
+// waiting ALTER TABLE out of it).
+const HELD_LOCK_ROWS = `FROM pg_locks l
+ JOIN pg_class c ON c.oid = l.relation
+ JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = '${SCHEMA}' AND c.relname = '${TABLE}'
+ AND l.mode = 'AccessExclusiveLock' AND l.granted`;
+
+async function waitForLockHeld(): Promise<void> {
+ await expect
+ .poll(() => querySql(env.databaseId, pgPort, `SELECT count(*) ${HELD_LOCK_ROWS}`), {
+ timeout: 15_000,
+ message: `lock on ${SCHEMA}.${TABLE} was not granted`,
+ })
+ .not.toBe("0");
+}
+
+function releaseLock(): void {
+ lockHolder?.kill();
+ lockHolder = undefined;
+}
+
+// The first `failures` executions time out on the held lock; the one after
+// them waits for the release, so the attempt count does not depend on timing.
+// The driver re-runs the whole sheet on each attempt, and it retries only on
+// this exact PostgreSQL error.
+const retriedSql = (failures: number) =>
+ [
+ `SELECT set_config('lock_timeout', CASE WHEN nextval('${SCHEMA}.${ATTEMPT_SEQUENCE}') <= ${failures} THEN '2s' ELSE '0' END, false);`,
+ `ALTER TABLE ${SCHEMA}.${TABLE} ADD COLUMN IF NOT EXISTS c${failures} TEXT;`,
+ ].join("\n");
+
+async function runRetriedPlan(
+ page: Page,
+ failures: number,
+): Promise<{ viewer: Locator; umbrella: Locator }> {
+ const planPage = new PlanDetailPage(page, env.baseURL);
+ execSqlScript(env.databaseId, pgPort, `ALTER SEQUENCE ${SCHEMA}.${ATTEMPT_SEQUENCE} RESTART WITH 1`);
+
+ const seeded = await seedReviewPlan(env, page, {
+ prefix: `E2E Log retry x${failures}`,
+ sql: retriedSql(failures),
+ runChecks: false,
+ });
+ await waitForRollout(env.api, seeded.planName);
+
+ await planPage.goto(projectId, seeded.planId);
+ await planPage.dismissModals();
+ await expect(planPage.headerRunStage).toBeVisible({ timeout: 20_000 });
+
+ // Held only from here, so seeding and plan checks never wait on it. hr_test
+ // carries a baseline changelog from the sample setup; a database without one
+ // would sync its schema before executing and block on this lock instead.
+ lockHolder = spawnPsql(
+ env.databaseId,
+ pgPort,
+ `BEGIN; LOCK TABLE ${SCHEMA}.${TABLE} IN ACCESS EXCLUSIVE MODE;`,
+ );
+ await waitForLockHeld();
+
+ await planPage.headerRunStage.click();
+ await planPage.confirmRunTaskDialog();
+
+ // The umbrella says so once the attempt after the last scheduled failure is
+ // waiting on the lock.
+ const viewer = page.getByTestId("task-run-log-viewer");
+ const umbrella = page.getByTestId("task-run-log-previous-attempts");
+ await expect(umbrella).toHaveText(
+ new RegExp(`Previous attempts\\s*${failures} attempts? · retrying`),
+ { timeout: 60_000 },
+ );
+
+ releaseLock();
+ await expect(planPage.headerStamp("Deployed")).toBeVisible({ timeout: 60_000 });
+ await expect(umbrella).toBeVisible({ timeout: 15_000 });
+ return { viewer, umbrella };
+}
+
+async function expectFoldedHistory(
+ viewer: Locator,
+ umbrella: Locator,
+ count: number,
+): Promise<void> {
+ await expect(umbrella).toHaveAttribute("aria-expanded", "false");
+ await expect(umbrella).toHaveText(new RegExp(`Previous attempts\\s*${count} attempts?`));
+ await expect(umbrella).not.toHaveText(/retrying/);
+ await expect(viewer.getByText(LOCK_TIMEOUT_TEXT)).toHaveCount(0);
+ await expect(viewer.locator("button[aria-expanded='true']")).toHaveCount(0);
+ const finalCommand = viewer.getByRole("button", { name: /Command Execute/ }).last();
+ await expect(finalCommand).toBeVisible();
+ const [umbrellaBox, finalBox] = await Promise.all([
+ umbrella.boundingBox(),
+ finalCommand.boundingBox(),
+ ]);
+ expect(umbrellaBox!.y).toBeLessThan(finalBox!.y);
+}
+
+const failedEntries = (viewer: Locator) =>
+ viewer.getByTestId("task-run-log-row").filter({ hasText: LOCK_TIMEOUT_TEXT });
+
+test.beforeAll(async () => {
+ env = loadTestEnv();
+ projectId = env.project.split("/").pop()!;
+ pgPort = await getInstancePgPort(env);
+
+ const project = await env.api.getProject(env.project);
+ originalSettings = {
+ requireIssueApproval: !!project.requireIssueApproval,
+ requirePlanCheckNoError: !!project.requirePlanCheckNoError,
+ enforceSqlReview: !!project.enforceSqlReview,
+ forceIssueLabels: !!project.forceIssueLabels,
+ executionRetryPolicy:
+ (project.executionRetryPolicy as { maximumRetries?: number } | undefined) ?? {
+ maximumRetries: 0,
+ },
+ };
+
+ await setPermissiveGates(env.api, env.project, {
+ executionRetryPolicy: { maximumRetries: RETRY_BUDGET },
+ });
+
+ execSqlScript(
+ env.databaseId,
+ pgPort,
+ [
+ `DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE;`,
+ `CREATE SCHEMA ${SCHEMA};`,
+ `CREATE TABLE ${SCHEMA}.${TABLE} (id INT PRIMARY KEY);`,
+ `CREATE SEQUENCE ${SCHEMA}.${ATTEMPT_SEQUENCE};`,
+ ].join("\n"),
+ );
+});
+
+// A failed test must not leave the next one waiting on its lock.
+test.afterEach(() => {
+ releaseLock();
+});
+
+test.afterAll(async () => {
+ // Settings first, so a psql failure below cannot leave them changed.
+ await env.api.updateProjectSettings(env.project, originalSettings);
+ execSqlScript(env.databaseId, pgPort, `DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
+});
+
+test("one previous attempt: the umbrella opens straight onto its sections, the failed one already open", async ({ page }) => {
+ const { viewer, umbrella } = await runRetriedPlan(page, 1);
+ await expectFoldedHistory(viewer, umbrella, 1);
+
+ await umbrella.click();
+ await expect(umbrella).toHaveAttribute("aria-expanded", "true");
+ await expect(page.getByTestId("task-run-log-attempt")).toHaveCount(0);
+ // The superseded Command Execute header comes first in the document.
+ await expect(
+ viewer.getByRole("button", { name: /Command Execute/ }).first(),
+ ).toHaveAttribute("aria-expanded", "true");
+ await expect(failedEntries(viewer).first()).toBeVisible();
+});
+
+test("several previous attempts: one nested row per attempt, each opening onto its sections", async ({ page }) => {
+ const { viewer, umbrella } = await runRetriedPlan(page, 2);
+ await expectFoldedHistory(viewer, umbrella, 2);
+
+ await umbrella.click();
+ const attemptRows = page.getByTestId("task-run-log-attempt");
+ await expect(attemptRows).toHaveCount(2);
+ for (const row of await attemptRows.all()) {
+ await expect(row).toHaveAttribute("aria-expanded", "false");
+ }
+ await expect(failedEntries(viewer)).toHaveCount(0);
+
+ await attemptRows.first().click();
+ await expect(attemptRows.first()).toHaveAttribute("aria-expanded", "true");
+ await expect(failedEntries(viewer).first()).toBeVisible();
+});
diff --git a/frontend/tests/e2e/plan-detail/plan-helpers.ts b/frontend/tests/e2e/plan-detail/plan-helpers.ts
index d0d59764e3..1f9aaef67d 100644
--- a/frontend/tests/e2e/plan-detail/plan-helpers.ts
+++ b/frontend/tests/e2e/plan-detail/plan-helpers.ts
@@ -169,3 +169,51 @@ export async function waitForPlanChecksDone(
`plan checks for ${planName} did not reach DONE within ${timeoutMs}ms (last status: ${lastStatus})`,
);
}
+
+// Poll until the plan's rollout exists; auto-creation follows issue creation
+// (or the last approval) asynchronously.
+export async function waitForRollout(
+ api: BytebaseApiClient,
+ planName: string,
+ timeoutMs = 30_000,
+): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ let lastError: unknown;
+ while (Date.now() < deadline) {
+ try {
+ if ((await withTimeout(api.getPlan(planName), 10_000, "getPlan")).hasRollout) {
+ return;
+ }
+ } catch (err) {
+ lastError = err;
+ }
+ await sleep(1000);
+ }
+ throw new Error(
+ `rollout was not auto-created for ${planName} in ${timeoutMs}ms` +
+ (lastError ? ` (last error: ${String(lastError)})` : ""),
+ );
+}
+
+// Gates off so a plan's rollout auto-creates and Run is offered at once.
+// A leftover approval rule from another spec would still force a pending
+// review even with requireIssueApproval=false, so the rules are cleared too.
+export async function setPermissiveGates(
+ api: BytebaseApiClient,
+ project: string,
+ extra: Parameters<BytebaseApiClient["updateProjectSettings"]>[1] = {},
+): Promise<void> {
+ await api.deletePolicy(project, "tag").catch(() => {});
+ await api.updateProjectSettings(project, {
+ requireIssueApproval: false,
+ requirePlanCheckNoError: false,
+ enforceSqlReview: false,
+ forceIssueLabels: false,
+ ...extra,
+ });
+ await api.upsertSetting(
+ "WORKSPACE_APPROVAL",
+ { workspaceApproval: { rules: [] } },
+ "value.workspace_approval",
+ );
+}