Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): e2e retries must not be structurally weaker than first attempts (#2421)

Nick Gomezc599f438a2e2
diff --git a/packages/app/package.json b/packages/app/package.json
index 0dc7eb915..a2066730d 100644
--- a/packages/app/package.json
+++ b/packages/app/package.json
@@ -26,7 +26,7 @@
"measure:fuzz": "bash scripts/measure-fuzz.sh",
"measure:stress": "bash scripts/measure-stress.sh",
"perf:compare": "bash scripts/perf-compare.sh",
- "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-compact-folders.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts",
+ "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-compact-folders.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts",
"test:e2e:install-browsers": "playwright install chromium webkit firefox",
"test:visual": "playwright test --config playwright.visual.config.ts",
"test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots",
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index 5f49c4319..c44a8d4af 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -162,7 +162,9 @@ async function checkApiConfig(baseURL: string, timeoutMs = 2_000): Promise<void>
* Three-phase readiness probe for the per-worker dev server. Fail-fast
* under sustained host load instead of letting each in-test browser
* also wait 30s for `Sync timed out`. Worst-case total budget:
- * 60 + 2 + 10 = 72s, within the worker fixture's 120s timeout below.
+ * 60 + 2 + 10 = 72s; phase 4 (`warmupAppFirstLoad` below) adds up to
+ * 60 (goto) + 60 (treeitem) = 120s, for a 192s worst case within the worker
+ * fixture's 240s timeout.
* (`checkCollabSync` lives in `./server-process.ts`, shared with the
* per-test fixtures.)
*/
@@ -177,6 +179,50 @@ async function waitForServerReady(baseURL: string, port: number): Promise<void>
await checkCollabSync(port);
}
+/**
+ * Phase 4 of the readiness probe: one awaited, throwaway real-browser first
+ * load, gated on the same app-interactive signal tests wait on (the
+ * fixture-seeded treeitem). The server-facing phases above leave Vite's
+ * per-process source-transform graph COLD — the per-run warm seed covers only
+ * the dependency-optimizer cache — so without this phase the app's first-load
+ * compile cost lands on whichever test runs first. On a retry that test is
+ * the retried test itself (Playwright discards the failed worker and boots a
+ * fresh replacement per retry), making retries structurally WEAKER than
+ * first attempts and inverting the retry-as-recovery design
+ * (`retries: 2` + `failOnFlakyTests: false` in playwright.config.ts).
+ * Contract pinned by `tests/stress/harness-app-warmth.e2e.ts`.
+ *
+ * Fail-closed on purpose: an app that cannot render inside this budget
+ * (2× the suite's 30s cold content budgets) would fail every test on the
+ * worker anyway — failing here routes through the fixture's
+ * readiness-failure path, which attaches the dev-server log tail instead
+ * of surfacing as a per-test generic timeout.
+ *
+ * The gate waits on the first entry of `REQUIRED_FIXTURE_ENTRY_NAMES`
+ * rendering in the sidebar tree. Any future `test.use({ workerServerEnv })`
+ * whose env hides or empties the default sidebar at first paint would stall
+ * every worker of that group here for the full budget — keep such envs
+ * compatible with the seeded fixture entries rendering.
+ */
+const APP_WARMUP_GOTO_TIMEOUT_MS = 60_000;
+const APP_WARMUP_TIMEOUT_MS = 60_000;
+
+async function warmupAppFirstLoad(
+ browser: import('@playwright/test').Browser,
+ baseURL: string,
+): Promise<void> {
+ const context = await browser.newContext();
+ try {
+ const page = await context.newPage();
+ await page.goto(`${baseURL}/`, { timeout: APP_WARMUP_GOTO_TIMEOUT_MS });
+ await page
+ .getByRole('treeitem', { name: REQUIRED_FIXTURE_ENTRY_NAMES[0], exact: true })
+ .waitFor({ state: 'visible', timeout: APP_WARMUP_TIMEOUT_MS });
+ } finally {
+ await context.close();
+ }
+}
+
/**
* Top-level content entries the worker fixture pre-seeds (see
* `seedRequiredFixtureFiles`). Tests that bulk-clear the shared worker
@@ -223,7 +269,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
*/
workerServerEnv: [{}, { scope: 'worker', option: true }],
workerServer: [
- async ({ workerServerEnv }, use, workerInfo) => {
+ async ({ workerServerEnv, browser }, use, workerInfo) => {
const port = await getFreePort();
const contentDir = mkdtempSync(join(tmpdir(), `ok-w${workerInfo.workerIndex}-`));
// Per-worker Vite optimized-dependency cacheDir — the third per-process
@@ -306,6 +352,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
try {
await waitForServerReady(baseURL, port);
+ await warmupAppFirstLoad(browser, baseURL);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
// Same guard shape as the teardown below: cleanup must run even if
@@ -337,7 +384,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
rmSync(viteCacheDir, { recursive: true, force: true });
}
},
- { scope: 'worker', timeout: 120_000 },
+ // 240s: server boot probe (≤72s) + app first-load warmup (≤60s goto +
+ // ≤60s treeitem = 120s) = 192s worst case, with 48s headroom under
+ // 4-worker CI contention.
+ { scope: 'worker', timeout: 240_000 },
],
// Override Playwright's built-in `baseURL` so `page.goto('/foo')` resolves
diff --git a/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts b/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts
index 512aba87b..a54a7c4c1 100644
--- a/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts
+++ b/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts
@@ -54,7 +54,12 @@ import type { Page } from '@playwright/test';
import { expect, test, waitForActiveProviderSynced } from './_helpers';
async function openFromSidebar(page: Page, filename: string) {
- await page.getByRole('treeitem', { name: filename, exact: true }).click({ timeout: 10_000 });
+ const row = page.getByRole('treeitem', { name: filename, exact: true });
+ // Render-gate before acting (mirrors docs-open.e2e.ts): a bare bounded
+ // click conflates "app/tree not rendered yet" with "row missing", and a
+ // hand-rolled budget sits below the config-owned expect budget.
+ await expect(row).toBeVisible();
+ await row.click();
}
// Same shape as docs-open.e2e.ts's DOC_A / DOC_B so this test exercises
diff --git a/packages/app/tests/stress/docs-open.e2e.ts b/packages/app/tests/stress/docs-open.e2e.ts
index def8b6501..ff562dd99 100644
--- a/packages/app/tests/stress/docs-open.e2e.ts
+++ b/packages/app/tests/stress/docs-open.e2e.ts
@@ -3,8 +3,9 @@
*
* Covers the core navigation UX properties of the hybrid render tree.
*
- * Requires: Playwright browsers installed. Dev server started by
- * playwright.config.ts webServer on VITE_PORT (or default 5173).
+ * Requires: Playwright browsers installed. Server provided per-worker by
+ * the `workerServer` fixture in `_helpers/fixtures.ts`; the `baseURL`
+ * fixture resolves `page.goto('/…')` against it.
*/
import { DOCUMENT_OPEN_BYTE_LIMIT } from '@inkeep/open-knowledge-core';
@@ -12,7 +13,13 @@ import type { Page } from '@playwright/test';
import { expect, test, waitForActiveProviderSynced } from './_helpers';
async function openFromSidebar(page: Page, filename: string) {
- await page.getByRole('treeitem', { name: filename, exact: true }).click({ timeout: 10_000 });
+ const row = page.getByRole('treeitem', { name: filename, exact: true });
+ // Render-gate before acting: a bare bounded click conflates "app/tree not
+ // rendered yet" with "row missing", and its hand-rolled budget sat below
+ // the config-owned expect budget the suite calibrates for CI contention.
+ // Gate visibility web-first (inherits the config budget), then click.
+ await expect(row).toBeVisible();
+ await row.click();
}
function sidebarItem(page: Page, filename: string) {
diff --git a/packages/app/tests/stress/harness-app-warmth.e2e.ts b/packages/app/tests/stress/harness-app-warmth.e2e.ts
new file mode 100644
index 000000000..de3131252
--- /dev/null
+++ b/packages/app/tests/stress/harness-app-warmth.e2e.ts
@@ -0,0 +1,94 @@
+/**
+ * Worker-fixture app-warmth contract.
+ *
+ * The suite's retry design assumes retries run at >= first-attempt strength
+ * ("retries absorb infra flake" — playwright.config.ts). Playwright boots a
+ * fresh replacement worker per retry, so every retry's `goto` is that
+ * worker's server's FIRST browser load. If the fixture's readiness guarantee
+ * ends at the server (HTTP + /api/config + /collab) without covering the
+ * app's first page load, that first load pays the full cold source-transform
+ * graph — the warm-cache seed covers only Vite's dependency-optimizer cache —
+ * and retries become structurally WEAKER than first attempts, inverting
+ * retry-as-recovery: a test that flakes on attempt 1 under contention then
+ * faces colder retries and hard-fails all attempts.
+ *
+ * Measurement shape (hardware-relative — absolute cold-load cost only
+ * manifests on CI-class hardware): three fresh browser contexts against this
+ * file's dedicated worker server. All pay a cold BROWSER cache, so the
+ * first-vs-warm differential is dominated by server-side transform warmth;
+ * common-mode runtime cost (CRDT /collab sync, tree render) inflates every
+ * sample alike, and its per-sample VARIANCE is why the warm baseline is the
+ * MIN of two samples (a single GC/scheduler spike in one warm sample must
+ * not manufacture a false cliff). Two guards keep this from flaking:
+ * - Absolute floor: a first load under 2.5s cannot blow any bounded
+ * first-interaction budget (>=10s in this suite) regardless of ratio, so
+ * sub-floor first loads pass outright — this deliberately ignores
+ * fast-hardware "cliffs" that are harmless in absolute terms.
+ * - Generous ratio: a genuinely cold first load measures >=10x a warm one
+ * (14.8x on idle 18-core hardware); 4x still catches it with wide margin
+ * while tolerating tail noise between warm samples.
+ */
+
+import { expect, REQUIRED_FIXTURE_ENTRY_NAMES, test } from './_helpers';
+
+// Dedicated worker for this file: option-value grouping gives it a private,
+// freshly-booted server whose only prior browser load is the fixture's own
+// warmup phase — no sibling test can add loads, so the measured first-vs-warm
+// differential reflects exactly the state the fixture hands every test (and
+// re-reddens if the warmup phase is removed: the first measure then becomes
+// the server's genuinely-cold first browser load). The env value is a pure
+// grouping token — nothing server-side reads it.
+test.use({ workerServerEnv: { OK_TEST_WARMTH_CONTRACT: '1' } });
+
+// Generous ceiling: the test MEASURES load time; it must not time out while
+// measuring the cold case it exists to catch.
+const APP_READY_TIMEOUT_MS = 90_000;
+// Ratio denominators below this floor say more about scheduler jitter than
+// server warmth; clamp so an anomalously fast warm load can't inflate the
+// ratio.
+const WARM_FLOOR_MS = 250;
+// First loads faster than this cannot produce the failure class this test
+// guards (bounded first-interaction budgets are >=10s suite-wide).
+const COLD_FLOOR_MS = 2_500;
+const MAX_COLD_TO_WARM_RATIO = 4;
+
+test('fixture hands the FIRST browser load a warm app (retries must not be structurally weaker than first attempts)', async ({
+ browser,
+ workerServer,
+}) => {
+ async function measureAppReadyMs(): Promise<number> {
+ const context = await browser.newContext();
+ try {
+ const page = await context.newPage();
+ const start = Date.now();
+ // Explicit goto ceiling matching the measurement budget: a regressed
+ // cold load must surface through the ratio assertion's diagnostic
+ // message, not a shorter default navigation timeout.
+ await page.goto(`${workerServer.baseURL}/`, { timeout: APP_READY_TIMEOUT_MS });
+ // App-interactive signal: the sidebar tree has rendered its
+ // fixture-seeded row (same entry the fixture warmup gates on).
+ await expect(
+ page.getByRole('treeitem', { name: REQUIRED_FIXTURE_ENTRY_NAMES[0], exact: true }),
+ ).toBeVisible({
+ timeout: APP_READY_TIMEOUT_MS,
+ });
+ return Date.now() - start;
+ } finally {
+ await context.close();
+ }
+ }
+
+ const firstLoadMs = await measureAppReadyMs();
+ const warmSampleA = await measureAppReadyMs();
+ const warmSampleB = await measureAppReadyMs();
+ const warmBaselineMs = Math.max(Math.min(warmSampleA, warmSampleB), WARM_FLOOR_MS);
+
+ const ratio = firstLoadMs / warmBaselineMs;
+ expect(
+ firstLoadMs < COLD_FLOOR_MS || ratio < MAX_COLD_TO_WARM_RATIO,
+ `first browser load ${firstLoadMs}ms vs warm baseline ${warmBaselineMs}ms ` +
+ `(min of ${warmSampleA}/${warmSampleB}ms; ratio ${ratio.toFixed(1)}x): the fixture ` +
+ `handed the first load a structurally colder app than a warm one — every retry IS ` +
+ `a first load on a fresh worker, so under CI contention this hard-fails all attempts`,
+ ).toBe(true);
+});
diff --git a/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts b/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts
index 582e8a783..358390753 100644
--- a/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts
+++ b/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts
@@ -396,7 +396,9 @@ test.describe('non-embedded UA', () => {
await page.waitForLoadState('domcontentloaded');
await expect
.poll(() => page.locator('text=With AI').count(), {
- timeout: 10_000,
+ // No inline timeout: inherit the config-owned expect budget — this is
+ // the first UI wait after goto, exactly the site class a hand-rolled
+ // sub-config budget exposes on cold/contended workers.
intervals: [200, 500, 1000],
message: 'With AI section visible in non-embedded empty state',
})