Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(open-knowledge): e2e retries must not be structurally weaker than first attempts (#2421)
Nick Gomezc599f438a2e2
diff --git a/packages/app/package.json b/packages/app/package.jsonindex 0dc7eb915..a2066730d 100644--- a/packages/app/package.json+++ b/packages/app/package.json@@ -26,7 +26,7 @@"measure:fuzz": "bash scripts/measure-fuzz.sh","measure:stress": "bash scripts/measure-stress.sh","perf:compare": "bash scripts/perf-compare.sh",- "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-compact-folders.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts",+ "test:e2e": "playwright test tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-compact-folders.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts","test:e2e:install-browsers": "playwright install chromium webkit firefox","test:visual": "playwright test --config playwright.visual.config.ts","test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots",diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.tsindex 5f49c4319..c44a8d4af 100644--- a/packages/app/tests/stress/_helpers/fixtures.ts+++ b/packages/app/tests/stress/_helpers/fixtures.ts@@ -162,7 +162,9 @@ async function checkApiConfig(baseURL: string, timeoutMs = 2_000): Promise<void>* Three-phase readiness probe for the per-worker dev server. Fail-fast* under sustained host load instead of letting each in-test browser* also wait 30s for `Sync timed out`. Worst-case total budget:- * 60 + 2 + 10 = 72s, within the worker fixture's 120s timeout below.+ * 60 + 2 + 10 = 72s; phase 4 (`warmupAppFirstLoad` below) adds up to+ * 60 (goto) + 60 (treeitem) = 120s, for a 192s worst case within the worker+ * fixture's 240s timeout.* (`checkCollabSync` lives in `./server-process.ts`, shared with the* per-test fixtures.)*/@@ -177,6 +179,50 @@ async function waitForServerReady(baseURL: string, port: number): Promise<void>await checkCollabSync(port);}+/**+ * Phase 4 of the readiness probe: one awaited, throwaway real-browser first+ * load, gated on the same app-interactive signal tests wait on (the+ * fixture-seeded treeitem). The server-facing phases above leave Vite's+ * per-process source-transform graph COLD — the per-run warm seed covers only+ * the dependency-optimizer cache — so without this phase the app's first-load+ * compile cost lands on whichever test runs first. On a retry that test is+ * the retried test itself (Playwright discards the failed worker and boots a+ * fresh replacement per retry), making retries structurally WEAKER than+ * first attempts and inverting the retry-as-recovery design+ * (`retries: 2` + `failOnFlakyTests: false` in playwright.config.ts).+ * Contract pinned by `tests/stress/harness-app-warmth.e2e.ts`.+ *+ * Fail-closed on purpose: an app that cannot render inside this budget+ * (2× the suite's 30s cold content budgets) would fail every test on the+ * worker anyway — failing here routes through the fixture's+ * readiness-failure path, which attaches the dev-server log tail instead+ * of surfacing as a per-test generic timeout.+ *+ * The gate waits on the first entry of `REQUIRED_FIXTURE_ENTRY_NAMES`+ * rendering in the sidebar tree. Any future `test.use({ workerServerEnv })`+ * whose env hides or empties the default sidebar at first paint would stall+ * every worker of that group here for the full budget — keep such envs+ * compatible with the seeded fixture entries rendering.+ */+const APP_WARMUP_GOTO_TIMEOUT_MS = 60_000;+const APP_WARMUP_TIMEOUT_MS = 60_000;++async function warmupAppFirstLoad(+ browser: import('@playwright/test').Browser,+ baseURL: string,+): Promise<void> {+ const context = await browser.newContext();+ try {+ const page = await context.newPage();+ await page.goto(`${baseURL}/`, { timeout: APP_WARMUP_GOTO_TIMEOUT_MS });+ await page+ .getByRole('treeitem', { name: REQUIRED_FIXTURE_ENTRY_NAMES[0], exact: true })+ .waitFor({ state: 'visible', timeout: APP_WARMUP_TIMEOUT_MS });+ } finally {+ await context.close();+ }+}+/*** Top-level content entries the worker fixture pre-seeds (see* `seedRequiredFixtureFiles`). Tests that bulk-clear the shared worker@@ -223,7 +269,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({*/workerServerEnv: [{}, { scope: 'worker', option: true }],workerServer: [- async ({ workerServerEnv }, use, workerInfo) => {+ async ({ workerServerEnv, browser }, use, workerInfo) => {const port = await getFreePort();const contentDir = mkdtempSync(join(tmpdir(), `ok-w${workerInfo.workerIndex}-`));// Per-worker Vite optimized-dependency cacheDir — the third per-process@@ -306,6 +352,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({try {await waitForServerReady(baseURL, port);+ await warmupAppFirstLoad(browser, baseURL);} catch (err) {const reason = err instanceof Error ? err.message : String(err);// Same guard shape as the teardown below: cleanup must run even if@@ -337,7 +384,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({rmSync(viteCacheDir, { recursive: true, force: true });}},- { scope: 'worker', timeout: 120_000 },+ // 240s: server boot probe (≤72s) + app first-load warmup (≤60s goto ++ // ≤60s treeitem = 120s) = 192s worst case, with 48s headroom under+ // 4-worker CI contention.+ { scope: 'worker', timeout: 240_000 },],// Override Playwright's built-in `baseURL` so `page.goto('/foo')` resolvesdiff --git a/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts b/packages/app/tests/stress/docs-open-scroll-restore.e2e.tsindex 512aba87b..a54a7c4c1 100644--- a/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts+++ b/packages/app/tests/stress/docs-open-scroll-restore.e2e.ts@@ -54,7 +54,12 @@ import type { Page } from '@playwright/test';import { expect, test, waitForActiveProviderSynced } from './_helpers';async function openFromSidebar(page: Page, filename: string) {- await page.getByRole('treeitem', { name: filename, exact: true }).click({ timeout: 10_000 });+ const row = page.getByRole('treeitem', { name: filename, exact: true });+ // Render-gate before acting (mirrors docs-open.e2e.ts): a bare bounded+ // click conflates "app/tree not rendered yet" with "row missing", and a+ // hand-rolled budget sits below the config-owned expect budget.+ await expect(row).toBeVisible();+ await row.click();}// Same shape as docs-open.e2e.ts's DOC_A / DOC_B so this test exercisesdiff --git a/packages/app/tests/stress/docs-open.e2e.ts b/packages/app/tests/stress/docs-open.e2e.tsindex def8b6501..ff562dd99 100644--- a/packages/app/tests/stress/docs-open.e2e.ts+++ b/packages/app/tests/stress/docs-open.e2e.ts@@ -3,8 +3,9 @@** Covers the core navigation UX properties of the hybrid render tree.*- * Requires: Playwright browsers installed. Dev server started by- * playwright.config.ts webServer on VITE_PORT (or default 5173).+ * Requires: Playwright browsers installed. Server provided per-worker by+ * the `workerServer` fixture in `_helpers/fixtures.ts`; the `baseURL`+ * fixture resolves `page.goto('/…')` against it.*/import { DOCUMENT_OPEN_BYTE_LIMIT } from '@inkeep/open-knowledge-core';@@ -12,7 +13,13 @@ import type { Page } from '@playwright/test';import { expect, test, waitForActiveProviderSynced } from './_helpers';async function openFromSidebar(page: Page, filename: string) {- await page.getByRole('treeitem', { name: filename, exact: true }).click({ timeout: 10_000 });+ const row = page.getByRole('treeitem', { name: filename, exact: true });+ // Render-gate before acting: a bare bounded click conflates "app/tree not+ // rendered yet" with "row missing", and its hand-rolled budget sat below+ // the config-owned expect budget the suite calibrates for CI contention.+ // Gate visibility web-first (inherits the config budget), then click.+ await expect(row).toBeVisible();+ await row.click();}function sidebarItem(page: Page, filename: string) {diff --git a/packages/app/tests/stress/harness-app-warmth.e2e.ts b/packages/app/tests/stress/harness-app-warmth.e2e.tsnew file mode 100644index 000000000..de3131252--- /dev/null+++ b/packages/app/tests/stress/harness-app-warmth.e2e.ts@@ -0,0 +1,94 @@+/**+ * Worker-fixture app-warmth contract.+ *+ * The suite's retry design assumes retries run at >= first-attempt strength+ * ("retries absorb infra flake" — playwright.config.ts). Playwright boots a+ * fresh replacement worker per retry, so every retry's `goto` is that+ * worker's server's FIRST browser load. If the fixture's readiness guarantee+ * ends at the server (HTTP + /api/config + /collab) without covering the+ * app's first page load, that first load pays the full cold source-transform+ * graph — the warm-cache seed covers only Vite's dependency-optimizer cache —+ * and retries become structurally WEAKER than first attempts, inverting+ * retry-as-recovery: a test that flakes on attempt 1 under contention then+ * faces colder retries and hard-fails all attempts.+ *+ * Measurement shape (hardware-relative — absolute cold-load cost only+ * manifests on CI-class hardware): three fresh browser contexts against this+ * file's dedicated worker server. All pay a cold BROWSER cache, so the+ * first-vs-warm differential is dominated by server-side transform warmth;+ * common-mode runtime cost (CRDT /collab sync, tree render) inflates every+ * sample alike, and its per-sample VARIANCE is why the warm baseline is the+ * MIN of two samples (a single GC/scheduler spike in one warm sample must+ * not manufacture a false cliff). Two guards keep this from flaking:+ * - Absolute floor: a first load under 2.5s cannot blow any bounded+ * first-interaction budget (>=10s in this suite) regardless of ratio, so+ * sub-floor first loads pass outright — this deliberately ignores+ * fast-hardware "cliffs" that are harmless in absolute terms.+ * - Generous ratio: a genuinely cold first load measures >=10x a warm one+ * (14.8x on idle 18-core hardware); 4x still catches it with wide margin+ * while tolerating tail noise between warm samples.+ */++import { expect, REQUIRED_FIXTURE_ENTRY_NAMES, test } from './_helpers';++// Dedicated worker for this file: option-value grouping gives it a private,+// freshly-booted server whose only prior browser load is the fixture's own+// warmup phase — no sibling test can add loads, so the measured first-vs-warm+// differential reflects exactly the state the fixture hands every test (and+// re-reddens if the warmup phase is removed: the first measure then becomes+// the server's genuinely-cold first browser load). The env value is a pure+// grouping token — nothing server-side reads it.+test.use({ workerServerEnv: { OK_TEST_WARMTH_CONTRACT: '1' } });++// Generous ceiling: the test MEASURES load time; it must not time out while+// measuring the cold case it exists to catch.+const APP_READY_TIMEOUT_MS = 90_000;+// Ratio denominators below this floor say more about scheduler jitter than+// server warmth; clamp so an anomalously fast warm load can't inflate the+// ratio.+const WARM_FLOOR_MS = 250;+// First loads faster than this cannot produce the failure class this test+// guards (bounded first-interaction budgets are >=10s suite-wide).+const COLD_FLOOR_MS = 2_500;+const MAX_COLD_TO_WARM_RATIO = 4;++test('fixture hands the FIRST browser load a warm app (retries must not be structurally weaker than first attempts)', async ({+ browser,+ workerServer,+}) => {+ async function measureAppReadyMs(): Promise<number> {+ const context = await browser.newContext();+ try {+ const page = await context.newPage();+ const start = Date.now();+ // Explicit goto ceiling matching the measurement budget: a regressed+ // cold load must surface through the ratio assertion's diagnostic+ // message, not a shorter default navigation timeout.+ await page.goto(`${workerServer.baseURL}/`, { timeout: APP_READY_TIMEOUT_MS });+ // App-interactive signal: the sidebar tree has rendered its+ // fixture-seeded row (same entry the fixture warmup gates on).+ await expect(+ page.getByRole('treeitem', { name: REQUIRED_FIXTURE_ENTRY_NAMES[0], exact: true }),+ ).toBeVisible({+ timeout: APP_READY_TIMEOUT_MS,+ });+ return Date.now() - start;+ } finally {+ await context.close();+ }+ }++ const firstLoadMs = await measureAppReadyMs();+ const warmSampleA = await measureAppReadyMs();+ const warmSampleB = await measureAppReadyMs();+ const warmBaselineMs = Math.max(Math.min(warmSampleA, warmSampleB), WARM_FLOOR_MS);++ const ratio = firstLoadMs / warmBaselineMs;+ expect(+ firstLoadMs < COLD_FLOOR_MS || ratio < MAX_COLD_TO_WARM_RATIO,+ `first browser load ${firstLoadMs}ms vs warm baseline ${warmBaselineMs}ms ` ++ `(min of ${warmSampleA}/${warmSampleB}ms; ratio ${ratio.toFixed(1)}x): the fixture ` ++ `handed the first load a structurally colder app than a warm one — every retry IS ` ++ `a first load on a fresh worker, so under CI contention this hard-fails all attempts`,+ ).toBe(true);+});diff --git a/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts b/packages/app/tests/stress/qa-sidebar-responsive.e2e.tsindex 582e8a783..358390753 100644--- a/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts+++ b/packages/app/tests/stress/qa-sidebar-responsive.e2e.ts@@ -396,7 +396,9 @@ test.describe('non-embedded UA', () => {await page.waitForLoadState('domcontentloaded');await expect.poll(() => page.locator('text=With AI').count(), {- timeout: 10_000,+ // No inline timeout: inherit the config-owned expect budget — this is+ // the first UI wait after goto, exactly the site class a hand-rolled+ // sub-config budget exposes on cold/contended workers.intervals: [200, 500, 1000],message: 'With AI section visible in non-embedded empty state',})