Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
perf(app): debounce PageListContext refetch storms (#2728)
Andrew Mikofalvy9c6f209ca224
diff --git a/.changeset/pagelist-push-refetch-coalescing.md b/.changeset/pagelist-push-refetch-coalescing.mdnew file mode 100644index 000000000..e6992b1c4--- /dev/null+++ b/.changeset/pagelist-push-refetch-coalescing.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++The sidebar's page directory no longer storms the server during bulk agent writes. File-change pushes now coalesce into a single trailing refetch (~300 ms window) instead of refiring the full page-list and document-list walk on every push, so an agent writing many files per second no longer triggers ~10 full corpus refetches and index rebuilds per second. The initial load and manual refreshes stay immediate.diff --git a/packages/app/src/components/PageListContext.push-coalescing.dom.test.tsx b/packages/app/src/components/PageListContext.push-coalescing.dom.test.tsxnew file mode 100644index 000000000..933f529f1--- /dev/null+++ b/packages/app/src/components/PageListContext.push-coalescing.dom.test.tsx@@ -0,0 +1,260 @@+/**+ * RTL mount test for the page-list push-coalescing contract.+ *+ * Pins the refetch cadence under a CC1 `files` push storm: a bulk agent+ * write can drive `files` pushes at up to ~10x/sec (one per the server's+ * 100 ms per-channel CC1 debounce window). Each push used to refire the full+ * `/api/pages` + `/api/documents` fetch pair immediately; the provider now+ * arms one trailing coalescing timer per quiet window instead, so a burst of+ * pushes produces exactly one refetch. The window is NOT reset by pushes+ * arriving while it is armed — a sustained storm still refreshes at the+ * window cadence rather than starving until the storm ends. The initial+ * mount load is never delayed by the window.+ *+ * Uses the REAL documents-events window bus (unlike the sibling+ * loading-stability test, which mocks it to a no-op) because the timer+ * sits between the bus subscription and the fetch.+ *+ * Invocation: `bun run test:dom` from `packages/app/`.+ */+import { afterEach, beforeEach, describe, expect, jest, mock, test } from 'bun:test';+import { act, cleanup, render, screen } from '@testing-library/react';++const setPageListCacheMock = mock((_payload: unknown) => {});++mock.module('@/editor/page-list-cache', () => ({+ buildPageIconsIndex: () => new Map<string, string>(),+ buildPagesBySlugIndex: (pages: ReadonlySet<string>, toSlug: (docName: string) => string) => {+ const index = new Map<string, string>();+ for (const docName of pages) {+ const slug = toSlug(docName);+ if (!index.has(slug)) index.set(slug, docName);+ }+ return index;+ },+ buildPagesByBasenameIndex: (pages: ReadonlySet<string>, toSlug: (docName: string) => string) => {+ const index = new Map<string, string>();+ for (const docName of [...pages].sort()) {+ const basename = docName.split('/').pop() ?? docName;+ const slug = toSlug(basename);+ if (!index.has(slug)) index.set(slug, docName);+ }+ return index;+ },+ setPageListCache: setPageListCacheMock,+}));++type ResponseResolver = (res: Response) => void;++let pageResolvers: ResponseResolver[] = [];+let docResolvers: ResponseResolver[] = [];+let originalFetch: typeof globalThis.fetch;++function jsonRes(body: unknown) {+ return { ok: true, json: async () => body } as Response;+}++function pagesBody(docNames: string[]) {+ return {+ pages: docNames.map((docName) => ({+ docName,+ title: docName,+ size: 1,+ modified: '2026-01-01T00:00:00.000Z',+ })),+ };+}++/** Resolve the most recent in-flight `/api/pages` + `/api/documents` pair. */+async function settleRound(docNames: string[]) {+ const pr = pageResolvers.shift();+ const dr = docResolvers.shift();+ if (!pr || !dr) throw new Error('settleRound: no in-flight fetch pair to resolve');+ await act(async () => {+ pr(jsonRes(pagesBody(docNames)));+ dr(jsonRes({ documents: [] }));+ await Promise.resolve();+ });+}++beforeEach(() => {+ pageResolvers = [];+ docResolvers = [];+ setPageListCacheMock.mockClear();+ __resetDocumentListInflightForTests();+ jest.useFakeTimers();+ originalFetch = globalThis.fetch;+ globalThis.fetch = ((input: RequestInfo | URL) => {+ const url = String(input);+ if (url.includes('/api/pages')) {+ return new Promise<Response>((resolve) => {+ pageResolvers.push(resolve);+ });+ }+ if (url.includes('/api/documents')) {+ return new Promise<Response>((resolve) => {+ docResolvers.push(resolve);+ });+ }+ return Promise.reject(new Error(`unexpected fetch: ${url}`));+ }) as typeof globalThis.fetch;+});++afterEach(() => {+ cleanup();+ jest.useRealTimers();+ globalThis.fetch = originalFetch;+});++function Probe() {+ const { loading, pages } = usePageList();+ if (loading) return <div data-testid="page-list-skeleton" />;+ return <div data-testid="page-list-content">{[...pages].sort().join(',')}</div>;+}++// Import AFTER the mocks above register so transitive dependencies bind to+// the stubs rather than the real modules.+const { PageListProvider, PUSH_REFRESH_COALESCE_MS, usePageList } = await import(+ './PageListContext'+);+const { emitDocumentsChanged } = await import('@/lib/documents-events');+const { __resetDocumentListInflightForTests } = await import('@/lib/documents-fetch');++function emitFilesPush() {+ act(() => {+ emitDocumentsChanged(['files']);+ });+}++describe('PageListContext push coalescing', () => {+ test('initial mount load fires immediately, not through the coalescing window', async () => {+ render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );++ // No timer advance: the cold-load fetch pair is already in flight.+ expect(pageResolvers.length).toBe(1);+ expect(docResolvers.length).toBe(1);++ await settleRound(['A']);+ expect(screen.getByTestId('page-list-content').textContent).toBe('A');+ });++ test('a burst of files pushes coalesces into one trailing refetch', async () => {+ render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );+ await settleRound(['A']);++ for (let i = 0; i < 5; i++) emitFilesPush();++ // Nothing fires inside the window.+ expect(pageResolvers.length).toBe(0);+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS - 1);+ });+ expect(pageResolvers.length).toBe(0);++ // The trailing edge flushes exactly one fetch pair for the whole burst.+ act(() => {+ jest.advanceTimersByTime(1);+ });+ expect(pageResolvers.length).toBe(1);+ expect(docResolvers.length).toBe(1);++ await settleRound(['A', 'B']);+ expect(screen.getByTestId('page-list-content').textContent).toBe('A,B');++ // A push after the flush arms a fresh window and fetches again.+ emitFilesPush();+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS);+ });+ expect(pageResolvers.length).toBe(1);+ });++ test('pushes arriving mid-window do not extend it (a storm cannot starve the refresh)', async () => {+ render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );+ await settleRound(['A']);++ emitFilesPush();+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS / 2);+ });+ emitFilesPush();+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS / 2);+ });++ // The flush fired at the ORIGINAL deadline even though the second push+ // landed only half a window ago.+ expect(pageResolvers.length).toBe(1);+ });++ test('non-files channels do not arm the window', async () => {+ render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );+ await settleRound(['A']);++ act(() => {+ emitDocumentsChanged(['backlinks', 'graph']);+ });+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS * 2);+ });+ expect(pageResolvers.length).toBe(0);+ });++ test('unmounting while the window is armed cancels the pending refetch', async () => {+ const { unmount } = render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );+ await settleRound(['A']);++ emitFilesPush();+ // Timer armed, nothing in flight yet.+ expect(pageResolvers.length).toBe(0);++ unmount();+ act(() => {+ jest.advanceTimersByTime(PUSH_REFRESH_COALESCE_MS * 2);+ });++ // Cleanup cleared the armed timer and disposed the scheduler, so the+ // torn-down provider never fires a refetch.+ expect(pageResolvers.length).toBe(0);+ });++ test('a focus refetch fires immediately even while a push window is armed', async () => {+ render(+ <PageListProvider>+ <Probe />+ </PageListProvider>,+ );+ await settleRound(['A']);++ emitFilesPush();+ // Push window armed but not yet flushed.+ expect(pageResolvers.length).toBe(0);++ act(() => {+ window.dispatchEvent(new Event('focus'));+ });+ // Focus refetches immediately through the scheduler — it must NOT be+ // deferred by the 300 ms push window.+ expect(pageResolvers.length).toBe(1);+ });+});diff --git a/packages/app/src/components/PageListContext.tsx b/packages/app/src/components/PageListContext.tsxindex 13dc09a1f..599858ad5 100644--- a/packages/app/src/components/PageListContext.tsx+++ b/packages/app/src/components/PageListContext.tsx@@ -10,8 +10,25 @@ import { subscribeToDocumentsChanged } from '@/lib/documents-events';import { fetchDocumentListShared } from '@/lib/documents-fetch';import { parseApiError } from '@/lib/parse-api-error';import { pageListReady } from '@/lib/perf/startup-marks';+import { createRefreshScheduler } from '@/lib/refresh-scheduler';import { deriveKnownFolderPaths } from './navigation-targets';+/**+ * Trailing coalescing window for CC1 `files`-push refetches. A bulk agent+ * write can drive `files` pushes at up to ~10×/sec (one per the server's+ * 100 ms per-channel CC1 debounce window); each would otherwise refire the+ * full `/api/pages` + `/api/documents` walk and rebuild every derived index.+ * The refetch already flows through `createRefreshScheduler` (at most one+ * fetch pair in flight), but the scheduler fires its first request eagerly —+ * so under a storm the scheduler alone still refetches as fast as each fetch+ * pair completes. This window sits in front of it: the first push in a quiet+ * period arms one timer; pushes landing while it is armed are absorbed by the+ * already-scheduled flush (no reset, so a sustained storm refreshes at the+ * window cadence instead of starving until the storm ends). Exported for+ * tests only.+ */+export const PUSH_REFRESH_COALESCE_MS = 300;+export interface PageMeta {size: number;modified: string;@@ -213,7 +230,7 @@ export function PageListProvider({ children }: { children: ReactNode }) {// page list is unchanged. The initial `useState(true)` covers the cold// load; background refetches keep serving the last-good `pages` and// reconcile in place when the fresh response lands.- void Promise.all([+ return Promise.all([loadPages(),loadDocumentListSummary().catch((err) => {logLoadAssetsError(err);@@ -278,20 +295,38 @@ export function PageListProvider({ children }: { children: ReactNode }) {// biome-ignore lint/correctness/useExhaustiveDependencies: run only on mountuseEffect(() => {- void refetch();+ // In-flight coalescing via `createRefreshScheduler`: at most one fetch+ // pair runs at a time, a request arriving mid-run collapses into a single+ // trailing re-run. Mount and focus/visibility call `request()` directly;+ // `files` pushes go through the arm-once timer below first (unlike+ // FileTree, which forwards every push straight to the scheduler). No abort+ // hook — the requestId latest-wins guard in `refetch` already discards+ // superseded responses, and `/api/documents` rides a shared single-flight+ // other consumers depend on.+ const scheduler = createRefreshScheduler(refetch);+ // Initial mount load fires immediately — only push-triggered refreshes+ // go through the coalescing timer below.+ scheduler.request();+ let pushFlushTimer: ReturnType<typeof setTimeout> | null = null;const handleResume = () => {if (document.visibilityState === 'visible') {- refetch();+ scheduler.request();}};window.addEventListener('focus', handleResume);window.addEventListener('visibilitychange', handleResume);const unsubscribe = subscribeToDocumentsChanged((channels) => {- if (channels.includes('files')) {- refetch();- }+ if (!channels.includes('files')) return;+ // Pushes landing while a flush is armed are covered by that flush.+ if (pushFlushTimer !== null) return;+ pushFlushTimer = setTimeout(() => {+ pushFlushTimer = null;+ scheduler.request();+ }, PUSH_REFRESH_COALESCE_MS);});return () => {+ if (pushFlushTimer !== null) clearTimeout(pushFlushTimer);+ scheduler.dispose();window.removeEventListener('focus', handleResume);window.removeEventListener('visibilitychange', handleResume);unsubscribe();