Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): bound the fresh-folder consent probe walk (PRD-8438) (#4370)
serafin-garcia9757d525490d
diff --git a/.changeset/bound-consent-probe-symlink-walk.md b/.changeset/bound-consent-probe-symlink-walk.mdnew file mode 100644index 000000000..85586744e--- /dev/null+++ b/.changeset/bound-consent-probe-symlink-walk.md@@ -0,0 +1,12 @@+---+"@inkeep/open-knowledge-desktop": patch+"@inkeep/open-knowledge": patch+---++Opening a folder whose content hides behind a directory symlink or junction no longer hands the setup preview an unbounded walk.++Before OpenKnowledge previews a folder you have picked, it runs a size check to decide whether the folder is small enough to list in full. That check stopped at directory symlinks and Windows junctions, while the preview it guards follows them. A folder holding a handful of real entries plus a link to a large tree passed the check, and the preview then walked the whole tree. On large trees the setup screen froze for seconds, and on trees whose links fan out into themselves the app could run out of memory and quit.++The size check now follows the same links the preview does. A folder that turns out to be too large is reported as truncated, and the preview is skipped, so the setup screen stays responsive.++One case reads worse than it is. The size check skips only links named `node_modules` and `.git`, while the preview also skips links named for about thirty build and cache directories such as `dist`, `build`, `.venv`, `.cache`, `coverage` and `Library`, plus anything your `.gitignore` or `.okignore` lists. A folder whose bulk sits behind a link with one of those names is counted by the size check even though the preview would have skipped that link, so it can read as "Found ≥ 50,000 markdown files" with no sample list. This does not stop you from opening the folder. Folders holding those directories for real already counted this way.diff --git a/knip.config.ts b/knip.config.tsindex 96d43e981..c58ed3f2e 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -139,9 +139,7 @@ export default {'packages/server': {entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],project: 'src/**',- ignoreDependencies: [- '@types/shell-quote',- ],+ ignoreDependencies: ['@types/shell-quote'],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],diff --git a/packages/app/src/components/ConsentDialogBody.dom.test.tsx b/packages/app/src/components/ConsentDialogBody.dom.test.tsxindex 659f55974..0d2380f7b 100644--- a/packages/app/src/components/ConsentDialogBody.dom.test.tsx+++ b/packages/app/src/components/ConsentDialogBody.dom.test.tsx@@ -1,4 +1,5 @@import { EDITOR_LABELS } from '@inkeep/open-knowledge-core';+import { setupI18n } from '@lingui/core';import * as actualLinguiMacro from '@lingui/react/macro';import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';import userEvent from '@testing-library/user-event';@@ -12,12 +13,14 @@ import type {} from '@/lib/desktop-bridge-types';import { renderLinguiTemplate } from '@/test-utils/lingui-mock';+const testI18n = setupI18n({ locale: 'en', messages: { en: {}, fr: {} } });+vi.doMock('@lingui/core/macro', () => ({ ...actualLinguiMacro, msg: renderLinguiTemplate }));vi.doMock('@lingui/react/macro', () => ({...actualLinguiMacro,Trans: ({ children }: { children: ReactNode }) => <>{children}</>,- useLingui: () => ({ t: renderLinguiTemplate }),+ useLingui: () => ({ t: renderLinguiTemplate, i18n: testI18n }),}));const { default: ConsentDialogBody } = await import('./ConsentDialogBody');@@ -132,6 +135,7 @@ async function expandAdvanced() {describe('ConsentDialogBody runtime form behavior', () => {afterEach(() => {cleanup();+ testI18n.activate('en');setBridge(undefined);vi.restoreAllMocks();});@@ -140,6 +144,59 @@ describe('ConsentDialogBody runtime form behavior', () => {expect(typeof ConsentDialogBody).toBe('function');});+ test.each([+ { locale: 'en', truncated: false, expected: '54,321' },+ { locale: 'en', truncated: true, expected: '≥ 54,321' },+ { locale: 'fr', truncated: false, expected: '54\u202f321' },+ { locale: 'fr', truncated: true, expected: '≥ 54\u202f321' },+ ])(+ 'shows the actual probe count in $locale with truncated=$truncated',+ async ({ locale, truncated, expected }) => {+ testI18n.activate(locale);+ setBridge({+ ...statusBridge([]),+ onboarding: {+ probeContent: async () => ({ ok: true, count: 54_321, sample: [], truncated }),+ },+ });+ renderConsentDialog();++ await waitFor(() => {+ expect(screen.getByTestId('consent-preview').textContent).toBe(+ `Found ${expected} markdown files`,+ );+ });+ },+ );++ test.each([+ { locale: 'en', headline: '5,000', remaining: '4,999' },+ { locale: 'fr', headline: '5\u202f000', remaining: '4\u202f999' },+ ])('formats the expanded remaining count in $locale', async ({ locale, headline, remaining }) => {+ testI18n.activate(locale);+ setBridge({+ ...statusBridge([]),+ onboarding: {+ probeContent: async () => ({+ ok: true,+ count: 5_000,+ sample: ['docs/README.md'],+ truncated: false,+ }),+ },+ });+ renderConsentDialog();++ const previewTrigger = await screen.findByRole('button', {+ name: `Found ${headline} markdown files`,+ });+ expect(previewTrigger.textContent).toBe(`Found ${headline} markdown files`);+ await userEvent.click(previewTrigger);++ expect(screen.getByText('docs/README.md')).not.toBeNull();+ expect(screen.getByText(/and .* more/).textContent).toBe(`and ${remaining} more`);+ });+test('advanced controls are collapsed by default and reveal on expand', async () => {renderConsentDialog();diff --git a/packages/app/src/components/ConsentDialogBody.tsx b/packages/app/src/components/ConsentDialogBody.tsxindex 2bb00a8f4..6686d715f 100644--- a/packages/app/src/components/ConsentDialogBody.tsx+++ b/packages/app/src/components/ConsentDialogBody.tsx@@ -470,7 +470,7 @@ function ConsentDialogForm({ payload, store, toast, detectionGraceMs }: ConsentD}function ProbePreview({ probe }: { probe: OkOnboardingProbeContentResult | null }) {- const { t } = useLingui();+ const { t, i18n } = useLingui();if (probe === null) {return (<p className="text-1sm text-muted-foreground" data-testid="consent-preview">@@ -486,7 +486,9 @@ function ProbePreview({ probe }: { probe: OkOnboardingProbeContentResult | null</p>);}- const countDisplay = probe.truncated ? '≥ 50,000' : String(probe.count);+ const numberFormat = new Intl.NumberFormat(i18n.locale);+ const formattedCount = numberFormat.format(probe.count);+ const countDisplay = probe.truncated ? `≥ ${formattedCount}` : formattedCount;const countLine = t`Found ${countDisplay} markdown files`;if (probe.sample.length === 0) {return (@@ -495,7 +497,8 @@ function ProbePreview({ probe }: { probe: OkOnboardingProbeContentResult | null</p>);}- const remaining = probe.truncated ? null : probe.count - probe.sample.length;+ const remainingCount = probe.truncated ? null : probe.count - probe.sample.length;+ const remaining = remainingCount === null ? null : numberFormat.format(remainingCount);return (<Collapsible data-testid="consent-preview"><CollapsibleTrigger className="flex items-center gap-1 text-1sm text-muted-foreground hover:text-foreground [&[data-state=open]>svg]:rotate-90">@@ -511,7 +514,7 @@ function ProbePreview({ probe }: { probe: OkOnboardingProbeContentResult | null<li key={path}>{path}</li>))}</ul>- {probe.truncated || (remaining !== null && remaining > 0) ? (+ {probe.truncated || (remainingCount !== null && remainingCount > 0) ? (<p className="mt-1 italic">{probe.truncated ? <Trans>and more</Trans> : <Trans>and {remaining} more</Trans>}</p>diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsindex a28cc2b18..8a60e5826 100644--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -1,4 +1,3 @@-import { readFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.tsindex 4fc2c11f9..6fcdc23f2 100644--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { normalizeBridge } from '@inkeep/open-knowledge-core';import { updateYFragment } from '@tiptap/y-tiptap';diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.tsindex 9f2f6feb6..40b5cdfd6 100644--- a/packages/app/tests/integration/interior-edit-freshness.test.ts+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.tsindex 2a085742b..93666af70 100644--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.tsindex 878d5196b..867598d34 100644--- a/packages/app/tests/integration/producer-guard-wiring.test.ts+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.tsindex 4ada3cf26..0970ac12f 100644--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterEach, describe, expect, test } from 'vitest';import { agentWriteMd, createTestServer, type TestServer } from './test-harness';diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.tsindex cbe2b705f..b632185e1 100644--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts@@ -1,4 +1,3 @@-import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';import { join } from 'node:path';diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.tsindex 89bc9b4d9..c530ce4ba 100644--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';import type * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.tsindex a78aeb0e4..d3767adc9 100644--- a/packages/app/tests/integration/undo-after-rollback.test.ts+++ b/packages/app/tests/integration/undo-after-rollback.test.ts@@ -1,4 +1,3 @@-import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.tsindex e7cc69c3f..b076f955c 100644--- a/packages/app/tests/integration/undo-recycle-reset.test.ts+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts@@ -1,4 +1,3 @@-import { writeFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.tsindex 9ddcd0730..7935e35f8 100644--- a/packages/app/tests/integration/undo-rederive-survival.test.ts+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts@@ -1,4 +1,3 @@-import { prependFrontmatter } from '@inkeep/open-knowledge-core';import { Plugin, PluginKey } from '@tiptap/pm/state';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';diff --git a/packages/cli/src/content/preview.test.ts b/packages/cli/src/content/preview.test.tsindex 33a14e4f3..dc6d45564 100644--- a/packages/cli/src/content/preview.test.ts+++ b/packages/cli/src/content/preview.test.ts@@ -143,6 +143,17 @@ describe('previewContent', () => {expect(result.warnings).toEqual([]);});+ it('warns on a self-referential symlink cycle instead of throwing', () => {+ writeFileSync(join(testDir, 'real.md'), '# Real');+ symlinkSync(join(testDir, 'selfie'), join(testDir, 'selfie'));++ const result = previewContent({ projectDir: testDir, contentDir: testDir });++ expect(result.totalCount).toBe(1);+ expect(result.warnings.length).toBe(1);+ expect(result.warnings[0]).toContain('broken or cyclic symlink');+ });+it('warns on broken symlinks instead of throwing', () => {writeFileSync(join(testDir, 'real.md'), '# Real');symlinkSync(join(testDir, 'nonexistent.md'), join(testDir, 'broken-link.md'));diff --git a/packages/cli/src/content/preview.unbounded-consumer.test.ts b/packages/cli/src/content/preview.unbounded-consumer.test.tsnew file mode 100644index 000000000..ca8b2ac16--- /dev/null+++ b/packages/cli/src/content/preview.unbounded-consumer.test.ts@@ -0,0 +1,38 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { previewContent } from './preview.ts';++const LINKED_FILE_COUNT = 1500;++let tmpRoot: string;+let fixtureRoot: string;++beforeEach(() => {+ tmpRoot = mkdtempSync(resolve(tmpdir(), 'ok-preview-unbounded-'));+ fixtureRoot = realpathSync(tmpRoot);+});++afterEach(() => {+ rmSync(tmpRoot, { recursive: true, force: true });+});++describe('previewContent — the CLI consumer descends directory symlinks', () => {+ test('counts every markdown file reachable through a directory symlink', () => {+ const bulk = resolve(fixtureRoot, 'bulk');+ const root = resolve(fixtureRoot, 'root');+ mkdirSync(bulk, { recursive: true });+ mkdirSync(root, { recursive: true });+ for (let i = 0; i < LINKED_FILE_COUNT; i += 1) {+ writeFileSync(join(bulk, `f${i}.md`), '# x');+ }+ writeFileSync(join(root, 'a.md'), '# a');+ symlinkSync(bulk, join(root, 'linked'), process.platform === 'win32' ? 'junction' : undefined);++ const result = previewContent({ projectDir: root, contentDir: root });++ expect(result.totalCount).toBe(LINKED_FILE_COUNT + 1);+ expect(result.warnings).toEqual([]);+ });+});diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex 2a597ec3d..f4cc8a286 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,4 +1,3 @@-import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex 01ef76a9f..8fb428b90 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,4 +1,3 @@-import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex 20a68d2a1..bfbd5a2e8 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex fa3cfcd86..bfd7e3109 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 8be632ba4..4d3250d3a 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,4 +1,3 @@-export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');diff --git a/packages/desktop/src/main/consent-dialog.symlink-descent.test.ts b/packages/desktop/src/main/consent-dialog.symlink-descent.test.tsnew file mode 100644index 000000000..511c185ad--- /dev/null+++ b/packages/desktop/src/main/consent-dialog.symlink-descent.test.ts@@ -0,0 +1,50 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { previewContent } from '@inkeep/open-knowledge';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { runProbe } from './consent-dialog.ts';++let tmpRoot: string;+let fixtureRoot: string;++beforeEach(() => {+ tmpRoot = mkdtempSync(resolve(tmpdir(), 'ok-probe-descent-'));+ fixtureRoot = realpathSync(tmpRoot);+});++afterEach(() => {+ rmSync(tmpRoot, { recursive: true, force: true });+});++function seedBulkBehindDirectorySymlink(fileCount: number, extension: string): string {+ const bulk = resolve(fixtureRoot, 'bulk');+ const root = resolve(fixtureRoot, 'root');+ mkdirSync(bulk, { recursive: true });+ mkdirSync(root, { recursive: true });+ for (let i = 0; i < fileCount; i += 1) {+ writeFileSync(join(bulk, `f${i}${extension}`), '');+ }+ writeFileSync(join(root, 'a.md'), '# a');+ symlinkSync(bulk, join(root, 'linked'), process.platform === 'win32' ? 'junction' : undefined);+ return root;+}++describe('runProbe — the guard verdict bounds the walk it authorises', () => {+ test('truncates rather than handing a symlinked over-cap tree to previewContent', async () => {+ const walkCap = 60;+ const root = seedBulkBehindDirectorySymlink(walkCap, '.txt');++ const result = await runProbe(+ previewContent,+ root,+ { contentDir: '.' },+ { walkCapForTests: walkCap },+ );++ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(result.truncated).toBe(true);+ expect(result.count).toBe(walkCap);+ });+});diff --git a/packages/desktop/src/main/consent-dialog.test.ts b/packages/desktop/src/main/consent-dialog.test.tsindex dd30d3edc..0f9b5f079 100644--- a/packages/desktop/src/main/consent-dialog.test.ts+++ b/packages/desktop/src/main/consent-dialog.test.ts@@ -1,4 +1,3 @@-import type { Dirent } from 'node:fs';import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';@@ -10,7 +9,6 @@ import {type PreviewContentFn,requestUserConsent,runProbe,- walkExceedsCap,} from './consent-dialog.ts';interface IpcStub extends ConsentIpcMainLike {@@ -447,50 +445,6 @@ describe('runProbe', () => {});});-describe('walkExceedsCap — async chunked yields', () => {- test('yields the event loop between chunks of entries', async () => {- const fakeEntries = Array.from({ length: 5000 }, (_, i) => ({- name: `f${i}.md`,- isDirectory: () => false,- })) as unknown as Dirent[];- const fakeReaddir = (path: string): Promise<readonly Dirent[]> =>- Promise.resolve(path === '/fake/root' ? fakeEntries : []);-- let yieldsDuringWalk = 0;- let walkCompleted = false;- const tickCounter = (): void => {- if (walkCompleted) return;- yieldsDuringWalk += 1;- setImmediate(tickCounter);- };- setImmediate(tickCounter);-- const truncated = await walkExceedsCap('/fake/root', 50_000, {- readdirImpl: fakeReaddir,- chunkYieldEvery: 500,- });- walkCompleted = true;-- expect(yieldsDuringWalk).toBeGreaterThanOrEqual(5);- expect(truncated).toBe(false);- });-- test('returns truncated=true when entry count exceeds cap', async () => {- const fakeEntries = Array.from({ length: 100 }, (_, i) => ({- name: `f${i}.md`,- isDirectory: () => false,- })) as unknown as Dirent[];- const fakeReaddir = (path: string): Promise<readonly Dirent[]> =>- Promise.resolve(path === '/fake/root' ? fakeEntries : []);-- const truncated = await walkExceedsCap('/fake/root', 10, {- readdirImpl: fakeReaddir,- chunkYieldEvery: 1000,- });- expect(truncated).toBe(true);- });-});-describe('requestUserConsent — probe-content pins to captured projectDir', () => {test('IPC probe-content uses captured payload.projectDir; renderer-supplied projectDir is ignored', async () => {const tmp = mkdtempSync(join(tmpdir(), 'ok-probe-pin-'));diff --git a/packages/desktop/src/main/consent-dialog.ts b/packages/desktop/src/main/consent-dialog.tsindex 9a1eb4108..f3e955857 100644--- a/packages/desktop/src/main/consent-dialog.ts+++ b/packages/desktop/src/main/consent-dialog.ts@@ -1,6 +1,4 @@-import type { Dirent } from 'node:fs';import { existsSync } from 'node:fs';-import { readdir } from 'node:fs/promises';import { join } from 'node:path';import { ALL_EDITOR_IDS } from '@inkeep/open-knowledge-core';import type { IpcMain, IpcMainInvokeEvent } from 'electron';@@ -16,6 +14,7 @@ import type {import { createHandler } from '../shared/ipc-handler.ts';import { type SendableWebContents, sendToRenderer } from '../shared/ipc-send.ts';import { getLogger } from './desktop-logger.ts';+import { walkExceedsCap } from './fs-walk-budget.ts';import { logIpcError } from './ipc-log.ts';export interface ConsentIpcMainLike extends Pick<IpcMain, 'handle' | 'removeHandler'> {}@@ -210,7 +209,15 @@ export function requestUserConsent(});return { ok: false, error: 'Probe must come from the dialog window.' };}+ logger.info('probing content dir', { contentDir: request.contentDir });+ const probeStartedAt = Date.now();const result = await runProbe(previewContent, payload.projectDir, request);+ logger.info('probe complete', {+ contentDir: request.contentDir,+ elapsedMs: Date.now() - probeStartedAt,+ ok: result.ok,+ truncated: result.ok ? result.truncated : undefined,+ });if (!result.ok) {logIpcError({event: 'ipc.error',@@ -309,7 +316,9 @@ export async function runProbe(previewContent: PreviewContentFn,projectDir: string,request: OnboardingProbeContentRequest,+ options: { readonly walkCapForTests?: number } = {},): Promise<OnboardingProbeContentResult> {+ const walkCap = options.walkCapForTests ?? PROBE_WALK_CAP;if (!isContentDirSafe(request.contentDir)) {return { ok: false, error: 'Content directory must be inside the project' };}@@ -322,9 +331,9 @@ export async function runProbe(}await new Promise<void>((r) => setImmediate(r));try {- const truncated = await walkExceedsCap(target, PROBE_WALK_CAP);+ const truncated = await walkExceedsCap(target, walkCap, { descendSymlinks: true });if (truncated) {- return { ok: true, count: PROBE_WALK_CAP, sample: [], truncated: true };+ return { ok: true, count: walkCap, sample: [], truncated: true };}const result = previewContent({projectDir,@@ -338,46 +347,7 @@ export async function runProbe(truncated: false,};} catch (err) {+ getLogger('consent-dialog').error({ err }, 'probe failed');return { ok: false, error: err instanceof Error ? err.message : 'probe failed' };}}--const CHUNK_YIELD_EVERY = 1000;--export async function walkExceedsCap(- root: string,- cap: number,- options: {- readonly readdirImpl?: (path: string) => Promise<readonly Dirent[]>;- readonly chunkYieldEvery?: number;- } = {},-): Promise<boolean> {- const readdirImpl = options.readdirImpl ?? ((p: string) => readdir(p, { withFileTypes: true }));- const chunkYieldEvery = options.chunkYieldEvery ?? CHUNK_YIELD_EVERY;- let count = 0;- const stack: string[] = [root];- while (stack.length > 0) {- const dir = stack.pop();- if (dir === undefined) break;- let entries: readonly Dirent[];- try {- entries = await readdirImpl(dir);- } catch (err) {- const code = (err as NodeJS.ErrnoException).code;- if (code === 'EMFILE' || code === 'ENFILE') return true;- continue;- }- for (const entry of entries) {- count += 1;- if (count > cap) return true;- if (count % chunkYieldEvery === 0) {- await new Promise<void>((r) => setImmediate(r));- }- if (entry.isDirectory()) {- if (entry.name === 'node_modules' || entry.name === '.git') continue;- stack.push(join(dir, entry.name));- }- }- }- return false;-}diff --git a/packages/desktop/src/main/folder-admission.size-probe-descent.test.ts b/packages/desktop/src/main/folder-admission.size-probe-descent.test.tsnew file mode 100644index 000000000..1908bfe3e--- /dev/null+++ b/packages/desktop/src/main/folder-admission.size-probe-descent.test.ts@@ -0,0 +1,80 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { type DiscoverProjectResult, discoverProject } from './folder-admission.ts';+import { createBootBudgetDirSizeProbe } from './fs-walk-budget.ts';++const ANCESTOR_SIZE_CAP = 100;++let tmpRoot: string;+let fakeHome: string;++beforeEach(() => {+ tmpRoot = mkdtempSync(resolve(tmpdir(), 'ok-size-probe-descent-'));+ fakeHome = resolve(realpathSync(tmpRoot), 'home');+ mkdirSync(fakeHome, { recursive: true });+});++afterEach(() => {+ rmSync(tmpRoot, { recursive: true, force: true });+});++function seedManagedAncestorWithLinkedBulk(bulkFileCount: number): string {+ const bulk = resolve(fakeHome, 'bulk');+ const ancestor = resolve(fakeHome, 'ancestor');+ const picked = resolve(ancestor, 'sub');+ mkdirSync(bulk, { recursive: true });+ mkdirSync(picked, { recursive: true });+ mkdirSync(resolve(ancestor, '.ok'), { recursive: true });+ writeFileSync(resolve(ancestor, '.ok/config.yml'), '$schema: x\n');+ for (let i = 0; i < bulkFileCount; i += 1) {+ writeFileSync(join(bulk, `f${i}.md`), '# x');+ }+ symlinkSync(+ bulk,+ resolve(ancestor, 'linked'),+ process.platform === 'win32' ? 'junction' : undefined,+ );+ return picked;+}++function seedManagedAncestorWithRealBulk(bulkFileCount: number): string {+ const ancestor = resolve(fakeHome, 'ancestor');+ const bulk = resolve(ancestor, 'bulk');+ const picked = resolve(ancestor, 'sub');+ mkdirSync(bulk, { recursive: true });+ mkdirSync(picked, { recursive: true });+ mkdirSync(resolve(ancestor, '.ok'), { recursive: true });+ writeFileSync(resolve(ancestor, '.ok/config.yml'), '$schema: x\n');+ for (let i = 0; i < bulkFileCount; i += 1) {+ writeFileSync(join(bulk, `f${i}.md`), '# x');+ }+ return picked;+}++async function discoverWithRealSizeProbe(picked: string): Promise<DiscoverProjectResult> {+ return discoverProject(picked, {+ homeDir: fakeHome,+ gitTopLevel: async () => null,+ dirSizeProbe: createBootBudgetDirSizeProbe(ANCESTOR_SIZE_CAP),+ });+}++describe('discoverProject — the ancestor size probe counts real directories only', () => {+ test('ancestor whose bulk hides behind a directory symlink still promotes silently', async () => {+ const picked = seedManagedAncestorWithLinkedBulk(ANCESTOR_SIZE_CAP + 50);++ const result = await discoverWithRealSizeProbe(picked);++ expect(result.kind).toBe('managed');+ });++ test('ancestor whose real directory bulk exceeds the cap requires confirmation', async () => {+ const picked = seedManagedAncestorWithRealBulk(ANCESTOR_SIZE_CAP + 50);++ const result = await discoverWithRealSizeProbe(picked);++ expect(result.kind).toBe('managed-requires-confirmation');+ });+});diff --git a/packages/desktop/src/main/fs-walk-budget.test.ts b/packages/desktop/src/main/fs-walk-budget.test.tsnew file mode 100644index 000000000..a47ed3417--- /dev/null+++ b/packages/desktop/src/main/fs-walk-budget.test.ts@@ -0,0 +1,216 @@+import type { Dirent } from 'node:fs';+import {+ mkdirSync,+ mkdtempSync,+ readdirSync,+ realpathSync,+ rmSync,+ statSync,+ symlinkSync,+ writeFileSync,+} from 'node:fs';+import { readdir } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { previewContent } from '@inkeep/open-knowledge';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { type TraversalEntry, walkExceedsCap } from './fs-walk-budget.ts';++let tmpRoot: string;+let fixtureRoot: string;++beforeEach(() => {+ tmpRoot = mkdtempSync(resolve(tmpdir(), 'ok-walk-budget-descent-'));+ fixtureRoot = realpathSync(tmpRoot);+});++afterEach(() => {+ rmSync(tmpRoot, { recursive: true, force: true });+});++function seedBulkBehindDirectorySymlink(fileCount: number, extension: string): string {+ const bulk = resolve(fixtureRoot, 'bulk');+ const root = resolve(fixtureRoot, 'root');+ mkdirSync(bulk, { recursive: true });+ mkdirSync(root, { recursive: true });+ for (let i = 0; i < fileCount; i += 1) {+ writeFileSync(join(bulk, `f${i}${extension}`), '');+ }+ writeFileSync(join(root, 'a.md'), '# a');+ symlinkSync(bulk, join(root, 'linked'), process.platform === 'win32' ? 'junction' : undefined);+ return root;+}++describe('walkExceedsCap — async chunked yields', () => {+ test('yields the event loop between chunks of entries', async () => {+ const fakeEntries: TraversalEntry[] = Array.from({ length: 5000 }, (_, i) => ({+ name: `f${i}.md`,+ isDirectory: () => false,+ isSymbolicLink: () => false,+ }));+ const fakeReaddir = (path: string): Promise<readonly TraversalEntry[]> =>+ Promise.resolve(path === '/fake/root' ? fakeEntries : []);++ let yieldsDuringWalk = 0;+ let walkCompleted = false;+ const tickCounter = (): void => {+ if (walkCompleted) return;+ yieldsDuringWalk += 1;+ setImmediate(tickCounter);+ };+ setImmediate(tickCounter);++ const truncated = await walkExceedsCap('/fake/root', 50_000, {+ descendSymlinks: false,+ readdirImpl: fakeReaddir,+ chunkYieldEvery: 500,+ });+ walkCompleted = true;++ expect(yieldsDuringWalk).toBeGreaterThanOrEqual(5);+ expect(truncated).toBe(false);+ });++ test('returns truncated=true when entry count exceeds cap', async () => {+ const fakeEntries: TraversalEntry[] = Array.from({ length: 100 }, (_, i) => ({+ name: `f${i}.md`,+ isDirectory: () => false,+ isSymbolicLink: () => false,+ }));+ const fakeReaddir = (path: string): Promise<readonly TraversalEntry[]> =>+ Promise.resolve(path === '/fake/root' ? fakeEntries : []);++ const truncated = await walkExceedsCap('/fake/root', 10, {+ descendSymlinks: false,+ readdirImpl: fakeReaddir,+ chunkYieldEvery: 1000,+ });+ expect(truncated).toBe(true);+ });+});++describe('walkExceedsCap — descent parity with the traversal it guards', () => {+ test('reports over-cap when the bulk is reachable only through a directory symlink', async () => {+ const root = seedBulkBehindDirectorySymlink(60, '.md');++ const admittedByConsumer = previewContent({ projectDir: root, contentDir: root }).totalCount;+ expect(admittedByConsumer).toBe(61);++ expect(await walkExceedsCap(root, admittedByConsumer - 1, { descendSymlinks: true })).toBe(+ true,+ );+ });++ test('stays under cap when the symlinked bulk fits inside it', async () => {+ const root = seedBulkBehindDirectorySymlink(60, '.md');++ expect(await walkExceedsCap(root, 500, { descendSymlinks: true })).toBe(false);+ });++ test('terminates after a bounded number of directory reads on self-referencing symlinks', async () => {+ const root = resolve(fixtureRoot, 'cyclic');+ mkdirSync(root, { recursive: true });+ writeFileSync(join(root, 'a.md'), '# a');+ const linkType = process.platform === 'win32' ? 'junction' : undefined;+ symlinkSync(root, join(root, 'self1'), linkType);+ symlinkSync(root, join(root, 'self2'), linkType);++ let directoriesRead = 0;+ const countingReaddir = async (path: string): Promise<readonly Dirent[]> => {+ directoriesRead += 1;+ return readdir(path, { withFileTypes: true });+ };++ const cap = 500;+ const entriesPerDirectory = 3;+ const symlinksPerDirectory = 2;+ const successfulReadsToExceedCap = Math.ceil((cap + 1) / entriesPerDirectory);+ const maxDirectoryReads = 1 + symlinksPerDirectory * successfulReadsToExceedCap;++ const verdict = await walkExceedsCap(root, cap, {+ descendSymlinks: true,+ readdirImpl: countingReaddir,+ });++ expect(verdict).toBe(true);+ expect(directoriesRead).toBeLessThanOrEqual(maxDirectoryReads);+ });++ test('a cyclic symlink does not by itself push a small tree over the cap', async () => {+ const root = resolve(fixtureRoot, 'cyclic-small');+ mkdirSync(root, { recursive: true });+ writeFileSync(join(root, 'a.md'), '# a');+ symlinkSync(root, join(root, 'self'), process.platform === 'win32' ? 'junction' : undefined);++ expect(await walkExceedsCap(root, 500, { descendSymlinks: true })).toBe(false);+ });+});++function seedBulk(name: string, fileCount: number): string {+ const bulk = resolve(fixtureRoot, name);+ mkdirSync(bulk, { recursive: true });+ for (let i = 0; i < fileCount; i += 1) {+ writeFileSync(join(bulk, `f${i}.md`), '');+ }+ return bulk;+}++function seedRootWithSymlinks(+ name: string,+ symlinks: readonly (readonly [string, string])[],+): string {+ const root = resolve(fixtureRoot, name);+ mkdirSync(root, { recursive: true });+ writeFileSync(join(root, 'a.md'), '# a');+ for (const [linkName, target] of symlinks) {+ const absoluteTarget = resolve(root, target);+ const targetIsDirectory =+ statSync(absoluteTarget, { throwIfNoEntry: false })?.isDirectory() ?? false;+ symlinkSync(+ absoluteTarget,+ join(root, linkName),+ process.platform === 'win32' ? (targetIsDirectory ? 'junction' : 'file') : undefined,+ );+ }+ return root;+}++describe('walkExceedsCap — symlinked entries the traversal does not count through', () => {+ test.each(['node_modules', '.git'])(+ 'does not count through a directory symlink named %s',+ async (excludedName) => {+ seedBulk('excluded-name-bulk', 60);+ const root = seedRootWithSymlinks('excluded-name-root', [+ [excludedName, '../excluded-name-bulk'],+ ]);++ expect(previewContent({ projectDir: root, contentDir: root }).totalCount).toBe(1);++ expect(await walkExceedsCap(root, readdirSync(root).length, { descendSymlinks: true })).toBe(+ false,+ );+ },+ );++ test('a symlink pointing at a file leaves the verdict unchanged', async () => {+ const bulkFileCount = 60;+ seedBulk('shared-bulk', bulkFileCount);+ const baseline = seedRootWithSymlinks('baseline-root', [['linked', '../shared-bulk']]);+ const withFileSymlink = seedRootWithSymlinks('file-symlink-root', [+ ['linked', '../shared-bulk'],+ ['alias.md', 'a.md'],+ ]);++ const cap = bulkFileCount + readdirSync(withFileSymlink).length;+ expect(+ previewContent({ projectDir: withFileSymlink, contentDir: withFileSymlink }).totalCount,+ ).toBeLessThanOrEqual(cap);++ const baselineVerdict = await walkExceedsCap(baseline, cap, { descendSymlinks: true });++ expect(baselineVerdict).toBe(false);+ expect(await walkExceedsCap(withFileSymlink, cap, { descendSymlinks: true })).toBe(+ baselineVerdict,+ );+ });+});diff --git a/packages/desktop/src/main/fs-walk-budget.ts b/packages/desktop/src/main/fs-walk-budget.tsnew file mode 100644index 000000000..f0c185b38--- /dev/null+++ b/packages/desktop/src/main/fs-walk-budget.ts@@ -0,0 +1,72 @@+import type { Dirent } from 'node:fs';+import { readdir } from 'node:fs/promises';+import { join } from 'node:path';+import { getLogger } from './desktop-logger.ts';++const CHUNK_YIELD_EVERY = 1000;++export type TraversalEntry = Pick<Dirent, 'name' | 'isDirectory' | 'isSymbolicLink'>;++function isTraversableDirent(entry: TraversalEntry): boolean {+ return entry.isDirectory() || entry.isSymbolicLink();+}++function isDirectoryDirent(entry: TraversalEntry): boolean {+ return entry.isDirectory();+}++export async function walkExceedsCap(+ root: string,+ cap: number,+ options: {+ readonly descendSymlinks: boolean;+ readonly readdirImpl?: (path: string) => Promise<readonly TraversalEntry[]>;+ readonly chunkYieldEvery?: number;+ },+): Promise<boolean> {+ const shouldDescend = options.descendSymlinks ? isTraversableDirent : isDirectoryDirent;+ const readdirImpl = options.readdirImpl ?? ((p: string) => readdir(p, { withFileTypes: true }));+ const chunkYieldEvery = options.chunkYieldEvery ?? CHUNK_YIELD_EVERY;+ let count = 0;+ const stack: string[] = [root];+ while (stack.length > 0) {+ const dir = stack.pop();+ if (dir === undefined) break;+ let entries: readonly TraversalEntry[];+ try {+ entries = await readdirImpl(dir);+ } catch (err) {+ const code = (err as NodeJS.ErrnoException).code;+ if (code === 'EMFILE' || code === 'ENFILE') return true;+ continue;+ }+ for (const entry of entries) {+ count += 1;+ if (count > cap) return true;+ if (count % chunkYieldEvery === 0) {+ await new Promise<void>((r) => setImmediate(r));+ }+ if (shouldDescend(entry)) {+ if (entry.name === 'node_modules' || entry.name === '.git') continue;+ stack.push(join(dir, entry.name));+ }+ }+ }+ return false;+}++export function createBootBudgetDirSizeProbe(+ cap: number,+): (dir: string) => Promise<{ readonly exceedsCap: boolean }> {+ return async (dir) => {+ try {+ return { exceedsCap: await walkExceedsCap(dir, cap, { descendSymlinks: false }) };+ } catch (err) {+ getLogger('project').warn(+ { err },+ 'project admission size probe failed, treating as over cap',+ );+ return { exceedsCap: true };+ }+ };+}diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.tsindex 79ca31252..1503aa16a 100644--- a/packages/desktop/src/main/index.ts+++ b/packages/desktop/src/main/index.ts@@ -223,7 +223,7 @@ import {runLoginShellProbe,runWindowsPathProbe,} from './claude-readiness.ts';-import { requestUserConsent, walkExceedsCap } from './consent-dialog.ts';+import { requestUserConsent } from './consent-dialog.ts';import { copyImageToClipboard } from './copy-image-clipboard.ts';import {type CrashDetection,@@ -271,6 +271,7 @@ import {} from './driver-boot-smoke.ts';import { EMBED_HOST_PATTERNS, rewriteEmbedRequestHeaders } from './embed-referer.ts';import { defaultGitTopLevel, discoverProject, validateFolderPick } from './folder-admission.ts';+import { createBootBudgetDirSizeProbe } from './fs-walk-budget.ts';import { ensureGitAvailable } from './git-preflight-handler.ts';import { readCanonicalGitHubRemoteUrl } from './git-remote.ts';import { classifyInstallShape } from './install-shape.ts';@@ -1621,6 +1622,7 @@ function logAiIntegrationOutcomes(result: ProjectAiIntegrationsResult): number {}const BOOT_BUDGET_FILE_CAP = 10_000;+const bootBudgetDirSizeProbe = createBootBudgetDirSizeProbe(BOOT_BUDGET_FILE_CAP);async function openProject(projectPath: string,@@ -1656,16 +1658,7 @@ async function openProject({ projectName: basename(dir), pickedName: basename(projectPath) },'probing ancestor size',);- try {- const exceedsCap = await walkExceedsCap(dir, BOOT_BUDGET_FILE_CAP);- return { exceedsCap };- } catch (err) {- getLogger('project').warn(- { err },- 'project admission size probe failed, treating as over cap',- );- return { exceedsCap: true };- }+ return bootBudgetDirSizeProbe(dir);},gitTopLevel: async (cwd) => {getLogger('project').info(diff --git a/packages/desktop/tests/smoke/_helpers/stale-build-guard.ts b/packages/desktop/tests/smoke/_helpers/stale-build-guard.tsindex 72cf58ead..18259a5db 100644--- a/packages/desktop/tests/smoke/_helpers/stale-build-guard.ts+++ b/packages/desktop/tests/smoke/_helpers/stale-build-guard.ts@@ -43,6 +43,7 @@ const CHECKS: BuildArtifactCheck[] = [resolve(DESKTOP_PKG, 'src/main/desktop-uninstall-result.ts'),resolve(DESKTOP_PKG, 'src/main/consent-dialog.ts'),resolve(DESKTOP_PKG, 'src/main/folder-admission.ts'),+ resolve(DESKTOP_PKG, 'src/main/fs-walk-budget.ts'),],},{