Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-9599: Show Files without waiting for startup indexing (#6827)
Andrew Mikofalvyf3dca92ba6c1
diff --git a/.changeset/files-sidebar-loads-before-indexing.md b/.changeset/files-sidebar-loads-before-indexing.mdnew file mode 100644index 000000000..edf1e1964--- /dev/null+++ b/.changeset/files-sidebar-loads-before-indexing.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++The Files sidebar now lists files while a large project is still indexing at startup, and its loading placeholder is visible on every color theme.diff --git a/packages/app/src/components/FileTree.tsx b/packages/app/src/components/FileTree.tsxindex a375d49d3..6d068333d 100644--- a/packages/app/src/components/FileTree.tsx+++ b/packages/app/src/components/FileTree.tsx@@ -3434,8 +3434,8 @@ function FileTreeSkeleton() {key={index}className="flex h-6 items-center gap-2">- <Skeleton className="h-3 w-3 shrink-0 rounded-sm" />- <Skeleton className={`h-3 ${width}`} />+ <Skeleton className="h-3 w-3 shrink-0 rounded-sm bg-sidebar-foreground/10" />+ <Skeleton className={`h-3 bg-sidebar-foreground/10 ${width}`} /></div>))}</div>diff --git a/packages/server/src/http/document-routes.test.ts b/packages/server/src/http/document-routes.test.tsindex 5e8f79dd4..dc387244d 100644--- a/packages/server/src/http/document-routes.test.ts+++ b/packages/server/src/http/document-routes.test.ts@@ -1,3 +1,4 @@+import { EventEmitter } from 'node:events';import {lstatSync,mkdirSync,@@ -16,6 +17,7 @@ import { join } from 'node:path';import { Hocuspocus } from '@hocuspocus/server';import { LOCAL_DIR } from '@inkeep/open-knowledge-core';import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { streamShowAllEntries, walkContentDirForShowAll } from '../api-extension.ts';import { makeCaptureRes, makeSyntheticReq } from '../composition-rig.test-helper.ts';import { createTestConflictAuthority } from '../conflict-authority.test-helper.ts';import type { ConflictAuthority } from '../conflict-authority.ts';@@ -181,6 +183,97 @@ describe('/api/documents?showAll single-flight walk', () => {});});+describe('/api/documents?showAll answers from disk before the boot ready gate settles', () => {+ function makeStreamingRes() {+ const emitter = new EventEmitter();+ const captured = { status: 0, headers: {} as Record<string, unknown>, body: '' };+ const res = Object.assign(emitter, {+ headersSent: false,+ writableEnded: false,+ destroyed: false,+ statusCode: 0,+ writeHead(status: number, headers?: Record<string, unknown>) {+ captured.status = status;+ Object.assign(captured.headers, headers);+ res.headersSent = true;+ return res;+ },+ setHeader(key: string, value: unknown) {+ captured.headers[key.toLowerCase()] = value;+ },+ getHeader(key: string) {+ return captured.headers[key.toLowerCase()];+ },+ write(chunk: string) {+ captured.body += chunk;+ return true;+ },+ end(chunk?: string) {+ if (chunk !== undefined) captured.body += chunk;+ if (captured.status === 0) captured.status = res.statusCode;+ res.writableEnded = true;+ },+ });+ return { res: res as unknown as ServerResponse, captured };+ }++ function listRootWhileBootIsPending(accept: string | undefined) {+ writeFileSync(join(projectDir, 'alpha.md'), '# Alpha\n');+ mkdirSync(join(projectDir, 'folder'));+ writeFileSync(join(projectDir, 'folder', 'beta.md'), '# Beta\n');+ const resolved = buildGroup(+ createTestConflictAuthority(projectDir),+ new Hocuspocus({ quiet: true }),+ {+ ready: new Promise<void>(() => {}),+ contentFilter: createContentFilter({ projectDir, contentDir: projectDir }),+ getShowAllMaxEntries: () => 10_000,+ streamShowAllEntries,+ walkContentDirForShowAll,+ },+ ).table.resolve('/api/documents');+ if (!resolved?.dispatch) throw new Error('no dispatch for /api/documents');+ const req = makeSyntheticReq({ url: '/api/documents?showAll=true&dir=&depth=1' });+ if (accept) req.headers.accept = accept;+ const { res, captured } = makeStreamingRes();+ const settled = Promise.race([+ resolved.dispatch(req, res).then(() => 'answered' as const),+ new Promise<'parked'>((resolve) => setTimeout(() => resolve('parked'), 2_000)),+ ]);+ return { settled, captured };+ }++ test('the JSON listing does not wait on boot', async () => {+ const { settled, captured } = listRootWhileBootIsPending(undefined);+ expect(await settled).toBe('answered');+ expect(captured.status).toBe(200);+ const body = JSON.parse(captured.body) as {+ documents: Array<{ kind: string; docName?: string; path?: string }>;+ };+ expect(body.documents.map((entry) => entry.docName ?? entry.path).sort()).toEqual([+ 'alpha',+ 'folder',+ ]);+ });++ test('the NDJSON listing does not wait on boot', async () => {+ const { settled, captured } = listRootWhileBootIsPending('application/x-ndjson');+ expect(await settled).toBe('answered');+ expect(captured.status).toBe(200);+ const lines = captured.body+ .trim()+ .split('\n')+ .map((line) => JSON.parse(line) as { type?: string; docName?: string; path?: string });+ expect(lines.at(-1)).toMatchObject({ type: 'complete', count: 2 });+ expect(+ lines+ .slice(0, -1)+ .map((entry) => entry.docName ?? entry.path)+ .sort(),+ ).toEqual(['alpha', 'folder']);+ });+});+test.each(['repeatable', 'single-pass'] as const)('regular group members and true symlinks survive both listing traversals of a %s inventory',async (shape) => {diff --git a/packages/server/src/http/document-routes.ts b/packages/server/src/http/document-routes.tsindex a6a2db703..4413a3be8 100644--- a/packages/server/src/http/document-routes.ts+++ b/packages/server/src/http/document-routes.ts@@ -237,14 +237,6 @@ export function createDocumentRoutes(deps: DocumentRouteDeps): DocumentRoutes {EmptyRequestSchema,async (req, res) => {try {- if (ready) {- await ready.catch((err: unknown) => {- log.warn(- { err, handler: 'document-list' },- '[api] ready gate rejected — responding with partial index',- );- });- }const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);const dir = url.searchParams.get('dir');const showAll = url.searchParams.get('showAll') === 'true';@@ -420,6 +412,14 @@ export function createDocumentRoutes(deps: DocumentRouteDeps): DocumentRoutes {return;}+ if (ready) {+ await ready.catch((err: unknown) => {+ log.warn(+ { err, handler: 'document-list' },+ '[api] ready gate rejected — responding with partial index',+ );+ });+ }const index = getFileIndex();const allFiles = [...getAllFilesIndex()];const folderIndex = getFolderIndex?.() ?? new Map<string, FolderIndexEntry>();