Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-9599: Show Files without waiting for startup indexing (#6827)

Andrew Mikofalvyf3dca92ba6c1
diff --git a/.changeset/files-sidebar-loads-before-indexing.md b/.changeset/files-sidebar-loads-before-indexing.md
new file mode 100644
index 000000000..edf1e1964
--- /dev/null
+++ b/.changeset/files-sidebar-loads-before-indexing.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+The Files sidebar now lists files while a large project is still indexing at startup, and its loading placeholder is visible on every color theme.
diff --git a/packages/app/src/components/FileTree.tsx b/packages/app/src/components/FileTree.tsx
index a375d49d3..6d068333d 100644
--- a/packages/app/src/components/FileTree.tsx
+++ b/packages/app/src/components/FileTree.tsx
@@ -3434,8 +3434,8 @@ function FileTreeSkeleton() {
key={index}
className="flex h-6 items-center gap-2"
>
- <Skeleton className="h-3 w-3 shrink-0 rounded-sm" />
- <Skeleton className={`h-3 ${width}`} />
+ <Skeleton className="h-3 w-3 shrink-0 rounded-sm bg-sidebar-foreground/10" />
+ <Skeleton className={`h-3 bg-sidebar-foreground/10 ${width}`} />
</div>
))}
</div>
diff --git a/packages/server/src/http/document-routes.test.ts b/packages/server/src/http/document-routes.test.ts
index 5e8f79dd4..dc387244d 100644
--- a/packages/server/src/http/document-routes.test.ts
+++ b/packages/server/src/http/document-routes.test.ts
@@ -1,3 +1,4 @@
+import { EventEmitter } from 'node:events';
import {
lstatSync,
mkdirSync,
@@ -16,6 +17,7 @@ import { join } from 'node:path';
import { Hocuspocus } from '@hocuspocus/server';
import { LOCAL_DIR } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { streamShowAllEntries, walkContentDirForShowAll } from '../api-extension.ts';
import { makeCaptureRes, makeSyntheticReq } from '../composition-rig.test-helper.ts';
import { createTestConflictAuthority } from '../conflict-authority.test-helper.ts';
import type { ConflictAuthority } from '../conflict-authority.ts';
@@ -181,6 +183,97 @@ describe('/api/documents?showAll single-flight walk', () => {
});
});
+describe('/api/documents?showAll answers from disk before the boot ready gate settles', () => {
+ function makeStreamingRes() {
+ const emitter = new EventEmitter();
+ const captured = { status: 0, headers: {} as Record<string, unknown>, body: '' };
+ const res = Object.assign(emitter, {
+ headersSent: false,
+ writableEnded: false,
+ destroyed: false,
+ statusCode: 0,
+ writeHead(status: number, headers?: Record<string, unknown>) {
+ captured.status = status;
+ Object.assign(captured.headers, headers);
+ res.headersSent = true;
+ return res;
+ },
+ setHeader(key: string, value: unknown) {
+ captured.headers[key.toLowerCase()] = value;
+ },
+ getHeader(key: string) {
+ return captured.headers[key.toLowerCase()];
+ },
+ write(chunk: string) {
+ captured.body += chunk;
+ return true;
+ },
+ end(chunk?: string) {
+ if (chunk !== undefined) captured.body += chunk;
+ if (captured.status === 0) captured.status = res.statusCode;
+ res.writableEnded = true;
+ },
+ });
+ return { res: res as unknown as ServerResponse, captured };
+ }
+
+ function listRootWhileBootIsPending(accept: string | undefined) {
+ writeFileSync(join(projectDir, 'alpha.md'), '# Alpha\n');
+ mkdirSync(join(projectDir, 'folder'));
+ writeFileSync(join(projectDir, 'folder', 'beta.md'), '# Beta\n');
+ const resolved = buildGroup(
+ createTestConflictAuthority(projectDir),
+ new Hocuspocus({ quiet: true }),
+ {
+ ready: new Promise<void>(() => {}),
+ contentFilter: createContentFilter({ projectDir, contentDir: projectDir }),
+ getShowAllMaxEntries: () => 10_000,
+ streamShowAllEntries,
+ walkContentDirForShowAll,
+ },
+ ).table.resolve('/api/documents');
+ if (!resolved?.dispatch) throw new Error('no dispatch for /api/documents');
+ const req = makeSyntheticReq({ url: '/api/documents?showAll=true&dir=&depth=1' });
+ if (accept) req.headers.accept = accept;
+ const { res, captured } = makeStreamingRes();
+ const settled = Promise.race([
+ resolved.dispatch(req, res).then(() => 'answered' as const),
+ new Promise<'parked'>((resolve) => setTimeout(() => resolve('parked'), 2_000)),
+ ]);
+ return { settled, captured };
+ }
+
+ test('the JSON listing does not wait on boot', async () => {
+ const { settled, captured } = listRootWhileBootIsPending(undefined);
+ expect(await settled).toBe('answered');
+ expect(captured.status).toBe(200);
+ const body = JSON.parse(captured.body) as {
+ documents: Array<{ kind: string; docName?: string; path?: string }>;
+ };
+ expect(body.documents.map((entry) => entry.docName ?? entry.path).sort()).toEqual([
+ 'alpha',
+ 'folder',
+ ]);
+ });
+
+ test('the NDJSON listing does not wait on boot', async () => {
+ const { settled, captured } = listRootWhileBootIsPending('application/x-ndjson');
+ expect(await settled).toBe('answered');
+ expect(captured.status).toBe(200);
+ const lines = captured.body
+ .trim()
+ .split('\n')
+ .map((line) => JSON.parse(line) as { type?: string; docName?: string; path?: string });
+ expect(lines.at(-1)).toMatchObject({ type: 'complete', count: 2 });
+ expect(
+ lines
+ .slice(0, -1)
+ .map((entry) => entry.docName ?? entry.path)
+ .sort(),
+ ).toEqual(['alpha', 'folder']);
+ });
+});
+
test.each(['repeatable', 'single-pass'] as const)(
'regular group members and true symlinks survive both listing traversals of a %s inventory',
async (shape) => {
diff --git a/packages/server/src/http/document-routes.ts b/packages/server/src/http/document-routes.ts
index a6a2db703..4413a3be8 100644
--- a/packages/server/src/http/document-routes.ts
+++ b/packages/server/src/http/document-routes.ts
@@ -237,14 +237,6 @@ export function createDocumentRoutes(deps: DocumentRouteDeps): DocumentRoutes {
EmptyRequestSchema,
async (req, res) => {
try {
- if (ready) {
- await ready.catch((err: unknown) => {
- log.warn(
- { err, handler: 'document-list' },
- '[api] ready gate rejected — responding with partial index',
- );
- });
- }
const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
const dir = url.searchParams.get('dir');
const showAll = url.searchParams.get('showAll') === 'true';
@@ -420,6 +412,14 @@ export function createDocumentRoutes(deps: DocumentRouteDeps): DocumentRoutes {
return;
}
+ if (ready) {
+ await ready.catch((err: unknown) => {
+ log.warn(
+ { err, handler: 'document-list' },
+ '[api] ready gate rejected — responding with partial index',
+ );
+ });
+ }
const index = getFileIndex();
const allFiles = [...getAllFilesIndex()];
const folderIndex = getFolderIndex?.() ?? new Map<string, FolderIndexEntry>();