Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-9500 Include pending editor edits in a saved version (#6580)
Andrew Mikofalvy2b3312e6b64a
diff --git a/.changeset/save-version-includes-recent-edits.md b/.changeset/save-version-includes-recent-edits.mdnew file mode 100644index 000000000..3359e2e39--- /dev/null+++ b/.changeset/save-version-includes-recent-edits.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Saving a version right after typing in the editor now includes that edit, instead of the text last saved to disk.diff --git a/packages/app/tests/integration/save-version-pending-edit.test.ts b/packages/app/tests/integration/save-version-pending-edit.test.tsnew file mode 100644index 000000000..a0b7fae18--- /dev/null+++ b/packages/app/tests/integration/save-version-pending-edit.test.ts@@ -0,0 +1,65 @@+import { execFileSync } from 'node:child_process';+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { SaveVersionSuccessSchema } from '@inkeep/open-knowledge-core';+import { resolveShadowDir } from '@inkeep/open-knowledge-core/shadow-repo-layout';+import { afterAll, beforeAll, expect, test } from 'vitest';+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';+import {+ awaitConvergedServerText,+ createTestClient,+ createTestServer,+ readTestDoc,+ type TestServer,+} from './test-harness';++let server: TestServer;++beforeAll(async () => {+ server = await createTestServer({+ debounce: 30_000,+ maxDebounce: 60_000,+ gitEnabled: true,+ commitDebounceMs: 30_000,+ });+}, HARNESS_BOOT_TIMEOUT_MS);++afterAll(async () => {+ await server.cleanup();+});++test('a version saved right after a client edit includes the edit', async () => {+ const docName = `pending-${crypto.randomUUID()}`;+ writeFileSync(join(server.contentDir, `${docName}.md`), '# Pending\n\nSaved line.\n', 'utf-8');+ const client = await createTestClient(server.port, docName);+ try {+ client.ytext.insert(client.ytext.length, '\nTyped just now.\n');+ await awaitConvergedServerText(server, client);+ expect(readTestDoc(server.contentDir, docName)).not.toContain('Typed just now.');++ const res = await fetch(`${server.baseUrl}/api/save-version`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({}),+ });+ expect(res.status).toBe(200);+ const { checkpointRef } = SaveVersionSuccessSchema.parse(await res.json());+ const version = checkpointRef.slice(checkpointRef.lastIndexOf('/') + 1);++ const git = (...args: string[]) =>+ execFileSync('git', [`--git-dir=${resolveShadowDir(server.contentDir)}`, ...args], {+ encoding: 'utf-8',+ });+ expect(git('show', `${version}:${docName}.md`)).toContain('Typed just now.');++ const parents = git('rev-list', '--parents', '-n', '1', version).trim().split(' ').slice(1);+ const parentsHoldingEdit = parents.filter((parent) =>+ git('ls-tree', '--name-only', parent).split('\n').includes(`${docName}.md`)+ ? git('show', `${parent}:${docName}.md`).includes('Typed just now.')+ : false,+ );+ expect(parentsHoldingEdit).toHaveLength(1);+ } finally {+ await client.cleanup();+ }+}, 60_000);diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex d20281eb1..25bcdb661 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -5518,6 +5518,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {computeLintViolations,log,flushDocToGit,+ flushGitCommit,commitOkArtifactWrite,isSafeDocName,shadowRef,diff --git a/packages/server/src/http/agent-write-routes.test.ts b/packages/server/src/http/agent-write-routes.test.tsindex a2bb6537e..1983e8454 100644--- a/packages/server/src/http/agent-write-routes.test.ts+++ b/packages/server/src/http/agent-write-routes.test.ts@@ -57,6 +57,7 @@ function buildGroup(overrides: Partial<Deps> = {}) {computeLintViolations: notDispatched,log: loggerFactory.getLogger('test'),flushDocToGit: notDispatched,+ flushGitCommit: undefined,commitOkArtifactWrite: notDispatched,isSafeDocName: notDispatched,shadowRef: undefined,diff --git a/packages/server/src/http/agent-write-routes.ts b/packages/server/src/http/agent-write-routes.tsindex 045ca48ae..7747e49c3 100644--- a/packages/server/src/http/agent-write-routes.ts+++ b/packages/server/src/http/agent-write-routes.ts@@ -212,6 +212,7 @@ export interface AgentWriteRouteDeps {) => Promise<LintViolationWarning[]>;log: import('../logger.ts').PinoLogger;flushDocToGit: (docName: string, label: string) => void;+ flushGitCommit: (() => Promise<void>) | undefined;commitOkArtifactWrite: (context: string) => Promise<unknown>;isSafeDocName: (docName: string) => boolean;shadowRef: ShadowRef | undefined;@@ -258,6 +259,7 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroupcomputeLintViolations,log,flushDocToGit,+ flushGitCommit,commitOkArtifactWrite,isSafeDocName,shadowRef,@@ -272,6 +274,18 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroupconst versionOpsService = createVersionOpsService({ getCurrentBranch, contentRoot });+ async function storePendingEdits(): Promise<void> {+ await Promise.all(+ [...hocuspocus.documents.values()]+ .filter((document) => !isSystemDoc(document.name) && !isConfigDoc(document.name))+ .map(async (document) => {+ await hocuspocus.debouncer.executeNow(`onStoreDocument-${document.name}`);+ await document.saveMutex.waitForUnlock();+ }),+ );+ await flushGitCommit?.();+ }+function getSubscriberCount(docName: string): number {try {const doc = hocuspocus.documents.get(docName);@@ -1440,6 +1454,7 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroupconst checkpointSummary = normalizeSummary(typeof body.summary === 'string' ? body.summary : undefined,);+ await storePendingEdits();const result = await versionOpsService.saveCheckpoint(shadow, {explicitWriters: writers,agentWriter,diff --git a/packages/server/src/persistence-flush-during-commit.test.ts b/packages/server/src/persistence-flush-during-commit.test.tsnew file mode 100644index 000000000..3b4be38ce--- /dev/null+++ b/packages/server/src/persistence-flush-during-commit.test.ts@@ -0,0 +1,133 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { setTimeout as sleep } from 'node:timers/promises';+import { afterEach, beforeEach, expect, test, vi } from 'vitest';+import * as Y from 'yjs';+import { recordContributor, swapContributors } from './contributor-tracker.ts';+import { DocumentDurabilityState } from './document-durability-state.ts';+import { createPersistenceExtension } from './persistence.ts';+import { initShadowRepo, type ShadowHandle, shadowGit } from './shadow-repo.ts';++const commitGate = vi.hoisted(() => ({+ next: undefined as undefined | { entered: () => void; release: Promise<void> },+}));++vi.mock('./shadow-repo.ts', async (importOriginal) => {+ const actual = await importOriginal<typeof import('./shadow-repo.ts')>();+ return {+ ...actual,+ commitWipFromTree: async (...args: Parameters<typeof actual.commitWipFromTree>) => {+ const gate = commitGate.next;+ commitGate.next = undefined;+ if (gate) {+ gate.entered();+ await gate.release;+ }+ return actual.commitWipFromTree(...args);+ },+ };+});++const EARLIER = { id: 'principal-earlier', name: 'Earlier' };+const TYPIST = { id: 'principal-typist', name: 'Typist' };++let tmpDir: string;+let contentDir: string;+let shadow: ShadowHandle;++beforeEach(async () => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-flush-during-commit-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ shadow = await initShadowRepo(tmpDir);+ swapContributors();+});++afterEach(() => {+ commitGate.next = undefined;+ swapContributors();+ rmSync(tmpDir, { recursive: true, force: true });+});++function replaceBody(document: Y.Doc, text: string): void {+ const fragment = document.getXmlFragment('default');+ const ytext = document.getText('source');+ document.transact(() => {+ fragment.delete(0, fragment.length);+ const paragraph = new Y.XmlElement('paragraph');+ paragraph.insert(0, [new Y.XmlText(text)]);+ fragment.insert(0, [paragraph]);+ ytext.delete(0, ytext.length);+ ytext.insert(0, `${text}\n`);+ });+}++async function wipHead(writerId: string): Promise<string> {+ return (+ await shadowGit(shadow).raw(+ 'for-each-ref',+ '--format=%(objectname)',+ `refs/wip/main/${writerId}`,+ )+ ).trim();+}++test.each([+ { timer: 'still armed', commitDebounceMs: 600_000, waitForTimer: false },+ { timer: 'already fired', commitDebounceMs: 300, waitForTimer: true },+])(+ 'a flush while a WIP commit runs commits the writers recorded after it began (timer $timer)',+ async ({ commitDebounceMs, waitForTimer }) => {+ writeFileSync(join(contentDir, 'notes.md'), 'initial\n');+ const persistence = createPersistenceExtension({+ contentDir,+ projectDir: tmpDir,+ contentRoot: 'content',+ shadowRef: { current: shadow },+ commitDebounceMs,+ durabilityState: new DocumentDurabilityState(),+ });+ const document = new Y.Doc();+ await persistence.extension.onLoadDocument?.({+ document,+ documentName: 'notes',+ context: {},+ } as never);++ let entered = () => {};+ let release = () => {};+ const reachedCommit = new Promise<void>((resolve) => {+ entered = resolve;+ });+ commitGate.next = {+ entered,+ release: new Promise<void>((resolve) => {+ release = resolve;+ }),+ };++ recordContributor('notes', EARLIER.id, EARLIER.name, EARLIER.id);+ replaceBody(document, 'first edit');+ await persistence.forceStore(document, 'notes');+ const runningCommit = persistence.flushPendingGitCommit();+ await reachedCommit;++ recordContributor('notes', TYPIST.id, TYPIST.name, TYPIST.id);+ replaceBody(document, 'second edit');+ await persistence.forceStore(document, 'notes');+ if (waitForTimer) await sleep(commitDebounceMs * 3);+ const flush = persistence.flushPendingGitCommit();+ release();+ await Promise.all([runningCommit, flush]);++ expect(await wipHead(EARLIER.id)).toMatch(/^[0-9a-f]{40}$/);+ const typistHead = await wipHead(TYPIST.id);+ expect(typistHead).toMatch(/^[0-9a-f]{40}$/);+ expect(await shadowGit(shadow).raw('show', `${typistHead}:content/notes.md`)).toBe(+ 'second edit\n',+ );++ document.destroy();+ },+);diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex 7580f7b87..76a7b0444 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -759,10 +759,15 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis}async function flushPendingGitCommit(): Promise<void> {+ if (commitInFlight && (gitCommitTimer || pendingAfterCommit)) {+ pendingAfterCommit = true;+ await commitInFlight;+ return flushPendingGitCommit();+ }if (gitCommitTimer) {clearTimeout(gitCommitTimer);gitCommitTimer = null;- commitInFlight ||= commitToWipRef().finally(() => {+ commitInFlight = commitToWipRef().finally(() => {commitInFlight = null;if (pendingAfterCommit) {pendingAfterCommit = false;