Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-9500 Include pending editor edits in a saved version (#6580)

Andrew Mikofalvy2b3312e6b64a
diff --git a/.changeset/save-version-includes-recent-edits.md b/.changeset/save-version-includes-recent-edits.md
new file mode 100644
index 000000000..3359e2e39
--- /dev/null
+++ b/.changeset/save-version-includes-recent-edits.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Saving a version right after typing in the editor now includes that edit, instead of the text last saved to disk.
diff --git a/packages/app/tests/integration/save-version-pending-edit.test.ts b/packages/app/tests/integration/save-version-pending-edit.test.ts
new file mode 100644
index 000000000..a0b7fae18
--- /dev/null
+++ b/packages/app/tests/integration/save-version-pending-edit.test.ts
@@ -0,0 +1,65 @@
+import { execFileSync } from 'node:child_process';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { SaveVersionSuccessSchema } from '@inkeep/open-knowledge-core';
+import { resolveShadowDir } from '@inkeep/open-knowledge-core/shadow-repo-layout';
+import { afterAll, beforeAll, expect, test } from 'vitest';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ awaitConvergedServerText,
+ createTestClient,
+ createTestServer,
+ readTestDoc,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer({
+ debounce: 30_000,
+ maxDebounce: 60_000,
+ gitEnabled: true,
+ commitDebounceMs: 30_000,
+ });
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+test('a version saved right after a client edit includes the edit', async () => {
+ const docName = `pending-${crypto.randomUUID()}`;
+ writeFileSync(join(server.contentDir, `${docName}.md`), '# Pending\n\nSaved line.\n', 'utf-8');
+ const client = await createTestClient(server.port, docName);
+ try {
+ client.ytext.insert(client.ytext.length, '\nTyped just now.\n');
+ await awaitConvergedServerText(server, client);
+ expect(readTestDoc(server.contentDir, docName)).not.toContain('Typed just now.');
+
+ const res = await fetch(`${server.baseUrl}/api/save-version`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({}),
+ });
+ expect(res.status).toBe(200);
+ const { checkpointRef } = SaveVersionSuccessSchema.parse(await res.json());
+ const version = checkpointRef.slice(checkpointRef.lastIndexOf('/') + 1);
+
+ const git = (...args: string[]) =>
+ execFileSync('git', [`--git-dir=${resolveShadowDir(server.contentDir)}`, ...args], {
+ encoding: 'utf-8',
+ });
+ expect(git('show', `${version}:${docName}.md`)).toContain('Typed just now.');
+
+ const parents = git('rev-list', '--parents', '-n', '1', version).trim().split(' ').slice(1);
+ const parentsHoldingEdit = parents.filter((parent) =>
+ git('ls-tree', '--name-only', parent).split('\n').includes(`${docName}.md`)
+ ? git('show', `${parent}:${docName}.md`).includes('Typed just now.')
+ : false,
+ );
+ expect(parentsHoldingEdit).toHaveLength(1);
+ } finally {
+ await client.cleanup();
+ }
+}, 60_000);
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index d20281eb1..25bcdb661 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -5518,6 +5518,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {
computeLintViolations,
log,
flushDocToGit,
+ flushGitCommit,
commitOkArtifactWrite,
isSafeDocName,
shadowRef,
diff --git a/packages/server/src/http/agent-write-routes.test.ts b/packages/server/src/http/agent-write-routes.test.ts
index a2bb6537e..1983e8454 100644
--- a/packages/server/src/http/agent-write-routes.test.ts
+++ b/packages/server/src/http/agent-write-routes.test.ts
@@ -57,6 +57,7 @@ function buildGroup(overrides: Partial<Deps> = {}) {
computeLintViolations: notDispatched,
log: loggerFactory.getLogger('test'),
flushDocToGit: notDispatched,
+ flushGitCommit: undefined,
commitOkArtifactWrite: notDispatched,
isSafeDocName: notDispatched,
shadowRef: undefined,
diff --git a/packages/server/src/http/agent-write-routes.ts b/packages/server/src/http/agent-write-routes.ts
index 045ca48ae..7747e49c3 100644
--- a/packages/server/src/http/agent-write-routes.ts
+++ b/packages/server/src/http/agent-write-routes.ts
@@ -212,6 +212,7 @@ export interface AgentWriteRouteDeps {
) => Promise<LintViolationWarning[]>;
log: import('../logger.ts').PinoLogger;
flushDocToGit: (docName: string, label: string) => void;
+ flushGitCommit: (() => Promise<void>) | undefined;
commitOkArtifactWrite: (context: string) => Promise<unknown>;
isSafeDocName: (docName: string) => boolean;
shadowRef: ShadowRef | undefined;
@@ -258,6 +259,7 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroup
computeLintViolations,
log,
flushDocToGit,
+ flushGitCommit,
commitOkArtifactWrite,
isSafeDocName,
shadowRef,
@@ -272,6 +274,18 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroup
const versionOpsService = createVersionOpsService({ getCurrentBranch, contentRoot });
+ async function storePendingEdits(): Promise<void> {
+ await Promise.all(
+ [...hocuspocus.documents.values()]
+ .filter((document) => !isSystemDoc(document.name) && !isConfigDoc(document.name))
+ .map(async (document) => {
+ await hocuspocus.debouncer.executeNow(`onStoreDocument-${document.name}`);
+ await document.saveMutex.waitForUnlock();
+ }),
+ );
+ await flushGitCommit?.();
+ }
+
function getSubscriberCount(docName: string): number {
try {
const doc = hocuspocus.documents.get(docName);
@@ -1440,6 +1454,7 @@ export function createAgentWriteRoutes(deps: AgentWriteRouteDeps): ApiRouteGroup
const checkpointSummary = normalizeSummary(
typeof body.summary === 'string' ? body.summary : undefined,
);
+ await storePendingEdits();
const result = await versionOpsService.saveCheckpoint(shadow, {
explicitWriters: writers,
agentWriter,
diff --git a/packages/server/src/persistence-flush-during-commit.test.ts b/packages/server/src/persistence-flush-during-commit.test.ts
new file mode 100644
index 000000000..3b4be38ce
--- /dev/null
+++ b/packages/server/src/persistence-flush-during-commit.test.ts
@@ -0,0 +1,133 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import * as Y from 'yjs';
+import { recordContributor, swapContributors } from './contributor-tracker.ts';
+import { DocumentDurabilityState } from './document-durability-state.ts';
+import { createPersistenceExtension } from './persistence.ts';
+import { initShadowRepo, type ShadowHandle, shadowGit } from './shadow-repo.ts';
+
+const commitGate = vi.hoisted(() => ({
+ next: undefined as undefined | { entered: () => void; release: Promise<void> },
+}));
+
+vi.mock('./shadow-repo.ts', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('./shadow-repo.ts')>();
+ return {
+ ...actual,
+ commitWipFromTree: async (...args: Parameters<typeof actual.commitWipFromTree>) => {
+ const gate = commitGate.next;
+ commitGate.next = undefined;
+ if (gate) {
+ gate.entered();
+ await gate.release;
+ }
+ return actual.commitWipFromTree(...args);
+ },
+ };
+});
+
+const EARLIER = { id: 'principal-earlier', name: 'Earlier' };
+const TYPIST = { id: 'principal-typist', name: 'Typist' };
+
+let tmpDir: string;
+let contentDir: string;
+let shadow: ShadowHandle;
+
+beforeEach(async () => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-flush-during-commit-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ shadow = await initShadowRepo(tmpDir);
+ swapContributors();
+});
+
+afterEach(() => {
+ commitGate.next = undefined;
+ swapContributors();
+ rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function replaceBody(document: Y.Doc, text: string): void {
+ const fragment = document.getXmlFragment('default');
+ const ytext = document.getText('source');
+ document.transact(() => {
+ fragment.delete(0, fragment.length);
+ const paragraph = new Y.XmlElement('paragraph');
+ paragraph.insert(0, [new Y.XmlText(text)]);
+ fragment.insert(0, [paragraph]);
+ ytext.delete(0, ytext.length);
+ ytext.insert(0, `${text}\n`);
+ });
+}
+
+async function wipHead(writerId: string): Promise<string> {
+ return (
+ await shadowGit(shadow).raw(
+ 'for-each-ref',
+ '--format=%(objectname)',
+ `refs/wip/main/${writerId}`,
+ )
+ ).trim();
+}
+
+test.each([
+ { timer: 'still armed', commitDebounceMs: 600_000, waitForTimer: false },
+ { timer: 'already fired', commitDebounceMs: 300, waitForTimer: true },
+])(
+ 'a flush while a WIP commit runs commits the writers recorded after it began (timer $timer)',
+ async ({ commitDebounceMs, waitForTimer }) => {
+ writeFileSync(join(contentDir, 'notes.md'), 'initial\n');
+ const persistence = createPersistenceExtension({
+ contentDir,
+ projectDir: tmpDir,
+ contentRoot: 'content',
+ shadowRef: { current: shadow },
+ commitDebounceMs,
+ durabilityState: new DocumentDurabilityState(),
+ });
+ const document = new Y.Doc();
+ await persistence.extension.onLoadDocument?.({
+ document,
+ documentName: 'notes',
+ context: {},
+ } as never);
+
+ let entered = () => {};
+ let release = () => {};
+ const reachedCommit = new Promise<void>((resolve) => {
+ entered = resolve;
+ });
+ commitGate.next = {
+ entered,
+ release: new Promise<void>((resolve) => {
+ release = resolve;
+ }),
+ };
+
+ recordContributor('notes', EARLIER.id, EARLIER.name, EARLIER.id);
+ replaceBody(document, 'first edit');
+ await persistence.forceStore(document, 'notes');
+ const runningCommit = persistence.flushPendingGitCommit();
+ await reachedCommit;
+
+ recordContributor('notes', TYPIST.id, TYPIST.name, TYPIST.id);
+ replaceBody(document, 'second edit');
+ await persistence.forceStore(document, 'notes');
+ if (waitForTimer) await sleep(commitDebounceMs * 3);
+ const flush = persistence.flushPendingGitCommit();
+ release();
+ await Promise.all([runningCommit, flush]);
+
+ expect(await wipHead(EARLIER.id)).toMatch(/^[0-9a-f]{40}$/);
+ const typistHead = await wipHead(TYPIST.id);
+ expect(typistHead).toMatch(/^[0-9a-f]{40}$/);
+ expect(await shadowGit(shadow).raw('show', `${typistHead}:content/notes.md`)).toBe(
+ 'second edit\n',
+ );
+
+ document.destroy();
+ },
+);
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index 7580f7b87..76a7b0444 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -759,10 +759,15 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
}
async function flushPendingGitCommit(): Promise<void> {
+ if (commitInFlight && (gitCommitTimer || pendingAfterCommit)) {
+ pendingAfterCommit = true;
+ await commitInFlight;
+ return flushPendingGitCommit();
+ }
if (gitCommitTimer) {
clearTimeout(gitCommitTimer);
gitCommitTimer = null;
- commitInFlight ||= commitToWipRef().finally(() => {
+ commitInFlight = commitToWipRef().finally(() => {
commitInFlight = null;
if (pendingAfterCommit) {
pendingAfterCommit = false;