Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): stop the duplication tripwire from eating a whole-doc paste (#2997)
Nick Gomeza5c3cec4d83a
diff --git a/.changeset/tripwire-whole-document-paste.md b/.changeset/tripwire-whole-document-paste.mdnew file mode 100644index 000000000..3fd4e7744--- /dev/null+++ b/.changeset/tripwire-whole-document-paste.md@@ -0,0 +1,7 @@+---+"@inkeep/open-knowledge": patch+---++Pasting a copy of an entire document no longer makes the paste disappear. Selecting all, copying, and pasting produces a document that is an exact doubling of what was just saved, which is the same shape a stale browser cache produces when it merges an old copy back in. The persistence guard that watches for that corruption could not tell the two apart, so about two seconds after the paste it refused to save and rolled the open document back to the copy on disk. The pasted content vanished from under the cursor, and undo could not bring it back because the rollback arrives as a remote change that neither the server nor the editor tracks as undoable.++The guard now distinguishes the two cases by when the doubling appears. A stale-cache merge shows up as soon as the document syncs, before the session has saved anything; a paste is an edit on a document that has already saved cleanly. Pastes are left alone and save normally. When the guard does still act on a genuine corruption, it now writes a recovery checkpoint of the pre-rollback document first, so that version stays restorable from version history instead of being lost.diff --git a/packages/app/src/components/TimelinePanel.dom.test.tsx b/packages/app/src/components/TimelinePanel.dom.test.tsxindex 72c5046e4..95e276709 100644--- a/packages/app/src/components/TimelinePanel.dom.test.tsx+++ b/packages/app/src/components/TimelinePanel.dom.test.tsx@@ -133,6 +133,12 @@ const SURFACED_KINDS: ParsedCheckpoint[] = [size: 12,metadata: { atRiskLines: 1, witnessAvailable: true },},+ {+ kind: 'persistence-duplication-reset',+ docName: 'notes',+ size: 12,+ metadata: { copies: 2, fragmentChildren: 18 },+ },];const AUTO_CONSOLIDATION: ParsedCheckpoint = {kind: 'auto-consolidation',diff --git a/packages/core/src/checkpoint-kind-registry.test.ts b/packages/core/src/checkpoint-kind-registry.test.tsindex 7036d4851..e06872040 100644--- a/packages/core/src/checkpoint-kind-registry.test.ts+++ b/packages/core/src/checkpoint-kind-registry.test.ts@@ -68,6 +68,12 @@ const SAMPLE_BY_KIND = {size: 12,metadata: { atRiskLines: 1, witnessAvailable: true },},+ 'persistence-duplication-reset': {+ kind: 'persistence-duplication-reset',+ docName: 'notes',+ size: 12,+ metadata: { copies: 2, fragmentChildren: 18 },+ },'auto-consolidation': {kind: 'auto-consolidation',docName: null,diff --git a/packages/core/src/checkpoint-kinds.ts b/packages/core/src/checkpoint-kinds.tsindex 391ec655d..6e4c339c9 100644--- a/packages/core/src/checkpoint-kinds.ts+++ b/packages/core/src/checkpoint-kinds.ts@@ -147,6 +147,23 @@ export type ParsedCheckpoint =size: number | null;metadata: { atRiskLines: number; witnessAvailable: boolean };}+ | {+ // The persistence structural-duplication tripwire refused a store whose+ // body was an integer doubling of the disk base and reset the live Y.Doc+ // from disk. That reset transacts under the file-watcher origin, which is+ // undo-eligible on neither the server UndoManager nor the client's, so+ // whatever the live document held is otherwise unrecoverable. Distinct+ // from `persistence-reconcile-loss` (the pre-write fragment rebuild) so an+ // operator can tell a duplication reset from a divergence repair by kind+ // alone, and from `observer-a-duplication` (the bridge layer's own gate)+ // so the two duplication classes keep separate budgets. `contents` is the+ // pre-reset document the reset discards — the restore anchor. Both+ // metadata fields are content-free counts.+ kind: 'persistence-duplication-reset';+ docName: string | null;+ size: number | null;+ metadata: { copies: number; fragmentChildren: number };+ }| {// Service-authored consolidation of dead/stale WIP chains.// GET /api/history excludes this kind by default so daily@@ -303,6 +320,23 @@ export function parseCheckpoint(body: string): ParsedCheckpoint | null {}return null;}+ if (kind === 'persistence-duplication-reset') {+ const m = metadata as { copies?: unknown; fragmentChildren?: unknown };+ if (+ typeof m.copies === 'number' &&+ Number.isFinite(m.copies) &&+ typeof m.fragmentChildren === 'number' &&+ Number.isFinite(m.fragmentChildren)+ ) {+ return {+ kind: 'persistence-duplication-reset',+ docName,+ size,+ metadata: { copies: m.copies, fragmentChildren: m.fragmentChildren },+ };+ }+ return null;+ }if (kind === 'auto-consolidation') {const m = metadata as { foldedRefs?: unknown; trigger?: unknown };if (@@ -470,6 +504,11 @@ export const CHECKPOINT_KIND_REGISTRY = {gcBucket: 'persistence-reconcile-loss',bundleExposure: 'metadata',},+ 'persistence-duplication-reset': {+ visibility: 'surfaced',+ gcBucket: 'persistence-duplication-reset',+ bundleExposure: 'metadata',+ },'auto-consolidation': {visibility: 'hidden',gcBucket: 'auto-consolidation',diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex 6c2ad092c..e06aca04e 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -316,6 +316,27 @@ export interface ReconciliationMetrics {* doc resting on a construct that diverges on every write-back lives here;* without the dedup it would evict its own useful anchors. */persistenceReconcileLossDeduped: number;+ /** Count of structural-duplication tripwire fires that BLOCKED the store and+ * reset the live document from disk. Increments on every reset, including+ * ones with no shadow wired, so a run whose anchors never landed does not+ * read as healthy. A nonzero value in a dogfood bundle means a user's live+ * content was replaced by the disk copy. */+ persistenceDuplicationReset: number;+ /** Successful `persistence-duplication-reset` checkpoint writes. With the+ * deduped sibling this closes the identity+ * `persistenceDuplicationReset = created + deduped + (anchors that never landed)`. */+ persistenceDuplicationResetCheckpointCreated: number;+ /** Resets whose pre-reset document was byte-identical to the anchor already on+ * the timeline for that doc, so no new checkpoint was minted. A client stuck+ * re-materializing the same duplicate lives here; without the dedup it would+ * evict its own useful anchors. */+ persistenceDuplicationResetDeduped: number;+ /** Count of duplication verdicts the tripwire declined to act on because the+ * document already had a settled write behind it, so the doubling is an+ * incremental edit (a paste) rather than a load-time materialization. The+ * false-positive class the guard used to destroy; a healthy signal, not a+ * loss. */+ persistenceDuplicationSpared: number;/** Count of Y.Text→XmlFragment re-derive-loop backstop trips — a run of* re-derive drains never reached a raw-byte fixed point and the B-direction* re-derive was frozen (checkpoint + ring event). A nonzero value in CI or a@@ -530,6 +551,10 @@ const counters: ReconciliationMetrics = {persistenceReconcileLoss: 0,persistenceReconcileLossCheckpointCreated: 0,persistenceReconcileLossDeduped: 0,+ persistenceDuplicationReset: 0,+ persistenceDuplicationResetCheckpointCreated: 0,+ persistenceDuplicationResetDeduped: 0,+ persistenceDuplicationSpared: 0,reDeriveBackstopTripped: 0,bridgeSplitBrainRederives: 0,bridgeSplitBrainRederivesSuppressed: 0,@@ -763,6 +788,22 @@ export function incrementPersistenceReconcileLossDeduped(): void {counters.persistenceReconcileLossDeduped++;}+export function incrementPersistenceDuplicationReset(): void {+ counters.persistenceDuplicationReset++;+}++export function incrementPersistenceDuplicationResetCheckpointCreated(): void {+ counters.persistenceDuplicationResetCheckpointCreated++;+}++export function incrementPersistenceDuplicationResetDeduped(): void {+ counters.persistenceDuplicationResetDeduped++;+}++export function incrementPersistenceDuplicationSpared(): void {+ counters.persistenceDuplicationSpared++;+}+export function incrementReDeriveBackstopTripped(): void {counters.reDeriveBackstopTripped++;}@@ -972,6 +1013,10 @@ export function resetMetrics(): void {counters.persistenceReconcileLoss = 0;counters.persistenceReconcileLossCheckpointCreated = 0;counters.persistenceReconcileLossDeduped = 0;+ counters.persistenceDuplicationReset = 0;+ counters.persistenceDuplicationResetCheckpointCreated = 0;+ counters.persistenceDuplicationResetDeduped = 0;+ counters.persistenceDuplicationSpared = 0;counters.reDeriveBackstopTripped = 0;counters.bridgeSplitBrainRederives = 0;counters.bridgeSplitBrainRederivesSuppressed = 0;diff --git a/packages/server/src/persistence-tripwire-paste.test.ts b/packages/server/src/persistence-tripwire-paste.test.tsnew file mode 100644index 000000000..6ed2396cb--- /dev/null+++ b/packages/server/src/persistence-tripwire-paste.test.ts@@ -0,0 +1,376 @@+/**+ * The persistence structural-duplication tripwire against a legitimate+ * whole-document paste, on the REAL `onStoreDocument` path.+ *+ * The measured failure: a user hand-types blank-line-separated paragraphs,+ * selects all, copies, pastes. The candidate body is now an exact doubling of+ * the base, so `classifyDuplication` returns `block` — the same verdict the+ * stale-cache merge produces. The tripwire then refuses the disk write AND+ * resets the live Y.Doc from disk under `FILE_WATCHER_ORIGIN`, which is not+ * undo-eligible on either side, so the paste is destroyed silently.+ *+ * The two classes are byte-identical at the classifier: both arrive under a+ * browser `source: 'connection'` origin and both duplicate with agreeing node+ * shapes, so neither transaction origin nor the Observer-A provenance/shape+ * test separates them. What does separate them is WHEN the doubling appears.+ * The stale-cache merge materializes at provider-sync time, before the session+ * has produced a settled write; a paste is an incremental edit on a document+ * that has already persisted cleanly. The tripwire therefore only acts+ * destructively while the document has no settled write behind it, and+ * checkpoints the live content before it does.+ *+ * Fidelity: real `createServer`, real `onStoreDocument`, real on-disk+ * contentDir, real Hocuspocus debounce, real shadow repo. The only test+ * affordance is `debounce`/`maxDebounce`, real `ServerOptions` fields the+ * sibling tripwire tests already use.+ */++import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { realpath } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';+import simpleGit from 'simple-git';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import type * as Y from 'yjs';+import { lossCaptureCurrentPath, parseLossCaptureLines } from './loss-capture.ts';+import { mdManager, schema } from './md-manager.ts';+import { getMetrics, resetMetrics } from './metrics.ts';+import { classifyDuplication } from './persistence-tripwire.ts';+import { createServer } from './server-factory.ts';+import { initShadowRepo, type ShadowHandle, shadowGit } from './shadow-repo.ts';+import { getDocumentHistory } from './timeline-query.ts';++const FIXTURE_DIR = resolve(import.meta.dirname, 'persistence-tripwire.fixtures');++/**+ * The reported document as it lands on disk. WYSIWYG Enter-Enter leaves an+ * EMPTY paragraph node between each typed paragraph, which serializes as a+ * doubled blank line — that is what makes this 51 bytes rather than the 43+ * the same five paragraphs occupy with single blank lines.+ */+const USER_DOC = 'hello\n\n\n\nkjnekandkjawnkjd\n\n\n\nwkajnd\n\n\n\nwk\n\n\n\nwwjwj\n';+const USER_DOC_LINE = 'kjnekandkjawnkjd';++const BROWSER_ORIGIN = {+ source: 'connection',+ connection: { context: { principalId: 'principal-test-paste' } },+} as const;++const P = (t: string) => ({ type: 'paragraph', content: [{ type: 'text', text: t }] });+const EMPTY = { type: 'paragraph' };++/** The nine children Enter-Enter typing of USER_DOC produces. */+const TYPED_CHILDREN = [+ P('hello'),+ EMPTY,+ P(USER_DOC_LINE),+ EMPTY,+ P('wkajnd'),+ EMPTY,+ P('wk'),+ EMPTY,+ P('wwjwj'),+];++function loadFixture(name: string): string {+ return readFileSync(join(FIXTURE_DIR, name), 'utf-8');+}++function occurrences(haystack: string, needle: string): number {+ let n = 0;+ let i = haystack.indexOf(needle);+ while (i !== -1) {+ n++;+ i = haystack.indexOf(needle, i + needle.length);+ }+ return n;+}++async function waitFor(+ predicate: () => boolean,+ { timeoutMs = 8_000, pollMs = 25 }: { timeoutMs?: number; pollMs?: number } = {},+): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ if (predicate()) return;+ await new Promise((r) => setTimeout(r, pollMs));+ }+ throw new Error(`waitFor timed out after ${timeoutMs}ms`);+}++async function expectStable<T>(+ read: () => T,+ { durationMs = 700, pollMs = 50 }: { durationMs?: number; pollMs?: number } = {},+): Promise<T> {+ const initial = read();+ const deadline = Date.now() + durationMs;+ while (Date.now() < deadline) {+ if (read() !== initial) throw new Error('value changed during stability window');+ await new Promise((r) => setTimeout(r, pollMs));+ }+ return initial;+}++interface Rig {+ tmpDir: string;+ shadow: ShadowHandle;+ cleanup: () => void;+}++async function setupRig(prefix: string): Promise<Rig> {+ // Realpath first: on macOS the tmpdir is a symlink, and persistence derives+ // its shadow tree prefix from `relative(projectDir, contentDir)` — two+ // spellings of the same directory resolve that to an escaping absolute path.+ const tmpDir = await realpath(mkdtempSync(join(tmpdir(), prefix)));+ const git = simpleGit({ baseDir: tmpDir });+ await git.init();+ // Pin the branch rather than inheriting `init.defaultBranch`. Checkpoint refs+ // are namespaced `refs/checkpoints/<branch>` off the real repo HEAD, so a+ // machine defaulting to `master` files the anchor somewhere this test would+ // otherwise not look.+ await git.raw('symbolic-ref', 'HEAD', 'refs/heads/main');+ await git.addConfig('user.name', 'Test User');+ await git.addConfig('user.email', '[email protected]');+ const shadow = await initShadowRepo(tmpDir);+ return {+ tmpDir,+ shadow,+ cleanup: () => rmSync(tmpDir, { recursive: true, force: true }),+ };+}++function replaceFragment(doc: Y.Doc, content: unknown[]): void {+ const xmlFragment = doc.getXmlFragment('default');+ doc.transact(() => {+ updateYFragment(doc, xmlFragment, schema.nodeFromJSON({ type: 'doc', content }), {+ mapping: new Map(),+ isOMark: new Map(),+ });+ }, BROWSER_ORIGIN);+}++/** The live fragment's children, as the clipboard would carry them. */+function liveChildren(frag: Y.XmlFragment): unknown[] {+ const json = yXmlFragmentToProseMirrorRootNode(frag, schema).toJSON() as {+ content?: unknown[];+ };+ return json.content ?? [];+}++function blockedEvents(warnSpy: { mock: { calls: unknown[][] } }): string[] {+ return warnSpy.mock.calls+ .map((call) => String(call[0] ?? ''))+ .filter((s) => s.includes('"event":"ok-persistence-duplication-blocked"'));+}++describe('persistence tripwire vs a whole-document paste', () => {+ let rig: Rig;++ beforeEach(() => {+ resetMetrics();+ });++ afterEach(() => {+ rig?.cleanup();+ });++ test('the classifier still reads a whole-document double as a block verdict', () => {+ // The discriminator lives at the tripwire call site, not in the classifier:+ // the shape genuinely IS an integer doubling, and the epoch-recovery guard+ // still needs that verdict. Pinning it here keeps a future "fix" from+ // weakening the classifier instead of the action it drives.+ expect(USER_DOC.length).toBe(51);+ expect(classifyDuplication(`${USER_DOC}\n${USER_DOC}`, USER_DOC)).toEqual({+ kind: 'block',+ reason: 'structural-duplication',+ copies: 2,+ });+ });++ test('select-all copy paste after a settled write survives and reaches disk', async () => {+ rig = await setupRig('ok-tripwire-paste-');+ const docName = 'Untitled';+ const docPath = join(rig.tmpDir, `${docName}.md`);+ writeFileSync(docPath, '', 'utf-8');++ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const server = createServer({+ contentDir: rig.tmpDir,+ projectDir: rig.tmpDir,+ quiet: true,+ debounce: 100,+ maxDebounce: 500,+ gitEnabled: false,+ shadowRepo: rig.shadow,+ });++ try {+ await server.ready;+ const conn = await server.hocuspocus.openDirectConnection(docName);+ const serverDoc = server.hocuspocus.documents.get(docName);+ expect(serverDoc).toBeDefined();+ if (!serverDoc) return;+ const frag = serverDoc.getXmlFragment('default');++ // The user hand-types five blank-line-separated paragraphs, and that+ // typing settles to disk as one clean copy. This settled write is what+ // marks the session as user-driven.+ replaceFragment(serverDoc, TYPED_CHILDREN);+ await waitFor(() => readFileSync(docPath, 'utf-8').length > 0);+ const baseline = readFileSync(docPath, 'utf-8');+ expect(baseline).toBe(USER_DOC);++ // Select all, copy, paste. Derived from the LIVE fragment, so this is+ // byte-for-byte what the user's own clipboard held.+ const kids = liveChildren(frag);+ replaceFragment(serverDoc, [...kids, ...kids]);+ expect(occurrences(serverDoc.getText('source').toString(), USER_DOC_LINE)).toBe(2);++ // The paste must reach disk rather than being refused as corruption.+ await waitFor(() => readFileSync(docPath, 'utf-8') !== baseline);+ const persisted = await expectStable(() => readFileSync(docPath, 'utf-8'));+ expect(occurrences(persisted, USER_DOC_LINE)).toBe(2);+ expect(persisted.length).toBeGreaterThan(baseline.length);++ // And the live document must still hold it — the destructive half of the+ // tripwire is what actually took the content away from the user.+ expect(occurrences(serverDoc.getText('source').toString(), USER_DOC_LINE)).toBe(2);+ expect(frag.length).toBeGreaterThan(TYPED_CHILDREN.length);++ // No block fired, so no checkpoint was needed and nothing was reset.+ expect(blockedEvents(warnSpy)).toHaveLength(0);+ expect(getMetrics().persistenceDuplicationReset).toBe(0);+ expect(getMetrics().persistenceDuplicationSpared).toBe(1);++ // The spared breadcrumb carries bounded-cardinality keys only. Pinning the+ // key SET, not just the values, is what stops a later change from adding a+ // raw-content field to an event that ships to telemetry.+ const spared = warnSpy.mock.calls+ .map((call) => String(call[0] ?? ''))+ .filter((s) => s.includes('"event":"ok-persistence-duplication-spared"'));+ expect(spared).toHaveLength(1);+ const sparedPayload = JSON.parse(spared[0] ?? '{}') as Record<string, unknown>;+ expect(new Set(Object.keys(sparedPayload))).toEqual(+ new Set([+ 'event',+ 'doc.name',+ 'candidateBytes',+ 'baseBytes',+ 'fragmentChildren',+ 'copies',+ 'reason',+ ]),+ );+ expect(sparedPayload['doc.name']).toBe(docName);+ expect(sparedPayload.copies).toBe(2);++ conn.disconnect();+ } finally {+ warnSpy.mockRestore();+ await server.destroy();+ }+ }, 30_000);++ test('a doubling with no settled write behind it still blocks, resets, and checkpoints', async () => {+ // The incident shape: the document loads from disk and is immediately+ // mutated to the doubled candidate, with no settled write in between. The+ // guard must still act — and the content it destroys must now be+ // recoverable from a checkpoint of its own kind.+ rig = await setupRig('ok-tripwire-incident-');+ const docName = 'incident-changeset-readme';+ const docPath = join(rig.tmpDir, `${docName}.md`);+ const baseMarkdown = loadFixture('incident-changeset-readme-doubled.base.md');+ const doubledMarkdown = loadFixture('incident-changeset-readme-doubled.candidate.md');+ writeFileSync(docPath, baseMarkdown, 'utf-8');+ const baselineBytes = readFileSync(docPath, 'utf-8');++ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const server = createServer({+ contentDir: rig.tmpDir,+ projectDir: rig.tmpDir,+ quiet: true,+ debounce: 100,+ maxDebounce: 500,+ gitEnabled: false,+ shadowRepo: rig.shadow,+ });++ try {+ await server.ready;+ const conn = await server.hocuspocus.openDirectConnection(docName);+ const serverDoc = server.hocuspocus.documents.get(docName);+ expect(serverDoc).toBeDefined();+ if (!serverDoc) return;++ const baseChildren = serverDoc.getXmlFragment('default').length;+ expect(baseChildren).toBeGreaterThan(0);++ const doubledJson = mdManager.parseWithFallback(doubledMarkdown) as { content?: unknown[] };+ replaceFragment(serverDoc, doubledJson.content ?? []);+ expect(serverDoc.getXmlFragment('default').length).toBe(baseChildren * 2);++ await waitFor(() => blockedEvents(warnSpy).length > 0);++ // Disk untouched, live document reset to the disk canonical state.+ await expectStable(() => readFileSync(docPath, 'utf-8'));+ expect(readFileSync(docPath, 'utf-8')).toBe(baselineBytes);+ await waitFor(() => serverDoc.getXmlFragment('default').length === baseChildren);+ await waitFor(() => serverDoc.getText('source').toString() === baselineBytes);+ expect(getMetrics().persistenceDuplicationReset).toBe(1);++ // The reset destroys live content, so it owes a restore anchor. Gate on+ // the completion counter: git creates the ref before the write promise+ // settles, so polling refs alone can observe a half-finished mint.+ await waitFor(() => getMetrics().persistenceDuplicationResetCheckpointCreated >= 1, {+ timeoutMs: 10_000,+ });+ // Listed across every branch namespace, not just `refs/checkpoints/main`,+ // so the assertion pins "exactly one anchor was minted" rather than+ // "an anchor landed on the branch this test happened to guess".+ const shas = (+ await shadowGit(rig.shadow).raw(+ 'for-each-ref',+ '--format=%(objectname)',+ 'refs/checkpoints',+ )+ )+ .toString()+ .split('\n')+ .map((s) => s.trim())+ .filter(Boolean);+ expect(shas).toHaveLength(1);+ const sha = shas[0] ?? '';++ // End to end: the anchor's blob holds the doubled content the reset took+ // away, and it surfaces as an ordinary restorable timeline row.+ const blob = (await shadowGit(rig.shadow).raw('show', `${sha}:${docName}`)).toString();+ expect(occurrences(blob, 'changeset')).toBeGreaterThan(+ occurrences(baselineBytes, 'changeset'),+ );+ const hist = await getDocumentHistory(rig.shadow, { docName }, '');+ const row = hist.entries.find((e) => e.sha === sha);+ expect(row?.checkpoint?.kind).toBe('persistence-duplication-reset');+ expect(row?.checkpoint?.metadata).toEqual({ copies: 2, fragmentChildren: baseChildren * 2 });++ // The reset threads `detect` into the paired intake, which is the whole+ // reason the injectable seam had to widen. Without a ring event here the+ // widening buys nothing, so pin the detector trip and its anchor.+ const ring = parseLossCaptureLines(+ readFileSync(lossCaptureCurrentPath(rig.tmpDir), 'utf-8'),+ ).filter((e) => e.site === 'persistence-duplication-reset');+ const trips = ring.filter((e) => e.event === 'detector-trip');+ expect(trips).toHaveLength(1);+ expect(trips[0]?.docName).toBe(docName);+ expect(trips[0]?.lostLen ?? 0).toBeGreaterThan(0);+ expect(ring.some((e) => e.event === 'checkpoint-write' && e.checkpointSha === sha)).toBe(+ true,+ );++ conn.disconnect();+ } finally {+ warnSpy.mockRestore();+ await server.destroy();+ }+ }, 30_000);+});diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex 1a34d7b34..bbcf1f4f7 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -45,6 +45,7 @@ import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiimport * as Y from 'yjs';import { LINEAGE_EPOCH_KEY } from './auth-token-schema.ts';import type { BacklinkIndex } from './backlink-index.ts';+import { type DeriveLossDetectOptions, detectPairedIntakeLoss } from './bridge-loss-detector.ts';import { getMsSinceLastUserTx, isDocQuiescent } from './bridge-quiescence.ts';import { assertBridgeInvariant, createDocCanonicalizer } from './bridge-watchdog.ts';import {@@ -74,6 +75,7 @@ import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';import {LOSS_EVENT_CHECKPOINT_WRITE,+ LOSS_EVENT_DETECTOR_TRIP,LOSS_EVENT_PERSISTENCE_HOLD,type LossCaptureRing,} from './loss-capture.ts';@@ -95,6 +97,10 @@ import {incrementGitWriterCommitFailure,incrementPersistenceDeferHold,incrementPersistenceDiskWrite,+ incrementPersistenceDuplicationReset,+ incrementPersistenceDuplicationResetCheckpointCreated,+ incrementPersistenceDuplicationResetDeduped,+ incrementPersistenceDuplicationSpared,incrementPersistenceForceFlushDuringBurst,incrementPersistenceReconcileLoss,incrementPersistenceReconcileLossCheckpointCreated,@@ -363,7 +369,21 @@ export interface PersistenceOptions {persistedMarkdown: string,previousMarkdown: string | null,) => void;- applyDiskContentToDoc?: (document: Y.Doc, content: string) => void;+ /**+ * Injectable disk-intake seam. Mirrors the full `applyDiskContentToDoc`+ * signature — narrowing it to `(document, content)` made the trailing+ * `detect` argument unreachable from every call site routed through the+ * option, which silently opted the tripwire reset out of paired-intake+ * derive-loss detection on the one path guaranteed to discard live content.+ */+ applyDiskContentToDoc?: (+ document: Y.Doc,+ content: string,+ resolveEmbed?: (basename: string, sourcePath: string) => string | null,+ sourcePath?: string,+ resolveSize?: (basename: string, sourcePath: string) => number | null,+ detect?: DeriveLossDetectOptions,+ ) => void;/*** Override `os.homedir()` for config-doc persistence. Tests scope* user-global writes (`__user__/config.yml`) to a tempdir; if unset,@@ -636,6 +656,26 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// `onStoreDocument` reads FM via `stripFrontmatter(ytext.toString())` and// writes Y.Text verbatim — no recompose step, no per-key cache, no L3 hook.const tripwireResetFailedDocs = new Set<string>();+ /**+ * Docs that have completed at least one store since they were loaded.+ *+ * The structural-duplication tripwire's two classes are indistinguishable at+ * the classifier: a stale-cache merge and a select-all copy paste both arrive+ * under a browser `source: 'connection'` origin and both duplicate with+ * agreeing node shapes, so neither the transaction origin nor Observer A's+ * provenance/shape race test separates them. What separates them is WHEN the+ * doubling appears. The stale-cache merge materializes at provider-sync time,+ * on a document that has not yet produced a settled write; a paste is an+ * incremental edit on a document that already persisted cleanly this session.+ *+ * Membership is therefore the tripwire's licence to act destructively, and+ * `onLoadDocument` clears it because a reload starts a fresh sync window.+ * A document that stays resident rarely loads again, so in practice the+ * licence lasts as long as the server process holds that document: the guard+ * is armed for the sync window after each load, which is the window the+ * stale-cache merge occupies.+ */+ const docsWithSettledWrite = new Set<string>();const applyDiskContent = options?.applyDiskContentToDoc ?? applyDiskContentToDoc;let pendingDeferredStoreFlushMode: 'within-branch' | 'discard-stale' | null = null;@@ -1111,6 +1151,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis* and the floor checkpoint so a bundle reader can pair them on one doc.*/const PERSISTENCE_PREWRITE_SITE = 'persistence-prewrite';+ const PERSISTENCE_DUPLICATION_SITE = 'persistence-duplication-reset';/*** Last floor-checkpoint payload minted per document. Some documents diverge@@ -1230,6 +1271,92 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis});}+ const lastDuplicationCheckpointPayload = new WeakMap<Y.Doc, string>();++ /**+ * Checkpoint-before-repair for the structural-duplication tripwire reset.+ *+ * `liveMarkdown` is the document the reset is about to discard. The reset+ * transacts under `FILE_WATCHER_ORIGIN`, which the server UndoManager+ * excludes by contract and which reaches browsers as a remote update outside+ * any local `trackedOrigins` set, so this anchor is the only way back to the+ * pre-reset state. Fire-and-forget, and the reset is never gated on it — the+ * dedup suppresses the MINT, not the repair.+ */+ function checkpointBeforeDuplicationReset(+ document: Y.Doc,+ documentName: string,+ liveMarkdown: string,+ copies: number,+ fragmentChildren: number,+ ): void {+ incrementPersistenceDuplicationReset();+ if (lastDuplicationCheckpointPayload.get(document) === liveMarkdown) {+ incrementPersistenceDuplicationResetDeduped();+ return;+ }+ // Claimed before the write so two trips can never race into two anchors;+ // released again if the write fails, so a transient shadow error still+ // retries on the next trip instead of permanently suppressing the anchor.+ lastDuplicationCheckpointPayload.set(document, liveMarkdown);+ const ring = options?.getLossRing?.();+ const shadow = shadowRef?.current;+ if (!shadow) {+ void ring?.record({+ event: LOSS_EVENT_CHECKPOINT_WRITE,+ docName: documentName,+ writerId: null,+ direction: 'b',+ site: PERSISTENCE_DUPLICATION_SITE,+ lostLen: liveMarkdown.length,+ });+ return;+ }+ const branch = getCurrentBranch?.() ?? 'main';+ queueMicrotask(() => {+ saveInMemoryCheckpoint(shadow, contentRoot, {+ kind: 'persistence-duplication-reset',+ docName: documentName,+ contents: liveMarkdown,+ label: `Before duplication reset @ ${new Date().toISOString()}`,+ branch,+ metadata: { copies, fragmentChildren },+ })+ .then((sha) => {+ incrementPersistenceDuplicationResetCheckpointCreated();+ void ring?.record({+ event: LOSS_EVENT_CHECKPOINT_WRITE,+ docName: documentName,+ writerId: null,+ direction: 'b',+ site: PERSISTENCE_DUPLICATION_SITE,+ lostLen: liveMarkdown.length,+ checkpointSha: sha,+ });+ console.warn(+ JSON.stringify({+ event: 'persistence-duplication-reset-checkpoint-created',+ docName: documentName,+ sha,+ kind: 'persistence-duplication-reset',+ timestamp: new Date().toISOString(),+ }),+ );+ })+ .catch((checkpointErr: unknown) => {+ if (lastDuplicationCheckpointPayload.get(document) === liveMarkdown) {+ lastDuplicationCheckpointPayload.delete(document);+ }+ const e =+ checkpointErr instanceof Error ? checkpointErr : new Error(String(checkpointErr));+ log.warn(+ { documentName, err: e },+ '[persistence] duplication-reset checkpoint write failed',+ );+ });+ });+ }+// Lazy-init histograms; safe to call in every hook. Meter is a no-op when OTel// SDK is disabled, so allocations are essentially free.let loadDurationHist: ReturnType<ReturnType<typeof getMeter>['createHistogram']> | null = null;@@ -1641,7 +1768,26 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// currentBase) bypass — there's nothing to duplicate yet.if (currentBase !== undefined) {const classification = classifyDuplication(markdown, currentBase);- if (classification.kind === 'block') {+ if (classification.kind === 'block' && docsWithSettledWrite.has(documentName)) {+ // This document already persisted cleanly this session, so the+ // doubling is an incremental edit — a whole-document paste, the+ // gesture the classifier cannot tell apart from a stale-cache+ // merge. Fall through to the normal write path rather than+ // refusing the store and recycling the live document out from+ // under the user.+ incrementPersistenceDuplicationSpared();+ console.warn(+ JSON.stringify({+ event: 'ok-persistence-duplication-spared',+ 'doc.name': documentName,+ candidateBytes: markdown.length,+ baseBytes: currentBase.length,+ fragmentChildren: document.getXmlFragment('default').length,+ copies: classification.copies,+ reason: classification.reason,+ }),+ );+ } else if (classification.kind === 'block') {if (tripwireResetFailedDocs.has(documentName)) {log.warn({ documentName },@@ -1716,10 +1862,55 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis} else {diskContent = currentBase;}+ // The reset discards the live document under an origin neither+ // undo stack can reach, so it owes a restore anchor first.+ checkpointBeforeDuplicationReset(+ document,+ documentName,+ markdown,+ classification.copies,+ fragmentChildren,+ );+ // The disk bytes are the shared ancestor the duplicate grew from,+ // so fragment content absent from them is exactly what this reset+ // destroys. That baseline is what makes the paired-intake+ // detector report the discarded copy instead of staying silent —+ // the precondition that lets the other persistence callers omit+ // `detect` (they re-apply bytes the fragment already derives+ // from) is false here by construction.+ const lossRing = options?.getLossRing?.();+ const detect: DeriveLossDetectOptions = {+ baselineFullMd: diskContent,+ report: (obs) => {+ // Never throws, per the reporter contract. This runs INSIDE+ // the reset transact, whose catch latches the per-doc tripwire+ // breaker — a diagnostic failure escaping here would strand+ // the doc on the breaker and log a reset failure for a reset+ // that actually succeeded.+ try {+ const dropped = detectPairedIntakeLoss(obs);+ if (dropped.length === 0) return;+ void lossRing?.record({+ event: LOSS_EVENT_DETECTOR_TRIP,+ docName: documentName,+ writerId: FILE_SYSTEM_WRITER.id,+ direction: 'b',+ site: PERSISTENCE_DUPLICATION_SITE,+ lostLen: dropped.reduce((n, s) => n + s.length, 0),+ digest: fnv1aDigest(dropped.join('\n')),+ });+ } catch (detectErr) {+ log.warn(+ { documentName, err: detectErr },+ '[persistence] duplication-reset loss detection failed',+ );+ }+ },+ };// Caller wraps for atomicity + paired-write origin identity// (precedent #24).document.transact(() => {- applyDiskContent(document, diskContent);+ applyDiskContent(document, diskContent, undefined, undefined, undefined, detect);}, FILE_WATCHER_ORIGIN);tripwireResetFailedDocs.delete(documentName);} catch (err) {@@ -1978,6 +2169,9 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis// Update reconciled base after successful storedurabilityState.setReconciledBase(documentName, markdown);+ // A settled write is what licences the duplication tripwire to stop+ // treating a later doubling as a load-time materialization.+ docsWithSettledWrite.add(documentName);tripwireResetFailedDocs.delete(documentName);persistenceDeferCounts.delete(documentName);@@ -2133,6 +2327,11 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persisconst extension: Extension = {async onLoadDocument({ document, documentName, context: _context }) {+ // A load opens a fresh provider-sync window, which is exactly when the+ // stale-cache duplication the tripwire guards against materializes. Drop+ // any settled-write licence carried over from a previous session so the+ // guard is armed again.+ docsWithSettledWrite.delete(documentName);if (isSystemDoc(documentName)) return;if (isConfigDoc(documentName)) {loadConfigDoc(document, documentName, configPersistenceCtx);diff --git a/packages/server/src/shadow-repo.test.ts b/packages/server/src/shadow-repo.test.tsindex e9b5ce4c4..5ab701742 100644--- a/packages/server/src/shadow-repo.test.ts+++ b/packages/server/src/shadow-repo.test.ts@@ -1581,6 +1581,16 @@ describe('gcCheckpointRefs (bridge-correctness SPEC §6 R7 + review iteration 5)metadata: { atRiskLines: 1, witnessAvailable: true },});return;+ case 'persistence-duplication-reset':+ await saveInMemoryCheckpoint(shadow, 'content/docs', {+ kind,+ docName: 'j.md',+ contents: `j${tag}\n`,+ label: 'l',+ date,+ metadata: { copies: 2, fragmentChildren: 18 },+ });+ return;case 'auto-consolidation':await writeAutoConsolidationCheckpoint(shadow, tag === '' ? 1 : Number(tag));return;@@ -1609,6 +1619,7 @@ describe('gcCheckpointRefs (bridge-correctness SPEC §6 R7 + review iteration 5)maxObserverAApplyLoss: 50,maxBridgeBackstopTrip: 50,maxPersistenceReconcileLoss: 50,+ maxPersistenceDuplicationReset: 50,maxAutoConsolidation: 50,ttlMs: 0,});diff --git a/packages/server/src/shadow-repo.ts b/packages/server/src/shadow-repo.tsindex 677ac5846..d230e22b7 100644--- a/packages/server/src/shadow-repo.ts+++ b/packages/server/src/shadow-repo.ts@@ -910,6 +910,14 @@ export type InMemoryCheckpointParams = (branch?: string;metadata: { rounds: number };}+ | {+ kind: 'persistence-duplication-reset';+ docName: string;+ contents: string;+ label: string;+ branch?: string;+ metadata: { copies: number; fragmentChildren: number };+ }) & {/*** Explicit commit timestamp (any git-parseable date, e.g. ISO 8601), applied@@ -1056,6 +1064,14 @@ async function saveInMemoryCheckpointInner(metadata: params.metadata,};break;+ case 'persistence-duplication-reset':+ parsed = {+ kind: 'persistence-duplication-reset',+ docName: params.docName,+ size,+ metadata: params.metadata,+ };+ break;}const bodyLine = formatCheckpointBodyLine(parsed);const message = `checkpoint: ${params.label}\n\n${bodyLine}`;@@ -1282,6 +1298,15 @@ export interface CheckpointRetentionPolicy {* is the first. Default 50.*/maxPersistenceReconcileLoss: number;+ /**+ * Maximum `persistence-duplication-reset` checkpoints to keep per branch.+ * Written when the structural-duplication tripwire refuses a doubled store+ * and resets the live document from disk. Its own budget so a client stuck+ * re-materializing a duplicate every debounce cannot evict merge-drop or+ * serializer-corruption recovery anchors — the budget is the second line of+ * defense; the mint-side dedup in `persistence.ts` is the first. Default 50.+ */+ maxPersistenceDuplicationReset: number;/*** Maximum `auto-consolidation` checkpoints to keep per branch.* These are service-authored when dead WIP chains are folded; left unbounded@@ -1313,6 +1338,7 @@ export const DEFAULT_CHECKPOINT_RETENTION: CheckpointRetentionPolicy = {maxObserverAApplyLoss: 50,maxBridgeBackstopTrip: 50,maxPersistenceReconcileLoss: 50,+ maxPersistenceDuplicationReset: 50,maxAutoConsolidation: 2,ttlMs: 30 * 24 * 60 * 60 * 1000,};@@ -1328,6 +1354,7 @@ export interface CheckpointGcResult {deletedObserverAApplyLoss: number;deletedBridgeBackstopTrip: number;deletedPersistenceReconcileLoss: number;+ deletedPersistenceDuplicationReset: number;deletedAutoConsolidation: number;retained: number;}@@ -1397,6 +1424,11 @@ const GC_BUCKET_POLICY = {counter: 'deletedPersistenceReconcileLoss',applyTtl: true,},+ 'persistence-duplication-reset': {+ limit: (p) => p.maxPersistenceDuplicationReset,+ counter: 'deletedPersistenceDuplicationReset',+ applyTtl: true,+ },// Count-only: TTL must never be able to reap every surviving// auto-checkpoint, or the chained consolidated history it anchors becomes// unreachable.@@ -1447,6 +1479,7 @@ async function gcCheckpointRefsInner(deletedObserverAApplyLoss: 0,deletedBridgeBackstopTrip: 0,deletedPersistenceReconcileLoss: 0,+ deletedPersistenceDuplicationReset: 0,deletedAutoConsolidation: 0,retained: 0,};