Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

convergence PR A (wip): content-evidence direction for git-gap, rows-behind and conversion; approval across time

garrytan115e231cd4f9
diff --git a/src/commands/sources-canonical.ts b/src/commands/sources-canonical.ts
index 874f1dea7e..3f63c19b6e 100644
--- a/src/commands/sources-canonical.ts
+++ b/src/commands/sources-canonical.ts
@@ -10,22 +10,26 @@ import { OperationError, opError } from '../core/ops/contract.ts';
import { cliRenderContext, toAgentError } from '../core/agent-output.ts';
import { registerSignalOwner, triggerCleanupAndExit } from '../core/process-cleanup.ts';
import { startMemoryTrace } from '../core/memory-trace.ts';
-import { applyGitConversion, applyMixedConversion, assertConversionCaller, CONVERSION_GROUP_MAX, conversionIncompleteFix, planGitConversion, planMixedConversion,
- type ConversionProgress, type GitConversionPreview, type GitConversionResult, type MixedConversionPreview } from '../core/persistence/git-conversion.ts';
+import { applyApprovedConversion, applyGitConversion, applyMixedConversion, assertConversionCaller, CONVERSION_GROUP_MAX, conversionApproval, conversionIncompleteFix, conversionRecovery,
+ conversionReviewList, planGitConversion, planMixedConversion, readConversionApproval, readConversionDecisions, writeConversionFile,
+ type ConversionProgress, type ConversionRecovery, type GitConversionPreview, type GitConversionResult, type MixedConversionPreview } from '../core/persistence/git-conversion.ts';
-const USAGE = 'Usage: gbrain sources canonical <source-id> --to git|mixed [--dry-run | --apply --expect <hash> [--no-embed] [--no-extract]] [--json]';
+const USAGE = 'Usage: gbrain sources canonical <source-id> --to git|mixed [--dry-run [--list-out <file>] [--approval-out <file>] | --apply (--expect <hash> | --approved <file>) [--no-embed] [--no-extract]] [--decisions <file>] [--json]';
const DOCS = 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion';
/** How often a running apply says where it stands (stderr), besides each change of phase. */
export const PROGRESS_EVERY_MS = 10_000;
const SIGNAL_EXIT: Record<string, number> = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 };
-export function parseCanonicalArgs(args: string[]): { sourceId: string; to: 'git' | 'mixed'; apply: boolean; expect: string | null; json: boolean; noEmbed: boolean; noExtract: boolean } {
+export function parseCanonicalArgs(args: string[]): { sourceId: string; to: 'git' | 'mixed'; apply: boolean; expect: string | null; json: boolean; noEmbed: boolean; noExtract: boolean;
+ decisions: string | null; listOut: string | null; approvalOut: string | null; approved: string | null } {
const positional: string[] = [];
let to: string | null = null, expect: string | null = null, apply = false;
+ const files: Record<'--decisions' | '--list-out' | '--approval-out' | '--approved', string | null> = { '--decisions': null, '--list-out': null, '--approval-out': null, '--approved': null };
for (let i = 0; i < args.length; i++) {
const arg = args[i]!;
if (arg === '--to') { to = args[++i] ?? null; continue; }
if (arg === '--expect') { expect = args[++i] ?? null; if (!expect) throw new OperationError('invalid_params', '--expect needs the preview hash.', USAGE); continue; }
+ if (arg in files) { const value = args[++i]; if (!value || value.startsWith('-')) throw new OperationError('invalid_params', `${arg} needs a file path.`, USAGE); files[arg as keyof typeof files] = value; continue; }
if (arg === '--apply') { apply = true; continue; }
if (arg === '--dry-run' || arg === '--json' || arg === '--no-embed' || arg === '--no-extract') continue;
if (arg.startsWith('-')) throw new OperationError('invalid_params', `Unknown sources canonical flag: ${arg}.`, USAGE);
@@ -34,8 +38,14 @@ export function parseCanonicalArgs(args: string[]): { sourceId: string; to: 'git
if (positional.length !== 1) throw new OperationError('invalid_params', 'sources canonical needs exactly one source id.', USAGE);
if (to !== 'git' && to !== 'mixed') throw new OperationError('invalid_params', `--to needs git or mixed; got ${to ?? 'nothing'}.`, USAGE);
if (apply && args.includes('--dry-run')) throw new OperationError('invalid_params', '--apply and --dry-run exclude each other.', USAGE);
- if (apply && !expect) throw new OperationError('invalid_params', '--apply needs --expect <hash> from the --dry-run preview.', USAGE);
- return { sourceId: positional[0]!, to, apply, expect, json: args.includes('--json'), noEmbed: args.includes('--no-embed'), noExtract: args.includes('--no-extract') };
+ const approved = files['--approved'];
+ if (apply && !expect && !approved) throw new OperationError('invalid_params', '--apply needs --expect <hash> from the --dry-run preview, or --approved <file> from its --approval-out.', USAGE);
+ if (expect && approved) throw new OperationError('invalid_params', '--expect and --approved exclude each other: an approval binds the pages, not the hash.', USAGE);
+ if (!apply && approved) throw new OperationError('invalid_params', '--approved is applied with --apply.', USAGE);
+ if (apply && (files['--list-out'] || files['--approval-out'])) throw new OperationError('invalid_params', '--list-out and --approval-out are written by the --dry-run preview.', USAGE);
+ if (to === 'mixed' && Object.values(files).some(Boolean)) throw new OperationError('invalid_params', '--decisions, --list-out, --approval-out and --approved belong to --to git.', USAGE);
+ return { sourceId: positional[0]!, to, apply, expect, json: args.includes('--json'), noEmbed: args.includes('--no-embed'), noExtract: args.includes('--no-extract'),
+ decisions: files['--decisions'], listOut: files['--list-out'], approvalOut: files['--approval-out'], approved };
}
const counts = (record: Record<string, number>) => Object.entries(record).filter(([, n]) => n > 0).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
@@ -54,18 +64,33 @@ export function describeGitPreview(p: GitConversionPreview): string {
`Parked effects: ${p.parked_effects.count} (retry ${p.parked_effects.retry}, reclassify as declared ${p.parked_effects.reclassify}; ${counts(p.parked_effects.by_code)}).`,
`The strict pass also imports ${p.index.imports} and deletes ${p.index.deletes} page(s) where the database holds nothing Git lacks.`,
`Database content that would be discarded: ${p.discard.count}${p.discard.count ? ` (${counts(p.discard.by)}); the apply refuses until it is 0` : ''}.`,
+ ...(p.evidence.count ? [`Pages with no import record, by content evidence: ${counts(p.evidence.by)}${p.discard.by.no_record_both_moved + p.discard.by.no_record_no_evidence
+ ? `; ${p.discard.by.no_record_both_moved + p.discard.by.no_record_no_evidence} wait on a decision (--list-out <file> writes the review list; fill in "decision" and pass it back with --decisions <file>)` : ''}.`] : []),
+ ...(p.decisions.digest ? [`Decisions: ${p.decisions.applied} applied, ${p.decisions.stale} stale (the page moved since), ${p.decisions.unused} unused${p.decisions.sample.length ? ` (${p.decisions.sample.join(', ')})` : ''}.`] : []),
...(causes.length ? [` By cause: ${causes.join(', ')}.`] : []), ...sample(p.discard.sample)];
lines.push(p.remaining === 0 && p.strict ? 'Nothing is left to convert.' : `Nothing changed. To apply: ${p.apply.join(' ')}`);
return lines.join('\n');
}
function describeResult(r: GitConversionResult): string {
+ if (r.status === 'delta') return [`Source ${r.source_id} is still mixed: ${r.committed.pages} unchanged approved page(s) committed in ${r.committed.groups} door commit(s), ${r.withheld.pages} withheld; ${r.delta!.count} item(s) left unapplied (${counts(r.delta!.by)}).`,
+ ...r.delta!.items.slice(0, 20).map(item => ` ${item.reason}: ${item.item}`), 'Relay this list to the owner; after their decision preview again with --approval-out and apply the new approval.'].join('\n');
const lines = [`Source ${r.source_id} ${r.status === 'converted' ? 'is git-canonical' : 'is git-canonical, with work left'}: ${r.committed.pages} page(s) committed in ${r.committed.groups} door commit(s), ${r.withheld.pages} private page(s) withheld (${r.withheld.commits.length} removal commit(s)), ${r.holds_cleared} hold(s) cleared, ${r.parked.retried} parked effect(s) retried and ${r.parked.reclassified} reclassified.`];
if (r.push.state === 'pending') lines.push(`Push pending (${r.push.code}): ${r.push.notice}`);
+ if (r.reclassified.length) lines.push(`Held instead of overwritten (the page moved since the plan): ${r.reclassified.length} (${r.reclassified.slice(0, 20).join(', ')}).`);
if (r.status === 'incomplete') lines.push(describeGitPreview(r.after));
return lines.join('\n');
}
+/** Item 6: where a failed apply left the source, and the non-destructive way back. */
+function describeRecovery(r: ConversionRecovery): string {
+ return [`Stopped in ${r.phase} with the source ${r.mode === 'git' ? 'git-canonical' : 'mixed'}: ${r.commits.length} door commit(s) made${r.commits.length ? ` (${r.commits.map(sha => sha.slice(0, 12)).join(', ')})` : ''}, push ${r.push?.state ?? 'not attempted'}; `
+ + `accepted writes: ${r.accepted.adopted} adopted, ${r.accepted.withheld} withheld, ${r.accepted.holds_cleared} hold(s) cleared, ${r.accepted.pass_writes} page write(s) by the strict passes.`,
+ ...(r.reclassified.length ? [`Held instead of overwritten: ${r.reclassified.slice(0, 20).join(', ')}.`] : []),
+ r.rollback.length ? `Rollback (nothing is rewritten; preview each, then apply its hash): ${r.rollback.map(argv => argv.join(' ')).join(', then ')}.`
+ : 'No rollback is needed: the source is still mixed, and the commits made hold the bytes the database already has.'].join('\n');
+}
+
function describeMixed(p: MixedConversionPreview): string {
return p.strict ? `Source ${p.source_id} is git-canonical; --to mixed removes canonical: git and changes nothing else (Git keeps every door commit).\nNothing changed. To apply: ${p.apply.join(' ')}`
: `Source ${p.source_id} is not git-canonical; there is nothing to change.`;
@@ -136,9 +161,31 @@ export async function runSourcesCanonical(engine: BrainEngine, args: string[]):
print(done, done.status === 'mixed' ? `Source ${parsed.sourceId} is mixed: canonical: git removed.` : `Source ${parsed.sourceId} was not git-canonical; nothing changed.`);
return;
}
- if (!parsed.apply) { const { preview } = await planGitConversion(engine, parsed.sourceId); print(preview, describeGitPreview(preview)); return; }
- const result = await withConversionReport(engine, parsed.sourceId, json,
- onProgress => applyGitConversion(engine, parsed.sourceId, parsed.expect!, { noEmbed: parsed.noEmbed, noExtract: parsed.noExtract, onProgress }));
+ const decisions = parsed.decisions ? await readConversionDecisions(engine, parsed.decisions, parsed.sourceId) : undefined;
+ if (!parsed.apply) {
+ const plan = await planGitConversion(engine, parsed.sourceId, { decisions });
+ const written = {
+ ...(parsed.listOut ? { review_list: await writeConversionFile(engine, parsed.listOut, '--list-out', await conversionReviewList(engine, plan)) } : {}),
+ ...(parsed.approvalOut ? { approval: await writeConversionFile(engine, parsed.approvalOut, '--approval-out', conversionApproval(plan)) } : {}) };
+ const files = Object.entries(written).map(([kind, path]) => `Wrote the ${kind.replace('_', ' ')} (mode 0600) to ${path}.`);
+ print({ ...plan.preview, ...(Object.keys(written).length ? { written } : {}) }, [describeGitPreview(plan.preview), ...files].join('\n'));
+ return;
+ }
+ const processing = { noEmbed: parsed.noEmbed, noExtract: parsed.noExtract, decisions };
+ const approval = parsed.approved ? await readConversionApproval(engine, parsed.approved, parsed.sourceId) : null;
+ const result = await withConversionReport(engine, parsed.sourceId, json, onProgress => approval
+ ? applyApprovedConversion(engine, parsed.sourceId, approval, { ...processing, onProgress })
+ : applyGitConversion(engine, parsed.sourceId, parsed.expect!, { ...processing, onProgress }));
+ if (result.status === 'delta') {
+ const error = opError('git_conversion_delta', `Source ${parsed.sourceId} is still mixed: ${result.delta!.count} item(s) changed since the approval or were not approved (${counts(result.delta!.by)}); only the unchanged approved pages were committed.`,
+ 'Relay the delta list and its counts to the owner; once they decide, preview again with --approval-out and apply the new approval.',
+ { docs: 'docs/guides/git-canonical.md#approval-across-time', detail: Object.keys(result.delta!.by).join(','), fix: { argv: ['gbrain', 'sources', 'canonical', parsed.sourceId, '--to', 'git', '--dry-run', '--json'],
+ consent: [], actor: 'user', requires_exclusive: false, why: 'Lists what is left; read-only.',
+ user_message: `${result.delta!.count} item(s) of ${parsed.sourceId} changed since you approved the conversion or were never approved, so they were not applied. Which should apply?` } });
+ print({ status: 'delta', result, ...toAgentError(error, { transport: 'cli', command: 'sources canonical', render: cliRenderContext() }) }, describeResult(result));
+ setCliExitVerdict(1);
+ return;
+ }
if (result.status === 'incomplete') {
const closing = result.closing_passes ? ` and ${result.closing_passes} closing pass(es), with Git still ahead at ${result.after.git_wins.count} path(s)` : '';
const error = opError('git_conversion_incomplete', `Source ${parsed.sourceId} is git-canonical, and ${result.after.remaining} item(s) were still open after its strict passes${closing}.`,
@@ -153,8 +200,9 @@ export async function runSourcesCanonical(engine: BrainEngine, args: string[]):
if (!(error instanceof OperationError)) throw error;
const env = toAgentError(error, { transport: 'cli', command: 'sources canonical', render: cliRenderContext() });
const at = stoppedAt.get(error);
- console.log(json ? JSON.stringify({ status: 'refused', ...env, ...(at ? { progress: at } : {}) })
- : `Conversion refused (${error.code}): ${error.message}\nFix: ${error.suggestion ?? USAGE}${env.fix?.command ? `\nNext: ${env.fix.command}` : ''}`);
+ const recovery = conversionRecovery(error);
+ console.log(json ? JSON.stringify({ status: 'refused', ...env, ...(at ? { progress: at } : {}), ...(recovery ? { recovery } : {}) })
+ : `Conversion refused (${error.code}): ${error.message}\nFix: ${error.suggestion ?? USAGE}${env.fix?.command ? `\nNext: ${env.fix.command}` : ''}${recovery ? `\n${describeRecovery(recovery)}` : ''}`);
setCliExitVerdict(1);
}
}
diff --git a/src/commands/sources-git-gap.ts b/src/commands/sources-git-gap.ts
index 6a4c1b3679..e6c52d00c8 100644
--- a/src/commands/sources-git-gap.ts
+++ b/src/commands/sources-git-gap.ts
@@ -26,8 +26,8 @@ import { isSourceDbOnlySlug } from '../core/persistence/source-storage.ts';
import { GIT_HOLD_OP, readSyncImportProvenances } from '../core/persistence/sync-holds.ts';
import { DERIVED_LINK_SOURCES } from '../core/persistence/twin-merge.ts';
import { scannerSlugRootMode, scannerSourcePath } from '../core/write-through.ts';
-import { compareInlineExportFile, compareSegmentedExportBlob, DECLARED_CLASSES, declaredDatabaseOnly, ignoredPaths, readBlobs, readHeadTree, worktreeBlobOid,
- type DeclaredClass, type ExportFileComparison, type HeadTree, type TreeBlob, type UnparseableReason } from '../core/persistence/git-gap.ts';
+import { compareInlineExportFile, compareSegmentedExportBlob, DECLARED_CLASSES, declaredDatabaseOnly, DIRECTION_EVIDENCE, ignoredPaths, readBlobs, readDirectionEvidence, readHeadTree, worktreeBlobOid,
+ type DeclaredClass, type DirectionEvidence, type Evidence, type ExportFileComparison, type HeadTree, type TreeBlob, type UnparseableReason } from '../core/persistence/git-gap.ts';
const USAGE = 'Usage: gbrain sources git-gap <source-id> [--sample <n>] [--json]';
export const GIT_GAP_DOCS = 'docs/designs/GIT_CANONICAL_MEMORY.md#the-git-gap-census';
@@ -42,9 +42,12 @@ export type WorktreeState = typeof WORKTREE_STATES[number];
export interface GitGapPage { id: string; slug: string; revision: string | null; path: string | null; state: PageState; segmented: boolean;
declared?: DeclaredClass; worktree?: WorktreeState; headBlob?: string; databaseAlsoMoved?: boolean; unparseable?: UnparseableReason;
/** A declared page: whether its row records a file (`source_path`), and the size of its committed blob. */
- recordsFile?: boolean; headSize?: number }
+ recordsFile?: boolean; headSize?: number;
+ /** A page that differs from its committed file with no import record, read with `evidence`: which side moved (persistence/git-gap.ts readDirectionEvidence). */
+ evidence?: Evidence }
-export interface GitGapSample { slug: string; path: string; state: PageState; fields?: string[]; declared?: DeclaredClass; basis?: 'import_record' | 'no_import_record'; worktree?: WorktreeState }
+export interface GitGapSample { slug: string; path: string; state: PageState; fields?: string[]; declared?: DeclaredClass; basis?: 'import_record' | 'no_import_record'; worktree?: WorktreeState;
+ evidence?: DirectionEvidence }
export interface GitGapCensus {
status: 'census';
@@ -54,13 +57,24 @@ export interface GitGapCensus {
head: string | null;
/** False when a doctor budget stopped the page scan; the counts cover the pages read so far. */
complete: boolean;
+ /** Pages the scan read; equals `live_pages` only when `complete`. */
pages: number;
+ /** The source's live pages, counted in SQL before the scan, so a partial scan's `pages` reads against the whole source. */
+ live_pages: number;
states: Record<PageState, number>;
declared: Record<DeclaredClass, number>;
/** Declared pages whose file is still committed at HEAD (for example a page that turned private). */
declared_with_head_blob: number;
- /** database_ahead pages with no import record, so the side that moved is unknown (see the design doc). */
+ /**
+ * Pages that differ from their committed file with no import record. Without `evidence` they are all database_ahead and
+ * the side that moved is unknown; with it, `direction_evidence` splits them.
+ */
database_ahead_without_import_record: number;
+ /**
+ * d4: those pages by the side content evidence says moved (adopt_db_moved and no_evidence are database_ahead,
+ * git_wins_file_moved is git_ahead, both_moved is git_ahead with the database also moved); null when evidence was not read.
+ */
+ direction_evidence: Record<DirectionEvidence, number> | null;
/** git_ahead pages whose row also moved since its last import: both sides changed. */
git_ahead_database_also_moved: number;
/** Pages of a source without a checkout that are not declared. Since G7a such a source is declared `database_only_source`, so this stays 0; kept for readers of the JSON. */
@@ -109,8 +123,8 @@ type Key = { id: string; slug: string; source_id: string; source_path: string |
const zeros = <K extends string>(keys: readonly K[]) => Object.fromEntries(keys.map(key => [key, 0])) as Record<K, number>;
function emptyCensus(sourceId: string, root: string | null, head: HeadTree | null): GitGapCensus {
- return { status: 'census', source_id: sourceId, checkout: root, head: head?.commit ?? null, complete: true, pages: 0, states: zeros(PAGE_STATES),
- declared: zeros(DECLARED_CLASSES), declared_with_head_blob: 0, database_ahead_without_import_record: 0, git_ahead_database_also_moved: 0, not_compared: 0,
+ return { status: 'census', source_id: sourceId, checkout: root, head: head?.commit ?? null, complete: true, pages: 0, live_pages: 0, states: zeros(PAGE_STATES),
+ declared: zeros(DECLARED_CLASSES), declared_with_head_blob: 0, database_ahead_without_import_record: 0, direction_evidence: null, git_ahead_database_also_moved: 0, not_compared: 0,
worktree: root ? zeros(WORKTREE_STATES) : null,
rows_without_git_home: { facts_unfenced: 0, facts_fence_page_missing: 0, facts_on_pages_without_file: { undeclared: 0, declared: 0 },
links: { total: 0, by_link_source: {} }, timeline_on_pages_without_file: { undeclared: 0, declared: 0 } },
@@ -123,8 +137,12 @@ interface Checkout { root: string; head: HeadTree; mode: Awaited<ReturnType<type
const refuse = (message: string, suggestion: string) => new OperationError('storage_error', message, suggestion, GIT_GAP_DOCS);
-/** `collect` receives each classified page, with the export snapshot the census compared for a database_ahead or git_ahead page. */
-export async function censusGitGap(engine: BrainEngine, sourceId: string, opts: { sample: number; deadlineAt?: number; collect?: GitGapCollect }): Promise<GitGapCensus> {
+/**
+ * `collect` receives each classified page, with the export snapshot the census compared for a database_ahead or git_ahead
+ * page (a page read with `evidence` comes without it, after the scan). `evidence` (d4) reads which side moved for the
+ * pages that differ with no import record, in one history walk after the scan; a scan the deadline stopped reads none.
+ */
+export async function censusGitGap(engine: BrainEngine, sourceId: string, opts: { sample: number; deadlineAt?: number; collect?: GitGapCollect; evidence?: boolean }): Promise<GitGapCensus> {
await resolveSourceId(engine, sourceId, undefined, { skipLocalSignals: true });
const [source] = await engine.executeRaw<{ local_path: string | null; incarnation: string }>('SELECT local_path, incarnation::text AS incarnation FROM sources WHERE id=$1', [sourceId]);
const root = source?.local_path && existsSync(source.local_path) ? source.local_path : null;
@@ -140,6 +158,9 @@ export async function censusGitGap(engine: BrainEngine, sourceId: string, opts:
checkout = { root, head, mode: await scannerSlugRootMode(engine, sourceId, root), scope };
}
const census = emptyCensus(sourceId, root, checkout?.head ?? null);
+ const [live] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND deleted_at IS NULL', [sourceId]);
+ census.live_pages = Number(live?.n ?? 0);
+ const pending: Unrecorded[] = [];
// G7a: a source with no checkout is a database-only source, one declared class for all its pages (outside strict mode).
const policy = { mirror, databaseOnlySource: root === null, dbOnly: (slug: string) => root !== null && isSourceDbOnlySlug(root, slug, 'refuse'), ignored: new Set<string>() };
for (let cursor = '0'; ;) {
@@ -149,11 +170,15 @@ export async function censusGitGap(engine: BrainEngine, sourceId: string, opts:
const batch = await tx.executeRaw<Key>(`SELECT id::text, slug, source_id, source_path, octet_length(source_path) AS source_path_bytes, source_uri, withheld_source_path, type, frontmatter,
database_only_reason, body_storage='segmented' AS segmented, knowledge_revision::text AS revision, NOT ${privacyPartSql('pages')} AS lineage_private
FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND pages.id > $2::bigint ORDER BY pages.id LIMIT ${BATCH}`, [sourceId, cursor]);
- if (batch.length) { await censusBatch(tx, batch, census, { checkout, policy, incarnation: source!.incarnation, sample: opts.sample, collect: opts.collect }); cursor = batch[batch.length - 1]!.id; }
+ if (batch.length) {
+ await censusBatch(tx, batch, census, { checkout, policy, incarnation: source!.incarnation, sample: opts.sample, collect: opts.collect, pending: opts.evidence ? pending : undefined });
+ cursor = batch[batch.length - 1]!.id;
+ }
return batch.length < BATCH;
});
if (done) break;
}
+ if (pending.length) await settleUnrecorded(engine, census, checkout!, pending, { sample: opts.sample, collect: opts.collect, read: census.complete });
await engine.transaction(async tx => {
await tx.executeRaw('SET TRANSACTION READ ONLY');
await countSourceRows(tx, census);
@@ -164,7 +189,34 @@ export async function censusGitGap(engine: BrainEngine, sourceId: string, opts:
export type GitGapCollect = (page: GitGapPage, snapshot?: ExportSnapshot) => void;
interface BatchContext { checkout: Checkout | null; policy: { mirror: boolean; databaseOnlySource: boolean; dbOnly: (slug: string) => boolean; ignored: Set<string> }; incarnation: string; sample: number;
- collect?: GitGapCollect }
+ collect?: GitGapCollect; pending?: Unrecorded[] }
+
+/** A page that differs from its committed file with no import record, kept for the evidence read after the scan. */
+interface Unrecorded { key: Key; rel: string; path: string; blob: TreeBlob; worktree: WorktreeState; fields: string[] }
+
+/**
+ * d4: the side each unrecorded page moved on, from content evidence (readDirectionEvidence), counted and collected as
+ * its state. With `read` false (a scan the deadline stopped) no evidence is read and each stays database_ahead, as before.
+ */
+async function settleUnrecorded(engine: BrainEngine, census: GitGapCensus, checkout: Checkout, pending: Unrecorded[],
+ opts: { sample: number; collect?: GitGapCollect; read: boolean }): Promise<void> {
+ const evidence = opts.read ? await readDirectionEvidence(engine, { sourceId: census.source_id, root: checkout.root, gitRoot: checkout.head.gitRoot, scope: checkout.scope,
+ tip: checkout.head.commit!, format: checkout.head.format, mode: checkout.mode,
+ subjects: pending.map(item => ({ id: item.key.id, slug: item.key.slug, revision: item.key.revision, rel: item.rel, importPath: scannerSourcePath(checkout.root, item.path, checkout.mode),
+ segmented: item.key.segmented === true, blob: item.blob })) }) : null;
+ if (evidence) census.direction_evidence = zeros(DIRECTION_EVIDENCE);
+ for (const item of pending) {
+ const found = evidence?.get(item.key.id);
+ const state: PageState = found?.kind === 'git_wins_file_moved' || found?.kind === 'both_moved' ? 'git_ahead' : 'database_ahead';
+ const databaseAlsoMoved = found?.kind === 'both_moved';
+ census.states[state]++;
+ census.database_ahead_without_import_record++;
+ if (found) census.direction_evidence![found.kind]++;
+ if (databaseAlsoMoved) census.git_ahead_database_also_moved++;
+ opts.collect?.(collected(item.key, item.rel, state, { worktree: item.worktree, headBlob: item.blob.oid, databaseAlsoMoved, ...(found ? { evidence: found } : {}) }));
+ note(census, opts.sample, { slug: item.key.slug, path: item.rel, state, fields: item.fields, basis: 'no_import_record', worktree: item.worktree, ...(found ? { evidence: found.kind } : {}) });
+ }
+}
const collected = (key: Key, path: string | null, state: PageState, extra: Partial<GitGapPage> = {}): GitGapPage =>
({ id: key.id, slug: key.slug, revision: key.revision, path, state, segmented: key.segmented === true, ...extra });
@@ -254,7 +306,7 @@ async function compareBatch(tx: BrainEngine, compare: Array<{ key: Key; path: st
if (!blobs) throw refuse(`Git could not read committed files of the checkout at ${checkout.root}.`,
`Nothing changed. Check the checkout with git -C ${checkout.root} fsck, then rerun gbrain sources git-gap ${sourceId}.`);
const withdrawals: ExportWithdrawal[] = await readExportWithdrawals(tx, sourceId);
- const differs: Array<{ key: Key; rel: string; blob: TreeBlob; worktree: WorktreeState; fields: string[]; snapshot: ExportSnapshot }> = [];
+ const differs: Array<{ key: Key; path: string; rel: string; blob: TreeBlob; worktree: WorktreeState; fields: string[]; snapshot: ExportSnapshot }> = [];
const snapshots = await readExportPages(tx, compare.map(item => item.key), withdrawals);
for (const [i, item] of compare.entries()) {
const snapshot = snapshots[i]!;
@@ -274,6 +326,7 @@ async function compareBatch(tx: BrainEngine, compare: Array<{ key: Key; path: st
for (const item of differs) {
const record = records.get(Number(item.key.id));
const imported = [record?.blob_oid, record?.disk_blob_oid].filter((oid): oid is string => typeof oid === 'string' && oid.length > 0);
+ if (!imported.length && ctx.pending) { ctx.pending.push({ key: item.key, rel: item.rel, path: item.path, blob: item.blob, worktree: item.worktree, fields: item.fields }); continue; }
const gitMoved = imported.length > 0 && !imported.includes(item.blob.oid);
const state: PageState = gitMoved ? 'git_ahead' : 'database_ahead';
census.states[state]++;
@@ -329,11 +382,12 @@ async function countSourceRows(tx: BrainEngine, census: GitGapCensus): Promise<v
const counts = (record: Record<string, number>) => Object.entries(record).filter(([, n]) => n > 0).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
export function describeGitGap(census: GitGapCensus): string {
- const lines = [`Git gap of source ${census.source_id} (read-only): ${census.pages} live page(s)${census.head ? `, compared with HEAD ${census.head.slice(0, 12)}` : ''}.`];
+ const lines = [`Git gap of source ${census.source_id} (read-only): ${census.complete ? `${census.pages} live page(s)` : `PARTIAL, ${census.pages} of ${census.live_pages} live page(s) read`}${census.head ? `, compared with HEAD ${census.head.slice(0, 12)}` : ''}.`];
if (!census.checkout) lines.push(`The source has no checkout: a database-only source, so its page(s) are declared database_only_source and none were compared.`);
else if (!census.head) lines.push('The checkout has no commit yet, so no page has a committed file.');
lines.push(`Pages: ${counts(census.states)}.`, `Declared database-only: ${counts(census.declared)}${census.declared_with_head_blob ? ` (${census.declared_with_head_blob} still committed at HEAD)` : ''}.`);
- if (census.database_ahead_without_import_record) lines.push(`${census.database_ahead_without_import_record} database_ahead page(s) have no import record, so the side that moved is unknown.`);
+ if (census.database_ahead_without_import_record && census.direction_evidence) lines.push(`${census.database_ahead_without_import_record} differing page(s) have no import record; by content evidence: ${counts(census.direction_evidence)}.`);
+ else if (census.database_ahead_without_import_record) lines.push(`${census.database_ahead_without_import_record} database_ahead page(s) have no import record, so the side that moved is unknown.`);
if (census.git_ahead_database_also_moved) lines.push(`${census.git_ahead_database_also_moved} git_ahead page(s) also changed in the database since their last import.`);
if (census.worktree) lines.push(`Working tree against HEAD: ${counts(census.worktree)}.`);
const rows = census.rows_without_git_home;
@@ -343,9 +397,9 @@ export function describeGitGap(census: GitGapCensus): string {
+ `(+${rows.timeline_on_pages_without_file.declared} declared).`);
lines.push(`Open sync holds: ${counts(census.holds)}. Parked effect targets: ${census.parked_effects.map(p => `${p.kind} ${p.code} ${p.targets}`).join(', ') || 'none'}.`);
for (const item of census.sample) {
- lines.push(` ${item.state}${item.declared ? ` (${item.declared})` : ''}: ${item.slug} (${item.path})${item.fields ? ` fields ${item.fields.join(', ')}` : ''}${item.worktree ? ` worktree ${item.worktree}` : ''}`);
+ lines.push(` ${item.state}${item.declared ? ` (${item.declared})` : ''}${item.evidence ? ` (${item.evidence})` : ''}: ${item.slug} (${item.path})${item.fields ? ` fields ${item.fields.join(', ')}` : ''}${item.worktree ? ` worktree ${item.worktree}` : ''}`);
}
- if (!census.complete) lines.push('PARTIAL: the scan stopped at its time budget, so the counts cover only the pages read.');
+ if (!census.complete) lines.push(`PARTIAL: the scan stopped at its time budget after ${census.pages} of ${census.live_pages} live page(s), so the counts cover only the pages read.`);
lines.push(census.next);
return lines.join('\n');
}
@@ -355,7 +409,7 @@ export async function runSourcesGitGap(engine: BrainEngine, args: string[]): Pro
if (args.includes('--help') || args.includes('-h')) { console.log(`${USAGE}\nRead-only: compares each page of the source with its committed file at HEAD (${GIT_GAP_DOCS}).`); return; }
try {
const parsed = parseGitGapArgs(args);
- const census = await censusGitGap(engine, parsed.sourceId, { sample: parsed.sample });
+ const census = await censusGitGap(engine, parsed.sourceId, { sample: parsed.sample, evidence: true });
console.log(json ? JSON.stringify(census) : describeGitGap(census));
} catch (error) {
if (!(error instanceof OperationError)) throw error;
diff --git a/src/commands/sync.ts b/src/commands/sync.ts
index 0d259c5ea6..c8fde25d5a 100644
--- a/src/commands/sync.ts
+++ b/src/commands/sync.ts
@@ -216,6 +216,8 @@ export interface SyncOpts {
canonicalOverride?: 'git';
/** #7866, internal (the conversion's strict passes): a full round without a `--full` episode, so a page equal to its blob is not re-imported. */
fullRound?: boolean;
+ /** PR A, internal (the conversion's strict passes): the Git verdicts over database versions the plan approved; any other is held (sync-pass.ts ConversionBinding). */
+ conversionBinding?: import('../core/persistence/sync-pass.ts').ConversionBinding;
/**
* v0.18.0 Step 5 — sync a specific named source. When set, sync reads
* local_path + last_commit from the sources table (not the global
diff --git a/src/core/error-registry.ts b/src/core/error-registry.ts
index dbf85164b5..841e148957 100644
--- a/src/core/error-registry.ts
+++ b/src/core/error-registry.ts
@@ -378,7 +378,9 @@ export const CODES = {
git_target_unsafe: { class: 'caller', summary: "Git cannot identify the existing canonical file by its native spelling." },
git_unavailable: { class: 'retryable', summary: "A git command did not complete." },
git_object_missing: { class: 'caller', summary: "A committed file's Git object is missing from the checkout and its working-tree copy differs or is gone, so managed sync holds that file alone.", why: "An interrupted gc, a partial clone without its promisor remote or disk corruption can drop an object a commit still names. Sync imports the working-tree bytes when they hash to the committed blob id; otherwise it imports nothing from a guess, and the rest of the source keeps syncing.", docs: 'docs/guides/write-refusals.md#git_object_missing', suggestion: 'Run git fsck in the checkout to see what is missing, restore it with git fetch (a partial or shallow clone) or from a backup, then sync again; every sync re-tests the hold.' },
- git_conversion_blocked: { class: 'consent', actor: 'user', summary: "Converting a source to git-canonical would discard database content or cannot place it, so the apply wrote nothing.", reasons: ['both_moved', 'held_drift_both', 'held_drift_unknown_differs', 'deleted_in_git', 'uncommitted_edit', 'ignored_target', 'unparseable', 'unsafe_path', 'segmented_page', 'export_mismatch', 'git_overwrites_database', 'ignore_status_unknown'], why: "gbrain sources canonical --to git applies only when every database row will be in Git or in a declared database-only class afterwards. The preview counts what would be lost or cannot be placed by category (detail lists them): both_moved and held_drift_both, a page Git and the database both changed; deleted_in_git, a page Git deleted that the database edited since; uncommitted_edit, a working-tree edit the adoption would overwrite; ignored_target, a page whose file Git ignores without a declared class; unparseable and unsafe_path, a committed file or path sync cannot read; segmented_page, a segmented page conversion does not adopt yet; export_mismatch, a page whose export does not read back equal; git_overwrites_database, a page version a strict pass would replace that nothing accepted; ignore_status_unknown, a target whose ignore status Git cannot read (beyond a symlink, outside the checkout, a timed-out check). There is no override.", docs: 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion', suggestion: 'Relay the categories and their paths (the --dry-run preview lists them) and ask the user how each should be resolved; preview again afterwards.' },
+ git_conversion_blocked: { class: 'consent', actor: 'user', summary: "Converting a source to git-canonical would discard database content or cannot place it, so the apply wrote nothing.", reasons: ['both_moved', 'held_drift_both', 'held_drift_unknown_differs', 'deleted_in_git', 'uncommitted_edit', 'ignored_target', 'unparseable', 'unsafe_path', 'segmented_page', 'export_mismatch', 'git_overwrites_database', 'ignore_status_unknown', 'no_record_both_moved', 'no_record_no_evidence'], why: "gbrain sources canonical --to git applies only when every database row will be in Git or in a declared database-only class afterwards. The preview counts what would be lost or cannot be placed by category (detail lists them): both_moved and held_drift_both, a page Git and the database both changed; no_record_both_moved and no_record_no_evidence, a page with no import record whose content history shows both sides moved, or shows nothing, until the owner's --decisions file decides it; deleted_in_git, a page Git deleted that the database edited since; uncommitted_edit, a working-tree edit the adoption would overwrite; ignored_target, a page whose file Git ignores without a declared class; unparseable and unsafe_path, a committed file or path sync cannot read; segmented_page, a segmented page conversion does not adopt yet; export_mismatch, a page whose export does not read back equal; git_overwrites_database, a page version a strict pass would replace that nothing accepted; ignore_status_unknown, a target whose ignore status Git cannot read (beyond a symlink, outside the checkout, a timed-out check). There is no override.", docs: 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion', suggestion: 'Relay the categories and their paths (the --dry-run preview lists them) and ask the user how each should be resolved; preview again afterwards.' },
+ git_conversion_delta: { class: 'consent', actor: 'user', summary: "An approved conversion found pages that changed since the approval, or decisions it did not approve, so only the unchanged pages were applied and the source stays mixed.", why: "gbrain sources canonical --to git --apply --approved applies only the pages whose slug, revision, committed blob, evidence and action are the ones the owner approved. A page that changed since, a new page, a database edit the plan would now discard, and any decision the approval did not hold stay unapplied and are listed (delta) with counts; nothing outside the approved set is applied without a new decision.", docs: 'docs/guides/git-canonical.md#approval-across-time', suggestion: 'Relay the delta list and its counts to the owner; once they decide, preview again with --approval-out and apply the new approval.' },
+ conversion_file_in_source_root: { class: 'caller', summary: "A conversion review list, decisions file or approval was refused because its path is inside a source's checkout.", why: "Those files name pages and paths. Written or read only outside every registered source root (mode 0600), they can never be synced, imported or committed.", docs: 'docs/guides/git-canonical.md#review-list-and-decisions', suggestion: 'Pass a path outside every source checkout, e.g. under ~/.gbrain.' },
git_conversion_incomplete: { class: 'retryable', summary: "A git-canonical conversion stopped because its strict pass did not finish, or finished with work left after the flag was set.", why: "The conversion runs a full strict-equivalent pass before it sets canonical: git, and one genuine strict pass after, then up to three closing strict passes while only door commits a live writer landed meanwhile are left (Git ahead of the database). A pass that ends partial, blocked or deferred stops the conversion where it is: before the flag the source stays as it was, after it the source stays strict (Git wins) and the plan lists what is left.", docs: 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion', suggestion: 'Wait until gbrain sync status --source <id> --json shows the source idle, then preview again with gbrain sources canonical <id> --to git --dry-run and apply the new hash.' },
git_home_required: { class: 'caller', summary: "A git-canonical source would store content in the database without a file in Git; nothing was written.", reasons: ['no_file_form', 'declared_target', 'facts_lane'], why: "On a source with canonical: git, page content reaches the database only after Git commits its file. The declared database-only classes (private, alias-private and lineage-private pages, storage.db_only directories, never-filed derive-phase pages, paths Git ignores, detached pages and origin twins) are the only exceptions. no_file_form: the write has no file and is in no declared class; declared_target: a publication would commit a file for a declared page; facts_lane: a writer with no file form yet.", docs: 'docs/guides/write-refusals.md#git_home_required', suggestion: 'Ask the user which git home to configure (a path Git tracks, or a storage.db_only directory in gbrain.yml), then submit the write again with a new request_id.' },
git_private_commit_refused: { class: 'caller', summary: "A git-canonical source refused to commit the file of a private, alias-private or lineage-private page; nothing was written and HEAD is unchanged.", reasons: ['private', 'alias_private', 'lineage_private'], why: "Private pages of a source with canonical: git are database-only: their bytes never enter Git history. A page turning private is published as the door commit that removes its file, never as a commit of its bytes.", docs: 'docs/guides/write-refusals.md#git_private_commit_refused', suggestion: 'The page stays database-only. Write it through put_page (which publishes a private page without a file), or remove visibility: private if it belongs in Git, then submit again with a new request_id.' },
diff --git a/src/core/hardened-git.ts b/src/core/hardened-git.ts
index 77d702653d..e39940ef64 100644
--- a/src/core/hardened-git.ts
+++ b/src/core/hardened-git.ts
@@ -90,6 +90,17 @@ export type HardenedGitResult = { ok: true; stdout: Buffer } | { ok: false; reas
export const REASON_LOG_OPTIONS: readonly string[] = ['-z', '--first-parent', '--diff-merges=first-parent', '--root', '--name-only', '--no-renames', '--no-relative',
'--no-ext-diff', '--no-textconv', '--no-show-signature', '--format=%x00%H%x00%(trailers:key=gbrain-reason,valueonly,unfold)'];
+/**
+ * The second `git log` gbrain runs in an untrusted checkout, the direction-evidence history read (persistence/git-gap.ts
+ * `readPathHistory`): a raw listing (modes and full blob ids, no patch) of every commit, merges against their first
+ * parent, with no renames, so only tree entries are compared and no filter, textconv or external diff runs. Every option
+ * is required.
+ */
+export const HISTORY_LOG_OPTIONS: readonly string[] = ['-z', '--raw', '--no-abbrev', '--full-history', '--root', '--diff-merges=first-parent', '--no-renames', '--no-relative',
+ '--no-ext-diff', '--no-textconv', '--no-show-signature', '--format=%x1e%H%x1f%ct'];
+/** Each `log` shape allowed: its options exactly, all of them. */
+const LOG_SHAPES: ReadonlyArray<readonly string[]> = [REASON_LOG_OPTIONS, HISTORY_LOG_OPTIONS];
+
/**
* The exact options each allowed plumbing subcommand may take; anything not
* listed here refuses.
@@ -98,13 +109,13 @@ const ALLOWED_OPTIONS: ReadonlyMap<string, ReadonlySet<string>> = new Map([
['rev-parse', new Set(['--show-toplevel', '--absolute-git-dir', '--verify', '--quiet', '--show-object-format'])],
['ls-files', new Set(['-s', '--stage', '--eol', '-z', '--cached', '--others', '--exclude-standard'])],
['ls-tree', new Set(['-r', '-l', '-z', '--full-tree'])],
- ['cat-file', new Set(['--batch'])],
+ ['cat-file', new Set(['--batch', '--batch-check'])],
['hash-object', new Set(['--no-filters', '--stdin-paths'])],
['check-attr', new Set(['-z', '--stdin'])],
['check-ignore', new Set(['-z', '--stdin'])],
['diff-index', new Set(['--cached', '--quiet'])],
['diff-tree', new Set(['-r', '-z', '--no-renames'])],
- ['log', new Set(REASON_LOG_OPTIONS)],
+ ['log', new Set(LOG_SHAPES.flat())],
]);
/** Options a subcommand must carry, so its output stays raw and machine-readable. */
@@ -112,7 +123,6 @@ const REQUIRED_OPTIONS: ReadonlyMap<string, readonly string[]> = new Map([
['hash-object', ['--no-filters']],
['diff-index', ['--cached']],
['diff-tree', ['-r', '-z', '--no-renames']],
- ['log', REASON_LOG_OPTIONS],
]);
/**
@@ -128,7 +138,8 @@ export function assertHardenedGitArgs(args: readonly string[]): void {
const end = args.findIndex(arg => arg === '--' || arg === '--end-of-options');
const options = args.slice(1, end < 0 ? undefined : end).filter(arg => arg.startsWith('-'));
const ok = allowed !== undefined && options.every(arg => allowed.has(arg))
- && (REQUIRED_OPTIONS.get(sub) ?? []).every(arg => options.includes(arg));
+ && (REQUIRED_OPTIONS.get(sub) ?? []).every(arg => options.includes(arg))
+ && (sub !== 'log' || LOG_SHAPES.some(shape => shape.every(arg => options.includes(arg)) && options.every(arg => shape.includes(arg))));
if (!ok) throw new Error(`hardened git refuses \`git ${args.slice(0, 2).join(' ')}\`: only filter-free plumbing may run in an untrusted checkout (see src/core/hardened-git.ts)`);
}
diff --git a/src/core/persistence/fault-points.ts b/src/core/persistence/fault-points.ts
index 5d8c1bcdc6..9a3b15acb5 100644
--- a/src/core/persistence/fault-points.ts
+++ b/src/core/persistence/fault-points.ts
@@ -104,8 +104,9 @@ export type FaultPoint = `publication:${PublicationBoundary}` | `effect:${Effect
| 'generation:published' | 'generation:retired' | 'generation:activated'
// GBRA-161 PR 1b: as a managed generation member (an overwrite or retirement after its activation committed) is prepared.
| 'generation:member'
- // G8a (git-conversion.ts): after a conversion group's door commit, and after the strict-equivalent pass, before the flag is set.
- | 'conversion:group_committed' | 'conversion:withhold_stamped' | 'conversion:withhold_committed' | 'conversion:before_flip';
+ // G8a (git-conversion.ts): after a conversion group's door commit, after the recheck (before the holds are cleared and the
+ // strict passes run), and after the strict-equivalent pass, before the flag is set.
+ | 'conversion:group_committed' | 'conversion:withhold_stamped' | 'conversion:withhold_committed' | 'conversion:after_recheck' | 'conversion:before_flip';
/**
* `signal` (#6278, `consumer:preparing` only): the preparation's abort signal, so
* a stalling hook can model both the reporter's cases: a preparer that honours
diff --git a/src/core/persistence/git-conversion.ts b/src/core/persistence/git-conversion.ts
index 4a57acde65..ad0862d68b 100644
--- a/src/core/persistence/git-conversion.ts
+++ b/src/core/persistence/git-conversion.ts
@@ -39,7 +39,7 @@
* only a plan with no work left is a completed conversion; otherwise the source stays strict (Git wins) and the
* remaining categories are reported. `--to mixed` removes the flag and changes nothing else.
*/
-import { lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { chmodSync, lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, relative, resolve, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts, SyncResult } from '../../commands/sync.ts';
@@ -54,7 +54,7 @@ import { currentSubmissionAuthority } from '../minions/submission-authority.ts';
import { inlineExportMarkdown, readExportPage, readExportPages, readExportWithdrawals, type ExportSnapshot } from '../export-snapshot.ts';
import { scannerSlugRootMode, scannerSourcePath } from '../write-through.ts';
import { censusGitGap, type GitGapCensus, type GitGapPage } from '../../commands/sources-git-gap.ts';
-import { compareInlineExportFile, ignoredPaths, readBlobs, type DeclaredClass, type ExportFileComparison } from './git-gap.ts';
+import { compareInlineExportFile, DIRECTION_EVIDENCE, ignoredPaths, readBlobs, type DeclaredClass, type DirectionEvidence, type Evidence, type ExportFileComparison } from './git-gap.ts';
import { commitThroughDoor } from './git-door.ts';
import { gitRootAhead, pushGitRoot } from './effect-git.ts';
import { DURABILITY_NOT_ENABLED_REASON, gitDurabilityPolicy } from './git-durability-policy.ts';
@@ -63,7 +63,7 @@ import { acquireWorktree, getWorktreeBinding, readManagedMode } from './ownershi
import { inspectDurabilityCheckout } from './git-durability-administration.ts';
import { classifyGitCheckout } from '../git-checkout.ts';
import { resolveManagedSyncContext, type ManagedSyncContext } from './sync-discovery.ts';
-import { readShadowLevel } from './sync-pass.ts';
+import { readShadowLevel, type ConversionBinding } from './sync-pass.ts';
import { isPrivacyClass, type PrivacyClass } from './git-declared.ts';
import { classifySyncPath } from './sync-classify.ts';
import { clearGitHold, fileDriftSide, readGitSourceHolds, readSyncImportProvenances, type GitHoldRecord } from './sync-holds.ts';
@@ -73,7 +73,7 @@ import { runPersistenceEffects } from './effects.ts';
import { faultPoint } from './fault-points.ts';
import { declarePersistenceProtocol } from './protocol.ts';
import { previewChangedError, previewHash } from './preview-approval.ts';
-import { sha256 } from './digest.ts';
+import { sha256, stableJson } from './digest.ts';
export const CONVERSION_GROUP_MAX = 200;
/** Adoption candidates whose export view the plan reads per read-only transaction (the census's batch size). */
@@ -107,11 +107,16 @@ type ConversionReporter = (patch: Partial<ConversionProgress>) => void;
export const ADOPT_STATES = ['database_ahead', 'undeclared_database_only', 'worktree_only'] as const;
export type AdoptState = typeof ADOPT_STATES[number];
-export const GIT_WINS_REASONS = ['git_ahead', 'deleted_by_commit', 'held_deletion', 'held_drift_unknown'] as const;
+/** `file_moved`: content evidence (d4) shows only the file moved since the row's last shared state; `decided`: the owner's decisions file chose Git. */
+export const GIT_WINS_REASONS = ['git_ahead', 'deleted_by_commit', 'held_deletion', 'held_drift_unknown', 'file_moved', 'decided'] as const;
export type GitWinsReason = typeof GIT_WINS_REASONS[number];
/** Database content the apply would discard or cannot place, by why: each must be zero before the apply runs. */
+/**
+ * `no_record_both_moved` and `no_record_no_evidence` (d4): a page with no import record whose content evidence shows both
+ * sides moved, or shows nothing; each stays a discard until the owner's decisions file (`--decisions`) decides it.
+ */
export const DISCARD_CATEGORIES = ['both_moved', 'held_drift_both', 'held_drift_unknown_differs', 'deleted_in_git', 'uncommitted_edit', 'ignored_target', 'unparseable', 'unsafe_path',
- 'segmented_page', 'export_mismatch', 'git_overwrites_database', 'ignore_status_unknown'] as const;
+ 'segmented_page', 'export_mismatch', 'git_overwrites_database', 'ignore_status_unknown', 'no_record_both_moved', 'no_record_no_evidence'] as const;
export type DiscardCategory = typeof DISCARD_CATEGORIES[number];
/** A private, alias-private or lineage-private page whose row still records a file: committed at HEAD, or absent from it. */
export const WITHHOLD_KINDS = ['file_committed', 'file_absent'] as const;
@@ -132,11 +137,16 @@ export interface DiscardDetail {
export_mismatch: Record<string, number>;
ignore_status_unknown: Record<string, number>;
}
-/** `disk`: sha256 of the working-tree bytes the preview saw (HEAD's or the export's), null when the file was absent. */
-export interface AdoptItem { id: string; source_id: string; slug: string; revision: string | null; path: string; state: AdoptState; disk: string | null }
+/** `disk`: sha256 of the working-tree bytes the preview saw (HEAD's or the export's), null when the file was absent; `blob`: the committed blob at HEAD, null when none; `evidence`: why a page with no import record is adopted (d4). */
+export interface AdoptItem { id: string; source_id: string; slug: string; revision: string | null; path: string; state: AdoptState; disk: string | null; blob?: string | null;
+ evidence?: DirectionEvidence | 'decided' }
/** `disk`: sha256 of the working-tree bytes of a committed file the preview saw, null when absent from the disk. */
export interface WithholdItem { id: string; slug: string; revision: string | null; path: string; kind: WithholdKind; declared: PrivacyClass; disk: string | null }
interface HoldItem { path: string; code: string; side: string | null; observed_at: string; action: 'git_wins' | 'rescreen' | 'blocked' }
+/** A path whose database version the plan lets Git replace: the page, revision and committed blob (null: Git deletes it) it approved (the admission binding, sync-pass.ts ConversionBinding). */
+export interface GitWinsItem { path: string; reason: GitWinsReason; id: string | null; slug: string | null; revision: string | null; blob: string | null; evidence?: DirectionEvidence }
+/** A page with no import record that the content evidence could not place (no_record_both_moved / no_record_no_evidence): what the owner's review list shows. */
+export interface ReviewItem { id: string; slug: string; path: string; revision: string | null; blob: string | null; evidence: Evidence }
interface ParkedItem { effect: string; request: string; kind: string; codes: string[]; slugs: string[]; action: 'retry' | 'reclassify' }
export interface GitConversionPreview {
@@ -160,19 +170,112 @@ export interface GitConversionPreview {
/** What a strict pass would import or delete that no page row loses (files with no row, removals Git committed). */
index: { imports: number; deletes: number };
discard: Counted<DiscardCategory> & { detail: DiscardDetail };
+ /**
+ * d4: the pages with no import record by the side content evidence says moved, and the owner's decisions: `applied`
+ * (a decision matched the page's slug, revision, blob and evidence), `stale` (it names the page at another revision,
+ * blob or evidence, so it is not applied) and `unused` (it names no page on the review list); `digest` binds the file.
+ */
+ evidence: Counted<DirectionEvidence>;
+ decisions: { digest: string | null; applied: number; stale: number; unused: number; sample: string[] };
/** Every item the apply still has to act on (adopt, withhold, git_wins, holds, parked retries, discard); 0 on a converted source. */
remaining: number;
preview_hash: string;
apply: string[];
}
-interface PageRevision { id: string; revision: string | null }
+interface PageRevision { id: string; revision: string | null; blob?: string | null }
/**
* `settled`: paths whose Git import replaces nothing (the census reads them equal, or the plan adopts, withholds or accepts
* them); `compared`: the paths settled because the page read equal to its committed file, with the revision compared.
*/
export interface GitConversionPlan { preview: GitConversionPreview; context: ManagedSyncContext; adopt: AdoptItem[]; withhold: WithholdItem[]; holds: HoldItem[]; parked: ParkedItem[];
- settled: Set<string>; compared: Map<string, PageRevision>; incarnation: string }
+ settled: Set<string>; compared: Map<string, PageRevision>; incarnation: string; gitWins: GitWinsItem[]; review: ReviewItem[]; decisions: ConversionDecision[] }
+
+/**
+ * d4: one owner decision of a `--decisions` file, for a page the content evidence could not place. It names the page as the
+ * review list showed it (slug, revision, committed blob, evidence kind) and applies only while the page is still that.
+ * `adopt` writes the database's page to its file; `git_wins` lets the committed file replace the row. A merged body is
+ * written first with gbrain repair file-drift; the page then reads equal and leaves the list.
+ */
+export interface ConversionDecision { slug: string; revision: string | null; blob: string | null; evidence: DirectionEvidence; decision: 'adopt' | 'git_wins' }
+export const DECISIONS_KIND = 'gbrain-conversion-decisions';
+export const APPROVAL_KIND = 'gbrain-conversion-approval';
+const DECISION_VALUES = ['adopt', 'git_wins'] as const;
+
+/**
+ * The review list and decision files hold page names and paths: each is written owner-only (0600) and refused anywhere
+ * under a registered source's root, so no sync or commit ever picks one up. Checked on the real path of the file's directory.
+ */
+export async function assertOutsideSourceRoots(engine: BrainEngine, file: string, flag: string): Promise<string> {
+ const target = resolve(file);
+ let directory: string;
+ try { directory = realpathSync(dirname(target)); } catch {
+ throw opError('invalid_params', `${flag} ${file}: its directory does not exist.`, `Pass a path in an existing directory outside every source's checkout, e.g. under ~/.gbrain.`);
+ }
+ const real = join(directory, target.slice(dirname(target).length + 1));
+ const roots = await engine.executeRaw<{ id: string; local_path: string }>('SELECT id, local_path FROM sources WHERE local_path IS NOT NULL');
+ for (const root of roots) {
+ let base: string;
+ try { base = realpathSync(root.local_path); } catch { base = resolve(root.local_path); }
+ const inside = relative(base, real);
+ if (inside === '' || (!inside.startsWith('..') && !inside.startsWith(sep) && !/^[A-Za-z]:/.test(inside))) {
+ throw opError('conversion_file_in_source_root', `${flag} ${file} is inside source ${root.id}'s root (${root.local_path}), so it was refused: a review or decisions file names pages and must never be synced or committed.`,
+ `Pass a path outside every source's checkout, e.g. under ~/.gbrain.`, { docs: DOCS });
+ }
+ }
+ return real;
+}
+
+/** Writes a review list or an approval owner-only (0600), outside every source root, replacing an older one but never through a symlink. */
+export async function writeConversionFile(engine: BrainEngine, file: string, flag: string, value: unknown): Promise<string> {
+ const real = await assertOutsideSourceRoots(engine, file, flag);
+ try { if (lstatSync(real).isSymbolicLink()) throw opError('invalid_params', `${flag} ${file} is a symlink, so it was not written.`, 'Pass the path of a regular file.'); }
+ catch (error) { if (error instanceof OperationError) throw error; }
+ const temp = `${real}.${process.pid}.tmp`;
+ try { writeFileSync(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' }); chmodSync(temp, 0o600); renameSync(temp, real); } finally { rmSync(temp, { force: true }); }
+ return real;
+}
+
+/** Reads a `--decisions` file (the review list with `decision` filled in): entries whose decision is null are skipped. */
+export async function readConversionDecisions(engine: BrainEngine, file: string, sourceId: string): Promise<ConversionDecision[]> {
+ const real = await assertOutsideSourceRoots(engine, file, '--decisions');
+ const bad = (why: string) => opError('invalid_params', `--decisions ${file}: ${why}`, `Fill in "decision" (adopt or git_wins) on entries of the review list gbrain sources canonical ${sourceId} --to git --dry-run --list-out <file> wrote, and pass that file unchanged otherwise.`);
+ let parsed: { kind?: unknown; version?: unknown; source_id?: unknown; pages?: unknown };
+ try { parsed = JSON.parse(readFileSync(real, 'utf8')); } catch { throw bad('it is not JSON.'); }
+ if (parsed?.kind !== DECISIONS_KIND || parsed.version !== 1) throw bad(`it is not a ${DECISIONS_KIND} file (version 1).`);
+ if (parsed.source_id !== sourceId) throw bad(`it is for source ${String(parsed.source_id)}, not ${sourceId}.`);
+ if (!Array.isArray(parsed.pages)) throw bad('it has no pages list.');
+ const out: ConversionDecision[] = [];
+ const seen = new Set<string>();
+ for (const entry of parsed.pages as Array<Record<string, unknown>>) {
+ if (entry?.decision == null) continue;
+ const { slug, revision, blob, evidence, decision } = entry;
+ if (typeof slug !== 'string' || !(DECISION_VALUES as readonly unknown[]).includes(decision) || !(DIRECTION_EVIDENCE as readonly unknown[]).includes(evidence)
+ || (revision !== null && typeof revision !== 'string') || (blob !== null && typeof blob !== 'string')) throw bad(`entry ${JSON.stringify(slug)} needs slug, revision, blob, evidence and a decision of adopt or git_wins.`);
+ if (seen.has(slug)) throw bad(`page ${slug} is decided twice.`);
+ seen.add(slug);
+ out.push({ slug, revision: revision as string | null, blob: blob as string | null, evidence: evidence as DirectionEvidence, decision: decision as ConversionDecision['decision'] });
+ }
+ return out;
+}
+
+/**
+ * The review list of a plan (`--list-out`): every page the evidence could not place, ordered newest first by the later of
+ * its database edit and its file's last commit (timestamps only order the list; they never decide a page), each with
+ * `decision: null` for the owner to fill in and pass back as `--decisions`.
+ */
+export async function conversionReviewList(engine: BrainEngine, plan: GitConversionPlan): Promise<{ kind: string; version: 1; source_id: string; head: string | null; pages: unknown[] }> {
+ const ids = plan.review.map(item => item.id);
+ const updated = new Map(ids.length ? (await engine.executeRaw<{ id: string; at: string | Date | null }>(
+ "SELECT id::text, updated_at AS at FROM pages WHERE id = ANY(string_to_array($1::text, ',')::bigint[])", [ids.join(',')])).map(row => [row.id, row.at ? new Date(row.at).getTime() : null]) : []);
+ const pages = plan.review.map(item => {
+ const database = updated.get(item.id) ?? null, file = item.evidence.committed_at === null ? null : item.evidence.committed_at * 1000;
+ return { order: Math.max(database ?? 0, file ?? 0), entry: { slug: item.slug, path: item.path, revision: item.revision, blob: item.blob, evidence: item.evidence.kind,
+ basis: item.evidence.basis, versions_compared: item.evidence.versions, blobs_compared: item.evidence.blobs,
+ database_updated_at: database === null ? null : new Date(database).toISOString(), file_committed_at: file === null ? null : new Date(file).toISOString(), decision: null } };
+ }).sort((a, b) => b.order - a.order).map(row => row.entry);
+ return { kind: DECISIONS_KIND, version: 1, source_id: plan.preview.source_id, head: plan.preview.head, pages };
+}
const counted = <K extends string>(keys: readonly K[]): Counted<K> => ({ count: 0, by: Object.fromEntries(keys.map(key => [key, 0])) as Record<K, number>, sample: [] });
const bump = (into: Record<string, number>, key: string) => { into[key] = (into[key] ?? 0) + 1; };
@@ -225,8 +328,12 @@ async function assertConversionCheckout(sourceId: string, root: string): Promise
/** The context a strict pass reads, for a source whose row does not say so yet. */
const strictContext = (context: ManagedSyncContext): ManagedSyncContext => ({ ...context, source: { ...context.source, config: { ...context.source.config, canonical: 'git' } } });
-/** The read-only plan of `--to git`; its preview is what the CLI prints and the apply binds to. */
-export async function planGitConversion(engine: BrainEngine, sourceId: string): Promise<GitConversionPlan> {
+/**
+ * The read-only plan of `--to git`; its preview is what the CLI prints and the apply binds to. `decisions` (d4, the
+ * owner's `--decisions` file) decide pages the content evidence could not place; each binds the page's slug, revision,
+ * committed blob and evidence kind, and its digest is bound into the preview hash.
+ */
+export async function planGitConversion(engine: BrainEngine, sourceId: string, opts: { decisions?: ConversionDecision[] } = {}): Promise<GitConversionPlan> {
const binding = await getWorktreeBinding(engine, sourceId);
if (binding?.local_path && binding.owner_host_id === hostIdFor(engine) && binding.state === 'active') await assertConversionCheckout(sourceId, resolve(binding.local_path, binding.relative_path));
const context = await resolveManagedSyncContext(engine, { sourceId, noPull: true, dryRun: true });
@@ -234,8 +341,8 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
const equal = new Map<string, PageRevision>();
const mode = await scannerSlugRootMode(engine, sourceId, context.root);
const exported = new ExportChecks(context.root, mode);
- const census = await censusGitGap(engine, sourceId, { sample: 0, collect: (page, snapshot) => {
- if (page.state === 'equal') { if (page.path) equal.set(page.path, { id: page.id, revision: page.revision }); return; }
+ const census = await censusGitGap(engine, sourceId, { sample: 0, evidence: true, collect: (page, snapshot) => {
+ if (page.state === 'equal') { if (page.path) equal.set(page.path, { id: page.id, revision: page.revision, blob: page.headBlob ?? null }); return; }
pages.push(page);
if (snapshot && page.state === 'database_ahead' && page.path && !page.segmented) exported.add(page.id, page.path, snapshot);
} });
@@ -248,9 +355,42 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
const holdRecords = (await readGitSourceHolds(engine, { sourceIds: [sourceId] }))[0]?.holds ?? [];
const held = new Map(holdRecords.map(hold => [hold.path, hold]));
const preview = emptyPreview(sourceId, context, census);
- const adopt: AdoptItem[] = [], withhold: WithholdItem[] = [], accepted = new Set<string>(), discarded = new Set<string>();
+ const adopt: AdoptItem[] = [], withhold: WithholdItem[] = [], accepted = new Map<string, GitWinsItem>(), discarded = new Set<string>(), review: ReviewItem[] = [];
const discard = (category: DiscardCategory, path: string) => { tally(preview.discard, category, path); discarded.add(path); };
- const gitWins = (reason: GitWinsReason, path: string) => { tally(preview.git_wins, reason, path); accepted.add(path); };
+ const gitWins = (reason: GitWinsReason, path: string, page?: { id: string; slug?: string; revision: string | null; blob: string | null; evidence?: DirectionEvidence }) => {
+ tally(preview.git_wins, reason, path);
+ accepted.set(path, { path, reason, id: page?.id ?? null, slug: page?.slug ?? null, revision: page?.revision ?? null, blob: page?.blob ?? null, ...(page?.evidence ? { evidence: page.evidence } : {}) });
+ };
+ const decisions = opts.decisions ?? [];
+ const bySlug = new Map(decisions.map(decision => [decision.slug, decision]));
+ const reviewed = new Set<string>();
+ if (decisions.length) preview.decisions.digest = sha256(Buffer.from(stableJson([...decisions].sort((a, b) => a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0))));
+ /** The owner's decision for a page the evidence could not place, when it names the page as it is now (slug, revision, committed blob, evidence). */
+ const decided = (page: GitGapPage): ConversionDecision['decision'] | null => {
+ reviewed.add(page.slug);
+ const decision = bySlug.get(page.slug);
+ if (!decision) return null;
+ if (decision.revision !== page.revision || decision.blob !== (page.headBlob ?? null) || decision.evidence !== page.evidence!.kind) {
+ preview.decisions.stale++;
+ if (preview.decisions.sample.length < SAMPLE) preview.decisions.sample.push(page.slug);
+ return null;
+ }
+ preview.decisions.applied++;
+ return decision.decision;
+ };
+ /** d4: where content evidence sends a page that differs from its committed file with no import record. */
+ const byEvidence = (page: GitGapPage, path: string) => {
+ const evidence = page.evidence!;
+ tally(preview.evidence, evidence.kind, path);
+ const subject = { id: page.id, slug: page.slug, revision: page.revision, blob: page.headBlob ?? null, evidence: evidence.kind };
+ if (evidence.kind === 'adopt_db_moved') { candidates.push({ page, state: 'database_ahead', evidence: evidence.kind }); return; }
+ if (evidence.kind === 'git_wins_file_moved') { gitWins('file_moved', path, subject); return; }
+ const decision = decided(page);
+ if (decision === 'adopt') { candidates.push({ page, state: 'database_ahead', evidence: 'decided' }); return; }
+ if (decision === 'git_wins') { gitWins('decided', path, subject); return; }
+ review.push({ id: page.id, slug: page.slug, path, revision: page.revision, blob: page.headBlob ?? null, evidence });
+ discard(evidence.kind === 'both_moved' ? 'no_record_both_moved' : 'no_record_no_evidence', path);
+ };
const declaredEqual = await equalDeclaredPaths(engine, sourceId, root, mode,
pages.filter(page => page.state === 'declared_database_only' && page.path && page.headBlob && !page.segmented));
preview.declared.committed_equal = declaredEqual.size;
@@ -258,13 +398,13 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
for (const hold of holds) {
tally(preview.holds, hold.side ? `${hold.code}:${hold.side}` : hold.code, hold.path);
if (hold.action === 'blocked') discard('held_drift_both', hold.path);
- else if (hold.side === 'unknown' && equal.has(hold.path)) gitWins('held_drift_unknown', hold.path);
+ else if (hold.side === 'unknown' && equal.has(hold.path)) gitWins('held_drift_unknown', hold.path, { ...equal.get(hold.path)!, blob: equal.get(hold.path)!.blob ?? null });
}
const noFile = pages.filter(page => page.state === 'undeclared_database_only' && page.path !== null);
const provenance = noFile.length ? await readDeletionProvenance(root, noFile.map(page => page.path!)) : new Map();
const records = noFile.length ? await readSyncImportProvenances(engine, sourceId, incarnation, noFile.map(page => Number(page.id))) : new Map();
- const candidates: Array<{ page: GitGapPage; state: AdoptState }> = [];
+ const candidates: Array<{ page: GitGapPage; state: AdoptState; evidence?: AdoptItem['evidence'] }> = [];
for (const page of pages) {
const path = page.path ?? page.slug;
const hold = page.path ? held.get(page.path) : undefined;
@@ -286,11 +426,13 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
case 'unparseable': discard('unparseable', path); bump(preview.discard.detail.unparseable, page.unparseable ?? 'unknown'); continue;
case 'unsafe_path': discard('unsafe_path', path); continue;
case 'git_ahead':
- if (side === 'unknown') discard('held_drift_unknown_differs', path);
+ if (page.evidence && side !== 'both') byEvidence(page, path);
+ else if (side === 'unknown') discard('held_drift_unknown_differs', path);
else if (page.databaseAlsoMoved && side !== 'both') discard('both_moved', path);
- else if (!page.databaseAlsoMoved) gitWins('git_ahead', path);
+ else if (!page.databaseAlsoMoved) gitWins('git_ahead', path, { id: page.id, slug: page.slug, revision: page.revision, blob: page.headBlob ?? null });
continue;
case 'database_ahead':
+ if (page.evidence && side !== 'both') { byEvidence(page, path); continue; }
if (side === 'unknown') { discard('held_drift_unknown_differs', path); continue; }
if (side === 'both') continue;
candidates.push({ page, state: 'database_ahead' });
@@ -299,21 +441,23 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
case 'undeclared_database_only': {
if (provenance.get(page.path!)?.kind !== 'deleted_by_commit') { candidates.push({ page, state: 'undeclared_database_only' }); continue; }
const record = records.get(Number(page.id));
- if (side === 'deleted') gitWins('held_deletion', path);
- else if (record?.revision != null && String(record.revision) === String(page.revision)) gitWins('deleted_by_commit', path);
+ const deleted = { id: page.id, slug: page.slug, revision: page.revision, blob: null };
+ if (side === 'deleted') gitWins('held_deletion', path, deleted);
+ else if (record?.revision != null && String(record.revision) === String(page.revision)) gitWins('deleted_by_commit', path, deleted);
else discard('deleted_in_git', path);
continue;
}
default: continue;
}
}
+ for (const decision of decisions) if (!reviewed.has(decision.slug)) { preview.decisions.unused++; if (preview.decisions.sample.length < SAMPLE) preview.decisions.sample.push(decision.slug); }
await planAdoption(engine, sourceId, root, candidates, exported, preview, adopt, discard);
const parked = await readParked(engine, sourceId);
for (const item of parked) for (const code of item.codes) preview.parked_effects.by_code[`${item.kind}:${code}`] = (preview.parked_effects.by_code[`${item.kind}:${code}`] ?? 0) + 1;
preview.parked_effects.count = parked.length;
preview.parked_effects.retry = parked.filter(item => item.action === 'retry').length;
preview.parked_effects.reclassify = parked.filter(item => item.action === 'reclassify').length;
- const settled = new Set([...equal.keys(), ...declaredEqual, ...adopt.map(item => item.path), ...withhold.map(item => item.path), ...accepted]);
+ const settled = new Set([...equal.keys(), ...declaredEqual, ...adopt.map(item => item.path), ...withhold.map(item => item.path), ...accepted.keys()]);
const placed = new Set([...settled, ...discarded]);
const unplaced = holds.filter(hold => !placed.has(hold.path));
for (const hold of unplaced) bump(preview.holds.unplaced.by, hold.side ? `${hold.code}:${hold.side}` : hold.code);
@@ -324,7 +468,10 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
preview.withhold.groups = Math.ceil(withhold.filter(item => item.kind === 'file_committed').length / CONVERSION_GROUP_MAX);
preview.remaining = preview.adopt.count + preview.withhold.count + preview.git_wins.count + preview.holds.count + preview.parked_effects.retry + preview.discard.count;
preview.preview_hash = previewHash({ source: sourceId, incarnation, to: 'git', strict: preview.strict, head: preview.head, states: census.states, declared: census.declared,
- adopt: adopt.map(item => [item.id, item.revision, item.path, item.state, item.disk]), git_wins: [...accepted].sort(),
+ adopt: adopt.map(item => [item.id, item.revision, item.path, item.state, item.disk]), git_wins: [...accepted.keys()].sort(),
+ // d4: bound only when a page had no import record, so the hash of a plan with none is the one earlier releases gave.
+ evidence: preview.evidence.count ? pages.flatMap(page => page.evidence ? [[page.id, page.revision, page.headBlob ?? null, page.evidence.kind, page.evidence.blob, page.evidence.version]] : []) : undefined,
+ decisions: preview.decisions.digest ?? undefined,
// ignore_status_unknown is bound only when present, so the hash of a plan with no such path is the one earlier releases gave.
discard: preview.discard.by.ignore_status_unknown ? preview.discard.by : { ...preview.discard.by, ignore_status_unknown: undefined },
// Bound only when present, so the hash of a plan with nothing to withhold is the one earlier releases gave.
@@ -333,14 +480,15 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
holds: holds.map(hold => [hold.path, hold.code, hold.side, hold.observed_at]), parked: parked.map(item => [item.effect, item.action, item.codes]), index: preview.index });
preview.apply = ['gbrain', 'sources', 'canonical', sourceId, '--to', 'git', '--apply', '--expect', preview.preview_hash];
const compared = new Map([...equal, ...pages.flatMap(page => page.path && declaredEqual.has(page.path) ? [[page.path, { id: page.id, revision: page.revision }] as const] : [])]);
- return { preview, context, adopt, withhold, holds, parked, settled, compared, incarnation };
+ return { preview, context, adopt, withhold, holds, parked, settled, compared, incarnation, gitWins: [...accepted.values()], review, decisions };
}
function emptyPreview(sourceId: string, context: ManagedSyncContext, census: GitGapCensus): GitConversionPreview {
return { status: 'preview', source_id: sourceId, to: 'git', strict: context.source.config?.canonical === 'git', head: census.head, pages: census.pages, equal: census.states.equal,
adopt: { ...counted(ADOPT_STATES), action: 'commit_through_door', groups: 0 }, declared: { ...counted<string>([]), action: 'none', committed_equal: 0 },
git_wins: { ...counted(GIT_WINS_REASONS), action: 'index_committed_blob' }, withhold: { ...counted(WITHHOLD_KINDS), action: 'commit_removal_and_withhold', groups: 0 }, holds: { ...counted<string>([]), action: 'clear', unplaced: { count: 0, by: {} } },
- parked_effects: { count: 0, retry: 0, reclassify: 0, by_code: {} }, index: { imports: 0, deletes: 0 }, discard: emptyDiscard(), remaining: 0, preview_hash: '', apply: [] };
+ parked_effects: { count: 0, retry: 0, reclassify: 0, by_code: {} }, index: { imports: 0, deletes: 0 }, discard: emptyDiscard(), evidence: counted(DIRECTION_EVIDENCE),
+ decisions: { digest: null, applied: 0, stale: 0, unused: 0, sample: [] }, remaining: 0, preview_hash: '', apply: [] };
}
/** A hold's fate: a held deletion or an unknown-side drift goes to Git (planGitConversion discards the drift when its row differs), a both-sides drift is database content Git would discard, the rest re-screen. */
@@ -370,7 +518,7 @@ class ExportChecks {
* database_ahead page's export view is the one the census compared; the other candidates are read ADOPTION_BATCH at a
* time in read-only transactions (#7617).
*/
-async function planAdoption(engine: BrainEngine, sourceId: string, root: string, candidates: Array<{ page: GitGapPage; state: AdoptState }>, exported: ExportChecks,
+async function planAdoption(engine: BrainEngine, sourceId: string, root: string, candidates: Array<{ page: GitGapPage; state: AdoptState; evidence?: AdoptItem['evidence'] }>, exported: ExportChecks,
preview: GitConversionPreview, adopt: AdoptItem[], discard: (category: DiscardCategory, path: string) => void): Promise<void> {
if (!candidates.length) return;
const status = await ignoredPaths(root, candidates.map(({ page }) => page.path!))
@@ -385,7 +533,7 @@ async function planAdoption(engine: BrainEngine, sourceId: string, root: string,
});
batch.forEach((page, i) => exported.add(page.id, page.path!, snapshots[i]!));
}
- for (const { page, state } of candidates) {
+ for (const { page, state, evidence } of candidates) {
const path = page.path!;
// A target Git cannot check (beyond a symlink, outside the checkout, a timed-out check) is never written; the census
// declares an ignored path with no file git_ignored, so ignored_target is a path whose rules changed since it read them.
@@ -406,7 +554,7 @@ async function planAdoption(engine: BrainEngine, sourceId: string, root: string,
bump(preview.discard.detail.uncommitted_edit, page.worktree ?? 'unknown');
continue;
}
- adopt.push({ id: page.id, source_id: sourceId, slug: page.slug, revision: page.revision, path, state, disk: disk ? sha256(disk) : null });
+ adopt.push({ id: page.id, source_id: sourceId, slug: page.slug, revision: page.revision, path, state, disk: disk ? sha256(disk) : null, blob: page.headBlob ?? null, ...(evidence ? { evidence } : {}) });
tally(preview.adopt, state, path);
}
}
@@ -485,7 +633,8 @@ async function planIndex(engine: BrainEngine, context: ManagedSyncContext, previ
}
export interface GitConversionResult {
- status: 'converted' | 'incomplete';
+ /** `delta` (item 7): an approved apply committed only the unchanged adoptions and withholds; `delta` lists the rest. */
+ status: 'converted' | 'incomplete' | 'delta';
source_id: string;
head: string | null;
committed: { groups: number; pages: number; commits: string[] };
@@ -499,6 +648,10 @@ export interface GitConversionResult {
closing_passes: number;
/** The plan after the flag was set and the strict passes ran: nothing left on a completed conversion. */
after: GitConversionPreview;
+ /** Item 3: paths a strict pass held because their database version moved since the plan (or no plan item covered it). */
+ reclassified: string[];
+ /** Item 7: what an approved apply left unapplied, by reason, for the owner. */
+ delta?: { count: number; by: Record<string, number>; items: ConversionDeltaItem[] };
}
/**
@@ -518,63 +671,197 @@ const blockedError = (preview: GitConversionPreview): OperationError => {
user_message: `Converting ${preview.source_id} to git-canonical would lose database content in ${categories.join(', ')}. Each path needs a decision before conversion can run.` } });
};
+/**
+ * Item 6: where a failed apply left the source, for the operator's recovery. `mode` is the source's canonical flag when it
+ * stopped; `commits` the door commits it made (each group's sha as commitThroughDoor returned it); `push` their push
+ * state; `accepted` the writes that landed (pages adopted and withheld, holds cleared, pages the strict passes wrote);
+ * `rollback` the non-destructive steps back to mixed: a strict source re-derives from Git, then `--to mixed` removes the
+ * flag. Nothing is rewritten: the adopted pages' files already hold the database's bytes.
+ */
+export interface ConversionRecovery { mode: 'mixed' | 'git'; phase: ConversionPhase; commits: string[]; push: ConversionPush | null;
+ accepted: { adopted: number; withheld: number; holds_cleared: number; pass_writes: number }; reclassified: string[]; rollback: string[][] }
+const recoveries = new WeakMap<object, ConversionRecovery>();
+/** The recovery report a failed apply attached to its error, if any. */
+export const conversionRecovery = (error: unknown): ConversionRecovery | undefined => error && typeof error === 'object' ? recoveries.get(error) : undefined;
+
+/** The non-destructive rollback of a source in `mode`: re-derive a strict source from Git, then return it to mixed. */
+export function conversionRollback(sourceId: string, mode: 'mixed' | 'git'): string[][] {
+ return mode === 'git' ? [['gbrain', 'sources', 'reindex', sourceId, '--from-git', '--dry-run'], ['gbrain', 'sources', 'canonical', sourceId, '--to', 'mixed', '--dry-run']] : [];
+}
+
/** The apply of `--to git` (see the header). Throws preview_changed or git_conversion_blocked before writing anything. */
export async function applyGitConversion(engine: BrainEngine, sourceId: string, expect: string,
- opts: { noEmbed?: boolean; noExtract?: boolean; onProgress?: (progress: ConversionProgress) => void } = {}): Promise<GitConversionResult> {
+ opts: { noEmbed?: boolean; noExtract?: boolean; onProgress?: (progress: ConversionProgress) => void; decisions?: ConversionDecision[] } = {}): Promise<GitConversionResult> {
+ const plan = await planGitConversion(engine, sourceId, { decisions: opts.decisions });
+ if (plan.preview.preview_hash !== expect) throw previewChangedError(expect, `gbrain sources canonical ${sourceId} --to git --dry-run`);
+ return runConversion(engine, plan, expect, opts);
+}
+
+/** The plan items an approval binds, by identity, each with the fingerprint that must still hold for it to apply (item 7). */
+function approvalItems(plan: GitConversionPlan): Record<string, string> {
+ const items: Record<string, string> = {};
+ for (const item of plan.adopt) items[`adopt:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.blob ?? null, item.evidence ?? null, item.state, item.disk]);
+ for (const item of plan.gitWins) items[`git_wins:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.blob, item.evidence ?? null, item.reason]);
+ for (const item of plan.withhold) items[`withhold:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.kind, item.declared, item.disk]);
+ for (const hold of plan.holds) items[`hold:${hold.path}`] = JSON.stringify([hold.code, hold.side, hold.observed_at, hold.action]);
+ for (const item of plan.parked) items[`parked:${item.effect}`] = JSON.stringify([item.action, item.codes]);
+ return items;
+}
+
+/**
+ * Item 7: an owner's approval of a plan, bound to the categories and to each page's slug, revision, committed blob and
+ * evidence rather than to HEAD. Written by `--approval-out` (0600, outside every source root) and applied with `--approved`.
+ */
+export interface ConversionApproval { kind: typeof APPROVAL_KIND; version: 1; source_id: string; incarnation: string; head: string | null; preview_hash: string;
+ categories: Record<string, Record<string, number>>; decisions: ConversionDecision[]; items: Record<string, string> }
+
+export function conversionApproval(plan: GitConversionPlan): ConversionApproval {
+ const p = plan.preview;
+ return { kind: APPROVAL_KIND, version: 1, source_id: p.source_id, incarnation: plan.incarnation, head: p.head, preview_hash: p.preview_hash,
+ categories: { adopt: p.adopt.by, git_wins: p.git_wins.by, withhold: p.withhold.by, holds: p.holds.by, evidence: p.evidence.by, discard: p.discard.by },
+ decisions: plan.decisions, items: approvalItems(plan) };
+}
+
+export async function readConversionApproval(engine: BrainEngine, file: string, sourceId: string): Promise<ConversionApproval> {
+ const real = await assertOutsideSourceRoots(engine, file, '--approved');
+ const bad = (why: string) => opError('invalid_params', `--approved ${file}: ${why}`, `Write the approval with gbrain sources canonical ${sourceId} --to git --dry-run --approval-out <file> and pass it unchanged.`);
+ let parsed: ConversionApproval;
+ try { parsed = JSON.parse(readFileSync(real, 'utf8')); } catch { throw bad('it is not JSON.'); }
+ if (parsed?.kind !== APPROVAL_KIND || parsed.version !== 1 || typeof parsed.items !== 'object' || !Array.isArray(parsed.decisions)) throw bad(`it is not a ${APPROVAL_KIND} file (version 1).`);
+ if (parsed.source_id !== sourceId) throw bad(`it is for source ${String(parsed.source_id)}, not ${sourceId}.`);
+ return parsed;
+}
+
+/** One item of an approved apply that stays unapplied: changed or new since the approval, a discard, or a decision the approval did not hold. */
+export interface ConversionDeltaItem { item: string; reason: 'changed' | 'new' | 'new_decision' | `discard:${DiscardCategory}` }
+
+/**
+ * Item 7: applies an approval across time. The apply plans again with the approval's decisions and compares each item with
+ * the approved one. With no difference the whole plan applies as `--expect` would. Otherwise only the unchanged adoptions
+ * and withholds are committed, nothing else (no hold is cleared, no strict pass runs, the flag is not set), and every
+ * changed or new item, every discard and every decision the approval did not hold is returned as the delta for the owner.
+ */
+export async function applyApprovedConversion(engine: BrainEngine, sourceId: string, approval: ConversionApproval,
+ opts: { noEmbed?: boolean; noExtract?: boolean; onProgress?: (progress: ConversionProgress) => void; decisions?: ConversionDecision[] } = {}): Promise<GitConversionResult> {
+ const plan = await planGitConversion(engine, sourceId, { decisions: approval.decisions });
+ if (plan.incarnation !== approval.incarnation) throw opError('source_changed', `Source ${sourceId} was replaced since the approval, so nothing was applied.`,
+ `Preview again with gbrain sources canonical ${sourceId} --to git --dry-run --approval-out <file> and have the owner approve it.`);
+ const approved = new Map(approval.decisions.map(decision => [decision.slug, JSON.stringify(decision)]));
+ const delta: ConversionDeltaItem[] = (opts.decisions ?? []).filter(decision => approved.get(decision.slug) !== JSON.stringify(decision))
+ .map(decision => ({ item: `decision:${decision.slug}`, reason: 'new_decision' as const }));
+ const current = approvalItems(plan);
+ for (const [item, fingerprint] of Object.entries(current)) if (approval.items[item] !== fingerprint) delta.push({ item, reason: approval.items[item] === undefined ? 'new' : 'changed' });
+ for (const category of DISCARD_CATEGORIES) if (plan.preview.discard.by[category]) delta.push({ item: `discard:${category}`, reason: `discard:${category}` });
+ if (!delta.length) return runConversion(engine, plan, plan.preview.preview_hash, opts);
+ const unchanged = (kind: string, path: string) => current[`${kind}:${path}`] === approval.items[`${kind}:${path}`];
+ const partial = { ...plan, adopt: plan.adopt.filter(item => unchanged('adopt', item.path)), withhold: plan.withhold.filter(item => unchanged('withhold', item.path)) };
+ const progress: ConversionProgress = { phase: 'adopt', groups_committed: 0, scanned: 0, total: partial.adopt.length };
+ const report: ConversionReporter = patch => { Object.assign(progress, patch); opts.onProgress?.({ ...progress }); };
+ const previewCommand = `gbrain sources canonical ${sourceId} --to git --dry-run --approval-out <file>`;
+ const committed = await adoptGroups(engine, partial, previewCommand, report);
+ report({ phase: 'withhold', scanned: 0, total: partial.withhold.length });
+ const withheld = await withholdGroups(engine, partial, previewCommand, groups => report({ scanned: progress.scanned + groups.pages, groups_committed: committed.groups + groups.commits }));
+ const push = await pushConversion(engine, plan.context);
+ const by: Record<string, number> = {};
+ for (const item of delta) by[item.reason] = (by[item.reason] ?? 0) + (item.reason.startsWith('discard:') ? plan.preview.discard.by[item.reason.slice(8) as DiscardCategory] : 1);
+ return { status: 'delta', source_id: sourceId, head: plan.preview.head, committed, withheld, holds_cleared: 0, parked: { retried: 0, reclassified: 0 }, push, closing_passes: 0,
+ after: (await planGitConversion(engine, sourceId, { decisions: approval.decisions })).preview, reclassified: [], delta: { count: Object.values(by).reduce((a, b) => a + b, 0), by, items: delta } };
+}
+
+/** The apply proper, on a plan already bound (by its hash or by an approval). */
+async function runConversion(engine: BrainEngine, plan: GitConversionPlan, expect: string,
+ opts: { noEmbed?: boolean; noExtract?: boolean; onProgress?: (progress: ConversionProgress) => void }): Promise<GitConversionResult> {
+ const sourceId = plan.preview.source_id;
const previewCommand = `gbrain sources canonical ${sourceId} --to git --dry-run`;
- const plan = await planGitConversion(engine, sourceId);
- if (plan.preview.preview_hash !== expect) throw previewChangedError(expect, previewCommand);
- if (plan.preview.discard.count) throw blockedError(plan.preview);
const progress: ConversionProgress = { phase: 'adopt', groups_committed: 0, scanned: 0, total: plan.adopt.length };
const report: ConversionReporter = patch => { Object.assign(progress, patch); opts.onProgress?.({ ...progress }); };
- report({});
- const committed = await adoptGroups(engine, plan, previewCommand, report);
- report({ phase: 'withhold', scanned: 0, total: plan.withhold.length });
- const withheld = await withholdGroups(engine, plan, previewCommand, groups => report({ scanned: progress.scanned + groups.pages, groups_committed: committed.groups + groups.commits }));
- report({ push: await pushConversion(engine, plan.context) });
- const parked = await settleParked(engine, sourceId, plan.parked);
- report({ phase: 'recheck', scanned: 0, total: 0 });
- // The planned holds are cleared only once this check passes, so a refusal keeps them: a held path is judged as the
- // strict pass will see it once its hold is cleared. A page the plan settled as equal to its file is judged again once it
- // moved since (its equality no longer holds).
- const held = new Set(plan.holds.map(hold => hold.path));
- const moved = await movedSincePlan(engine, sourceId, plan.compared);
- const recheck = await readShadowLevel(engine, { sourceId, full: true, context: strictContext(plan.context) });
- const preview = { ...plan.preview, discard: emptyDiscard() };
- for (const entry of recheck?.paths ?? []) {
- const path = sourcePath(recheck!.scope, entry.path);
- if (plan.settled.has(path) && !moved.has(path)) continue;
- const result = held.has(path) ? classifySyncPath({ ...entry.state, hold: null }, { strict: true }) : entry.result;
- if (!result.settled && !result.inFlight && result.gitWins) tally(preview.discard, 'git_overwrites_database', path);
- }
- if (preview.discard.count) throw blockedError(preview);
- report({ phase: 'holds', scanned: 0, total: plan.holds.length });
- let holdsCleared = 0;
- for (const hold of plan.holds) {
- if (await engine.transaction(tx => clearGitHold(tx, { sourceId, incarnation: plan.incarnation, path: hold.path, observedAt: hold.observed_at }))) holdsCleared++;
- report({ scanned: progress.scanned + 1 });
- }
- const processing = { noEmbed: opts.noEmbed, noExtract: opts.noExtract };
- const planned = recheck?.paths.length ?? 0;
- const pass = (phase: 'strict_pass_1' | 'strict_pass_2' | 'strict_pass_closing', total = planned) => {
- report({ phase, scanned: 0, total, ...ROUND_RESET });
- return strictPass(engine, sourceId, { sourceId, noPull: true, fullRound: phase !== 'strict_pass_closing', ...processing,
- ...(phase === 'strict_pass_1' ? { canonicalOverride: 'git' as const } : {}), onProgress: passProgress(report) });
- };
- passOrThrow(sourceId, 'strict_pass_1', await pass('strict_pass_1'), progress);
- await faultPoint('conversion:before_flip', { sourceId });
- await recordConversion(engine, sourceId, plan.incarnation, 'git', expect, plan.preview.head);
- passOrThrow(sourceId, 'strict_pass_2', await pass('strict_pass_2'), progress);
- const final = await pushConversion(engine, plan.context);
- const push = final.reason === 'nothing_to_push' ? progress.push! : final;
- report({ push });
- let after = (await planGitConversion(engine, sourceId)).preview, closing = 0;
- for (; after.strict && after.remaining > 0 && after.remaining === after.git_wins.count && closing < CLOSING_PASSES; closing++) {
- passOrThrow(sourceId, 'strict_pass_closing', await pass('strict_pass_closing', after.git_wins.count), progress);
- after = (await planGitConversion(engine, sourceId)).preview;
+ const recovery: ConversionRecovery = { mode: plan.preview.strict ? 'git' : 'mixed', phase: 'adopt', commits: [], push: null,
+ accepted: { adopted: 0, withheld: 0, holds_cleared: 0, pass_writes: 0 }, reclassified: [], rollback: [] };
+ // Item 3: the strict passes may replace a database version only where the plan approved Git replacing that page at that
+ // revision and blob (an adopted page at its planned revision, its blob being the one just committed); any other is held.
+ const binding: ConversionBinding = { expect: new Map<string, { id: string; revision: string | null; blob?: string | null }>([...plan.gitWins.flatMap(item => item.id ? [[item.path, { id: item.id, revision: item.revision, blob: item.blob }] as const] : []),
+ ...plan.adopt.map(item => [item.path, { id: item.id, revision: item.revision, blob: undefined }] as const)]), reclassified: new Set() };
+ try {
+ if (plan.preview.discard.count) throw blockedError(plan.preview);
+ report({});
+ const committed = await adoptGroups(engine, plan, previewCommand, report, recovery.commits);
+ recovery.accepted.adopted = committed.pages;
+ report({ phase: 'withhold', scanned: 0, total: plan.withhold.length }); recovery.phase = 'withhold';
+ const withheld = await withholdGroups(engine, plan, previewCommand, groups => report({ scanned: progress.scanned + groups.pages, groups_committed: committed.groups + groups.commits }), recovery.commits);
+ recovery.accepted.withheld = withheld.pages;
+ report({ push: await pushConversion(engine, plan.context) }); recovery.push = progress.push!;
+ const parked = await settleParked(engine, sourceId, plan.parked);
+ report({ phase: 'recheck', scanned: 0, total: 0 }); recovery.phase = 'recheck';
+ // The planned holds are cleared only once this check passes, so a refusal keeps them: a held path is judged as the
+ // strict pass will see it once its hold is cleared. A page the plan settled as equal to its file is judged again once it
+ // moved since (its equality no longer holds).
+ const held = new Set(plan.holds.map(hold => hold.path));
+ const moved = await movedSincePlan(engine, sourceId, plan.compared);
+ const recheck = await readShadowLevel(engine, { sourceId, full: true, context: strictContext(plan.context) });
+ const preview = { ...plan.preview, discard: emptyDiscard() };
+ for (const entry of recheck?.paths ?? []) {
+ const path = sourcePath(recheck!.scope, entry.path);
+ if (plan.settled.has(path) && !moved.has(path)) continue;
+ const result = held.has(path) ? classifySyncPath({ ...entry.state, hold: null }, { strict: true }) : entry.result;
+ if (!result.settled && !result.inFlight && result.gitWins) tally(preview.discard, 'git_overwrites_database', path);
+ }
+ if (preview.discard.count) throw blockedError(preview);
+ await faultPoint('conversion:after_recheck', { sourceId });
+ report({ phase: 'holds', scanned: 0, total: plan.holds.length }); recovery.phase = 'holds';
+ let holdsCleared = 0;
+ for (const hold of plan.holds) {
+ if (await engine.transaction(tx => clearGitHold(tx, { sourceId, incarnation: plan.incarnation, path: hold.path, observedAt: hold.observed_at }))) holdsCleared++;
+ recovery.accepted.holds_cleared = holdsCleared;
+ report({ scanned: progress.scanned + 1 });
+ }
+ const processing = { noEmbed: opts.noEmbed, noExtract: opts.noExtract };
+ const planned = recheck?.paths.length ?? 0;
+ const pass = async (phase: 'strict_pass_1' | 'strict_pass_2' | 'strict_pass_closing', total = planned) => {
+ report({ phase, scanned: 0, total, ...ROUND_RESET }); recovery.phase = phase;
+ const result = await strictPass(engine, sourceId, { sourceId, noPull: true, fullRound: phase !== 'strict_pass_closing', ...processing, conversionBinding: binding,
+ ...(phase === 'strict_pass_1' ? { canonicalOverride: 'git' as const } : {}), onProgress: passProgress(report) });
+ recovery.accepted.pass_writes += result.added + result.modified + result.deleted + result.renamed;
+ recovery.reclassified = [...binding.reclassified].sort();
+ return result;
+ };
+ const first = await pass('strict_pass_1');
+ if (binding.reclassified.size) throw changedSincePlan(sourceId, binding.reclassified, 'mixed');
+ passOrThrow(sourceId, 'strict_pass_1', first, progress);
+ await faultPoint('conversion:before_flip', { sourceId });
+ await recordConversion(engine, sourceId, plan.incarnation, 'git', expect, plan.preview.head);
+ recovery.mode = 'git';
+ passOrThrow(sourceId, 'strict_pass_2', await pass('strict_pass_2'), progress);
+ const final = await pushConversion(engine, plan.context);
+ const push = final.reason === 'nothing_to_push' ? progress.push! : final;
+ report({ push }); recovery.push = push;
+ let after = (await planGitConversion(engine, sourceId)).preview, closing = 0;
+ for (; after.strict && after.remaining > 0 && after.remaining === after.git_wins.count && closing < CLOSING_PASSES && !binding.reclassified.size; closing++) {
+ passOrThrow(sourceId, 'strict_pass_closing', await pass('strict_pass_closing', after.git_wins.count), progress);
+ after = (await planGitConversion(engine, sourceId)).preview;
+ }
+ return { status: after.remaining === 0 && !binding.reclassified.size ? 'converted' : 'incomplete', source_id: sourceId, head: after.head, committed, withheld, holds_cleared: holdsCleared, parked, push,
+ closing_passes: closing, after, reclassified: [...binding.reclassified].sort() };
+ } catch (error) {
+ if (error instanceof OperationError) {
+ recovery.phase = progress.phase;
+ recovery.mode = (await engine.executeRaw<{ strict: boolean }>("SELECT COALESCE(config->>'canonical','')='git' AS strict FROM sources WHERE id=$1", [sourceId]).catch(() => []))[0]?.strict ? 'git' : recovery.mode;
+ recovery.push ??= progress.push ?? null;
+ recovery.rollback = conversionRollback(sourceId, recovery.mode);
+ recoveries.set(error, recovery);
+ }
+ throw error;
}
- return { status: after.remaining === 0 ? 'converted' : 'incomplete', source_id: sourceId, head: after.head, committed, withheld, holds_cleared: holdsCleared, parked, push,
- closing_passes: closing, after };
+}
+
+/**
+ * Item 3: a strict pass held pages whose database version moved since the plan (or that no plan item covered) instead of
+ * letting Git overwrite them. Before the flag the source stays mixed; the next preview judges each page again.
+ */
+function changedSincePlan(sourceId: string, paths: ReadonlySet<string>, mode: 'mixed' | 'git'): OperationError {
+ const sample = [...paths].sort().slice(0, SAMPLE);
+ return opError('git_conversion_incomplete', `${paths.size} page(s) of source ${sourceId} changed since the conversion was planned (${sample.join(', ')}${paths.size > sample.length ? ', …' : ''}); the strict pass held them instead of overwriting them, and the conversion stopped there${mode === 'mixed' ? ' with the source still mixed' : ''}.`,
+ `Preview again with gbrain sources canonical ${sourceId} --to git --dry-run: each held page is classified again by its content evidence; apply the new hash.`,
+ { docs: DOCS, reason: 'changed_since_preview', detail: 'changed_since_preview', fix: conversionIncompleteFix(sourceId) });
}
/**
@@ -671,7 +958,7 @@ function passOrThrow(sourceId: string, phase: ConversionPhase, result: SyncResul
* tree moved since the preview stops the apply (preview_changed) before anything else changes: the groups already
* committed hold the bytes the database has, so the rerun preview reads them equal.
*/
-async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string, report: ConversionReporter): Promise<GitConversionResult['committed']> {
+async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string, report: ConversionReporter, made: string[] = []): Promise<GitConversionResult['committed']> {
const { context, adopt } = plan;
const root = context.root;
const commits: string[] = [];
@@ -692,7 +979,7 @@ async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, preview
}
for (const file of files) writeAtomically(root, file.path, file.bytes);
const commit = await commitThroughDoor(root, files.map(file => ({ path: file.path, planned: sha256(file.bytes) })), lock);
- if (commit) commits.push(commit);
+ if (commit) { commits.push(commit); made.push(commit); }
} finally { await lock.release(); }
report({ scanned: start + group.length, groups_committed: commits.length });
await faultPoint('conversion:group_committed', { sourceId: context.sourceId, round: start / CONVERSION_GROUP_MAX + 1 });
@@ -712,7 +999,7 @@ async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, preview
* the rerun lists only what is left.
*/
async function withholdGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string,
- onGroup: (done: { pages: number; commits: number }) => void): Promise<GitConversionResult['withheld']> {
+ onGroup: (done: { pages: number; commits: number }) => void, made: string[] = []): Promise<GitConversionResult['withheld']> {
const { context } = plan;
const root = context.root;
const commits: string[] = [];
@@ -746,7 +1033,7 @@ async function withholdGroups(engine: BrainEngine, plan: GitConversionPlan, prev
removeConfined(root, target);
}
const commit = await commitThroughDoor(root, group.map(item => ({ path: join(root, item.path), planned: null })), lock);
- if (commit) commits.push(commit);
+ if (commit) { commits.push(commit); made.push(commit); }
await faultPoint('conversion:withhold_committed', { sourceId: context.sourceId, round: start / CONVERSION_GROUP_MAX + 1 });
} finally { await lock.release(); }
onGroup({ pages: group.length, commits: commits.length });
diff --git a/src/core/persistence/git-gap.ts b/src/core/persistence/git-gap.ts
index 66ac64248d..6c9bded850 100644
--- a/src/core/persistence/git-gap.ts
+++ b/src/core/persistence/git-gap.ts
@@ -9,10 +9,11 @@
*/
import { createHash } from 'node:crypto';
import { createReadStream, lstatSync, realpathSync } from 'node:fs';
-import { join, posix } from 'node:path';
+import { join, posix, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
-import { hardenedGitStream, hardenedGitSync, NulRecordSplitter } from '../hardened-git.ts';
-import { exportBodyLedger, segmentedExportHead, segmentedExportMarkdown, type ExportSnapshot, type ExportWithdrawal } from '../export-snapshot.ts';
+import type { Page } from '../types.ts';
+import { hardenedGitStream, hardenedGitSync, HISTORY_LOG_OPTIONS, NulRecordSplitter } from '../hardened-git.ts';
+import { exportBodyLedger, readExportPages, readExportWithdrawals, segmentedExportHead, segmentedExportMarkdown, type ExportSnapshot, type ExportWithdrawal } from '../export-snapshot.ts';
import { parseMarkdown } from '../markdown.ts';
import { isPrivacyDerivationPrivate, isPrivacyPrivatePage } from '../search/private-visibility.ts';
import { alignWithStored } from './align-stored.ts';
@@ -21,6 +22,9 @@ import { isDeclaredReason, isNeverFiledDerivedPage } from './git-declared.ts';
import { classifyDrift } from './reconcile-additive.ts';
import { reconcileCanonical } from './reconcile-merge.ts';
import { reconcileFileReading } from './reconcile-state.ts';
+import { SYNC_READ_BOUND } from './sync-blobs.ts';
+import { RestoreTargetError, restoreFilePath } from '../restore-target.ts';
+import type { SlugRootMode } from '../sync-anchor.ts';
/** Why a page has no tracked file on purpose, in the order `declaredDatabaseOnly` tests them. */
export const DECLARED_CLASSES = ['mirror_source', 'database_only_source', 'private', 'alias_private', 'lineage_private', 'database_only', 'db_only_prefix', 'derive_folder', 'git_ignored'] as const;
@@ -126,6 +130,16 @@ export type ExportFileComparison = { state: 'equal' } | { state: 'differs'; path
*/
export function compareInlineExportFile(raw: Buffer, importPath: string, snapshot: ExportSnapshot): ExportFileComparison {
const { page, tags } = snapshot;
+ const file = canonicalFile(raw, importPath, page);
+ if (file.state !== 'parsed') return file;
+ const database = reconcileCanonical(page, tags);
+ if (digest(file.canonical) === digest(database)) return { state: 'equal' };
+ return { state: 'differs', paths: classifyDrift(file.canonical, database).paths.map(p => p.path) };
+}
+
+/** A committed file parsed the way reconcile parses a canonical file, aligned with the stored page (the census contract's file side). */
+function canonicalFile(raw: Buffer, importPath: string, page: ExportSnapshot['page']): { state: 'parsed'; canonical: ReturnType<typeof reconcileCanonical> }
+ | { state: 'differs'; paths: string[] } | { state: 'unparseable'; reason: UnparseableReason } {
const text = raw.toString('utf8');
if (!Buffer.from(text).equals(raw)) return { state: 'unparseable', reason: 'not_utf8' };
const reading = reconcileFileReading(text, importPath, page);
@@ -133,9 +147,7 @@ export function compareInlineExportFile(raw: Buffer, importPath: string, snapsho
if (parsed.errors?.some(e => !['MISSING_OPEN', 'EMPTY_FRONTMATTER'].includes(e.code))) return { state: 'unparseable', reason: 'frontmatter' };
if (parsed.slug !== page.slug) return { state: 'differs', paths: ['/slug'] };
alignWithStored(parsed, page, reading.inferred);
- const file = reconcileCanonical(parsed, parsed.tags), database = reconcileCanonical(page, tags);
- if (digest(file) === digest(database)) return { state: 'equal' };
- return { state: 'differs', paths: classifyDrift(file, database).paths.map(p => p.path) };
+ return { state: 'parsed', canonical: reconcileCanonical(parsed, parsed.tags) };
}
export type ObjectFormat = 'sha1' | 'sha256';
@@ -255,3 +267,219 @@ export async function worktreeBlobOid(path: string, format: ObjectFormat): Promi
for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
return hash.digest('hex');
}
+
+/**
+ * d4 of the convergence plan: which side moved for a page whose export view differs from its committed file and whose row
+ * no import record explains (so neither the record's blob nor its revision says which side changed). Content evidence
+ * only, never a timestamp: the page's versions (page_versions, the export view each write archived) and the blobs its
+ * path held in Git are compared newest first, EVIDENCE_DEPTH of each, under the census contract, to find the last state
+ * both sides shared.
+ * - adopt_db_moved: an older version of the page equals the committed file, so only the database moved since.
+ * - git_wins_file_moved: the row equals an older committed blob of the path, so only the file moved since.
+ * - both_moved: the newest shared state is older on both sides, or the history matches on both sides at once.
+ * - no_evidence: nothing shared within the depth (pruned versions, a path with no history, an unreadable history).
+ * A page stored in segments has no comparable older version: it is git_wins_file_moved only when its streamed export's
+ * blob id equals an older blob of the path, and no_evidence otherwise. Renames are followed through the origins the
+ * page's versions recorded (`page_versions.source_path`): their paths' blobs count as the page's history too.
+ */
+export const DIRECTION_EVIDENCE = ['adopt_db_moved', 'git_wins_file_moved', 'both_moved', 'no_evidence'] as const;
+export type DirectionEvidence = typeof DIRECTION_EVIDENCE[number];
+export type EvidenceBasis = 'older_version_matches_file' | 'row_matches_older_file' | 'conflicting_history' | 'shared_older_state' | 'no_shared_state'
+ | 'segmented' | 'history_unreadable' | 'changed_during_read';
+/** Versions and blobs compared per page, newest first. */
+export const EVIDENCE_DEPTH = 32;
+/** Pages whose versions, export views and blobs one evidence read holds at a time. */
+const EVIDENCE_BATCH = 64;
+
+/**
+ * `blob`: the older committed blob the row equals (git_wins_file_moved) or the shared state's blob (both_moved);
+ * `version`: the page_versions id that equals the committed file (adopt_db_moved) or the shared state's version;
+ * `committed_at`: the commit time (Unix seconds) of the newest commit that touched the page's paths, for ordering a review list.
+ */
+export interface Evidence { kind: DirectionEvidence; basis: EvidenceBasis; blob: string | null; version: number | null; versions: number; blobs: number; committed_at: number | null }
+
+/** A page the evidence read classifies: its id, slug, revision as the census read it, its path (source-root relative) and blob at `tip`, and its path as sync keys it (scannerSourcePath). */
+export interface EvidenceSubject { id: string; slug: string; revision: string | null; rel: string; importPath: string; segmented: boolean; blob: TreeBlob }
+
+interface HistoryEntry { oid: string; at: number; seq: number }
+
+/**
+ * One filter-free `git log --raw` walk of `tip` under `scope` (Git-root relative; '' is the whole repository): per path of
+ * `interest` (Git-root relative), the distinct blobs it held, newest first, at most `depth` + 1 each (the tip's own blob
+ * is usually the first). Null when Git could not answer.
+ */
+export async function readPathHistory(root: string, tip: string, scope: string, interest: ReadonlySet<string>, depth = EVIDENCE_DEPTH): Promise<Map<string, HistoryEntry[]> | null> {
+ const history = new Map<string, HistoryEntry[]>();
+ if (!interest.size) return history;
+ const splitter = new NulRecordSplitter();
+ let at = 0, seq = 0, raw: string | null = null;
+ const result = await hardenedGitStream(root, ['log', ...HISTORY_LOG_OPTIONS, '--end-of-options', tip, '--', ...(scope ? [`:(literal)${scope}`] : [])], {
+ timeoutMs: 600_000, onChunk: chunk => {
+ for (const record of splitter.push(chunk)) {
+ if (raw !== null) {
+ const path = record.toString('utf8');
+ const [, mode = '', , oid = '', status = ''] = raw.slice(1).split(' ');
+ raw = null;
+ if (!interest.has(path) || status === 'D' || (mode !== '100644' && mode !== '100755')) continue;
+ const list = history.get(path) ?? [];
+ if (list.length <= depth && !list.some(entry => entry.oid === oid)) list.push({ oid, at, seq: seq++ });
+ history.set(path, list);
+ continue;
+ }
+ const token = record.toString('utf8').replace(/^\n+/, '');
+ if (token.startsWith('\x1e')) at = Number(token.slice(1).split('\x1f')[1] ?? 0);
+ else if (token.startsWith(':')) raw = token;
+ }
+ } });
+ return result.ok && splitter.atRecordBoundary ? history : null;
+}
+
+/** The sizes of blobs by id (`cat-file --batch-check`); a missing object is left out. Null when Git could not answer. */
+export async function readBlobSizes(root: string, oids: readonly string[]): Promise<Map<string, number> | null> {
+ const sizes = new Map<string, number>();
+ if (!oids.length) return sizes;
+ const pieces: Buffer[] = [];
+ const result = await hardenedGitStream(root, ['cat-file', '--batch-check'], { stdin: oids.map(oid => oid + '\n').join(''), timeoutMs: 300_000,
+ maxBytes: oids.length * 160 + 1024, onChunk: piece => { pieces.push(piece); } });
+ if (!result.ok) return null;
+ for (const line of Buffer.concat(pieces).toString('utf8').split('\n')) {
+ const [oid, type, size] = line.split(' ');
+ if (oid && type === 'blob') sizes.set(oid, Number(size));
+ }
+ return sizes;
+}
+
+/**
+ * The direction pure decision: `row` and `head` are the canonical digests of the page's export view and of its file at
+ * the tip (null when the file does not parse), `versions` the page's older versions and `blobs` the path's older blobs
+ * (the tip's excluded), each newest first with its digest (null when it does not parse).
+ */
+export function decideDirection(row: string, head: string | null, versions: ReadonlyArray<{ id: number; digest: string }>,
+ blobs: ReadonlyArray<{ oid: string; digest: string | null }>): Pick<Evidence, 'kind' | 'basis' | 'blob' | 'version'> {
+ const fileMoved = blobs.find(blob => blob.digest === row);
+ const databaseMoved = head === null ? undefined : versions.find(version => version.digest === head);
+ if (fileMoved && databaseMoved) return { kind: 'both_moved', basis: 'conflicting_history', blob: fileMoved.oid, version: databaseMoved.id };
+ if (fileMoved) return { kind: 'git_wins_file_moved', basis: 'row_matches_older_file', blob: fileMoved.oid, version: null };
+ if (databaseMoved) return { kind: 'adopt_db_moved', basis: 'older_version_matches_file', blob: null, version: databaseMoved.id };
+ const digests = new Map<string, string>();
+ for (const blob of blobs) if (blob.digest !== null && !digests.has(blob.digest)) digests.set(blob.digest, blob.oid);
+ const shared = versions.find(version => digests.has(version.digest));
+ if (shared) return { kind: 'both_moved', basis: 'shared_older_state', blob: digests.get(shared.digest)!, version: shared.id };
+ return { kind: 'no_evidence', basis: 'no_shared_state', blob: null, version: null };
+}
+
+type VersionRow = { id: number; page_id: string; compiled_truth: string | null; frontmatter: unknown; timeline: string | null; title: string | null; type: string | null; tags: unknown;
+ segmented: boolean; is_deleted: boolean | null };
+const json = <T>(value: unknown, fallback: T): T => value == null ? fallback : typeof value === 'string' ? JSON.parse(value) as T : value as T;
+
+/**
+ * The evidence of every subject, read against the commit `tip` of the checkout at `root` (`gitRoot` its top level,
+ * `scope` the source's directory in it, `mode` its slug-root mode). One history walk for all of them, then
+ * EVIDENCE_BATCH pages at a time: their export views and versions in a read-only transaction, their older blobs through
+ * `cat-file --batch`. A page whose revision moved since the census read it is no_evidence (`changed_during_read`), so a
+ * later preview judges it again. Read-only.
+ */
+export async function readDirectionEvidence(engine: BrainEngine, input: { sourceId: string; root: string; gitRoot: string; scope: string; tip: string; format: ObjectFormat;
+ mode: SlugRootMode; subjects: EvidenceSubject[] }): Promise<Map<string, Evidence>> {
+ const { sourceId, root, scope, tip, format, mode, subjects } = input;
+ const out = new Map<string, Evidence>();
+ if (!subjects.length) return out;
+ const gitPath = (rel: string) => scope ? `${scope}/${rel}` : rel;
+ const ids = subjects.map(subject => subject.id);
+ const origins = await engine.executeRaw<{ page_id: string; slug: string; source_path: string }>(`SELECT DISTINCT v.page_id::text AS page_id, p.slug, v.source_path
+ FROM page_versions v JOIN pages p ON p.id=v.page_id WHERE v.page_id = ANY(string_to_array($1::text, ',')::int[]) AND v.source_path IS NOT NULL AND v.source_path <> ''`, [ids.join(',')]);
+ const paths = new Map(subjects.map(subject => [subject.id, new Set([gitPath(subject.rel)])]));
+ for (const origin of origins) {
+ try {
+ const target = restoreFilePath(root, origin.slug, origin.source_path, Buffer.byteLength(origin.source_path), mode);
+ const rel = target.slice(root.length + 1).split(sep).join('/');
+ if (!rel.startsWith('../') && target.startsWith(root + sep)) paths.get(origin.page_id)?.add(gitPath(rel));
+ } catch (error) { if (!(error instanceof RestoreTargetError)) throw error; }
+ }
+ const history = await readPathHistory(root, tip, scope, new Set([...paths.values()].flatMap(set => [...set])));
+ const none = (basis: EvidenceBasis, committed_at: number | null = null): Evidence => ({ kind: 'no_evidence', basis, blob: null, version: null, versions: 0, blobs: 0, committed_at });
+ if (!history) { for (const subject of subjects) out.set(subject.id, none('history_unreadable')); return out; }
+ const older = (subject: EvidenceSubject) => {
+ const merged = [...paths.get(subject.id)!].flatMap(path => history.get(path) ?? []).sort((a, b) => a.seq - b.seq);
+ const seen = new Set<string>([subject.blob.oid]), list: HistoryEntry[] = [];
+ for (const entry of merged) if (!seen.has(entry.oid) && list.length < EVIDENCE_DEPTH) { seen.add(entry.oid); list.push(entry); }
+ return { list, committed_at: merged[0]?.at ?? null };
+ };
+ const withdrawals = await readExportWithdrawals(engine, sourceId);
+ for (let start = 0; start < subjects.length; start += EVIDENCE_BATCH) {
+ const batch = subjects.slice(start, start + EVIDENCE_BATCH);
+ const read = await engine.transaction(async tx => {
+ await tx.executeRaw('SET TRANSACTION READ ONLY');
+ const live = new Map((await tx.executeRaw<{ id: string; revision: string | null }>(`SELECT id::text, knowledge_revision::text AS revision FROM pages
+ WHERE id = ANY(string_to_array($1::text, ',')::bigint[]) AND deleted_at IS NULL`, [batch.map(subject => subject.id).join(',')])).map(row => [row.id, row.revision]));
+ const current = batch.filter(subject => live.has(subject.id) && live.get(subject.id) === subject.revision);
+ const snapshots = await readExportPages(tx, current.map(subject => ({ id: subject.id, source_id: sourceId, slug: subject.slug })), withdrawals);
+ const versions = current.length ? await tx.executeRaw<VersionRow>(`SELECT id, page_id::text AS page_id, compiled_truth, frontmatter, timeline, title, type, tags,
+ COALESCE(body_storage,'inline')='segmented' AS segmented, is_deleted FROM (SELECT v.*, row_number() OVER (PARTITION BY v.page_id ORDER BY v.id DESC) AS rank FROM page_versions v
+ WHERE v.page_id = ANY(string_to_array($1::text, ',')::int[])) ranked WHERE rank <= $2 ORDER BY page_id, id DESC`, [current.map(subject => subject.id).join(','), EVIDENCE_DEPTH]) : [];
+ const segmentedOids = new Map<string, Set<string>>();
+ for (const [i, subject] of current.entries()) {
+ if (!subject.segmented) continue;
+ const sizes = await readBlobSizes(root, older(subject).list.map(entry => entry.oid));
+ segmentedOids.set(subject.id, sizes ? await segmentedExportBlobIds(tx, snapshots[i]!, withdrawals, new Set(sizes.values()), format) : new Set());
+ }
+ return { current, snapshots, versions, segmentedOids };
+ });
+ for (const subject of batch) if (!read.current.includes(subject)) out.set(subject.id, none('changed_during_read', older(subject).committed_at));
+ const byPage = new Map<string, VersionRow[]>();
+ for (const row of read.versions) byPage.set(row.page_id, [...byPage.get(row.page_id) ?? [], row]);
+ const inline = read.current.filter(subject => !subject.segmented);
+ const oids = [...new Set(inline.flatMap(subject => [subject.blob.oid, ...older(subject).list.map(entry => entry.oid)]))];
+ const sizes = await readBlobSizes(root, oids) ?? new Map<string, number>();
+ const readable = oids.flatMap(oid => { const size = sizes.get(oid); return size !== undefined && size <= SYNC_READ_BOUND ? [{ oid, size, mode: '100644' }] : []; });
+ const fileDigests = new Map<string, string | null>();
+ for (let from = 0; from < readable.length;) {
+ let to = from, bytes = 0;
+ while (to < readable.length && (to === from || bytes + readable[to]!.size <= CAT_FILE_BATCH_BYTES)) bytes += readable[to++]!.size;
+ const chunk = readable.slice(from, to);
+ const blobs = await readBlobs(root, chunk);
+ for (const subject of inline) {
+ const i = read.current.indexOf(subject), page = read.snapshots[i]!.page;
+ for (const blob of chunk) {
+ const key = `${subject.id}:${blob.oid}`;
+ if (fileDigests.has(key) || !blobs?.has(blob.oid)) continue;
+ if (blob.oid !== subject.blob.oid && !older(subject).list.some(entry => entry.oid === blob.oid)) continue;
+ const file = canonicalFile(blobs.get(blob.oid)!, subject.importPath, page);
+ fileDigests.set(key, file.state === 'parsed' ? digest(file.canonical) : null);
+ }
+ }
+ from = to;
+ }
+ for (const [i, subject] of read.current.entries()) {
+ const { list, committed_at } = older(subject);
+ if (subject.segmented) {
+ const match = list.find(entry => read.segmentedOids.get(subject.id)?.has(entry.oid));
+ out.set(subject.id, match ? { kind: 'git_wins_file_moved', basis: 'row_matches_older_file', blob: match.oid, version: null, versions: 0, blobs: list.length, committed_at }
+ : { ...none('segmented', committed_at), blobs: list.length });
+ continue;
+ }
+ const snapshot = read.snapshots[i]!;
+ const versions = (byPage.get(subject.id) ?? []).filter(row => !row.segmented && !row.is_deleted && row.compiled_truth !== null).map(row => ({ id: Number(row.id),
+ digest: digest(reconcileCanonical({ type: (row.type ?? snapshot.page.type) as Page['type'], title: row.title ?? snapshot.page.title, compiled_truth: row.compiled_truth!,
+ timeline: row.timeline ?? '', frontmatter: json(row.frontmatter, {}) }, json<string[]>(row.tags, []))) }));
+ const decided = decideDirection(digest(reconcileCanonical(snapshot.page, snapshot.tags)), fileDigests.get(`${subject.id}:${subject.blob.oid}`) ?? null, versions,
+ list.map(entry => ({ oid: entry.oid, digest: fileDigests.get(`${subject.id}:${entry.oid}`) ?? null })));
+ out.set(subject.id, { ...decided, versions: versions.length, blobs: list.length, committed_at });
+ }
+ }
+ return out;
+}
+
+/** The Git blob ids a segmented page's streamed export would have at each of `sizes` (only the one equal to its length can match). */
+async function segmentedExportBlobIds(tx: BrainEngine, snapshot: ExportSnapshot, withdrawals: ExportWithdrawal[], sizes: ReadonlySet<number>, format: ObjectFormat): Promise<Set<string>> {
+ const hashes = new Map([...sizes].map(size => [size, createHash(format).update(`blob ${size}\0`)]));
+ const max = Math.max(0, ...sizes);
+ let bytes = 0;
+ for await (const piece of segmentedExportMarkdown(tx, snapshot.page, exportBodyLedger(snapshot.page.slug, withdrawals), segmentedExportHead(snapshot))) {
+ bytes += typeof piece === 'string' ? Buffer.byteLength(piece) : piece.length;
+ if (bytes > max) return new Set();
+ for (const hash of hashes.values()) hash.update(piece);
+ }
+ const hash = hashes.get(bytes);
+ return hash ? new Set([hash.digest('hex')]) : new Set();
+}
diff --git a/src/core/persistence/rows-behind.ts b/src/core/persistence/rows-behind.ts
index 461372a49d..0c1995331f 100644
--- a/src/core/persistence/rows-behind.ts
+++ b/src/core/persistence/rows-behind.ts
@@ -38,7 +38,8 @@ import { canonicalPath, sourceContext } from '../ghost-pages.ts';
import { privatePagesFilterFragment, readHonorAliases } from '../search/private-visibility.ts';
import { scannerSourcePath } from '../write-through.ts';
import { managedPersistenceEnabled } from './ownership.ts';
-import { compareInlineExportFile, compareSegmentedExportBlob, declaredDatabaseOnly, readBlobs, readHeadTree, type ExportFileComparison, type TreeBlob } from './git-gap.ts';
+import { compareInlineExportFile, compareSegmentedExportBlob, declaredDatabaseOnly, DIRECTION_EVIDENCE, readBlobs, readDirectionEvidence, readHeadTree,
+ type DirectionEvidence, type Evidence, type EvidenceSubject, type ExportFileComparison, type TreeBlob } from './git-gap.ts';
import { prepareRepairPublication, repairScreenConfig } from './file-repair.ts';
import { SYNC_READ_BOUND } from './sync-blobs.ts';
import { heldGitPaths, readSyncImportProvenances } from './sync-holds.ts';
@@ -77,6 +78,8 @@ export interface RowBehind {
overlay?: true;
/** The digest of the page the import of the checkpoint bytes publishes (`prepareRepairPublication`). */
digest?: string;
+ /** A row with no import record, scanned with `evidence`: which side moved since the row and its file last agreed (d4, git-gap.ts readDirectionEvidence). */
+ evidence?: Evidence;
}
export interface RowsBehindScan {
@@ -89,14 +92,18 @@ export interface RowsBehindScan {
complete: boolean;
/** The page id the deadline stopped the scan at (not compared), or null. */
stopped_at: number | null;
- /** Live file-backed pages decided. */
+ /** Live file-backed pages decided; equals `live_pages` only when `complete`. */
pages: number;
+ /** The source's live file-backed pages, counted in SQL before the scan, so a partial scan's `pages` reads against the whole source. */
+ live_pages: number;
behind: RowBehind[];
unproven: RowBehind[];
/** More rows than ROWS_BEHIND_LIST_LIMIT in a class; `behind_count` / `unproven_count` keep counting. */
truncated: boolean;
behind_count: number;
unproven_count: number;
+ /** d4: `unproven_count` split by content evidence (they sum to it); null when the scan read no evidence. */
+ unproven_by: Record<DirectionEvidence, number> | null;
counts: Partial<Record<RowsBehindSkip, number>>;
}
@@ -104,12 +111,14 @@ type Key = { id: string; slug: string; source_id: string; source_path: string; s
frontmatter: unknown; database_only_reason: string | null; segmented: boolean; lineage_private: boolean; revision: string };
const empty = (sourceId: string, status: RowsBehindScan['status'], reason?: string): RowsBehindScan => ({ source_id: sourceId, status, ...(reason ? { reason } : {}),
- checkpoint: null, head: null, complete: status !== 'unknown', stopped_at: null, pages: 0, behind: [], unproven: [], truncated: false, behind_count: 0, unproven_count: 0, counts: {} });
+ checkpoint: null, head: null, complete: status !== 'unknown', stopped_at: null, pages: 0, live_pages: 0, behind: [], unproven: [], truncated: false, behind_count: 0, unproven_count: 0,
+ unproven_by: null, counts: {} });
const isMarkdown = (path: string) => /\.mdx?$/i.test(path);
+/** `evidence` (d4): read which side moved for every unproven row once the scan completes (`unproven_by`, each row's `evidence`). */
export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
- opts: { deadlineAt?: number; confirmUnproven?: boolean; now?: () => number; batchBytes?: number } = {}): Promise<RowsBehindScan> {
+ opts: { deadlineAt?: number; confirmUnproven?: boolean; now?: () => number; batchBytes?: number; evidence?: boolean } = {}): Promise<RowsBehindScan> {
if (!(await managedPersistenceEnabled(engine))) return empty(sourceId, 'not_applicable', 'not_managed');
const context = await sourceContext(engine, sourceId);
if (!context.ok) return empty(sourceId, context.status, context.reason);
@@ -131,6 +140,9 @@ export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
const { eligible, metafileHeads } = await loadSyncEligibility(engine, { root, gitRoot: checkpoint.gitRoot, sourceId, source: { last_commit: source.last_commit, config: source.config ?? {} } });
const policy = { mirror: false, databaseOnlySource: false, dbOnly: (slug: string) => isSourceDbOnlySlug(root, slug, 'refuse'), ignored: new Set<string>() };
const scan: RowsBehindScan = { ...empty(sourceId, 'known'), checkpoint: checkpoint.commit, head: head.commit };
+ const [live] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND source_path IS NOT NULL', [sourceId]);
+ scan.live_pages = Number(live?.n ?? 0);
+ const subjects: EvidenceSubject[] = [];
const count = (why: RowsBehindSkip) => { scan.counts[why] = (scan.counts[why] ?? 0) + 1; };
const batchBytes = opts.batchBytes ?? ROWS_BEHIND_BATCH_BYTES;
let screen: Awaited<ReturnType<typeof repairScreenConfig>> | undefined;
@@ -220,6 +232,8 @@ export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
if (typeof row === 'string') { count(row); continue; }
const list = row.record === 'recorded' ? scan.behind : scan.unproven;
if (row.record === 'recorded') scan.behind_count++; else scan.unproven_count++;
+ if (row.record !== 'recorded' && opts.evidence) subjects.push({ id: candidate.key.id, slug: candidate.key.slug, revision: candidate.key.revision, rel: candidate.rel,
+ importPath: scannerSourcePath(root, `${root}/${candidate.rel}`, context.mode), segmented: candidate.key.segmented === true, blob: candidate.blob });
if (list.length < ROWS_BEHIND_LIST_LIMIT) list.push(row); else scan.truncated = true;
}
for (const [id, why] of skipped) if (stop === null || id < stop) count(why);
@@ -227,6 +241,12 @@ export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
if (stop !== null) { scan.complete = false; scan.stopped_at = stop; break; }
if (done) break;
}
+ if (opts.evidence && scan.complete) {
+ const evidence = await readDirectionEvidence(engine, { sourceId, root, gitRoot: checkpoint.gitRoot, scope, tip: checkpoint.commit, format: checkpoint.format, mode: context.mode, subjects });
+ scan.unproven_by = Object.fromEntries(DIRECTION_EVIDENCE.map(kind => [kind, 0])) as Record<DirectionEvidence, number>;
+ for (const subject of subjects) scan.unproven_by[evidence.get(subject.id)!.kind]++;
+ for (const row of scan.unproven) { const found = evidence.get(String(row.page_id)); if (found) row.evidence = found; }
+ }
return scan;
}
diff --git a/src/core/persistence/sync-pass-run.ts b/src/core/persistence/sync-pass-run.ts
index 43fe82e6ca..ad4978c208 100644
--- a/src/core/persistence/sync-pass-run.ts
+++ b/src/core/persistence/sync-pass-run.ts
@@ -878,7 +878,8 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Pr
writeIds: value.obligations.converged?.writeback ?? [] })) return synced(true);
const levelInput = { sourceId, context, workingTree: mode === 'working',
pass: { noSchemaPack: processingOptions.noSchemaPack, episodeFullId: episode?.full_id ?? null, retryConsumed: round > 1 ? retryConsumed : undefined, exclude: opts.exclude, includeHidden: opts.includeHidden, strategy: opts.strategy, bootstrap: markersOwed(),
- slugMode: value.obligations.slug_mode, cache: levelCache, levelReuse, passId, recorded: applied.recorded, onProgress: beat.report, assertActive } };
+ slugMode: value.obligations.slug_mode, cache: levelCache, levelReuse, passId, recorded: applied.recorded, onProgress: beat.report, assertActive,
+ ...(opts.conversionBinding ? { conversion: opts.conversionBinding } : {}) } };
const level = await readLevelReporting(engine, levelInput, round, opts.onProgress);
await faultPoint('sync:level_read', { sourceId, round, level: level ? { statements: level.statements, confirm: level.confirm ?? null, reused: level.reused ?? 0, shadow: level, input: levelInput } : undefined });
assertActive();
diff --git a/src/core/persistence/sync-pass.ts b/src/core/persistence/sync-pass.ts
index 26ba9662e8..ebb03eaf05 100644
--- a/src/core/persistence/sync-pass.ts
+++ b/src/core/persistence/sync-pass.ts
@@ -85,7 +85,19 @@ export interface PassLevelInput {
onProgress?: () => void;
/** Throws once the pass must stop (a job deadline, a lost drain lease): checked at the read's pauses and while its origin history walk runs. */
assertActive?: () => void;
+ /** The conversion apply's strict passes (SyncOpts.conversionBinding, persistence/git-conversion.ts): the only Git verdicts that may replace a database version. */
+ conversion?: ConversionBinding;
}
+
+/**
+ * PR A admission binding: the paths (source-root relative) whose database version the conversion plan approved Git
+ * replacing, each with the page, revision and committed blob (null for a deletion; unset when any blob will do, as for a
+ * page the apply itself adopted) it approved. In a pass that carries
+ * it, any other Git verdict that would replace a database version (`gitWins`), or one whose page, revision or blob
+ * moved since the plan, is reclassified as the drift (or held deletion) a mixed pass gives it, so the pass holds the
+ * path instead of overwriting the page; `reclassified` collects those paths for the apply.
+ */
+export interface ConversionBinding { expect: ReadonlyMap<string, { id: string; revision: string | null; blob?: string | null }>; reclassified: Set<string> }
/** PGLite's keyset page size (design v3.2 §2.1). */
const PGLITE_KEYSET_ROWS = 2_000;
/**
@@ -906,6 +918,7 @@ export async function readShadowLevel(engine: BrainEngine, input: { sourceId: st
}
timings.level_bytes = t();
}
+ if (strict && input.pass?.conversion) bindConversion(paths, scope, input.pass.conversion);
const kind = syncPassKind({ converged: null, head, parserFp: '', pack: pack?.identity ?? null, selectionFp: '', now: Date.now(), openEpisode: input.full === true, owedTail: false,
retryRow: retry.size > 0, notConvergedStreak: 0, mismatchStreak: 0, requestNewer: false, openAtStampFinished: false, workingChange: false,
@@ -1071,6 +1084,23 @@ async function confirmPages(exec: Exec, input: { kind: 'postgres' | 'pglite'; so
* #6926, folded into 4b: what a folded checkpoint needs of the round's `I⋈P` (S1): its mark, its page count and its largest
* page id (a page past it is one the pass created). Null when the round left no marked `I⋈P` (working mode, a failed mark).
*/
+/** See ConversionBinding: an unapproved or moved Git verdict over a database version becomes the hold a mixed pass gives it. */
+function bindConversion(paths: ShadowPath[], scope: string, binding: ConversionBinding): void {
+ for (const [i, entry] of paths.entries()) {
+ const r = entry.result;
+ if (!r.gitWins || r.settled || r.inFlight || (r.action !== 'import' && r.action !== 'delete')) continue;
+ const path = scope && entry.path.startsWith(`${scope}/`) ? entry.path.slice(scope.length + 1) : entry.path;
+ const expected = binding.expect.get(path);
+ if (expected && String(entry.pageId) === expected.id && (entry.revision ?? null) === expected.revision && (expected.blob === undefined || (entry.state.level ?? null) === expected.blob)) continue;
+ binding.reclassified.add(path);
+ const result: SyncClassified = r.action === 'delete'
+ ? { cls: 'delete_drift', settled: false, action: 'hold', side: 'deleted', state: 'file_gone', inFlight: false, classAction: 'repair' }
+ : { cls: 'drift', settled: false, action: 'hold', side: entry.state.record?.revision != null ? 'both' : 'unknown', inFlight: false, classAction: 'repair' };
+ paths[i] = { path: entry.path, slug: entry.slug, pageId: entry.pageId, revision: entry.revision, state: entry.state, result,
+ ...(entry.legacyPageless ? { legacyPageless: entry.legacyPageless } : {}), ...(entry.ambiguousOrigin ? { ambiguousOrigin: entry.ambiguousOrigin } : {}) };
+ }
+}
+
export function passFoldBase(cache: PassLevelCache): { mark: LevelMark; pages: number; maxId: string } | null {
const table = cache.pages?.table;
if (!table || !cache.pages) return null;
diff --git a/test/e2e/repair-file-drift-deleted-side-d5-postgres.test.ts b/test/e2e/repair-file-drift-deleted-side-d5-postgres.test.ts
new file mode 100644
index 0000000000..17486631c2
--- /dev/null
+++ b/test/e2e/repair-file-drift-deleted-side-d5-postgres.test.ts
@@ -0,0 +1,3 @@
+import { registerPostgresTests } from '../helpers/test-backends.ts';
+
+await registerPostgresTests(() => import('../repair-file-drift-deleted-side-d5.test.ts'));
diff --git a/test/repair-file-drift-deleted-side-d5.test.ts b/test/repair-file-drift-deleted-side-d5.test.ts
new file mode 100644
index 0000000000..5432f27c16
--- /dev/null
+++ b/test/repair-file-drift-deleted-side-d5.test.ts
@@ -0,0 +1,204 @@
+/**
+ * PR A item 5: the deleted side of `gbrain repair file-drift --held --side deleted --state <state> [--provenance <p>] --prefer file|database`.
+ * - a live page whose file a commit removed, held by sync as `file_gone` (provenance `deleted_by_commit`), previews
+ * --prefer database as `recreate_file` ("recreate the file"), and its --apply --expect <hash> writes the file back from
+ * the page: the file parses back to the page and the page stays live with the same content; the apply leaves the
+ * hold to sync (an uncommitted recreated file keeps it), and committing the file then syncing clears it;
+ * - every other deleted-side Decision previews its own verb (fileDriftActionVerb, sync-holds.ts): tombstone_edited
+ * database "keep the deletion and remove the file" / file "restore and import"; purged_differs database "remove the
+ * file" / file "import as a new page"; file_gone file "soft-delete the page"; a never-committed file kept
+ * database-only (`detach_page`, "keep the page database-only (no file)"); a deletion HEAD still holds lists
+ * `deletion_uncommitted` and resolves nothing;
+ * - a preview writes nothing: the checkout, the pages and the holds are byte-for-byte as they were.
+ * Real PGLite (and Postgres when DATABASE_URL is set), real managed syncs and real Git: every hold here is written by a
+ * sync that held the deletion, none by hand.
+ */
+import { describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import type { OperationContext } from '../src/core/ops/contract.ts';
+import { parseMarkdown } from '../src/core/markdown.ts';
+import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
+import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';
+import { performManagedSync } from '../src/core/persistence/sync-run.ts';
+import { repairRunner } from '../src/core/repair/registry.ts';
+import { resolveRepairScope } from '../src/core/repair/core.ts';
+import { managedBrain, settleManagedGitEffects } from './helpers/managed-brain.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
+
+const logger = { info() {}, warn() {}, error() {} };
+const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]',
+ '-c', 'commit.gpgsign=false', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const commit = (root: string, message: string) => { git(root, 'add', '-A'); git(root, 'commit', '-qm', message); };
+const note = (title: string, body: string) => `---\ntype: concept\ntitle: ${title}\n---\n\n${body}\n`;
+const OPTS = { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true };
+const NAMES = ['alpha', 'beta', 'gamma', 'delta', 'epsilon'];
+
+type Hold = { path: string; code: string; observed_at: string; meta: Record<string, unknown> };
+async function holds(engine: BrainEngine): Promise<Record<string, Hold>> {
+ const rows = await engine.executeRaw<{ record: Hold }>("SELECT completed_keys->0 AS record FROM op_checkpoints WHERE op='sync-hold'");
+ return Object.fromEntries(rows.map(({ record }) => [record.path, record]));
+}
+/** What a preview must leave alone: every file of the checkout, HEAD and the index, every page row and every hold. */
+async function state(engine: BrainEngine, root: string) {
+ const files = Object.fromEntries(readdirSync(join(root, 'notes')).sort().map(name => [name, readFileSync(join(root, 'notes', name), 'utf8')]));
+ const pages = await engine.executeRaw(`SELECT slug, knowledge_revision::text AS revision, deleted_at::text AS deleted_at, compiled_truth, source_path, database_only_reason
+ FROM pages WHERE source_id='default' AND slug LIKE 'notes/%' ORDER BY slug`);
+ return { files, head: git(root, 'rev-parse', 'HEAD'), status: git(root, 'status', '--porcelain'), pages, holds: await holds(engine) };
+}
+
+/** A page an older import wrote: no import record, no write request, last written an hour ago (before any deleting commit). */
+async function olderImport(engine: BrainEngine, slug: string): Promise<void> {
+ const coordinated = (sql: string, params: unknown[]) => engine.transaction(tx => withCoordinatedWrite(tx, ['default'], () => tx.executeRaw(sql, params), TEST_WRITE_ATTRIBUTION));
+ await coordinated(`DELETE FROM op_checkpoints WHERE op='sync-import-provenance'
+ AND completed_keys->0->>'page_id' IN (SELECT id::text FROM pages WHERE source_id='default' AND slug=$1)`, [slug]);
+ await coordinated(`UPDATE pages SET knowledge_revision=$1::uuid, revision_write_request_id=NULL, updated_at=now() - interval '1 hour'
+ WHERE source_id='default' AND slug=$2`, [randomUUID(), slug]);
+}
+
+for (const backend of testBackends()) describe(`[${backend}] PR A item 5: repair file-drift --held --side deleted`, () => {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+ const setup = ({ root }: { root: string }) => {
+ mkdirSync(join(root, 'notes'), { recursive: true }); git(root, 'init', '-q');
+ for (const name of NAMES) writeFileSync(join(root, `notes/${name}-example.md`), note(`notes/${name}-example`, `The committed body of notes/${name}-example.`));
+ commit(root, 'notes');
+ };
+ const run = async (engine: BrainEngine, opts: { state: string; prefer: 'file' | 'database'; provenance?: string; only?: string[]; expect?: string }) =>
+ (await repairRunner(engine, { apply: opts.expect !== undefined, logger })).run('file-drift', await resolveRepairScope(engine, 'default'),
+ { explicit: true, sourceFlag: 'default', held: true, side: 'deleted', state: opts.state, prefer: opts.prefer,
+ ...(opts.provenance ? { provenance: opts.provenance } : {}), ...(opts.only ? { only: opts.only } : {}), ...(opts.expect ? { expect: opts.expect } : {}) });
+ type Page = { slug: string; action?: string; verb?: string; state?: string; path?: string; provenance?: { kind: string; subject?: string } };
+ const pages = (result: Awaited<ReturnType<typeof run>>) => (result.details?.pages ?? []) as Page[];
+ const listed = (result: Awaited<ReturnType<typeof run>>) => (result.listing ?? []).map(entry => [entry.item, entry.class]);
+
+ test('file_gone deleted_by_commit: --prefer database previews recreate_file, its apply writes the file back from the page, and commit + sync clears the hold', () => managedBrain(async ({ engine, root }) => {
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ await olderImport(engine, 'notes/alpha-example');
+ git(root, 'rm', '-q', 'notes/alpha-example.md'); git(root, 'commit', '-qm', 'cull alpha');
+ await performManagedSync(engine, OPTS);
+ const path = 'notes/alpha-example.md';
+ expect((await holds(engine))[path]).toMatchObject({ code: 'file_drift', meta: { side: 'deleted', state: 'file_gone' } });
+ expect((await holds(engine))[path]!.meta.reason).toBeUndefined();
+ const before = await state(engine, root);
+ const page = (await engine.readPageSnapshot('notes/alpha-example', { sourceId: 'default' }))!;
+
+ const preview = await run(engine, { state: 'file_gone', prefer: 'database', provenance: 'deleted_by_commit' });
+ expect(preview).toMatchObject({ mode: 'dry_run', affected: 1 });
+ expect(pages(preview)).toEqual([expect.objectContaining({ slug: 'notes/alpha-example', action: 'recreate_file', verb: 'recreate the file',
+ provenance: expect.objectContaining({ kind: 'deleted_by_commit', subject: 'cull alpha' }) })]);
+ expect(listed(preview)).toEqual([['default:notes/alpha-example', 'prefer_database']]);
+ expect(preview.listing![0]!.detail).toContain(`recreates its missing file ${path} from the page`);
+ expect(preview.listing![0]!.detail).toContain(`deleted by commit ${git(root, 'rev-parse', 'HEAD').slice(0, 12)}`);
+ expect(preview.apply_command).toContain('--provenance deleted_by_commit --prefer database --apply --expect');
+ expect(await state(engine, root)).toEqual(before);
+ expect(existsSync(join(root, path))).toBe(false);
+
+ const applied = await run(engine, { state: 'file_gone', prefer: 'database', provenance: 'deleted_by_commit', expect: preview.preview_hash! });
+ expect(applied.outcomes).toEqual({ repaired: 1 });
+ const parsed = parseMarkdown(readFileSync(join(root, path), 'utf8'), path);
+ const after = (await engine.readPageSnapshot('notes/alpha-example', { sourceId: 'default' }))!;
+ expect(after.page.deleted_at).toBeFalsy();
+ expect(after.page.compiled_truth).toBe(page.page.compiled_truth);
+ expect(after.page.title).toBe(page.page.title);
+ expect(parsed.compiled_truth).toBe(page.page.compiled_truth.trim());
+ expect(parsed.title).toBe(page.page.title);
+ expect(parsed.type).toBe(page.page.type);
+ // The apply recreates the file uncommitted (Git durability is off here, as the preview warns) and leaves the hold to the
+ // sync that re-screens it: an uncommitted file keeps the hold and the page; committing it and syncing clears the hold.
+ expect(preview.warnings).toEqual([expect.stringContaining('unless the recreated files are committed')]);
+ expect(preview.details!.undone_by_next_sync_unless_committed).toEqual({ recreated: 1, removed: 0 });
+ expect(git(root, 'status', '--porcelain')).toBe(`?? ${path}`);
+ expect((await holds(engine))[path]).toMatchObject({ code: 'file_drift', meta: { state: 'file_gone' } });
+ await performManagedSync(engine, OPTS);
+ expect((await holds(engine))[path]).toMatchObject({ code: 'file_drift', meta: { state: 'file_gone' } });
+ expect((await engine.readPageSnapshot('notes/alpha-example', { sourceId: 'default' }))!.page.deleted_at).toBeFalsy();
+ git(root, 'add', '--', path); git(root, 'commit', '-qm', 'gbrain: repair file drift in default');
+ await performManagedSync(engine, OPTS);
+ expect((await holds(engine))[path]).toBeUndefined();
+ const synced = (await engine.readPageSnapshot('notes/alpha-example', { sourceId: 'default' }))!;
+ expect(synced.page.deleted_at).toBeFalsy();
+ expect(synced.page.compiled_truth).toBe(page.page.compiled_truth);
+ expect(parseMarkdown(readFileSync(join(root, path), 'utf8'), path).compiled_truth).toBe(page.page.compiled_truth.trim());
+ }, { databaseUrl, setup }), 300_000);
+
+ test('every other deleted-side Decision previews its own verb, and no preview writes anything', () => managedBrain(async ({ engine, ctx, root }) => {
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ const snapshot = (slug: string) => engine.readPageSnapshot(slug, { sourceId: 'default', includeDeleted: true });
+ const remove = async (c: OperationContext, slug: string, purge: boolean) => submitPageMutation(c, { operation: 'delete_page',
+ params: { slug, request_id: randomUUID(), expected_revision: (await snapshot(slug))!.revision, ...(purge ? { purge: true } : {}) } });
+ // file_gone: a commit removed notes/alpha's file; notes/delta's too, then a commit restored it and the worktree lost it again (HEAD holds it).
+ for (const slug of ['notes/alpha-example', 'notes/delta-example']) await olderImport(engine, slug);
+ git(root, 'rm', '-q', 'notes/alpha-example.md', 'notes/delta-example.md'); git(root, 'commit', '-qm', 'cull alpha and delta');
+ // tombstone_edited: notes/beta deleted in the brain, then its file edited and committed; purged_differs: notes/gamma purged, then a different file committed.
+ await remove(ctx, 'notes/beta-example', false);
+ await remove(ctx, 'notes/gamma-example', true);
+ await settleManagedGitEffects(engine);
+ writeFileSync(join(root, 'notes/beta-example.md'), note('notes/beta-example', 'Edited after the delete.'));
+ writeFileSync(join(root, 'notes/gamma-example.md'), note('notes/gamma-example', 'A different file at the purged origin.'));
+ commit(root, 'edit beta, re-add gamma');
+ await performManagedSync(engine, OPTS);
+ git(root, 'checkout', '-q', 'HEAD~2', '--', 'notes/delta-example.md'); commit(root, 'restore delta');
+ rmSync(join(root, 'notes/delta-example.md'));
+ // never_committed: a working-tree page no commit ever held, then removed from the disk.
+ await engine.setConfig('sync.include_working_tree', 'true');
+ writeFileSync(join(root, 'notes/zeta-example.md'), note('notes/zeta-example', 'An untracked note only the brain holds.'));
+ await performManagedSync(engine, OPTS);
+ await engine.setConfig('sync.include_working_tree', 'false');
+ rmSync(join(root, 'notes/zeta-example.md'));
+ await performManagedSync(engine, OPTS);
+
+ const held = await holds(engine);
+ expect(Object.fromEntries(Object.entries(held).map(([path, hold]) => [path, [hold.code, hold.meta.side, hold.meta.state, hold.meta.reason ?? null]]))).toEqual({
+ 'notes/alpha-example.md': ['file_drift', 'deleted', 'file_gone', null],
+ 'notes/beta-example.md': ['file_drift', 'deleted', 'tombstone_edited', null],
+ 'notes/delta-example.md': ['file_drift', 'deleted', 'file_gone', null],
+ 'notes/gamma-example.md': ['file_drift', 'deleted', 'purged_differs', null],
+ 'notes/zeta-example.md': ['file_drift', 'deleted', 'file_gone', 'never_committed'],
+ });
+ const before = await state(engine, root);
+ const preview = async (opts: Parameters<typeof run>[1]) => {
+ const result = await run(engine, opts);
+ expect(result.mode).toBe('dry_run');
+ expect(await state(engine, root)).toEqual(before);
+ return result;
+ };
+
+ const tombDb = await preview({ state: 'tombstone_edited', prefer: 'database' });
+ expect(pages(tombDb)).toEqual([expect.objectContaining({ slug: 'notes/beta-example', action: 'remove_file', state: 'tombstone_edited', verb: 'keep the deletion and remove the file' })]);
+ expect(listed(tombDb)).toEqual([['default:notes/beta-example', 'prefer_database']]);
+ const tombFile = await preview({ state: 'tombstone_edited', prefer: 'file' });
+ expect(pages(tombFile)).toEqual([expect.objectContaining({ slug: 'notes/beta-example', action: 'import_file', state: 'tombstone_edited', verb: 'restore and import' })]);
+ expect(listed(tombFile)).toEqual([['default:notes/beta-example', 'prefer_file']]);
+
+ const purgedDb = await preview({ state: 'purged_differs', prefer: 'database' });
+ expect(pages(purgedDb)).toEqual([expect.objectContaining({ slug: 'notes/gamma-example', action: 'remove_file', state: 'purged_differs', verb: 'remove the file' })]);
+ expect(listed(purgedDb)).toEqual([['default:notes/gamma-example', 'prefer_database']]);
+ const purgedFile = await preview({ state: 'purged_differs', prefer: 'file' });
+ expect(pages(purgedFile)).toEqual([expect.objectContaining({ slug: 'notes/gamma-example', action: 'import_file', state: 'purged_differs', verb: 'import as a new page' })]);
+ expect(listed(purgedFile)).toEqual([['default:notes/gamma-example', 'prefer_file']]);
+
+ const goneFile = await preview({ state: 'file_gone', prefer: 'file', provenance: 'deleted_by_commit' });
+ expect(pages(goneFile)).toEqual([expect.objectContaining({ slug: 'notes/alpha-example', action: 'soft_delete', verb: 'soft-delete the page',
+ provenance: expect.objectContaining({ kind: 'deleted_by_commit' }) })]);
+ expect(listed(goneFile)).toEqual([['default:notes/alpha-example', 'prefer_file']]);
+ expect(goneFile.residuals).toMatchObject({ other_provenance: 2 });
+
+ const never = await preview({ state: 'file_gone', prefer: 'database', provenance: 'never_committed' });
+ expect(pages(never)).toEqual([expect.objectContaining({ slug: 'notes/zeta-example', action: 'detach_page', verb: 'keep the page database-only (no file)',
+ provenance: { kind: 'never_committed' } })]);
+ expect(listed(never)).toEqual([['default:notes/zeta-example', 'prefer_database']]);
+
+ for (const prefer of ['database', 'file'] as const) {
+ const uncommitted = await preview({ state: 'file_gone', prefer, provenance: 'deletion_uncommitted' });
+ expect(uncommitted.affected).toBe(0);
+ expect(pages(uncommitted)).toEqual([]);
+ expect(listed(uncommitted)).toEqual([['default:notes/delta-example', 'deletion_uncommitted']]);
+ expect(uncommitted.residuals).toMatchObject({ deletion_uncommitted: 1, other_provenance: 2 });
+ expect(uncommitted.listing![0]!.detail).toContain('deletion uncommitted: HEAD still holds the file');
+ }
+ }, { databaseUrl, setup }), 300_000);
+});