Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

test(convergence): busy-agent soak cell (modes h-m) on a shared convergence harness; informational nightly job (L5, GST-1502)

garrytan0dce824a3f44
diff --git a/.github/workflows/convergence-nightly.yml b/.github/workflows/convergence-nightly.yml
index ef25799058..18f6aed4e6 100644
--- a/.github/workflows/convergence-nightly.yml
+++ b/.github/workflows/convergence-nightly.yml
@@ -9,6 +9,11 @@ name: Convergence nightly
# concurrency group, and a red lands in the nightly-red issue (nightly-watch.yml).
# Every job name carries its seed, and a failure writes the seed, the modes, the
# fault plan, the fired faults and the one-line repro to the step summary.
+# The busy-agent job (L5, test/busy-agent-soak.serial.test.ts) runs the same harness
+# under a busy agent's workload, from the committed generic profile
+# test/fixtures/busy-agent-profile.json. It is informational (continue-on-error) for
+# its first seven nights; after that a release touching persistence, sync or Git
+# needs it green on the release's exact sha.
on:
schedule:
@@ -115,3 +120,89 @@ jobs:
path: .context/convergence
include-hidden-files: true
if-no-files-found: warn
+
+ busy-agent:
+ name: Busy-agent soak cell (informational) / Bun ${{ matrix.bun }} / seed ${{ inputs.seed || github.run_id }}
+ runs-on: ubicloud-standard-4-ubuntu-2404
+ timeout-minutes: 150
+ continue-on-error: true
+ strategy:
+ fail-fast: false
+ matrix:
+ bun: ['1.4.4']
+ services:
+ postgres:
+ image: pgvector/pgvector:pg16
+ env:
+ POSTGRES_USER: gbrain_test
+ POSTGRES_PASSWORD: gbrain_test
+ POSTGRES_DB: gbrain_test
+ ports: ['5432:5432']
+ options: >-
+ --health-cmd "pg_isready -U gbrain_test -d gbrain_test"
+ --health-interval 5s --health-timeout 5s --health-retries 20
+ pgbouncer:
+ image: edoburu/pgbouncer:latest
+ env:
+ DB_HOST: postgres
+ DB_PORT: '5432'
+ DB_USER: gbrain_test
+ DB_PASSWORD: gbrain_test
+ POOL_MODE: transaction
+ AUTH_TYPE: plain
+ MAX_CLIENT_CONN: '200'
+ DEFAULT_POOL_SIZE: '10'
+ IGNORE_STARTUP_PARAMETERS: extra_float_digits,statement_timeout,idle_in_transaction_session_timeout,search_path
+ ports: ['55433:5432']
+ env:
+ GBRAIN_TEST_BUSY_AGENT_LOG_DIR: ${{ github.workspace }}/.context/busy-agent
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
+ with:
+ bun-version: ${{ matrix.bun }}
+ - name: Install dependencies (bounded, retried once)
+ timeout-minutes: 8
+ run: bun scripts/ci-bun-install.ts --frozen-lockfile --ignore-scripts
+ - name: Run the busy-agent modes (h)-(m) from the generic profile (50 ms)
+ timeout-minutes: 120
+ env:
+ GBRAIN_TEST_BUSY_AGENT: '1'
+ GBRAIN_PGBOUNCER_URL: postgres://gbrain_test:[email protected]:55433/gbrain_test
+ GBRAIN_PGBOUNCER_DIRECT_URL: postgres://gbrain_test:[email protected]:5432/gbrain_test
+ GBRAIN_TEST_BUSY_AGENT_LATENCY_MS: '50'
+ GBRAIN_TEST_BUSY_AGENT_SEED: ${{ inputs.seed || github.run_id }}
+ GBRAIN_TEST_BUSY_AGENT_SYNC_TIMEOUT_MS: '1800000'
+ GBRAIN_TEST_BUSY_AGENT_SCENARIO_TIMEOUT_MS: '3600000'
+ run: |
+ set -o pipefail
+ mkdir -p "$GBRAIN_TEST_BUSY_AGENT_LOG_DIR"
+ bun --no-env-file test --timeout="$GBRAIN_TEST_BUSY_AGENT_SCENARIO_TIMEOUT_MS" test/busy-agent-soak.serial.test.ts 2>&1 | tee "$GBRAIN_TEST_BUSY_AGENT_LOG_DIR/bun-test.log"
+ - name: Write the seed-carrying failure summary
+ if: failure() || cancelled()
+ env:
+ SEED: ${{ inputs.seed || github.run_id }}
+ BUN_CELL: ${{ matrix.bun }}
+ run: |
+ log="$GBRAIN_TEST_BUSY_AGENT_LOG_DIR/bun-test.log"
+ {
+ echo "### Busy-agent soak cell red (informational): Bun $BUN_CELL, seed $SEED"
+ echo
+ echo '```'
+ grep -E '^\(fail\)|^seed [0-9]|^fault plan:|^fired:|^repro ' "$log" 2>/dev/null | head -60 || true
+ [ -s "$log" ] || echo "no test output: the step timed out or never started"
+ echo '```'
+ echo
+ echo "Replay: gh workflow run convergence-nightly.yml -f seed=$SEED (or the repro line above, locally)."
+ } >> "$GITHUB_STEP_SUMMARY"
+ - name: Stop harness processes a timed-out run left behind
+ if: always()
+ run: pkill -9 -f 'src/cli\.ts (serve|sync|put|call|extract|sources|git|doctor)' || true
+ - name: Upload the busy-agent evidence (profile, fault plan, every writer, serve and refresh log)
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+ with:
+ name: busy-agent-nightly-bun-${{ matrix.bun }}
+ path: .context/busy-agent
+ include-hidden-files: true
+ if-no-files-found: warn
diff --git a/package.json b/package.json
index ca841dff89..5e5b3828df 100644
--- a/package.json
+++ b/package.json
@@ -102,6 +102,7 @@
"test:profile": "bash scripts/profile-tests.sh",
"test:serial": "bash scripts/run-serial-tests.sh",
"test:convergence": "GBRAIN_TEST_CONVERGENCE=1 bun --no-env-file test --timeout=3600000 test/sync-convergence.serial.test.ts",
+ "test:busy-agent": "GBRAIN_TEST_BUSY_AGENT=1 bun --no-env-file test --timeout=3600000 test/busy-agent-soak.serial.test.ts",
"test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts drain-lease && bun scripts/spec/mutants.ts refresh-barrier",
"ci:local": "bash scripts/ci-local.sh",
"ci:local:diff": "bash scripts/ci-local.sh --diff",
diff --git a/scripts/ci-convergence-scope.sh b/scripts/ci-convergence-scope.sh
index 64a9650efc..5dd99cb386 100755
--- a/scripts/ci-convergence-scope.sh
+++ b/scripts/ci-convergence-scope.sh
@@ -11,7 +11,7 @@
# runs never call this; they always run the gate.
set -euo pipefail
-CONVERGENCE_PATHS='^(bun\.lock$|vendor/postgres/|src/core/persistence/|src/commands/sync/|src/core/page-state/|src/core/engine-sql/|src/core/storage/|test/fixtures/convergence-fault-kill\.ts$|test/sync-convergence\.serial\.test\.ts$|scripts/ci-convergence-scope\.sh$|\.github/workflows/(persistence-validation|convergence-nightly)\.yml$)|^src/commands/sync[^/]*\.ts$|^src/core/(sync[^/]*|import-file|markdown|write-through|timeline-write-through|storage|storage-config|page-lock|ghost-pages|postgres-engine)\.ts$'
+CONVERGENCE_PATHS='^(bun\.lock$|vendor/postgres/|src/core/persistence/|src/commands/sync/|src/core/page-state/|src/core/engine-sql/|src/core/storage/|test/fixtures/convergence-fault-kill\.ts$|test/helpers/convergence-brain\.ts$|test/sync-convergence\.serial\.test\.ts$|scripts/ci-convergence-scope\.sh$|\.github/workflows/(persistence-validation|convergence-nightly)\.yml$)|^src/commands/sync[^/]*\.ts$|^src/core/(sync[^/]*|import-file|markdown|write-through|timeline-write-through|storage|storage-config|page-lock|ghost-pages|postgres-engine)\.ts$'
files=$(cat) || { echo run; exit 0; }
# Consume the full input: grep -q closes an early match's pipe and printf
diff --git a/scripts/persistence/README.md b/scripts/persistence/README.md
index 628942232e..c6643abb39 100644
--- a/scripts/persistence/README.md
+++ b/scripts/persistence/README.md
@@ -659,6 +659,49 @@ summary (every failing test with its seed, fault plan, fired faults and repro
line) and the upload of `LOG_DIR` still run. The nightly watcher
(`nightly-watch.yml`) files its failures like every other scheduled workflow.
+### Busy-agent soak cell
+
+`test/busy-agent-soak.serial.test.ts` (L5) runs the convergence gate's
+real-process brain (`test/helpers/convergence-brain.ts`, shared with
+`test/sync-convergence.serial.test.ts`: Postgres behind transaction-mode
+PgBouncer through latency proxies, real `src/cli.ts` processes, a resident
+serve, the fault seam, the acknowledged-write ledger and the oracle) under the
+workload a busy personal agent puts on one managed checkout. Its counts and
+cadences come from a profile: the committed generic one,
+`test/fixtures/busy-agent-profile.json`, unless
+`GBRAIN_TEST_BUSY_AGENT_PROFILE` names another (a profile derived from a real
+deployment stays private). A profile holds exactly the test's `PROFILE_SHAPE`
+keys, each a positive integer, plus an optional `description`; it never holds
+content.
+
+| Mode | What happens | Expected end state |
+| --- | --- | --- |
+| (h) refresh ticks | With `persistence.refresh_barrier` on, scheduled `sources refresh` ticks pull upstream commits while foreground `put`s land inside the fence, some on notes the tick changes, and an agent writes across every tick | every tick's refresh completes and releases its fence; every write that reached the refreshing checkout is answered pending with a receipt naming the refresh, never refused, and commits after it, or ends `revision_conflict` with nothing applied when the refresh changed its file |
+| (i) foreign writers | Plain-git committers and an uncommitted-file writer beside gbrain | `doctor` `foreign_writers` names every commit gbrain did not make and none it made, and the aged uncommitted files |
+| (j) legacy rows | Rows with no import record in every content-evidence direction | `sources git-gap` classifies each by its direction evidence; `both_moved` and `no_evidence` rows end held `file_drift` |
+| (k) strict conversion | `sources canonical --to git` over several adopt groups, stalled after the first (`conversion:group_committed`) while another clone pushes, door writes land and a refresh runs | the conversion ends `converted`, strict, with nothing left to plan; every door write and refresh is accepted or refused with a registered code; the other agent's commit reaches HEAD |
+| (l) segmented staging | A segmented write whose staging expires or is aborted at `segmented:manifest_staged` | the write refuses `body_staging_expired`, the page keeps its body, a resend commits and the sweep leaves no live staging |
+| (m) lost update | A file-writing cron (`git commit-push --base`) and an MCP `edit_page` writer race on one page | `edit_page` refuses `revision_conflict`, `commit-push --base` refuses `commit_base_moved`, and no acknowledged write is silently reverted |
+
+Every scenario then gets the convergence gate's oracle (above). It skips
+unless `GBRAIN_TEST_BUSY_AGENT=1`; with the Postgres and PgBouncer of the
+gate above:
+
+```bash
+GBRAIN_TEST_BUSY_AGENT_LATENCY_MS=0 bun run test:busy-agent # every mode, fast local loop
+GBRAIN_TEST_BUSY_AGENT_MODES=k bun run test:busy-agent # one mode
+```
+
+Knobs (`GBRAIN_TEST_BUSY_AGENT_*`): `SEED` (default 8800), `MODES`,
+`PROFILE`, `LATENCY_MS` (default 50), `SYNC_TIMEOUT_MS` (default 900000),
+`SCENARIO_TIMEOUT_MS` (default 3600000) and `LOG_DIR`.
+
+CI: the `busy-agent` job of `.github/workflows/convergence-nightly.yml`, on
+the same runner class, at 50 ms with the run id as the seed. It is
+informational (`continue-on-error`) for its first seven nights; after that a
+release touching persistence, sync or Git needs it green on the release's
+exact sha.
+
## Engine graduation tests
Graduation (`gbrain migrate --to postgres`) is tested against two fixtures.
diff --git a/scripts/serial-files.tsv b/scripts/serial-files.tsv
index 610c3b4587..65fe3e411d 100644
--- a/scripts/serial-files.tsv
+++ b/scripts/serial-files.tsv
@@ -398,6 +398,7 @@ test/spawn-seam-batch1-4992.serial.test.ts R1 R1: mutates process.env (GBRAIN_HO
test/stdio-stderr-redirect.serial.test.ts global-state global-state: patches process stdout/stderr/console
test/subagent-put-page-rejection.serial.test.ts global-state global-state: process-wide test seams __setChatTransportForTests; configureGateway changes the process-wide AI gateway; spyOn putPage.handler
test/sync-convergence.serial.test.ts global-state global-state: binds local servers/ports (two latency proxies per scenario) and spawns serve/sync/put/call processes against each scenario's brain; its oracle tests read the chaos scenario their describe's beforeAll ran, across it() boundaries
+test/busy-agent-soak.serial.test.ts global-state global-state: binds local servers/ports (two latency proxies per scenario) and spawns serve/sync/put/sources/git/doctor processes against each scenario's brain; its mode tests read the scenario their describe's beforeAll ran, across it() boundaries
test/sync-cost-gate.serial.test.ts global-state global-state: process-wide test seams __setEmbedTransportForTests; configureGateway changes the process-wide AI gateway; patches process stdout/stderr/console
test/sync-default-write-guard.serial.test.ts global-state global-state: spyOn console.error, console.log, process.exit
test/sync-delegated-options-6696.serial.test.ts global-state global-state: process-wide test seams __resetDelegatedSyncForTests and the embedding transport; binds local servers/ports; patches process stdout/stderr/console
diff --git a/scripts/serial-weights.json b/scripts/serial-weights.json
index 965f2ff5f4..e15edaf29c 100644
--- a/scripts/serial-weights.json
+++ b/scripts/serial-weights.json
@@ -415,6 +415,7 @@
"test/stdio-stderr-redirect.serial.test.ts": 0,
"test/subagent-put-page-rejection.serial.test.ts": 3,
"test/sync-convergence.serial.test.ts": 3,
+ "test/busy-agent-soak.serial.test.ts": 1,
"test/sync-cost-gate.serial.test.ts": 18,
"test/sync-default-write-guard.serial.test.ts": 4,
"test/sync-delegated-options-6696.serial.test.ts": 11,
diff --git a/test/busy-agent-soak.serial.test.ts b/test/busy-agent-soak.serial.test.ts
new file mode 100644
index 0000000000..ed9c57e02b
--- /dev/null
+++ b/test/busy-agent-soak.serial.test.ts
@@ -0,0 +1,772 @@
+/**
+ * L5 (GST-1502): the busy-agent soak cell, the convergence harness under the workload a busy personal agent puts on
+ * one managed checkout. It shares the real-process brain of the managed-sync convergence gate
+ * (test/helpers/convergence-brain.ts: Postgres behind transaction-mode PgBouncer through latency proxies, real
+ * `src/cli.ts` processes, a resident serve, the acknowledged-write ledger and the oracle), and its counts and cadences
+ * come from a committed generic profile (test/fixtures/busy-agent-profile.json; GBRAIN_TEST_BUSY_AGENT_PROFILE points
+ * at another, such as a private one derived from a real deployment).
+ *
+ * Modes (MODES below; GBRAIN_TEST_BUSY_AGENT_MODES=h,m runs exactly those):
+ * (h) scheduled `sources refresh` ticks with foreground writes inside the fence: with persistence.refresh_barrier on,
+ * a write that reaches the checkout while a refresh holds it is answered pending and commits after the refresh,
+ * or ends revision_conflict when the refresh changed its file;
+ * (i) plain-git committers and uncommitted-file writers beside gbrain: doctor foreign_writers names every commit
+ * gbrain did not make and none it made;
+ * (j) legacy rows with no import record in every direction: `sources git-gap` classifies each by content evidence;
+ * (k) a strict conversion with live door writers and a refresh between adopt groups;
+ * (l) a segmented page whose staging expires between staging and publication: the write refuses
+ * body_staging_expired, the page keeps its body, a resend commits and the sweep leaves no live staging;
+ * (m) the lost-update race between a file-writing cron and an MCP write to the same page: edit_page refuses
+ * revision_conflict and `git commit-push --base` refuses commit_base_moved, and no acknowledged write is
+ * silently reverted.
+ *
+ * Every scenario ends with the convergence gate's oracle (a fresh serve, at most two plain syncs, disk == database,
+ * no open request, registered terminal codes, the cursor at HEAD, the ledger and the status oracle). The cell is
+ * informational in .github/workflows/convergence-nightly.yml for seven nights, then gates releases that touch
+ * persistence, sync or Git. Setup: scripts/persistence/README.md ("Busy-agent soak cell"). It is skipped unless
+ * GBRAIN_TEST_BUSY_AGENT=1.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, utimesSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { FOREIGN_DIRTY_WARN_MS } from '../src/commands/doctor/checks/foreign-writers.ts';
+import { inlineExportMarkdown, readExportPage, readExportWithdrawals } from '../src/core/export-snapshot.ts';
+import { GIT_HOLD_OP, SYNC_IMPORT_PROVENANCE_OP } from '../src/core/persistence/sync-holds.ts';
+import { CODES } from '../src/core/error-registry.ts';
+import { CONVERSION_GROUP_MAX } from '../src/core/persistence/git-conversion.ts';
+import { sweepSegmentStorage } from '../src/core/persistence/journal.ts';
+import { withEnv } from './helpers/with-env.ts';
+import {
+ Brain as HarnessBrain, SOURCE, StdioServe, as, fileHash, lastJson, noteMarker, noteSlug, noteText, oracleTests, until,
+ type Fault, type HarnessKnobs, type SyncRun,
+} from './helpers/convergence-brain.ts';
+
+const REPO_ROOT = resolve(import.meta.dir, '..');
+const knob = (name: string, fallback: number): number => {
+ const raw = process.env[`GBRAIN_TEST_BUSY_AGENT_${name}`];
+ const value = raw === undefined || raw === '' ? fallback : Number(raw);
+ if (!Number.isInteger(value) || value < 0) throw new Error(`GBRAIN_TEST_BUSY_AGENT_${name} must be a non-negative integer, got '${raw}'.`);
+ return value;
+};
+const SEED = knob('SEED', 8800);
+const LATENCY_MS = knob('LATENCY_MS', 50);
+const SYNC_TIMEOUT_MS = knob('SYNC_TIMEOUT_MS', 900_000);
+const SCENARIO_TIMEOUT_MS = knob('SCENARIO_TIMEOUT_MS', 3_600_000);
+const PROFILE_PATH = process.env.GBRAIN_TEST_BUSY_AGENT_PROFILE || join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json');
+const pooledTarget = process.env.GBRAIN_PGBOUNCER_URL;
+const directTarget = process.env.GBRAIN_PGBOUNCER_DIRECT_URL;
+const enabled = process.env.GBRAIN_TEST_BUSY_AGENT === '1';
+if (enabled && !(pooledTarget && directTarget)) {
+ throw new Error('GBRAIN_TEST_BUSY_AGENT=1 needs GBRAIN_PGBOUNCER_URL (transaction-mode PgBouncer) and GBRAIN_PGBOUNCER_DIRECT_URL (the Postgres behind it).');
+}
+const LOG_DIR = process.env.GBRAIN_TEST_BUSY_AGENT_LOG_DIR;
+
+// ---------------------------------------------------------------- the profile
+
+/** Counts and cadences only: the shape of a busy agent's day, never its content. */
+const PROFILE_SHAPE = {
+ notes: 'pages in the checkout',
+ refresh_ticks: '(h) scheduled refresh ticks',
+ upstream_commits_per_tick: '(h) notes another clone changes and pushes before each tick',
+ fence_writes_per_tick: '(h) foreground puts of unchanged notes while the tick holds the fence',
+ fence_conflicts_per_tick: '(h) foreground puts of notes the tick changes, while it holds the fence',
+ background_write_interval_ms: '(h) cadence of the agent puts that run across every tick',
+ plain_git_committers: '(i) plain-git committers beside gbrain',
+ plain_git_commits_each: '(i) commits per plain-git committer',
+ uncommitted_files: '(i) page files an uncommitted-file writer leaves behind',
+ legacy_rows_per_direction: '(j) legacy rows per evidence direction',
+ conversion_groups: '(k) adopt groups of the strict conversion',
+ conversion_door_writes: '(k) door writes during the conversion',
+ segmented_page_kib: '(l) size of the segmented page',
+ segmented_writes: '(l) writes to the segmented page after the expired one',
+ lost_update_rounds: '(m) free-running rounds of the cron and the MCP writer',
+ cron_interval_ms: '(m) cadence of the file-writing cron',
+ agent_interval_ms: '(m) cadence of the MCP writer',
+} as const;
+type Profile = Record<keyof typeof PROFILE_SHAPE, number>;
+/** The profile file: a JSON object of exactly PROFILE_SHAPE's keys, each a positive integer, plus an optional `description`. */
+function readProfile(raw: string, where: string): Profile {
+ const parsed = JSON.parse(raw) as Record<string, unknown>;
+ const problems = [
+ ...Object.keys(parsed).filter(key => key !== 'description' && !(key in PROFILE_SHAPE)).map(key => `unknown key ${key}`),
+ ...Object.keys(PROFILE_SHAPE).filter(key => !Number.isInteger(parsed[key]) || (parsed[key] as number) < 1).map(key => `${key} must be a positive integer, got ${JSON.stringify(parsed[key])}`),
+ ];
+ if (problems.length) throw new Error(`busy-agent profile ${where}: ${problems.join('; ')}`);
+ return Object.fromEntries(Object.keys(PROFILE_SHAPE).map(key => [key, parsed[key]])) as Profile;
+}
+const PROFILE = readProfile(readFileSync(PROFILE_PATH, 'utf8'), PROFILE_PATH);
+
+// ---------------------------------------------------------------- modes
+
+const MODES = {
+ h: { title: 'scheduled sources refresh ticks with foreground writes inside the fence', skip: null },
+ i: { title: 'plain-git committers and uncommitted-file writers beside gbrain', skip: null },
+ j: { title: 'legacy rows with no import record in every direction', skip: null },
+ k: { title: 'a strict conversion with live door writers and a refresh between adopt groups', skip: null },
+ l: { title: 'a segmented page whose staging expires between staging and publication', skip: null },
+ m: { title: 'the lost-update race between a file-writing cron and an MCP write to the same page', skip: null },
+} as const satisfies Record<string, { title: string; skip: string | null }>;
+type Mode = keyof typeof MODES;
+const requestedModes = process.env.GBRAIN_TEST_BUSY_AGENT_MODES?.split(',').map(mode => mode.trim()).filter(Boolean);
+for (const mode of requestedModes ?? []) if (!(mode in MODES)) throw new Error(`GBRAIN_TEST_BUSY_AGENT_MODES names unknown mode '${mode}' (known: ${Object.keys(MODES).join(', ')}).`);
+const runs = (mode: Mode): boolean => requestedModes ? requestedModes.includes(mode) : MODES[mode].skip === null;
+
+const KNOBS: HarnessKnobs = {
+ tag: 'busy-agent', seed: SEED, latencyMs: LATENCY_MS, syncTimeoutMs: SYNC_TIMEOUT_MS, logDir: LOG_DIR,
+ pooledTarget: pooledTarget ?? '', directTarget: directTarget ?? '', summary: `profile ${PROFILE_PATH}; latency ${LATENCY_MS} ms`,
+ repro: modes => `GBRAIN_TEST_BUSY_AGENT=1 GBRAIN_TEST_BUSY_AGENT_SEED=${SEED} GBRAIN_TEST_BUSY_AGENT_MODES=${modes.join(',')} GBRAIN_TEST_BUSY_AGENT_LATENCY_MS=${LATENCY_MS} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun test test/busy-agent-soak.serial.test.ts`,
+};
+class Brain extends HarnessBrain { constructor(name: string, modes: Mode[]) { super(name, modes, KNOBS); } }
+
+/** A describe that runs when its mode runs; skipped (with its skip reason in the title) otherwise. */
+function scenario(mode: Mode, title: string, body: () => void): void {
+ const skip = MODES[mode].skip;
+ (enabled && runs(mode) ? describe : describe.skip)(`${title} [(${mode})]${!(enabled && runs(mode)) && skip ? ` (skipped until ${skip})` : ''}`, body);
+}
+
+// ---------------------------------------------------------------- shared steps
+
+interface Receipt { state: string | null; error_code: string | null; deferred_refresh_id: string | null; completed_at: string | null }
+/** A request's row by the caller's request id, read from the journal (null before it was journaled or when it was refused at admission). */
+async function receipt(b: Brain, requestId: string): Promise<Receipt | null> {
+ const [row] = await b.engine!.executeRaw<Receipt>(
+ `SELECT state, error_code, deferred_refresh_id::text, completed_at::text FROM persistence_requests WHERE request_id=$1::uuid ORDER BY sequence DESC LIMIT 1`, [requestId]);
+ return row ?? null;
+}
+const TERMINAL = new Set(['committed', 'conflict', 'failed', 'cancelled']);
+/** The error code a CLI refusal printed (`(code)` or `"code":`), for reports. */
+function printedCode(r: { out: string; err: string }): string {
+ const json = lastJson(r.out) ?? lastJson(r.err);
+ const code = (json?.error as { code?: string } | undefined)?.code ?? json?.code;
+ if (typeof code === 'string') return code;
+ return /\b([a-z]+(?:_[a-z]+)+)\b/.exec(r.err)?.[1] ?? 'unknown';
+}
+function put(b: Brain, slug: string, content: string, requestId: string) {
+ return b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '2000'], { stdin: content });
+}
+/**
+ * Waits for a stalled child to write its fault marker. A child that exits first, or never gets there, fails the
+ * scenario with its output kept (`<tag>-unreached.log`) instead of leaving a stall nobody releases.
+ */
+async function reached(b: Brain, tag: string, run: SyncRun, label: string): Promise<void> {
+ let exited = false;
+ void run.result.then(() => { exited = true; });
+ try {
+ await until(() => exited || existsSync(b.marker(tag)), label, SYNC_TIMEOUT_MS);
+ } catch { run.proc.kill('SIGKILL'); }
+ if (existsSync(b.marker(tag))) return;
+ const r = await run.result;
+ b.keepLog(`${tag}-unreached.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ throw new Error(`${label}: the child ${exited ? `exited ${r.code}` : 'timed out'} first\n${r.out.slice(-2000)}\n${r.err.slice(-4000)}\n${b.context()}`);
+}
+/** Commits files in a clone that is not the managed checkout (the upstream another agent pushes to). */
+function cloneGit(root: string, ...args: string[]): string {
+ return execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args],
+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+}
+function writeInto(root: string, slug: string, content: string): void {
+ mkdirSync(dirname(join(root, `${slug}.md`)), { recursive: true });
+ writeFileSync(join(root, `${slug}.md`), content);
+}
+/**
+ * A busy agent's checkout is Git-durable: the owner commits every page write through its Git effect. The harness records
+ * the setting the way `gbrain sources writer git-durability --enable` does, without its push probe (a checkout with no
+ * remote commits locally). Each scenario's database holds this one checkout's binding.
+ */
+async function durable(b: Brain): Promise<void> {
+ await b.engine!.executeRaw("UPDATE persistence_host_bindings SET git_durability='enabled'");
+}
+/** Seeds the notes, publishes them to a bare origin the checkout tracks, and clones that origin as the upstream writer. */
+function seedTracked(b: Brain, notes: number): { origin: string; upstream: string } {
+ const origin = join(b.home, 'origin.git'), upstream = join(b.home, 'upstream');
+ b.writeNotes(Array.from({ length: notes }, (_, i) => i), 'Initial', 'fixture');
+ execFileSync('git', ['init', '-q', '--bare', origin]);
+ b.git('remote', 'add', 'origin', origin);
+ b.git('push', '-q', '-u', 'origin', 'HEAD');
+ execFileSync('git', ['clone', '-q', origin, upstream], { stdio: 'ignore' });
+ return { origin, upstream };
+}
+
+// ---------------------------------------------------------------- always-on calibration
+
+describe('busy-agent cell calibration (L5, GST-1502)', () => {
+ test('the committed profile names every count and cadence, and nothing else', () => {
+ const committed = readProfile(readFileSync(join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json'), 'utf8'), 'committed');
+ expect(Object.keys(committed).sort()).toEqual(Object.keys(PROFILE_SHAPE).sort());
+ });
+ test('a profile with an unknown key, a missing count or a non-integer is refused', () => {
+ const good = readFileSync(join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json'), 'utf8');
+ const edit = (patch: (p: Record<string, unknown>) => void) => { const p = JSON.parse(good) as Record<string, unknown>; patch(p); return JSON.stringify(p); };
+ expect(() => readProfile(edit(p => { p.page_titles = 3; }), 't')).toThrow('unknown key page_titles');
+ expect(() => readProfile(edit(p => { delete p.notes; }), 't')).toThrow('notes must be a positive integer');
+ expect(() => readProfile(edit(p => { p.refresh_ticks = 1.5; }), 't')).toThrow('refresh_ticks must be a positive integer');
+ });
+ test('every mode runs: none is skipped', () => {
+ expect(Object.entries(MODES).filter(([, mode]) => mode.skip !== null).map(([name]) => name)).toEqual([]);
+ });
+});
+
+// ---------------------------------------------------------------- (h) refresh ticks with writes inside the fence
+
+scenario('h', 'writes that reach a refreshing checkout wait for the refresh and then commit or conflict', () => {
+ const b = new Brain('refresh-fence', ['h']);
+ interface FenceWrite { tick: number; slug: string; requestId: string; expect: 'committed' | 'revision_conflict'; exit: number; printed: string; start: number; receipt: Receipt | null }
+ const fenceWrites: FenceWrite[] = [];
+ const refreshes: Array<{ tick: number; exit: number; outcome: string; state: string | null; released: string | null }> = [];
+ beforeAll(async () => {
+ let upstream = '';
+ await b.setup(brain => { upstream = seedTracked(brain, PROFILE.notes).upstream; }, { 'persistence.refresh_barrier': 'true' });
+ await durable(b);
+ const serve = new StdioServe(b);
+ await serve.start();
+ // The agent writes the first half of the notes; the other clone changes the second half, so a refresh never has to
+ // merge two edits of one file. Only the fence conflicts write a note the tick changes, and those apply nothing.
+ const half = Math.floor(PROFILE.notes / 2);
+ let done = false;
+ const background = (async () => {
+ for (let k = 0; !done; k++) {
+ const slug = noteSlug(b.between(0, half - 1)), marker = `Agent put ${k} of ${slug} (${randomUUID().slice(0, 8)})`;
+ const requestId = randomUUID(), start = performance.now();
+ const r = await put(b, slug, `---\ntype: concept\ntitle: Agent ${k}\n---\n\n${marker}.\n`, requestId).result;
+ b.keepLog(`agent-put-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });
+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });
+ await Bun.sleep(PROFILE.background_write_interval_ms);
+ }
+ })();
+ for (let tick = 1; tick <= PROFILE.refresh_ticks; tick++) {
+ const pick = (lo: number, hi: number, n: number) => [...new Set(Array.from({ length: 4 * n }, () => b.between(lo, hi - 1)))].slice(0, n);
+ const changed = pick(half, PROFILE.notes, PROFILE.upstream_commits_per_tick);
+ const untouched = pick(0, half, PROFILE.fence_writes_per_tick);
+ const start = performance.now();
+ cloneGit(upstream, 'pull', '-q', '--no-rebase', '--no-edit');
+ for (const i of changed) writeInto(upstream, noteSlug(i), noteText(i, `Upstream tick ${tick}`));
+ cloneGit(upstream, 'add', '--', ...changed.map(i => `${noteSlug(i)}.md`));
+ cloneGit(upstream, 'commit', '-qm', `tick ${tick}: another agent's notes`);
+ cloneGit(upstream, 'push', '-q', 'origin', 'HEAD');
+ const ack = performance.now();
+ for (const i of changed) b.ledger.record({ kind: 'commit', slug: noteSlug(i), marker: noteMarker(i, `Upstream tick ${tick}`), touches: ['db', 'file'], start, ack });
+ // The tick's member sync stalls at its first round, so the refresh holds the fence (state syncing) while the writes arrive.
+ const tag = `tick-${tick}`, release = join(b.home, `release-${tag}`);
+ const fault: Fault = { point: 'sync:round_start', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 };
+ b.plan.push(`tick ${tick}: stall the refresh's member sync at sync:round_start #1`);
+ const refresh = b.spawnCli(['sources', 'refresh', SOURCE, '--json'], { fault });
+ await reached(b, tag, refresh, `tick ${tick}'s refresh to reach its member sync`);
+ b.recordFault(tag, `tick ${tick} sync:round_start stall`);
+ const writes = [
+ ...untouched.map(i => ({ i, expect: 'committed' as const })),
+ ...changed.slice(0, PROFILE.fence_conflicts_per_tick).map(i => ({ i, expect: 'revision_conflict' as const })),
+ ];
+ await Promise.all(writes.map(async ({ i, expect: want }) => {
+ const slug = noteSlug(i), requestId = randomUUID(), started = performance.now();
+ const marker = `Fence write tick ${tick} of ${slug}`;
+ const r = await put(b, slug, `---\ntype: concept\ntitle: Fence ${tick}\n---\n\n${marker}.\n`, requestId).result;
+ b.keepLog(`fence-${tick}-${slug.replaceAll('/', '_')}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ fenceWrites.push({ tick, slug, requestId, expect: want, exit: r.code, printed: r.code === 0 || r.code === 10 ? '' : printedCode(r), start: started, receipt: null });
+ }));
+ writeFileSync(release, '');
+ const r = await refresh.result;
+ b.keepLog(`refresh-${tick}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ const [row] = await b.engine!.executeRaw<{ state: string; released_at: string | null }>(
+ `SELECT state, released_at::text FROM persistence_worktree_refreshes ORDER BY created_at DESC LIMIT 1`);
+ refreshes.push({ tick, exit: r.code, outcome: String(lastJson(r.out)?.status ?? lastJson(r.out)?.error ?? 'none'), state: row?.state ?? null, released: row?.released_at ?? null });
+ b.note(`tick ${tick}: refresh exit ${r.code} state ${row?.state}; ${writes.length} write(s) inside the fence`);
+ for (const w of fenceWrites.filter(f => f.tick === tick)) {
+ await until(async () => TERMINAL.has((await receipt(b, w.requestId))?.state ?? ''), `fence write ${w.slug} of tick ${tick} to settle`, 120_000).catch(() => {});
+ w.receipt = await receipt(b, w.requestId);
+ const marker = `Fence write tick ${tick} of ${w.slug}`;
+ if (w.receipt?.state === 'committed') b.ledger.record({ kind: 'put', slug: w.slug, marker, touches: ['db', 'file'], start: w.start, ack: performance.now() });
+ }
+ }
+ done = true;
+ await background;
+ await serve.stop();
+ await b.recover();
+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test('every tick\'s refresh completed and released its fence', () => b.verdict(refreshes.filter(r => r.exit !== 0 || r.state !== 'completed' || !r.released)
+ .map(r => `tick ${r.tick}: refresh exit ${r.exit} outcome ${r.outcome} state ${r.state} released ${r.released}`)));
+ test(as('wedge', 'every write that reached the refreshing checkout was answered pending with a receipt naming the refresh, never refused'), () =>
+ b.verdict(fenceWrites.filter(w => w.exit !== 10 || !w.receipt?.deferred_refresh_id)
+ .map(w => `tick ${w.tick} ${w.slug}: exit ${w.exit}${w.printed ? ` (${w.printed})` : ''}, deferred_refresh_id ${w.receipt?.deferred_refresh_id ?? 'none'}`)));
+ test(as('lost_write', 'a fence write to a file the refresh left alone committed after the refresh'), () =>
+ b.verdict(fenceWrites.filter(w => w.expect === 'committed' && w.receipt?.state !== 'committed')
+ .map(w => `tick ${w.tick} ${w.slug}: ${w.receipt?.state ?? 'no receipt'} ${w.receipt?.error_code ?? ''}`)));
+ test(as('lost_write', 'a fence write to a file the refresh changed ended revision_conflict with nothing applied'), () =>
+ b.verdict(fenceWrites.filter(w => w.expect === 'revision_conflict' && !(w.receipt?.state === 'conflict' && w.receipt.error_code === 'revision_conflict'))
+ .map(w => `tick ${w.tick} ${w.slug}: ${w.receipt?.state ?? 'no receipt'} ${w.receipt?.error_code ?? ''}`)));
+ test('every tick put writes inside the fence, so the barrier was exercised', () =>
+ b.verdict(fenceWrites.length >= PROFILE.refresh_ticks ? [] : [`${fenceWrites.length} fence write(s) over ${PROFILE.refresh_ticks} tick(s)`]));
+ oracleTests(b, () => ({ bigPage: false }));
+});
+
+// ---------------------------------------------------------------- (i) plain-git and uncommitted-file writers
+
+scenario('i', 'doctor foreign_writers names every commit made beside gbrain, none gbrain made, and every stale uncommitted page', () => {
+ const b = new Brain('foreign-writers', ['i']);
+ const plain: Array<{ sha: string; slug: string }> = [];
+ const dropped: string[] = [];
+ let gbrainShas: string[] = [];
+ let check: { status?: string; details?: { foreign?: Array<{ sha: string }>; dirty?: Array<{ path: string; age_ms: number }>; dirty_count?: number; partial?: boolean } } | undefined;
+ beforeAll(async () => {
+ await b.setup(brain => brain.writeNotes(Array.from({ length: PROFILE.notes }, (_, i) => i), 'Initial', 'fixture'));
+ await durable(b);
+ const fixtureHead = b.git('rev-parse', 'HEAD');
+ const serve = new StdioServe(b);
+ await serve.start();
+ // The agent writes the first half of the notes through gbrain (its door commits start the commit ledger); the
+ // plain-git committers write the second half, so no plain commit races a gbrain write of the same file.
+ const half = Math.floor(PROFILE.notes / 2);
+ const agentPut = async (k: number) => {
+ const slug = noteSlug(k % half), marker = `Agent put ${k} of ${slug} (${randomUUID().slice(0, 8)})`, start = performance.now();
+ const requestId = randomUUID();
+ const r = await b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '20000'],
+ { stdin: `---\ntype: concept\ntitle: Agent ${k}\n---\n\n${marker}.\n` }).result;
+ b.keepLog(`agent-put-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });
+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });
+ return r.code;
+ };
+ if (await agentPut(0) !== 0) throw new Error(`the first agent put did not commit, so gbrain's commit ledger may not exist\n${b.context()}`);
+ await until(() => b.git('rev-list', '--count', `${fixtureHead}..HEAD`) !== '0', 'the first door commit', 120_000);
+ let done = false;
+ const agent = (async () => { for (let k = 1; !done; k++) { await agentPut(k); await Bun.sleep(PROFILE.background_write_interval_ms); } })();
+ const committer = async (c: number) => {
+ for (let k = 0; k < PROFILE.plain_git_commits_each; k++) {
+ const i = half + b.between(0, PROFILE.notes - half - 1), slug = noteSlug(i), version = `Plain git ${c}.${k}`;
+ if (plain.some(p => p.slug === slug)) continue;
+ const start = performance.now();
+ writeInto(b.repo, slug, noteText(i, version));
+ for (let attempt = 1; ; attempt++) {
+ try { b.git('add', '--', `${slug}.md`); b.git('commit', '-qm', `committer ${c}: ${version}`, '--', `${slug}.md`); break; }
+ catch (error) { if (attempt >= 10) throw error; await Bun.sleep(500); }
+ }
+ plain.push({ sha: b.git('log', '-1', '--format=%H', '--', `${slug}.md`), slug });
+ b.ledger.record({ kind: 'commit', slug, marker: noteMarker(i, version), touches: ['db', 'file'], start, ack: performance.now() });
+ await Bun.sleep(b.between(300, 1500));
+ }
+ };
+ await Promise.all(Array.from({ length: PROFILE.plain_git_committers }, (_, c) => committer(c)));
+ // The uncommitted-file writer leaves new page files behind, last touched past the warn window.
+ const stale = (Date.now() - FOREIGN_DIRTY_WARN_MS - 5 * 60_000) / 1000;
+ for (let k = 0; k < PROFILE.uncommitted_files; k++) {
+ const slug = `inbox/dropped-${k}`;
+ writeInto(b.repo, slug, `---\ntype: note\ntitle: Dropped ${k}\n---\n\nLeft uncommitted by another writer.\n`);
+ utimesSync(join(b.repo, `${slug}.md`), stale, stale);
+ dropped.push(slug);
+ }
+ done = true;
+ await agent;
+ await until(async () => (await b.openRequests()).length === 0, 'the agent writes and their door commits to settle', 300_000).catch(() => {});
+ const r = await b.spawnCli(['doctor', '--only', 'foreign_writers', '--json']).result;
+ b.keepLog('doctor-foreign-writers.json', `exit ${r.code}\n${r.out}\n${r.err}`);
+ check = ((lastJson(r.out)?.checks ?? []) as Array<typeof check & { name: string }>).find(c => c?.name === 'foreign_writers');
+ gbrainShas = b.git('rev-list', `${fixtureHead}..HEAD`).split('\n').filter(sha => sha && !plain.some(p => p.sha === sha));
+ b.note(`${plain.length} plain commit(s), ${gbrainShas.length} gbrain commit(s), ${dropped.length} uncommitted file(s); foreign_writers ${check?.status}`);
+ await serve.stop();
+ await b.recover();
+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test('foreign_writers fails and names every plain-git commit', () => {
+ const named = new Set((check?.details?.foreign ?? []).map(c => c.sha));
+ b.verdict([
+ ...(check?.status === 'fail' ? [] : [`foreign_writers status ${check?.status ?? 'missing'}, expected fail`]),
+ ...(check?.details?.partial ? ['foreign_writers read only part of the checkout (partial)'] : []),
+ ...plain.filter(p => !named.has(p.sha)).map(p => `plain commit ${p.sha.slice(0, 12)} (${p.slug}) not named`),
+ ]);
+ });
+ test('foreign_writers names no commit gbrain made', () => {
+ const gbrain = new Set(gbrainShas);
+ b.verdict((check?.details?.foreign ?? []).filter(c => gbrain.has(c.sha)).map(c => `gbrain commit ${c.sha.slice(0, 12)} named as foreign`));
+ });
+ test('every stale uncommitted page file is listed past the warn window', () => {
+ const listed = new Map((check?.details?.dirty ?? []).map(d => [d.path.replace(/\.md$/, ''), d.age_ms]));
+ b.verdict(dropped.filter(slug => ![...listed.keys()].some(path => path.endsWith(slug)) || [...listed.entries()].find(([path]) => path.endsWith(slug))![1] < FOREIGN_DIRTY_WARN_MS)
+ .map(slug => `${slug}: not listed as a stale uncommitted page file`));
+ });
+ oracleTests(b, () => ({ exclude: new Set(dropped), bigPage: false }));
+});
+
+// ---------------------------------------------------------------- (j) legacy rows with no import record
+
+scenario('j', 'legacy rows with no import record are classified by content evidence in every direction', () => {
+ const b = new Brain('legacy-rows', ['j']);
+ const DIRECTIONS = ['adopt_db_moved', 'git_wins_file_moved', 'both_moved', 'no_evidence'] as const;
+ const targets = new Map<string, typeof DIRECTIONS[number]>();
+ let census: Record<string, unknown> | null = null;
+ const after = new Map<string, { file: string; db: string; held: string | null }>();
+ beforeAll(async () => {
+ await b.setup(brain => brain.writeNotes(Array.from({ length: PROFILE.notes }, (_, i) => i), 'Initial', 'fixture'));
+ await durable(b);
+ const pool = [...new Set(Array.from({ length: 8 * PROFILE.legacy_rows_per_direction }, () => b.between(0, PROFILE.notes - 1)))];
+ DIRECTIONS.forEach((direction, d) => { for (let k = 0; k < PROFILE.legacy_rows_per_direction; k++) targets.set(noteSlug(pool[d * PROFILE.legacy_rows_per_direction + k]!), direction); });
+ const slugs = [...targets.keys()];
+ // Each file becomes the page's own export, imported once, so the page and its committed blob share a state.
+ const rendered = async (slug: string) => b.engine!.transaction(async tx => {
+ const [key] = await tx.executeRaw<{ id: string; source_id: string; slug: string }>('SELECT id::text, source_id, slug FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, slug]);
+ return inlineExportMarkdown(await readExportPage(tx, key!, await readExportWithdrawals(tx, SOURCE)));
+ });
+ for (const slug of slugs) writeFileSync(join(b.repo, `${slug}.md`), await rendered(slug));
+ b.git('add', '--', ...slugs.map(slug => `${slug}.md`)); b.git('commit', '-qm', 'files equal their pages', '--', ...slugs.map(slug => `${slug}.md`));
+ const imported = await b.sync(['--no-pull', '--no-delegate', '--json']).result;
+ if (imported.code !== 0) throw new Error(`the import of the rendered files exited ${imported.code}\n${imported.err}\n${b.context()}`);
+ // The rows lose their import records (imported before records existed), then each side moves as its direction says.
+ await withEnv({ GBRAIN_HOME: b.home }, async () => {
+ await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ try {
+ await b.engine!.executeRaw(`DELETE FROM op_checkpoints WHERE op=$1 AND completed_keys->0->>'page_id' IN (SELECT id::text FROM pages WHERE source_id=$2 AND slug = ANY($3::text[]))`,
+ [SYNC_IMPORT_PROVENANCE_OP, SOURCE, slugs]);
+ for (const [slug, direction] of targets) {
+ if (direction === 'git_wins_file_moved') continue;
+ if (direction !== 'no_evidence') await b.engine!.createVersion(slug, { sourceId: SOURCE });
+ const start = performance.now();
+ await b.engine!.putPage(slug, { type: 'concept', title: slug.split('/')[1]!, compiled_truth: `Edited in the database (${direction}).`, timeline: '' }, { sourceId: SOURCE });
+ b.ledger.record({ kind: 'db_only', slug, marker: `Edited in the database (${direction}).`, touches: ['db'], start, ack: performance.now() });
+ }
+ } finally { await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1'); }
+ });
+ const fileMoved = [...targets].filter(([, d]) => d !== 'adopt_db_moved').map(([slug]) => slug);
+ for (const slug of fileMoved) writeFileSync(join(b.repo, `${slug}.md`), noteText(Number(slug.slice(-4)), `Plain git edit (${targets.get(slug)})`));
+ const editStart = performance.now();
+ b.git('add', '--', ...fileMoved.map(slug => `${slug}.md`)); b.git('commit', '-qm', 'plain git edits', '--', ...fileMoved.map(slug => `${slug}.md`));
+ // A held file never reaches its page, so a plain edit of a row whose page also moved lands in the file alone.
+ for (const slug of fileMoved) b.ledger.record({ kind: 'commit', slug, marker: noteMarker(Number(slug.slice(-4)), `Plain git edit (${targets.get(slug)})`),
+ touches: targets.get(slug) === 'git_wins_file_moved' ? ['db', 'file'] : ['file'], start: editStart, ack: performance.now() });
+ const r = await b.spawnCli(['sources', 'git-gap', SOURCE, '--json', '--sample', String(4 * targets.size + 20)]).result;
+ b.keepLog('git-gap.json', `exit ${r.code}\n${r.out}\n${r.err}`);
+ census = lastJson(r.out);
+ b.note(`git-gap: direction_evidence ${JSON.stringify(census?.direction_evidence)}`);
+ await b.recover();
+ const disk = b.diskPages(), db = await b.dbPages();
+ const holds = await b.engine!.executeRaw<{ record: Record<string, unknown> }>(`SELECT completed_keys->0 AS record FROM op_checkpoints WHERE op=$1`, [GIT_HOLD_OP]);
+ for (const slug of slugs) after.set(slug, { file: disk.get(slug) ?? '', db: db.get(slug)?.body ?? '', held: (holds.find(h => h.record.slug === slug)?.record.code as string | undefined) ?? null });
+ for (const [slug, state] of after) b.note(`${targets.get(slug)} ${slug}: held ${state.held}; db ${JSON.stringify(state.db.slice(0, 60))}; file ${JSON.stringify(state.file.slice(-60))}`);
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test('sources git-gap classifies every legacy row by the side that moved', () => {
+ const samples = new Map(((census?.samples ?? census?.sample ?? []) as Array<{ slug: string; evidence?: string }>).map(s => [s.slug, s.evidence]));
+ b.verdict([
+ ...(census?.complete === true ? [] : [`git-gap census incomplete or missing: ${JSON.stringify(census)?.slice(0, 300)}`]),
+ ...[...targets].filter(([slug, want]) => samples.get(slug) !== want).map(([slug, want]) => `${slug}: evidence ${samples.get(slug) ?? 'none'}, expected ${want}`),
+ ...DIRECTIONS.filter(d => (census?.direction_evidence as Record<string, number> | null)?.[d] !== PROFILE.legacy_rows_per_direction)
+ .map(d => `direction_evidence.${d} is ${(census?.direction_evidence as Record<string, number> | null)?.[d]}, expected ${PROFILE.legacy_rows_per_direction}`),
+ ]);
+ });
+ test(as('lost_write', 'the sync after the census overwrote no database move: a legacy row whose page moved keeps it'), () =>
+ b.verdict([...targets].filter(([slug, d]) => d !== 'git_wins_file_moved' && !after.get(slug)?.db.includes(`Edited in the database (${d}).`))
+ .map(([slug, d]) => `${d} ${slug}: the page lost its database edit (${JSON.stringify(after.get(slug)?.db.slice(0, 80))})`)));
+ test('a row only the file moved imports the file; a row both sides moved is held file_drift for a person', () =>
+ b.verdict([...targets].flatMap(([slug, d]) => {
+ const state = after.get(slug)!;
+ if (d === 'git_wins_file_moved') return state.db.includes('Plain git edit') && !state.held ? [] : [`${slug}: the file move was not imported (held ${state.held})`];
+ if (d === 'adopt_db_moved') return state.held ? [`${slug}: held ${state.held} although its file never changed`] : [];
+ return state.held === 'file_drift' ? [] : [`${d} ${slug}: held ${state.held}, expected file_drift`];
+ })));
+ oracleTests(b, () => {
+ const held = [...targets].filter(([, d]) => d === 'both_moved' || d === 'no_evidence').map(([slug]) => slug);
+ return { exclude: new Set([...targets].filter(([, d]) => d !== 'git_wins_file_moved').map(([slug]) => slug)), bigPage: false,
+ status: { holds: held.map(slug => ({ code: 'file_drift', path: `${slug}.md` })), needsHuman: held.length > 0 } };
+ });
+});
+
+// ---------------------------------------------------------------- (k) a strict conversion under live writers
+
+scenario('k', 'a strict conversion with live door writers and a refresh between adopt groups converts and loses nothing', () => {
+ const b = new Brain('strict-conversion', ['k']);
+ const adoptCount = (PROFILE.conversion_groups - 1) * CONVERSION_GROUP_MAX + 10;
+ const notes = Math.max(PROFILE.notes, adoptCount + 40);
+ const upstreamNotes = [adoptCount, adoptCount + 1, adoptCount + 2], doorBase = adoptCount + 20;
+ let preview: Record<string, unknown> | null = null;
+ const applies: Array<{ exit: number; status: string; code: string | null; closing: unknown }> = [];
+ const refreshes: Array<{ when: string; exit: number; status: string; code: string }> = [];
+ const doorWrites: Array<{ slug: string; exit: number; code: string }> = [];
+ let upstreamHead = '', strictAfter = false, remainingAfter: unknown = null;
+ const canonical = (args: string[], fault?: Fault) => b.spawnCli(['sources', 'canonical', SOURCE, '--to', 'git', ...args, '--json'], { fault });
+ const refresh = async (when: string) => {
+ const r = await b.spawnCli(['sources', 'refresh', SOURCE, '--json']).result;
+ b.keepLog(`refresh-${when}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ refreshes.push({ when, exit: r.code, status: String(lastJson(r.out)?.status ?? 'none'), code: r.code === 0 ? '' : String(lastJson(r.out)?.code ?? printedCode(r)) });
+ };
+ const apply = async (hash: string, fault?: Fault, tag?: string) => {
+ const run = canonical(['--apply', '--expect', hash, '--no-extract'], fault);
+ return { run, done: async () => {
+ const r = await run.result;
+ b.keepLog(`apply-${applies.length + 1}.log`, `exit ${r.code}\n${r.out.slice(-20000)}\n${r.err.slice(-20000)}`);
+ const out = lastJson(r.out);
+ applies.push({ exit: r.code, status: String(out?.status ?? 'none'), code: (out?.code as string | undefined) ?? null, closing: (out?.result as Record<string, unknown> | undefined)?.closing_passes ?? null });
+ if (tag) b.recordFault(tag, 'apply: conversion:group_committed stall');
+ } };
+ };
+ const previewHash = async () => {
+ const r = await canonical(['--dry-run']).result;
+ b.keepLog(`preview-${applies.length}.json`, `exit ${r.code}\n${r.out.slice(-20000)}\n${r.err.slice(-4000)}`);
+ const printed = lastJson(r.out);
+ const out = (printed?.preview as Record<string, unknown> | undefined) ?? printed;
+ return { out, hash: String(out?.preview_hash ?? '') };
+ };
+ beforeAll(async () => {
+ let upstream = '';
+ await b.setup(brain => { upstream = seedTracked(brain, notes).upstream; }, { 'persistence.refresh_barrier': 'true' });
+ await durable(b);
+ // Rows the database moved since their import: the conversion adopts them into Git, CONVERSION_GROUP_MAX per commit.
+ await withEnv({ GBRAIN_HOME: b.home }, async () => {
+ await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ try {
+ for (let i = 0; i < adoptCount; i++) {
+ const start = performance.now(), marker = `Database edit of note ${i}`;
+ await b.engine!.putPage(noteSlug(i), { type: 'concept', title: `Note ${i} Example`, compiled_truth: `${marker}, adopted by the conversion.`, timeline: '' }, { sourceId: SOURCE });
+ b.ledger.record({ kind: 'db_only', slug: noteSlug(i), marker, touches: ['db', 'file'], start, ack: performance.now() });
+ }
+ } finally { await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1'); }
+ });
+ const serve = new StdioServe(b);
+ await serve.start();
+ const first = await previewHash();
+ preview = first.out;
+ // Another agent pushes while the conversion runs; the refresh between adopt groups brings it in.
+ const pushStart = performance.now();
+ cloneGit(upstream, 'pull', '-q', '--no-rebase', '--no-edit');
+ for (const i of upstreamNotes) writeInto(upstream, noteSlug(i), noteText(i, 'Upstream during conversion'));
+ cloneGit(upstream, 'add', '--', ...upstreamNotes.map(i => `${noteSlug(i)}.md`));
+ cloneGit(upstream, 'commit', '-qm', 'another agent during the conversion');
+ cloneGit(upstream, 'push', '-q', 'origin', 'HEAD');
+ upstreamHead = cloneGit(upstream, 'rev-parse', 'HEAD');
+ for (const i of upstreamNotes) b.ledger.record({ kind: 'commit', slug: noteSlug(i), marker: noteMarker(i, 'Upstream during conversion'), touches: ['db', 'file'], start: pushStart, ack: performance.now() });
+ const tag = 'between-groups', release = join(b.home, `release-${tag}`);
+ b.plan.push('stall the apply after its first adopt group commits (conversion:group_committed #1); refresh and door writes there');
+ const stalled = await apply(first.hash, { point: 'conversion:group_committed', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 }, tag);
+ await reached(b, tag, stalled.run, 'the apply to commit its first adopt group');
+ const writers = Array.from({ length: PROFILE.conversion_door_writes }, async (_, k) => {
+ const i = doorBase + k, slug = noteSlug(i), marker = `Door write ${k} during the conversion`, start = performance.now(), requestId = randomUUID();
+ const r = await b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '20000'],
+ { stdin: `---\ntype: concept\ntitle: Door ${k}\n---\n\n${marker}.\n` }).result;
+ b.keepLog(`door-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ doorWrites.push({ slug, exit: r.code, code: r.code === 0 || r.code === 10 ? '' : printedCode(r) });
+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });
+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });
+ });
+ await refresh('between adopt groups');
+ writeFileSync(release, '');
+ await Promise.all(writers);
+ await stalled.done();
+ // An apply that ends incomplete names its rerun (git_conversion_incomplete): one more preview and apply, no fault.
+ if (applies.at(-1)?.status !== 'converted') { const again = await previewHash(); await (await apply(again.hash)).done(); }
+ if (!refreshes.some(r => r.exit === 0)) await refresh('after the conversion');
+ strictAfter = (await b.engine!.executeRaw("SELECT 1 FROM sources WHERE id=$1 AND config->>'canonical'='git'", [SOURCE])).length === 1;
+ remainingAfter = (await previewHash()).out?.remaining ?? null;
+ b.note(`adopt ${adoptCount} row(s) in ${JSON.stringify(preview?.adopt)}; applies ${JSON.stringify(applies)}; refreshes ${JSON.stringify(refreshes)}; door writes ${JSON.stringify(doorWrites)}; strict ${strictAfter}; remaining ${remainingAfter}`);
+ await serve.stop();
+ await b.recover();
+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test('the conversion had more adopt groups than one, so the refresh landed between two of them', () => {
+ const groups = (preview?.adopt as { groups?: number } | undefined)?.groups ?? 0;
+ b.verdict(groups >= PROFILE.conversion_groups ? [] : [`the preview planned ${groups} adopt group(s), expected at least ${PROFILE.conversion_groups}`]);
+ });
+ test(as('wedge', 'the conversion converts the source to strict Git-canonical with nothing left to plan'), () =>
+ b.verdict([...(applies.at(-1)?.status === 'converted' ? [] : [`the last apply ended ${JSON.stringify(applies.at(-1))}`]),
+ ...(strictAfter ? [] : ['the source is not git-canonical']), ...(remainingAfter === 0 ? [] : [`the preview after the conversion has ${remainingAfter} remaining`])]));
+ test(as('wedge', 'the refresh between adopt groups, or the one after the conversion, brought in the other agent\'s commit'), () =>
+ b.verdict(refreshes.some(r => r.exit === 0) && b.git('merge-base', '--is-ancestor', upstreamHead, 'HEAD') === '' ? [] : [`refreshes ${JSON.stringify(refreshes)}; HEAD does not contain ${upstreamHead.slice(0, 12)}`]));
+ test(as('untyped_error', 'every door write and refresh during the conversion was accepted or refused with a registered code'), () =>
+ b.verdict([...doorWrites.filter(w => w.exit !== 0 && w.exit !== 10 && !(w.code in CODES)).map(w => `${w.slug}: exit ${w.exit} ${w.code}`),
+ ...refreshes.filter(r => r.exit !== 0 && !(r.code in CODES)).map(r => `refresh ${r.when}: exit ${r.exit} ${r.code}`)]));
+ oracleTests(b, () => ({ bigPage: false }));
+});
+
+// ---------------------------------------------------------------- (l) segmented staging expiry
+
+scenario('l', 'a segmented write whose staging expires or aborts before publication publishes nothing, and a resend commits', () => {
+ const b = new Brain('segmented-staging', ['l']);
+ const PAGE = 'archive/segmented-log';
+ const archive = (version: string) => `---\ntype: note\ntitle: Segmented log\n---\n\n# Segmented log (${version})\n\n` +
+ Array.from({ length: Math.ceil(PROFILE.segmented_page_kib * 1024 / 96) }, (_, i) => `Entry ${i} of the ${version} archive, a steady line of synthetic operating detail.\n`).join('');
+ const injected: Array<{ how: 'expired' | 'aborted'; staged: string | null; exit: number; code: string; receipt: Receipt | null; kept: boolean }> = [];
+ let storage: string | null = null;
+ let liveAfterSweep: unknown[] = [];
+ let tiling: { status?: string } | undefined;
+ /** A segmented page keeps its body in segment rows, not compiled_truth, so the page is compared by content hash. */
+ const pageIs = async (version: string) => (await b.dbPages()).get(PAGE)?.hash === fileHash(PAGE, archive(version));
+ const putArchive = (version: string, requestId: string, fault?: Fault) =>
+ b.spawnCli(['put', PAGE, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '30000'], { stdin: archive(version), fault });
+ const sweep = () => withEnv({ GBRAIN_HOME: b.home }, () => sweepSegmentStorage(b.engine!));
+ beforeAll(async () => {
+ await b.setup(brain => brain.commit('fixture', [
+ ...Array.from({ length: Math.min(PROFILE.notes, 20) }, (_, i) => ({ slug: noteSlug(i), content: noteText(i, 'Initial'), marker: noteMarker(i, 'Initial') })),
+ { slug: PAGE, content: archive('v0'), marker: 'Segmented log (v0)' }]), { 'pages.segmented_storage': 'on' });
+ await durable(b);
+ for (const ack of b.ledger.acks) if (ack.slug === PAGE) ack.touches = ['file'];
+ storage = (await b.engine!.executeRaw<{ body_storage: string }>('SELECT body_storage FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, PAGE]))[0]?.body_storage ?? null;
+ // Each write stalls once its manifest is staged; the harness then ends its staging the way a sweep (expired) or a
+ // fact purge (aborted) does, and releases it into publication.
+ for (const how of ['expired', 'aborted'] as const) {
+ const tag = `staging-${how}`, release = join(b.home, `release-${tag}`), requestId = randomUUID();
+ b.plan.push(`stall the ${how} write at segmented:manifest_staged #1`);
+ const write = putArchive(`v-${how}`, requestId, { point: 'segmented:manifest_staged', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 });
+ await reached(b, tag, write, `the ${how} write to stage its manifest`);
+ b.recordFault(tag, `${how}: segmented:manifest_staged stall`);
+ if (how === 'expired') {
+ await b.engine!.executeRaw(`UPDATE page_body_stagings SET idle_expires_at=now() - interval '1 hour', hard_expires_at=now() - interval '1 minute'
+ WHERE source_id=$1 AND slug=$2 AND state IN ('open','sealed','prepared')`, [SOURCE, PAGE]);
+ await sweep();
+ } else {
+ await b.engine!.executeRaw(`UPDATE page_body_stagings SET state='aborted', updated_at=now() WHERE source_id=$1 AND slug=$2 AND state IN ('open','sealed','prepared')`, [SOURCE, PAGE]);
+ }
+ const [staged] = await b.engine!.executeRaw<{ state: string }>('SELECT state FROM page_body_stagings WHERE source_id=$1 AND slug=$2 ORDER BY created_at DESC LIMIT 1', [SOURCE, PAGE]);
+ writeFileSync(release, '');
+ const r = await write.result;
+ b.keepLog(`${tag}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ await until(async () => TERMINAL.has((await receipt(b, requestId))?.state ?? ''), `the ${how} write to settle`, 120_000).catch(() => {});
+ injected.push({ how, staged: staged?.state ?? null, exit: r.code, code: r.code === 0 ? '' : printedCode(r), receipt: await receipt(b, requestId), kept: await pageIs('v0') });
+ }
+ // The resend, then the profile's later writes, each a new version of the whole page.
+ for (let k = 0; k <= PROFILE.segmented_writes; k++) {
+ const version = `v${k + 1}`, start = performance.now();
+ const r = await putArchive(version, randomUUID()).result;
+ b.keepLog(`write-${version}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ // The ledger reads the file; disk == database (by content hash) carries the page side of a segmented write.
+ if (r.code === 0) b.ledger.record({ kind: 'put', slug: PAGE, marker: `Segmented log (${version})`, touches: ['file'], start, ack: performance.now() });
+ else b.note(`write ${version}: exit ${r.code} ${printedCode(r)}`);
+ }
+ await sweep();
+ liveAfterSweep = await b.engine!.executeRaw(`SELECT id::text, kind, state FROM page_body_stagings WHERE state IN ('open','sealed','prepared')`);
+ const doctor = await b.spawnCli(['doctor', '--only', 'segmented_tiling', '--json']).result;
+ b.keepLog('doctor-segmented-tiling.json', `exit ${doctor.code}\n${doctor.out}\n${doctor.err}`);
+ tiling = ((lastJson(doctor.out)?.checks ?? []) as Array<{ name: string; status?: string }>).find(c => c.name === 'segmented_tiling');
+ b.note(`storage ${storage}; injected ${JSON.stringify(injected.map(i => ({ how: i.how, staged: i.staged, exit: i.exit, code: i.code, receipt: i.receipt?.error_code ?? i.receipt?.state })))}; tiling ${tiling?.status}`);
+ await b.recover();
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test('the page is stored in segments, so the staging path is the one under test', () => b.verdict(storage === 'segmented' ? [] : [`body_storage is ${storage}`]));
+ test(as('lost_write', 'a write whose staging expired or aborted before publication refuses body_staging_expired and the page keeps its body'), () =>
+ b.verdict(injected.flatMap(i => [
+ ...(i.staged === i.how ? [] : [`${i.how}: the staging was ${i.staged} when the write was released`]),
+ ...((i.code === 'body_staging_expired' || i.receipt?.error_code === 'body_staging_expired') && i.receipt?.state !== 'committed' ? [] : [`${i.how}: exit ${i.exit} ${i.code}, receipt ${i.receipt?.state} ${i.receipt?.error_code}`]),
+ ...(i.kept ? [] : [`${i.how}: the page lost its previous body`]),
+ ])));
+ test('the sweep leaves no live staging behind', () => b.verdict(liveAfterSweep.map(row => `live staging after the sweep: ${JSON.stringify(row)}`)));
+ test(as('projection_drift', 'doctor segmented_tiling finds every segmented page tiled'), () => b.verdict(tiling?.status === 'ok' ? [] : [`segmented_tiling is ${tiling?.status ?? 'missing'}`]));
+ oracleTests(b, () => ({ bigPage: false }));
+});
+
+// ---------------------------------------------------------------- (m) the lost-update race
+
+scenario('m', 'a file-writing cron and an MCP writer racing on one page lose no acknowledged write', () => {
+ const b = new Brain('lost-update', ['m']);
+ const BOARD = 'ops/daily-board', PATH = `${BOARD}.md`;
+ const board = (entries: string[]) => `---\ntype: note\ntitle: Daily board\n---\n\n# Daily board\n\n${entries.map(e => `- ${e}\n`).join('')}END\n`;
+ /**
+ * Every refusal either writer met, by code. The race may only produce refusals that apply nothing: commit_base_moved
+ * for the cron; for the agent revision_conflict, or source_changed while the file holds the cron's commit before the
+ * sync imports it (pending_import) or the cron's bytes before it commits them (an uncoordinated local edit).
+ */
+ const refusals: Array<{ writer: 'cron' | 'agent'; code: string }> = [];
+ const acked: Array<{ writer: 'cron' | 'agent'; marker: string }> = [];
+ const forced: Record<'A' | 'B', string | null> = { A: null, B: null };
+ let serve!: StdioServe;
+ const mcp = async (name: string, args: Record<string, unknown>) => {
+ const msg = await serve.request('tools/call', { name, arguments: args });
+ const result = msg.result as { isError?: boolean; content?: Array<{ text?: string }> } | undefined;
+ const text = result?.content?.[0]?.text ?? JSON.stringify(msg.error ?? null);
+ let body: Record<string, unknown> = {};
+ try { body = JSON.parse(text) as Record<string, unknown>; } catch { body = { text }; }
+ return { isError: result?.isError === true || !!msg.error, body };
+ };
+ const revision = async () => String((await mcp('get_page', { slug: BOARD, include_content: true, source_id: SOURCE })).body.revision ?? '');
+ /** The agent's MCP edit at a revision it read: committed, or the code that refused it (a pending write is followed to its receipt). */
+ const agentEdit = async (marker: string, rev: string): Promise<string> => {
+ const requestId = randomUUID();
+ const r = await mcp('edit_page', { slug: BOARD, source_id: SOURCE, expected_revision: rev, request_id: requestId, edits: [{ old_text: '\nEND\n', new_text: `\n- ${marker}\nEND\n` }] });
+ if (r.isError) return String(r.body.code ?? r.body.error ?? 'unknown');
+ if (r.body.status === 'pending') {
+ await until(async () => TERMINAL.has((await receipt(b, requestId))?.state ?? ''), `the agent edit ${marker} to settle`, 300_000);
+ const settled = await receipt(b, requestId);
+ if (settled?.state !== 'committed') return settled?.error_code ?? settled?.state ?? 'unknown';
+ }
+ return 'committed';
+ };
+ /** The cron's commit of the file it read at HEAD (blob `base`), bound to that blob with --base; a refusal discards its stale bytes. */
+ const cronCommit = async (marker: string, read: { base: string; content: string }): Promise<string> => {
+ writeFileSync(join(b.repo, PATH), read.content.replace('\nEND\n', `\n- ${marker}\nEND\n`));
+ const r = await b.spawnCli(['git', 'commit-push', '--path', b.repo, '--base', `${PATH}=${read.base}`, '-m', `cron: ${marker}`, '--json', '--', PATH]).result;
+ b.keepLog(`cron-${marker.replaceAll(' ', '_')}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ if (r.code === 0) return 'committed';
+ b.git('checkout', '--', PATH);
+ return printedCode(r);
+ };
+ const cronRead = () => ({ base: b.git('rev-parse', `HEAD:${PATH}`), content: b.git('show', `HEAD:${PATH}`) + '\n' });
+ const tick = () => b.sync(['--no-pull', '--json']).result;
+ /**
+ * One writer's acknowledged append, retried from a fresh read after each refusal, at most 20 times. An agent refused
+ * because the file holds the cron's commit before the next sync imports it waits for that sync like any client.
+ */
+ const append = async (writer: 'cron' | 'agent', marker: string) => {
+ for (let attempt = 1; attempt <= 20; attempt++) {
+ const outcome = writer === 'agent' ? await agentEdit(marker, await revision()) : await cronCommit(marker, cronRead());
+ if (outcome === 'committed') { acked.push({ writer, marker }); return; }
+ refusals.push({ writer, code: outcome });
+ await Bun.sleep(b.between(300, 1500));
+ }
+ };
+ beforeAll(async () => {
+ await b.setup(brain => brain.commit('fixture', [
+ ...Array.from({ length: Math.min(PROFILE.notes, 20) }, (_, i) => ({ slug: noteSlug(i), content: noteText(i, 'Initial'), marker: noteMarker(i, 'Initial') })),
+ { slug: BOARD, content: board(['seeded']), marker: '- seeded' }]), { 'persistence.refresh_barrier': 'true' });
+ await durable(b);
+ serve = new StdioServe(b);
+ await serve.start();
+ // Forced order A: the agent reads, the cron commits and the sync imports it, then the agent writes at its stale revision.
+ const staleRev = await revision();
+ if (await cronCommit('cron forced A', cronRead()) !== 'committed') throw new Error(`forced A: the cron commit was refused\n${b.context()}`);
+ acked.push({ writer: 'cron', marker: 'cron forced A' });
+ await tick();
+ await until(async () => (await revision()) !== staleRev, 'the cron commit to be imported', 120_000);
+ forced.A = await agentEdit('agent forced A', staleRev);
+ // Forced order B: the cron reads the blob, the agent's edit commits through the door, then the cron commits on its stale read.
+ const staleRead = cronRead();
+ if (await agentEdit('agent forced B', await revision()) !== 'committed') throw new Error(`forced B: the agent edit was refused\n${b.context()}`);
+ acked.push({ writer: 'agent', marker: 'agent forced B' });
+ await until(() => b.git('rev-parse', `HEAD:${PATH}`) !== staleRead.base, 'the agent edit\'s door commit', 120_000);
+ forced.B = await cronCommit('cron forced B', staleRead);
+ // Free-running: both writers on their cadences, and the agent's scheduled sync importing the cron's commits.
+ let done = false;
+ const ticks = (async () => { while (!done) { await tick(); await Bun.sleep(PROFILE.cron_interval_ms); } })();
+ await Promise.all([
+ (async () => { for (let k = 0; k < PROFILE.lost_update_rounds; k++) { await append('cron', `cron ${k}`); await Bun.sleep(PROFILE.cron_interval_ms); } })(),
+ (async () => { for (let k = 0; k < PROFILE.lost_update_rounds; k++) { await append('agent', `agent ${k}`); await Bun.sleep(PROFILE.agent_interval_ms); } })(),
+ ]);
+ done = true;
+ await ticks;
+ b.note(`forced A: ${forced.A}; forced B: ${forced.B}; ${acked.length} acknowledged append(s); refusals ${JSON.stringify(Object.fromEntries([...new Set(refusals.map(r => `${r.writer}:${r.code}`))].map(k => [k, refusals.filter(r => `${r.writer}:${r.code}` === k).length])))}`);
+ await serve.stop();
+ await b.recover();
+ }, SCENARIO_TIMEOUT_MS);
+ afterAll(() => b.teardown());
+
+ test(as('lost_write', 'an MCP edit at a revision the cron\'s imported commit moved refuses revision_conflict'), () =>
+ b.verdict(forced.A === 'revision_conflict' ? [] : [`forced order A ended ${forced.A}`]));
+ test(as('lost_write', 'a commit-push bound to a blob the agent\'s commit moved refuses commit_base_moved'), () =>
+ b.verdict(forced.B === 'commit_base_moved' ? [] : [`forced order B ended ${forced.B}`]));
+ test(as('lost_write', 'no acknowledged append is silently reverted: every one is in the file and the page'), async () => {
+ const file = b.diskPages().get(BOARD) ?? '', page = (await b.dbPages()).get(BOARD)?.body ?? '';
+ b.verdict(acked.flatMap(a => [...(file.includes(`- ${a.marker}\n`) ? [] : [`${a.writer} append ${a.marker}: missing from the file`]),
+ ...(page.includes(`- ${a.marker}`) ? [] : [`${a.writer} append ${a.marker}: missing from the page`])]));
+ });
+ test('every refusal the free-running writers met is the race\'s own and applied nothing', () =>
+ b.verdict(refusals.filter(r => !(r.writer === 'agent' ? ['revision_conflict', 'source_changed'].includes(r.code) : r.code === 'commit_base_moved'))
+ .map(r => `${r.writer} refused ${r.code}`)));
+ test('both writers got every append in', () =>
+ b.verdict((['cron', 'agent'] as const).filter(w => acked.filter(a => a.writer === w && /^\w+ \d+$/.test(a.marker)).length !== PROFILE.lost_update_rounds)
+ .map(w => `${w}: ${acked.filter(a => a.writer === w && /^\w+ \d+$/.test(a.marker)).length} of ${PROFILE.lost_update_rounds} appends acknowledged`)));
+ oracleTests(b, () => ({ bigPage: false }));
+});
diff --git a/test/fixtures/busy-agent-profile.json b/test/fixtures/busy-agent-profile.json
new file mode 100644
index 0000000000..7bb8c6cd4b
--- /dev/null
+++ b/test/fixtures/busy-agent-profile.json
@@ -0,0 +1,20 @@
+{
+ "description": "A generic busy-agent workload for the busy-agent soak cell (test/busy-agent-soak.serial.test.ts): counts and cadences only, no content. A profile derived from a real deployment stays private and is passed with GBRAIN_TEST_BUSY_AGENT_PROFILE.",
+ "notes": 120,
+ "refresh_ticks": 4,
+ "upstream_commits_per_tick": 3,
+ "fence_writes_per_tick": 4,
+ "fence_conflicts_per_tick": 2,
+ "background_write_interval_ms": 1500,
+ "plain_git_committers": 2,
+ "plain_git_commits_each": 3,
+ "uncommitted_files": 2,
+ "legacy_rows_per_direction": 2,
+ "conversion_groups": 3,
+ "conversion_door_writes": 4,
+ "segmented_page_kib": 1536,
+ "segmented_writes": 2,
+ "lost_update_rounds": 12,
+ "cron_interval_ms": 900,
+ "agent_interval_ms": 700
+}
diff --git a/test/helpers/convergence-brain.ts b/test/helpers/convergence-brain.ts
new file mode 100644
index 0000000000..e0f22c0dd2
--- /dev/null
+++ b/test/helpers/convergence-brain.ts
@@ -0,0 +1,590 @@
+/**
+ * The real-process brain harness shared by the managed-sync convergence gate (test/sync-convergence.serial.test.ts)
+ * and the busy-agent soak cell (test/busy-agent-soak.serial.test.ts): one isolated Postgres database behind
+ * transaction-mode PgBouncer, reached by every child process through latency proxies, a Git worktree bound to one
+ * managed source, real `src/cli.ts` child processes (optionally under the fault preload
+ * test/fixtures/convergence-fault-kill.ts), a resident stdio serve, the acknowledged-write ledger and the oracle reads.
+ * Each suite passes its own knobs (seed, latency, log directory, repro line). Setup: scripts/persistence/README.md.
+ */
+import { test } from 'bun:test';
+import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, relative, resolve } from 'node:path';
+import postgres from '#postgres';
+import { PostgresEngine } from '../../src/core/postgres-engine.ts';
+import { claimWorktree } from '../../src/core/persistence/ownership.ts';
+import { parseMarkdown } from '../../src/core/markdown.ts';
+import { parseFactsFence } from '../../src/core/facts-fence.ts';
+import { contentHash } from '../../src/core/utils.ts';
+import { CODES } from '../../src/core/error-registry.ts';
+import type { FaultPoint } from '../../src/core/persistence/fault-points.ts';
+import { assertSafeE2eDatabaseUrl } from './db-guard.ts';
+import { keylessBrainEnv } from './provider-env.ts';
+import { withEnv } from './with-env.ts';
+import { LatencyProxy } from './latency-proxy.ts';
+import type { ViolationClass } from '../../scripts/persistence/model.ts';
+
+const REPO_ROOT = resolve(import.meta.dir, '../..');
+const FAULT_PRELOAD = join(REPO_ROOT, 'test/fixtures/convergence-fault-kill.ts');
+export const SOURCE = 'workspace';
+export const BIG = 'companies/acme-example';
+export const ENTITY = 'people/alice-example';
+export const MAX_RECOVERY_RUNS = 2;
+
+/** What a suite passes to every brain: the knobs its failures print, where its logs go, and the targets behind the proxies. */
+export interface HarnessKnobs {
+ /** Prefixes every progress line (`[<tag> <scenario>]`) and names the log file `<tag>.log` in logDir. */
+ tag: string;
+ seed: number;
+ latencyMs: number;
+ /** Per child process (a sync or a writer call) unless a call passes its own. */
+ syncTimeoutMs: number;
+ logDir: string | undefined;
+ pooledTarget: string;
+ directTarget: string;
+ /** The suite's other knobs, printed after the seed and modes in every failure. */
+ summary: string;
+ /** The local command that reruns these modes, printed in every failure. */
+ repro: (modes: string[]) => string;
+}
+
+/** mulberry32: a fixed seed gives the same fault plan and writer targets on every run. */
+export function prng(seed: number): () => number {
+ let a = seed >>> 0;
+ return () => {
+ a = (a + 0x6d2b79f5) >>> 0;
+ let t = a;
+ t = Math.imul(t ^ (t >>> 15), t | 1);
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+// ---------------------------------------------------------------- the status oracle
+
+/** Every `gbrain sync status --json` field the operator loop reads (sync-status.ts `SyncStatus`). */
+export const STATUS_FIELDS = ['source_id', 'run_id', 'cursor', 'committed_last_10m', 'live_job_id', 'holds', 'last_error', 'needs_human',
+ 'resume_argv', 'checkpoint_blockers', 'cursors', 'terminal_failures', 'next'] as const;
+export interface StatusExpectation { holds?: Array<{ code: string; path: string }>; needsHuman?: boolean }
+/**
+ * What a converged source's status must say. Field-present assertions first: a renderer that drops a field fails
+ * here, not in an agent's loop. `checkpoint_blockers` and `cursors` are arrays on every status (#6712 PR 0 made
+ * `checkpoint_blockers` always present). The harness reads `sync status --json --detail`: since #6712 PR 2 the
+ * terminal summary, blocker list and cursor join run only behind the flag.
+ */
+/** The default (no `--detail`) status after the chaos: every skipped heavy read says so and names the `--detail` command (#6712 PR 2, DX F1). */
+export function fastStatusProblems(status: Record<string, unknown> | null): string[] {
+ if (!status) return ['sync status --json printed no JSON document'];
+ return (['checkpoint_blockers_read', 'cursors_read', 'terminal_failures_read'] as const).flatMap(field => {
+ const read = status[field] as { status?: string; fix?: { argv?: string[] } } | undefined;
+ return read?.status === 'not_read' && read.fix?.argv?.includes('--detail') ? [] : [`sync status --json: ${field} is ${JSON.stringify(read)}, expected not_read with a --detail fix`];
+ });
+}
+
+export function statusProblems(status: Record<string, unknown> | null, expected: StatusExpectation = {}): string[] {
+ if (!status) return ['sync status --json printed no JSON document'];
+ const out = STATUS_FIELDS.filter(field => !(field in status)).map(field => `sync status --json: field ${field} missing`);
+ if (out.length) return out;
+ const cursor = status.cursor as { done?: boolean } | null;
+ if (!Array.isArray(status.cursors)) out.push('sync status --json: cursors is not an array');
+ if (!Array.isArray(status.checkpoint_blockers)) out.push('sync status --json: checkpoint_blockers is not an array');
+ else if (status.checkpoint_blockers.length) out.push(`sync status --json: ${status.checkpoint_blockers.length} checkpoint blocker(s)`);
+ // A finished run's cursor is retired, so a converged source shows none (null) or a done one.
+ if (cursor && cursor.done !== true) out.push(`sync status --json: cursor not done (${JSON.stringify(cursor)})`);
+ const terminal = status.terminal_failures as { status?: string; active_blockers?: unknown[] } | null;
+ if (terminal?.status !== 'ok') out.push(`sync status --json: terminal_failures.status is ${terminal?.status}`);
+ else if (!Array.isArray(terminal.active_blockers)) out.push('sync status --json: terminal_failures.active_blockers is not an array');
+ else if (terminal.active_blockers.length) out.push(`sync status --json: ${terminal.active_blockers.length} active terminal blocker(s)`);
+ if (status.needs_human !== (expected.needsHuman ?? false)) out.push(`sync status --json: needs_human is ${status.needs_human}, expected ${expected.needsHuman ?? false}`);
+ const holds = Array.isArray(status.holds) ? (status.holds as Array<{ code: string; path: string }>).map(h => `${h.code}:${h.path}`).sort() : ['holds is not an array'];
+ const want = (expected.holds ?? []).map(h => `${h.code}:${h.path}`).sort();
+ if (JSON.stringify(holds) !== JSON.stringify(want)) out.push(`sync status --json: holds [${holds.join(', ')}], expected [${want.join(', ')}]`);
+ if (status.last_error !== null) out.push(`sync status --json: last_error ${JSON.stringify(status.last_error)}`);
+ if (!expected.holds?.length && status.next !== null) out.push(`sync status --json: next ${JSON.stringify(status.next)}`);
+ return out;
+}
+
+// ---------------------------------------------------------------- the acknowledged-write ledger
+
+/**
+ * One writer operation that returned success, recorded by the harness at the moment it saw the success (never read
+ * back from the brain), or a write the brain accepted (exit 10) that had not settled when its writer stopped, recorded
+ * when its receipt, read after recovery, said committed (`Brain.settlePending`). `touches` is where the write landed: the page in the database, its file, or (for `remember`)
+ * an additive fact row. `marker` is text that only this write put there; `null` on a page location is a deletion.
+ */
+export interface Ack { kind: 'commit' | 'put' | 'remember' | 'maintenance' | 'db_only' | 'git_mv' | 'git_rm' | 'unlink'; slug: string;
+ marker: string | null; touches: Array<'db' | 'file'> | 'fact'; start: number; ack: number }
+export class Ledger {
+ acks: Ack[] = [];
+ record(ack: Ack): void { this.acks.push(ack); }
+ /**
+ * For every slug and location, the writes no later acknowledged write causally supersedes (none started after it
+ * was acknowledged); the final state must show one of them. An additive fact survives unless a page write on its
+ * slug may have been ordered after it.
+ */
+ problems(disk: Map<string, string>, db: Map<string, { body: string } | null>, facts: Set<string>): string[] {
+ const out: string[] = [];
+ const pageWrites = this.acks.filter(a => a.touches !== 'fact' && a.kind !== 'maintenance');
+ for (const slug of new Set(pageWrites.map(a => a.slug))) {
+ for (const where of ['db', 'file'] as const) {
+ const writes = pageWrites.filter(a => a.slug === slug && (a.touches as string[]).includes(where));
+ if (!writes.length) continue;
+ const maximal = writes.filter(w => !writes.some(later => later.start > w.ack));
+ const seen = where === 'file' ? disk.get(slug) ?? null : db.get(slug)?.body ?? null;
+ if (maximal.some(w => w.marker === null ? seen === null : seen?.includes(w.marker))) continue;
+ out.push(`${where} ${slug}: lost acknowledged write; has ${seen === null ? 'nothing' : JSON.stringify(seen.slice(0, 80))}, expected one of ${maximal.map(w => `${w.kind}@${Math.round(w.ack)}:${w.marker === null ? '(absent)' : JSON.stringify(w.marker)}`).join(' | ')}`);
+ }
+ }
+ for (const fact of this.acks.filter(a => a.touches === 'fact')) {
+ const excused = pageWrites.some(w => w.slug === fact.slug && (w.touches as string[]).includes('db') && w.ack > fact.start);
+ if (!excused && !facts.has(fact.marker!)) out.push(`fact on ${fact.slug}: lost acknowledged remember ${JSON.stringify(fact.marker)}`);
+ }
+ return out;
+ }
+}
+
+// ---------------------------------------------------------------- the fixture
+
+export function noteText(i: number, version: string): string {
+ return `---\ntype: concept\ntitle: Note ${i} Example\n---\n\n${version} body for note ${i}, an observation about widget-co and acme-example.\n\n${'Supporting detail sentence. '.repeat(8 + (i % 5))}\n`;
+}
+export const noteMarker = (i: number, version: string) => `${version} body for note ${i},`;
+export const noteSlug = (i: number) => `notes/note-${String(i).padStart(4, '0')}`;
+export function bigPage(facts: number, timeline: number, version: string): string {
+ const day = (n: number) => new Date(Date.UTC(2018, 0, 1) + n * 86_400_000).toISOString().slice(0, 10);
+ const paragraphs = Array.from({ length: 30 }, (_, i) => `Section ${i} (${version}): acme-example ${'expanded with a steady stream of synthetic operating detail. '.repeat(6)}`);
+ const rows = Array.from({ length: facts }, (_, i) =>
+ `| ${i + 1} | Synthetic claim ${i + 1} about acme-example | fact | 0.9 | world | medium | ${day(i)} | | fixture | |`);
+ const bullets = Array.from({ length: timeline }, (_, i) => `- **${day(i)}** | Synthetic event ${i + 1} at acme-example`);
+ return [
+ '---', 'type: company', 'title: Acme Example', '---', '', '# Acme Example', '', ...paragraphs.flatMap(p => [p, '']),
+ '## Facts', '', '<!--- gbrain:facts:begin -->',
+ '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |',
+ '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|',
+ ...rows, '<!--- gbrain:facts:end -->', '', '<!-- timeline -->', '', ...bullets, '',
+ ].join('\n');
+}
+export const entityText = '---\ntype: person\ntitle: Alice Example\n---\n\nAlice Example works with acme-example.\n';
+
+// ---------------------------------------------------------------- one brain
+
+export interface Fault { point: FaultPoint; nth: number; marker: string; operation?: string; path?: string; action?: 'kill' | 'edit' | 'stall' | 'throw';
+ edit?: { file: string; content: string }; release?: string; ms?: number; sqlstate?: string;
+ prime?: { point: FaultPoint; nth: number; path?: string; edit: { file: string; content: string } } }
+export interface Run { code: number; out: string; err: string; ms: number }
+export interface SyncRun { proc: ReturnType<typeof Bun.spawn>; result: Promise<Run> }
+
+export async function until(check: () => boolean | Promise<boolean>, label: string, timeoutMs = 300_000): Promise<void> {
+ const deadline = performance.now() + timeoutMs;
+ while (performance.now() < deadline) {
+ if (await check()) return;
+ await Bun.sleep(100);
+ }
+ throw new Error(`timed out waiting for ${label}`);
+}
+export function lastJson(out: string): Record<string, unknown> | null {
+ try { return JSON.parse(out) as Record<string, unknown>; } catch { /* several documents */ }
+ for (const line of out.trim().split('\n').reverse()) { try { return JSON.parse(line) as Record<string, unknown>; } catch { /* not the envelope */ } }
+ return null;
+}
+/** The longest `stalled Ns` progress line a sync printed: how long it waited on a predecessor's leftovers. */
+export function worstStall(err: string): number { return Math.max(0, ...[...err.matchAll(/stalled (\d+)s/g)].map(m => Number(m[1]))); }
+
+/**
+ * One scenario's brain: its database, home, Git worktree, proxies, child processes, ledger and log. `context()` is
+ * what every failure prints: seed, modes, the fault plan and the one-line repro.
+ */
+export class Brain {
+ home = ''; repo = ''; database = '';
+ admin: ReturnType<typeof postgres> | undefined;
+ engine: PostgresEngine | undefined;
+ pooledProxy: LatencyProxy | undefined; directProxy: LatencyProxy | undefined;
+ pooledUrl = ''; directUrl = '';
+ serves: StdioServe[] = [];
+ ledger = new Ledger();
+ /** Writes the brain accepted (exit 10) that had not settled when their writer stopped; `settlePending` reads their receipts after recovery. */
+ pending: Array<{ write: Omit<Ack, 'ack'>; requestId: string }> = [];
+ plan: string[] = [];
+ fired: Record<string, string | null> = {};
+ killedPids = new Set<number>();
+ recoveryRuns: Array<Run & { envelope: Record<string, unknown> | null }> = [];
+ elapsed: Record<string, number> = {};
+ calibration: string[] = [];
+ readonly rand: () => number;
+ constructor(readonly name: string, readonly modes: string[], readonly knobs: HarnessKnobs) { this.rand = prng(knobs.seed ^ [...name].reduce((h, c) => Math.imul(h, 31) + c.charCodeAt(0), 7)); }
+
+ between(lo: number, hi: number): number { return lo + Math.floor(this.rand() * (hi - lo + 1)); }
+ note(line: string): void {
+ console.error(`[${this.knobs.tag} ${this.name}] ${line}`);
+ if (this.knobs.logDir) { mkdirSync(this.knobs.logDir, { recursive: true }); appendFileSync(join(this.knobs.logDir, `${this.knobs.tag}.log`), `[${this.name}] ${line}\n`); }
+ }
+ keepLog(name: string, text: string): void { if (this.knobs.logDir) { mkdirSync(join(this.knobs.logDir, this.name), { recursive: true }); writeFileSync(join(this.knobs.logDir, this.name, name), text); } }
+ context(): string {
+ return [`seed ${this.knobs.seed}; modes ${this.modes.join(',')}; ${this.knobs.summary}`,
+ `fault plan: ${this.plan.join('; ') || 'none'}`,
+ `fired: ${Object.entries(this.fired).map(([k, v]) => `${k} => ${v ?? 'did not fire'}`).join('; ') || 'none'}`,
+ `repro (Postgres + PgBouncer from scripts/persistence/README.md): ${this.knobs.repro(this.modes)}`].join('\n');
+ }
+ /** Throws with the problems and the failure context; a passing check prints nothing. */
+ verdict(problems: string[]): void {
+ if (problems.length) throw new Error(`${problems.length} problem(s):\n ${problems.slice(0, 40).join('\n ')}\n${this.context()}`);
+ }
+ marker(tag: string): string { return join(this.home, `fault-${tag}.json`); }
+ recordFault(tag: string, label: string): string | null {
+ const at = existsSync(this.marker(tag)) ? readFileSync(this.marker(tag), 'utf8') : null;
+ this.fired[label] = at;
+ if (at) { const fired = JSON.parse(at) as { pid: number; action: string }; if (fired.action === 'kill') this.killedPids.add(fired.pid); }
+ return at;
+ }
+
+ git(...args: string[]): string { return execFileSync('git', ['-C', this.repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
+ /**
+ * Writes files and commits exactly those paths, as the shared-checkout rules require of every committer; each write
+ * is acknowledged when the commit returns. Sweeping the worktree (`git add -A`) would commit a writer's in-flight
+ * write-through bytes as an out-of-band Git change of a page that writer is still writing.
+ */
+ commit(message: string, files: Array<{ slug: string; content: string; marker: string }>): void {
+ const start = performance.now();
+ for (const f of files) { mkdirSync(dirname(join(this.repo, `${f.slug}.md`)), { recursive: true }); writeFileSync(join(this.repo, `${f.slug}.md`), f.content); }
+ const paths = files.map(f => `${f.slug}.md`);
+ this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);
+ const ack = performance.now();
+ for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });
+ }
+ writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {
+ this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);
+ }
+ bigFile(facts: number, timeline: number, version: string) { return { slug: BIG, content: bigPage(facts, timeline, version), marker: `Section 0 (${version})` }; }
+
+ childEnv(extra: Record<string, string | undefined> = {}): Record<string, string> {
+ return keylessBrainEnv(process.env, this.home, {
+ DATABASE_URL: this.pooledUrl, GBRAIN_DATABASE_URL: undefined, GBRAIN_DIRECT_DATABASE_URL: this.directUrl,
+ GBRAIN_BRAIN_ID: 'host', GBRAIN_SOURCE: SOURCE, GBRAIN_SWEEP: '0', GBRAIN_HOOKS: undefined,
+ GBRAIN_SKIP_STARTUP_HOOKS: '1', GBRAIN_NO_SKILL_NAG: '1', GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS: '300',
+ GBRAIN_SYNC_NO_DELEGATE: undefined, GBRAIN_TEST_CONVERGENCE_FAULT: undefined,
+ ...extra,
+ });
+ }
+ faultArgs(fault?: Fault): { preload: string[]; env: Record<string, string | undefined> } {
+ if (!fault) return { preload: [], env: {} };
+ return { preload: ['--preload', FAULT_PRELOAD], env: { GBRAIN_TEST_CONVERGENCE_FAULT: JSON.stringify(fault) } };
+ }
+ spawnCli(args: string[], opts: { fault?: Fault; stdin?: string; timeoutMs?: number } = {}): SyncRun {
+ const { preload, env } = this.faultArgs(opts.fault);
+ const started = performance.now();
+ const proc = Bun.spawn([process.execPath, '--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), ...args], {
+ cwd: REPO_ROOT, env: this.childEnv(env), stdin: opts.stdin === undefined ? 'ignore' : new Blob([opts.stdin]), stdout: 'pipe', stderr: 'pipe',
+ });
+ const timer = setTimeout(() => proc.kill('SIGKILL'), opts.timeoutMs ?? this.knobs.syncTimeoutMs);
+ const result = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])
+ .then(([out, err, code]) => ({ code, out, err, ms: performance.now() - started }))
+ .finally(() => clearTimeout(timer));
+ return { proc, result };
+ }
+ sync(args: string[], fault?: Fault): SyncRun { return this.spawnCli(['sync', '--source', SOURCE, ...args], { fault }); }
+ async committedSince(at: string): Promise<number> {
+ return (await this.engine!.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM persistence_requests WHERE state='committed' AND completed_at > $1::timestamptz AND operation='submit_job'`, [at]))[0].n;
+ }
+ async now(): Promise<string> { return (await this.engine!.executeRaw<{ at: string }>('SELECT now()::text AS at'))[0].at; }
+
+ async setup(seed: (b: Brain) => void, config: Record<string, string> = {}): Promise<void> {
+ assertSafeE2eDatabaseUrl(this.knobs.directTarget);
+ this.home = mkdtempSync(join(tmpdir(), 'gb-conv-'));
+ mkdirSync(join(this.home, '.gbrain'));
+ this.database = `gbrain_test_${this.knobs.tag.replaceAll('-', '_')}_${randomUUID().replaceAll('-', '')}`;
+ this.admin = postgres(this.knobs.directTarget, { max: 1, prepare: false });
+ await this.admin.unsafe(`CREATE DATABASE ${this.database}`);
+ const direct = new URL(this.knobs.directTarget); direct.pathname = `/${this.database}`;
+ const pooled = new URL(this.knobs.pooledTarget); pooled.pathname = `/${this.database}`;
+ this.pooledProxy = new LatencyProxy({ host: pooled.hostname, port: Number(pooled.port) || 5432 }, this.knobs.latencyMs / 2);
+ this.directProxy = new LatencyProxy({ host: direct.hostname, port: Number(direct.port) || 5432 }, this.knobs.latencyMs / 2);
+ await Promise.all([this.pooledProxy.listen(), this.directProxy.listen()]);
+ const viaPooled = new URL(pooled); viaPooled.hostname = '127.0.0.1'; viaPooled.port = String(this.pooledProxy.port); viaPooled.searchParams.set('prepare', 'false');
+ const viaDirect = new URL(direct); viaDirect.hostname = '127.0.0.1'; viaDirect.port = String(this.directProxy.port);
+ this.pooledUrl = viaPooled.toString(); this.directUrl = viaDirect.toString();
+ this.engine = new PostgresEngine();
+ await withEnv({ GBRAIN_HOME: this.home, GBRAIN_DIRECT_DATABASE_URL: undefined }, async () => {
+ await this.engine!.connect({ database_url: direct.toString(), poolSize: 4 });
+ await this.engine!.initSchema();
+ });
+ writeFileSync(join(this.home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: this.pooledUrl, embedding_disabled: true }));
+ this.repo = mkdtempSync(join(tmpdir(), 'gb-conv-repo-'));
+ this.git('init', '-q'); this.git('config', 'user.email', '[email protected]'); this.git('config', 'user.name', 'Example');
+ seed(this);
+ await withEnv({ GBRAIN_HOME: this.home }, async () => {
+ await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ await this.engine!.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [SOURCE, this.repo]);
+ await claimWorktree(this.engine!, SOURCE, this.repo);
+ for (const [key, value] of Object.entries(config)) await this.engine!.setConfig(key, value);
+ await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
+ });
+ this.note(`seed=${this.knobs.seed} latency=${this.knobs.latencyMs}ms round trip via proxies (pooled -> ${pooled.host}, direct -> ${direct.host}); modes ${this.modes.join(',')}`);
+ // Phase 0: a fault-free import, so the oracle is calibrated before any chaos.
+ const started = performance.now();
+ const baseline = await this.sync(['--no-pull', '--no-delegate', '--json']).result;
+ this.elapsed.baseline_import_ms = performance.now() - started;
+ if (baseline.code !== 0) throw new Error(`baseline sync exited ${baseline.code}\n${baseline.out}\n${baseline.err}\n${this.context()}`);
+ this.calibration = await this.diskDbMismatches(new Set());
+ if (existsSync(join(this.repo, `${BIG}.md`))) {
+ const rows = await this.childRowProblems();
+ this.calibration.push(...rows.problems, ...(rows.timelineDuplicates || rows.timelineStrays ? [`timeline: ${rows.timelineDuplicates} duplicate and ${rows.timelineStrays} stray rows after a fault-free import`] : []));
+ }
+ this.note(`phase 0: fault-free import in ${(this.elapsed.baseline_import_ms / 1000).toFixed(1)}s; oracle calibration problems: ${this.calibration.length}`);
+ }
+
+ /** A fresh resident serve and at most MAX_RECOVERY_RUNS plain syncs, no operator step. */
+ async recover(): Promise<void> {
+ const started = performance.now();
+ const serve = new StdioServe(this);
+ await serve.start();
+ for (let run = 1; run <= MAX_RECOVERY_RUNS; run++) {
+ const r = await this.sync(['--no-pull', '--json']).result;
+ const envelope = lastJson(r.out);
+ this.recoveryRuns.push({ ...r, envelope });
+ this.keepLog(`recovery-${run}-client.log`, `${r.out}\n${r.err}`);
+ this.note(`recovery run ${run}: exit ${r.code} in ${(r.ms / 1000).toFixed(1)}s outcome=${String(envelope?.outcome ?? envelope?.sync_status ?? envelope?.error ?? 'none')}`);
+ if (r.code === 0 && envelope?.outcome === 'synced') break;
+ }
+ this.elapsed.recovery_ms = performance.now() - started;
+ this.keepLog('recovery-serve.log', serve.stderr);
+ this.note(`elapsed: ${Object.entries(this.elapsed).map(([k, ms]) => `${k.replace(/_ms$/, '')} ${(ms / 1000).toFixed(1)}s`).join(', ')}; recovery took ${this.recoveryRuns.length} sync run(s)`);
+ }
+
+ /**
+ * #6841: an accepted write still queued when its writer gave up may commit later, after a write the ledger holds.
+ * Once the brain has converged, each one's receipt is read the way the write protocol tells a client to after exit
+ * 10 (get_write_request). A committed write is acknowledged then: it may have landed any time since its start, so it
+ * is concurrent with every write in that window. Any other state means it never landed and stays out of the ledger.
+ */
+ async settlePending(receiptState = async (requestId: string) =>
+ lastJson((await this.spawnCli(['call', '--source', SOURCE, 'get_write_request', JSON.stringify({ request_id: requestId })]).result).out)?.state): Promise<void> {
+ for (const { write, requestId } of this.pending) {
+ const state = await receiptState(requestId);
+ if (state === 'committed') this.ledger.record({ ...write, ack: performance.now() });
+ this.note(`accepted ${write.kind} of ${write.slug} (request ${requestId}) unsettled when its writer stopped: receipt ${String(state ?? 'unreadable')}`);
+ }
+ }
+
+ async teardown(): Promise<void> {
+ for (const serve of this.serves) await serve.stop().catch(() => {});
+ try { await this.engine?.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1'); } catch { /* best effort */ }
+ await this.engine?.disconnect().catch(() => {});
+ await this.pooledProxy?.close().catch(() => {});
+ await this.directProxy?.close().catch(() => {});
+ if (this.admin) {
+ try { if (this.database) await this.admin.unsafe(`DROP DATABASE IF EXISTS ${this.database} WITH (FORCE)`); } finally { await this.admin.end(); }
+ }
+ for (const dir of [this.home, this.repo]) if (dir) rmSync(dir, { recursive: true, force: true });
+ }
+
+ // ------------------------------------------------------------ oracle reads
+
+ diskPages(): Map<string, string> {
+ const out = new Map<string, string>();
+ const walk = (dir: string) => {
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
+ if (entry.name.startsWith('.')) continue;
+ const full = join(dir, entry.name);
+ if (entry.isDirectory()) walk(full);
+ else if (entry.name.endsWith('.md')) out.set(relative(this.repo, full).replace(/\.md$/, ''), readFileSync(full, 'utf8'));
+ }
+ };
+ walk(this.repo);
+ return out;
+ }
+ async dbPages(): Promise<Map<string, { hash: string | null; body: string }>> {
+ const rows = await this.engine!.executeRaw<{ slug: string; content_hash: string | null; body: string }>(
+ `SELECT slug, content_hash, coalesce(compiled_truth,'') || E'\\n' || coalesce(timeline,'') AS body FROM pages WHERE source_id=$1 AND deleted_at IS NULL`, [SOURCE]);
+ return new Map(rows.map(r => [r.slug, { hash: r.content_hash, body: r.body }]));
+ }
+ async diskDbMismatches(exclude: Set<string>): Promise<string[]> {
+ const disk = this.diskPages(), db = await this.dbPages(), out: string[] = [];
+ for (const [slug, content] of disk) {
+ if (exclude.has(slug)) continue;
+ if (!db.has(slug)) out.push(`${slug}: file exists, no live page`);
+ else if (db.get(slug)!.hash !== fileHash(slug, content)) out.push(`${slug}: page content_hash differs from the file`);
+ }
+ for (const slug of db.keys()) if (!disk.has(slug) && !exclude.has(slug)) out.push(`${slug}: live page, no file`);
+ return out;
+ }
+ async ledgerProblems(): Promise<string[]> {
+ const facts = new Set((await this.engine!.executeRaw<{ fact: string }>('SELECT fact FROM facts WHERE source_id=$1 AND expired_at IS NULL', [SOURCE])).map(r => r.fact));
+ return this.ledger.problems(this.diskPages(), await this.dbPages(), facts);
+ }
+ async openRequests(): Promise<unknown[]> {
+ const open = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; recovery: boolean; blocked_reason: string | null }>(
+ `SELECT request_id::text, slug, state, recovery IS NOT NULL AS recovery, blocked_reason FROM persistence_requests
+ WHERE state IN ('queued','deferred','running','recovering') OR recovery IS NOT NULL ORDER BY sequence`);
+ const worktrees = await this.engine!.executeRaw<{ id: string; state: string }>(`SELECT id::text, state FROM persistence_worktrees WHERE state<>'active'`);
+ return [...open.map(r => `open request ${JSON.stringify(r)}`), ...worktrees.map(w => `worktree ${w.id} is ${w.state}`)];
+ }
+ async unexplainedTerminal(exclude: Set<string>): Promise<string[]> {
+ const rows = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; error_code: string | null; operation: string }>(
+ `SELECT request_id::text, slug, state, error_code, operation FROM persistence_requests WHERE state IN ('conflict','failed','cancelled') ORDER BY sequence`);
+ const disk = this.diskPages(), db = await this.dbPages();
+ this.note(`terminal non-committed requests: ${rows.length} (${[...new Set(rows.map(r => `${r.state}:${r.error_code}`))].join(', ') || 'none'})`);
+ return rows.filter(row => {
+ const content = disk.get(row.slug);
+ const superseded = exclude.has(row.slug) || (content !== undefined && db.get(row.slug)?.hash === fileHash(row.slug, content));
+ return !superseded && !(row.error_code && row.error_code in CODES);
+ }).map(row => `unexplained ${row.state} request ${row.request_id} (${row.slug}, ${row.error_code ?? 'no code'})`);
+ }
+ async cursorProblems(): Promise<string[]> {
+ const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);
+ const head = this.git('rev-parse', 'HEAD');
+ return source.last_commit === head ? [] : [`sources.last_commit ${source.last_commit} is not HEAD ${head}`];
+ }
+ async status(detail = true): Promise<Record<string, unknown> | null> {
+ const r = await this.spawnCli(['sync', 'status', '--source', SOURCE, '--json', ...(detail ? ['--detail'] : [])]).result;
+ this.keepLog(detail ? 'status.json' : 'status-fast.json', `${r.out}\n${r.err}`);
+ return lastJson(r.out);
+ }
+ recoveryProblems(): string[] {
+ const last = this.recoveryRuns.at(-1);
+ return last?.code === 0 && last.envelope?.outcome === 'synced' ? []
+ : [`no plain sync succeeded within ${MAX_RECOVERY_RUNS} runs: ${this.recoveryRuns.map(r => `exit ${r.code} outcome ${String(r.envelope?.outcome ?? r.envelope?.error ?? 'none')}`).join('; ')}`];
+ }
+ /**
+ * The large page's child rows against its file. Facts are keyed by fence row, so
+ * loss, duplication and strays are all problems. Timeline rows are judged for
+ * loss only; duplicate and stray timeline rows come back as a measured count,
+ * because derived timeline rows are never pruned (a known defect outside #6650).
+ */
+ async childRowProblems(): Promise<{ problems: string[]; timelineDuplicates: number; timelineStrays: number }> {
+ const content = this.diskPages().get(BIG);
+ if (!content) return { problems: [`${BIG}: no file`], timelineDuplicates: 0, timelineStrays: 0 };
+ const out: string[] = [];
+ const expectedTimeline = [...content.matchAll(/^- \*\*(\d{4}-\d{2}-\d{2})\*\* \| (.+)$/gm)].map(m => `${m[1]}|${m[2]}`);
+ const timeline = await this.engine!.executeRaw<{ key: string; n: number }>(
+ `SELECT t.date::text || '|' || t.summary AS key, count(*)::int AS n FROM timeline_entries t JOIN pages p ON p.id=t.page_id
+ WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL AND t.event_page_id IS NULL GROUP BY 1`, [SOURCE, BIG]);
+ const have = new Map(timeline.map(r => [r.key, r.n]));
+ const lostTimeline = expectedTimeline.filter(k => !have.has(k));
+ const dupTimeline = timeline.filter(r => r.n > 1);
+ const extraTimeline = timeline.filter(r => !expectedTimeline.includes(r.key));
+ if (lostTimeline.length) out.push(`timeline: ${lostTimeline.length} of ${expectedTimeline.length} file entries missing (first: ${lostTimeline[0]})`);
+ const expectedFacts = parseFactsFence(content).facts.map(f => `${f.rowNum}|${f.claim}`);
+ const facts = await this.engine!.executeRaw<{ key: string; n: number }>(
+ `SELECT row_num::text || '|' || fact AS key, count(*)::int AS n FROM facts
+ WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL GROUP BY 1`, [SOURCE, BIG]);
+ const haveFacts = new Map(facts.map(r => [r.key, r.n]));
+ const lostFacts = expectedFacts.filter(k => !haveFacts.has(k));
+ const dupFacts = facts.filter(r => r.n > 1);
+ const extraFacts = facts.filter(r => !expectedFacts.includes(r.key));
+ if (lostFacts.length) out.push(`facts: ${lostFacts.length} of ${expectedFacts.length} fence rows missing (first: ${lostFacts[0]})`);
+ if (dupFacts.length) out.push(`facts: ${dupFacts.length} rows duplicated (first: ${dupFacts[0].key} x${dupFacts[0].n})`);
+ if (extraFacts.length) out.push(`facts: ${extraFacts.length} rows not in the fence (first: ${extraFacts[0].key})`);
+ return { problems: out, timelineDuplicates: dupTimeline.reduce((n, r) => n + r.n - 1, 0), timelineStrays: extraTimeline.length };
+ }
+}
+
+export function fileHash(slug: string, content: string): string {
+ const parsed = parseMarkdown(content, `${slug}.md`);
+ return contentHash({ title: parsed.title, type: parsed.type, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline, frontmatter: parsed.frontmatter, tags: [...parsed.tags].sort() });
+}
+
+/** One stdio MCP session against a spawned `gbrain serve`; the serve is the host's resident persistence owner. */
+export class StdioServe {
+ readonly child: ChildProcess;
+ stderr = '';
+ exited: Promise<number | null>;
+ private nextId = 1;
+ private pending = new Map<number, (msg: Record<string, unknown>) => void>();
+ private buf = '';
+ constructor(private brain: Brain, fault?: Fault) {
+ const { preload, env } = brain.faultArgs(fault);
+ this.child = spawn(process.execPath, ['--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), 'serve'], { cwd: REPO_ROOT, env: brain.childEnv(env), stdio: ['pipe', 'pipe', 'pipe'] });
+ this.exited = new Promise(done => this.child.once('exit', code => done(code)));
+ this.child.stderr!.on('data', (d: Buffer) => { this.stderr += d.toString('utf8'); });
+ this.child.stdin!.on('error', () => {});
+ this.child.stdout!.on('data', (d: Buffer) => {
+ this.buf += d.toString('utf8');
+ let idx: number;
+ while ((idx = this.buf.indexOf('\n')) !== -1) {
+ const line = this.buf.slice(0, idx).trim(); this.buf = this.buf.slice(idx + 1);
+ if (!line) continue;
+ try {
+ const msg = JSON.parse(line) as Record<string, unknown>;
+ const id = msg.id as number | undefined;
+ if (id !== undefined && this.pending.has(id)) { this.pending.get(id)!(msg); this.pending.delete(id); }
+ } catch { /* serve chatter */ }
+ }
+ });
+ brain.serves.push(this);
+ }
+ get pid(): number { return this.child.pid!; }
+ get alive(): boolean { return this.child.exitCode === null && this.child.signalCode === null; }
+ request(method: string, params: Record<string, unknown>, timeoutMs = 240_000): Promise<Record<string, unknown>> {
+ const id = this.nextId++;
+ return new Promise((done, reject) => {
+ const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`rpc ${method} timed out`)); }, timeoutMs);
+ this.pending.set(id, msg => { clearTimeout(t); done(msg); });
+ this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
+ });
+ }
+ async start(): Promise<void> {
+ await this.request('initialize', { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'convergence', version: '1' } });
+ this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
+ await until(async () => (await this.brain.engine!.executeRaw<{ n: number }>(
+ `SELECT count(*)::int AS n FROM persistence_consumers WHERE pid=$1 AND kind='serve' AND renewed_at > now() - interval '30 seconds'`, [this.pid]))[0].n > 0,
+ `serve ${this.pid} heartbeat`, 120_000);
+ }
+ async stop(): Promise<void> {
+ if (!this.alive) return;
+ this.child.kill('SIGTERM');
+ const timer = setTimeout(() => this.child.kill('SIGKILL'), 10_000);
+ await this.exited; clearTimeout(timer);
+ }
+}
+
+/**
+ * A writer call that retries an unsettled attempt (exit 10, a lost owner) with the same request id and a terminal refusal with a new one.
+ * When the attempts or the chaos run out on an unsettled attempt, the request it names may still commit, so it comes back `pending`.
+ */
+export async function acknowledged(b: Brain, label: string, call: (requestId: string) => SyncRun, stop: () => boolean): Promise<{ start: number; ack: number } | { pending: string } | null> {
+ let requestId = randomUUID();
+ let unsettled = false;
+ for (let attempt = 1; attempt <= 20 && !stop(); attempt++) {
+ const start = performance.now();
+ const r = await call(requestId).result;
+ b.keepLog(`writer-${label.replaceAll('/', '_')}-${attempt}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
+ if (r.code === 0) return { start, ack: performance.now() };
+ const terminal = /\((conflict|failed|cancelled)\)/.test(r.err);
+ b.note(`writer ${label} attempt ${attempt} exit ${r.code}${terminal ? ' (terminal; next attempt uses a new request id)' : ''}: ${r.err.trim().split('\n').slice(-1)[0] ?? ''}`);
+ unsettled = !terminal;
+ if (terminal) requestId = randomUUID();
+ await Bun.sleep(3000);
+ }
+ return unsettled ? { pending: requestId } : null;
+}
+export const outcome = (ok: { ack: number } | { pending: string } | null) => !ok ? 'unacknowledged' : 'pending' in ok ? `pending ${ok.pending}` : 'ok';
+
+/** Oracle tests are named by the persistence gate's violation class (scripts/persistence/model.ts) they would report. */
+export const as = (violation: ViolationClass, title: string): string => `[${violation}] ${title}`;
+
+// ---------------------------------------------------------------- the shared oracle
+
+export interface EndState { exclude?: Set<string>; status?: StatusExpectation; bigPage?: boolean }
+/** The oracle tests every scenario ends with; `end()` is read when the tests run (after the scenario's beforeAll). */
+export function oracleTests(b: Brain, end: () => EndState): void {
+ test('the oracle is calibrated: a fault-free import satisfies it', () => b.verdict(b.calibration));
+ test('every planned fault fired, so the chaos actually happened', () => b.verdict(Object.entries(b.fired).filter(([, at]) => at === null).map(([fault]) => `${fault} did not fire`)));
+ test(as('wedge', `a plain sync succeeds within ${MAX_RECOVERY_RUNS} runs after the chaos, with no operator step`), () => b.verdict(b.recoveryProblems()));
+ test(as('projection_drift', 'disk == database for every page the end state does not name'), async () => b.verdict(await b.diskDbMismatches(end().exclude ?? new Set())));
+ test(as('wedge', 'no request blocks the checkpoint or a root'), async () => b.verdict((await b.openRequests()).map(String)));
+ test(as('untyped_error', 'every non-committed terminal request is superseded or carries a registered error code'), async () => b.verdict(await b.unexplainedTerminal(end().exclude ?? new Set())));
+ test(as('wedge', 'the cursor reached HEAD'), async () => b.verdict(await b.cursorProblems()));
+ test(as('lost_write', 'every acknowledged write survives or is causally superseded by a later acknowledged write'), async () => {
+ b.note(`ledger: ${b.ledger.acks.length} acknowledged writes (${[...new Set(b.ledger.acks.map(a => a.kind))].map(kind => `${kind} ${b.ledger.acks.filter(a => a.kind === kind).length}`).join(', ')})`);
+ b.verdict(await b.ledgerProblems());
+ });
+ test(as('wedge', 'sync status --json reports the converged state with every field present'), async () => b.verdict(statusProblems(await b.status(), end().status)));
+ test(as('wedge', 'plain sync status --json names every heavy read it skipped instead of reading as all-clear'), async () => b.verdict(fastStatusProblems(await b.status(false))));
+ if (end().bigPage !== false) test(as('lost_write', 'the large page keeps every fence fact exactly once and loses no timeline entry'), async () => {
+ const { problems, timelineDuplicates, timelineStrays } = await b.childRowProblems();
+ b.note(`large page timeline (measured, not asserted): ${timelineDuplicates} duplicate rows, ${timelineStrays} rows not in the file`);
+ b.verdict(problems);
+ });
+}
diff --git a/test/scripts/ci-pr-scope.test.ts b/test/scripts/ci-pr-scope.test.ts
index 71870fa31d..6c336b7e2e 100644
--- a/test/scripts/ci-pr-scope.test.ts
+++ b/test/scripts/ci-pr-scope.test.ts
@@ -225,7 +225,7 @@ esac
for (const path of ['src/core/persistence/sync-run.ts', 'src/core/persistence/fault-points.ts', 'src/commands/sync/run.ts', 'src/commands/sync.ts',
'src/core/sync-holds.ts', 'src/core/page-state/projections.ts', 'src/core/engine-sql/links.ts', 'src/core/storage/local.ts', 'src/core/storage-config.ts',
'src/core/import-file.ts', 'src/core/markdown.ts', 'src/core/write-through.ts', 'src/core/postgres-engine.ts',
- 'test/sync-convergence.serial.test.ts', 'test/fixtures/convergence-fault-kill.ts', 'scripts/ci-convergence-scope.sh', '.github/workflows/persistence-validation.yml',
+ 'test/sync-convergence.serial.test.ts', 'test/fixtures/convergence-fault-kill.ts', 'test/helpers/convergence-brain.ts', 'scripts/ci-convergence-scope.sh', '.github/workflows/persistence-validation.yml',
'.github/workflows/convergence-nightly.yml', 'bun.lock', 'vendor/postgres/src/connection.js']) {
expect(convergenceScope(`README.md\n${path}\n`), path).toBe('run');
}
diff --git a/test/sync-convergence.serial.test.ts b/test/sync-convergence.serial.test.ts
index 46764984a3..998c72dd5b 100644
--- a/test/sync-convergence.serial.test.ts
+++ b/test/sync-convergence.serial.test.ts
@@ -56,29 +56,19 @@
* skipped unless GBRAIN_TEST_CONVERGENCE=1.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
-import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
+import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
-import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
-import { dirname, join, relative, resolve } from 'node:path';
-import postgres from '#postgres';
-import { PostgresEngine } from '../src/core/postgres-engine.ts';
-import { claimWorktree } from '../src/core/persistence/ownership.ts';
-import { parseMarkdown } from '../src/core/markdown.ts';
-import { parseFactsFence } from '../src/core/facts-fence.ts';
-import { contentHash } from '../src/core/utils.ts';
-import { CODES } from '../src/core/error-registry.ts';
+import { join } from 'node:path';
import { CONSUMER_LIVE_MS, CONSUMER_RENEWAL_MS } from '../src/core/persistence/consumer-heartbeat.ts';
import type { FaultPoint } from '../src/core/persistence/fault-points.ts';
import { PUBLICATION_CEILING_DEFAULT_MS, publicationLeaseMs } from '../src/core/persistence/publication-deadline.ts';
-import { assertSafeE2eDatabaseUrl } from './helpers/db-guard.ts';
-import { keylessBrainEnv } from './helpers/provider-env.ts';
-import { withEnv } from './helpers/with-env.ts';
-import { LatencyProxy } from './helpers/latency-proxy.ts';
-import type { ViolationClass } from '../scripts/persistence/model.ts';
+import {
+ Brain as HarnessBrain, ENTITY, Ledger, SOURCE, STATUS_FIELDS, StdioServe, acknowledged, as, entityText, fastStatusProblems, lastJson,
+ noteMarker, noteSlug, noteText, oracleTests, outcome, prng, statusProblems, until, worstStall, type Ack, type Fault, type HarnessKnobs,
+} from './helpers/convergence-brain.ts';
-const REPO_ROOT = resolve(import.meta.dir, '..');
-const FAULT_PRELOAD = join(REPO_ROOT, 'test/fixtures/convergence-fault-kill.ts');
const knob = (name: string, fallback: number): number => {
const raw = process.env[`GBRAIN_TEST_CONVERGENCE_${name}`];
const value = raw === undefined || raw === '' ? fallback : Number(raw);
@@ -120,9 +110,15 @@ const requestedModes = process.env.GBRAIN_TEST_CONVERGENCE_MODES?.split(',').map
for (const mode of requestedModes ?? []) if (!(mode in MODES)) throw new Error(`GBRAIN_TEST_CONVERGENCE_MODES names unknown mode '${mode}' (known: ${Object.keys(MODES).join(', ')}).`);
const runs = (mode: Mode): boolean => requestedModes ? requestedModes.includes(mode) : MODES[mode].skip === null;
-const SOURCE = 'workspace';
-const BIG = 'companies/acme-example';
-const ENTITY = 'people/alice-example';
+const KNOBS: HarnessKnobs = {
+ tag: 'convergence', seed: SEED, latencyMs: LATENCY_MS, syncTimeoutMs: SYNC_TIMEOUT_MS, logDir: LOG_DIR,
+ pooledTarget: pooledTarget ?? '', directTarget: directTarget ?? '', summary: `notes ${NOTES}; latency ${LATENCY_MS} ms`,
+ repro: modes => `${[`GBRAIN_TEST_CONVERGENCE_SEED=${SEED}`, `GBRAIN_TEST_CONVERGENCE_MODES=${modes.join(',')}`, `GBRAIN_TEST_CONVERGENCE_NOTES=${NOTES}`,
+ `GBRAIN_TEST_CONVERGENCE_KILLS=${KILLS}`, `GBRAIN_TEST_CONVERGENCE_LATENCY_MS=${LATENCY_MS}`].join(' ')} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun run test:convergence`,
+};
+/** One scenario's brain: the shared harness brain (test/helpers/convergence-brain.ts) with this gate's knobs. */
+class Brain extends HarnessBrain { constructor(name: string, modes: Mode[]) { super(name, modes, KNOBS); } }
+
/**
* The kill loop's fault points: the seams a managed sync import crosses. Its pages are already in Git, so the
* file-publication pair (`before_publication`/`after_publication`) never fires for it. Per-page seams are counted
@@ -145,9 +141,6 @@ const KILL_POINTS = [
// #6712 PR 2 (L5): after the checkpoint committed and the lease was released, before the owed tail runs (the next sync runs it).
{ point: 'sync:tail_started', per: 'group' },
] as const satisfies ReadonlyArray<{ point: FaultPoint; per: 'page' | 'group' | 'save' | 'held' | 'deferred' }>;
-const MAX_RECOVERY_RUNS = 2;
-/** Oracle tests are named by the persistence gate's violation class (scripts/persistence/model.ts) they would report. */
-const as = (violation: ViolationClass, title: string): string => `[${violation}] ${title}`;
/**
* The longest a sync may sit with no progress after an owner was killed: the
* window in which a dead owner's heartbeat still reads live, one renewal
@@ -156,520 +149,8 @@ const as = (violation: ViolationClass, title: string): string => `[${violation}]
*/
const OWNER_KILL_STALL_BOUND_S = (CONSUMER_LIVE_MS + CONSUMER_RENEWAL_MS + 30_000) / 1000;
-/** mulberry32: a fixed seed gives the same fault plan and writer targets on every run. */
-function prng(seed: number): () => number {
- let a = seed >>> 0;
- return () => {
- a = (a + 0x6d2b79f5) >>> 0;
- let t = a;
- t = Math.imul(t ^ (t >>> 15), t | 1);
- t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
- return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
- };
-}
-
-// ---------------------------------------------------------------- the status oracle
-
-/** Every `gbrain sync status --json` field the operator loop reads (sync-status.ts `SyncStatus`). */
-const STATUS_FIELDS = ['source_id', 'run_id', 'cursor', 'committed_last_10m', 'live_job_id', 'holds', 'last_error', 'needs_human',
- 'resume_argv', 'checkpoint_blockers', 'cursors', 'terminal_failures', 'next'] as const;
-interface StatusExpectation { holds?: Array<{ code: string; path: string }>; needsHuman?: boolean }
-/**
- * What a converged source's status must say. Field-present assertions first: a renderer that drops a field fails
- * here, not in an agent's loop. `checkpoint_blockers` and `cursors` are arrays on every status (#6712 PR 0 made
- * `checkpoint_blockers` always present). The harness reads `sync status --json --detail`: since #6712 PR 2 the
- * terminal summary, blocker list and cursor join run only behind the flag.
- */
-/** The default (no `--detail`) status after the chaos: every skipped heavy read says so and names the `--detail` command (#6712 PR 2, DX F1). */
-function fastStatusProblems(status: Record<string, unknown> | null): string[] {
- if (!status) return ['sync status --json printed no JSON document'];
- return (['checkpoint_blockers_read', 'cursors_read', 'terminal_failures_read'] as const).flatMap(field => {
- const read = status[field] as { status?: string; fix?: { argv?: string[] } } | undefined;
- return read?.status === 'not_read' && read.fix?.argv?.includes('--detail') ? [] : [`sync status --json: ${field} is ${JSON.stringify(read)}, expected not_read with a --detail fix`];
- });
-}
-
-function statusProblems(status: Record<string, unknown> | null, expected: StatusExpectation = {}): string[] {
- if (!status) return ['sync status --json printed no JSON document'];
- const out = STATUS_FIELDS.filter(field => !(field in status)).map(field => `sync status --json: field ${field} missing`);
- if (out.length) return out;
- const cursor = status.cursor as { done?: boolean } | null;
- if (!Array.isArray(status.cursors)) out.push('sync status --json: cursors is not an array');
- if (!Array.isArray(status.checkpoint_blockers)) out.push('sync status --json: checkpoint_blockers is not an array');
- else if (status.checkpoint_blockers.length) out.push(`sync status --json: ${status.checkpoint_blockers.length} checkpoint blocker(s)`);
- // A finished run's cursor is retired, so a converged source shows none (null) or a done one.
- if (cursor && cursor.done !== true) out.push(`sync status --json: cursor not done (${JSON.stringify(cursor)})`);
- const terminal = status.terminal_failures as { status?: string; active_blockers?: unknown[] } | null;
- if (terminal?.status !== 'ok') out.push(`sync status --json: terminal_failures.status is ${terminal?.status}`);
- else if (!Array.isArray(terminal.active_blockers)) out.push('sync status --json: terminal_failures.active_blockers is not an array');
- else if (terminal.active_blockers.length) out.push(`sync status --json: ${terminal.active_blockers.length} active terminal blocker(s)`);
- if (status.needs_human !== (expected.needsHuman ?? false)) out.push(`sync status --json: needs_human is ${status.needs_human}, expected ${expected.needsHuman ?? false}`);
- const holds = Array.isArray(status.holds) ? (status.holds as Array<{ code: string; path: string }>).map(h => `${h.code}:${h.path}`).sort() : ['holds is not an array'];
- const want = (expected.holds ?? []).map(h => `${h.code}:${h.path}`).sort();
- if (JSON.stringify(holds) !== JSON.stringify(want)) out.push(`sync status --json: holds [${holds.join(', ')}], expected [${want.join(', ')}]`);
- if (status.last_error !== null) out.push(`sync status --json: last_error ${JSON.stringify(status.last_error)}`);
- if (!expected.holds?.length && status.next !== null) out.push(`sync status --json: next ${JSON.stringify(status.next)}`);
- return out;
-}
-
-// ---------------------------------------------------------------- the acknowledged-write ledger
-
-/**
- * One writer operation that returned success, recorded by the harness at the moment it saw the success (never read
- * back from the brain), or a write the brain accepted (exit 10) that had not settled when its writer stopped, recorded
- * when its receipt, read after recovery, said committed (`Brain.settlePending`). `touches` is where the write landed: the page in the database, its file, or (for `remember`)
- * an additive fact row. `marker` is text that only this write put there; `null` on a page location is a deletion.
- */
-interface Ack { kind: 'commit' | 'put' | 'remember' | 'maintenance' | 'db_only' | 'git_mv' | 'git_rm' | 'unlink'; slug: string;
- marker: string | null; touches: Array<'db' | 'file'> | 'fact'; start: number; ack: number }
-class Ledger {
- acks: Ack[] = [];
- record(ack: Ack): void { this.acks.push(ack); }
- /**
- * For every slug and location, the writes no later acknowledged write causally supersedes (none started after it
- * was acknowledged); the final state must show one of them. An additive fact survives unless a page write on its
- * slug may have been ordered after it.
- */
- problems(disk: Map<string, string>, db: Map<string, { body: string } | null>, facts: Set<string>): string[] {
- const out: string[] = [];
- const pageWrites = this.acks.filter(a => a.touches !== 'fact' && a.kind !== 'maintenance');
- for (const slug of new Set(pageWrites.map(a => a.slug))) {
- for (const where of ['db', 'file'] as const) {
- const writes = pageWrites.filter(a => a.slug === slug && (a.touches as string[]).includes(where));
- if (!writes.length) continue;
- const maximal = writes.filter(w => !writes.some(later => later.start > w.ack));
- const seen = where === 'file' ? disk.get(slug) ?? null : db.get(slug)?.body ?? null;
- if (maximal.some(w => w.marker === null ? seen === null : seen?.includes(w.marker))) continue;
- out.push(`${where} ${slug}: lost acknowledged write; has ${seen === null ? 'nothing' : JSON.stringify(seen.slice(0, 80))}, expected one of ${maximal.map(w => `${w.kind}@${Math.round(w.ack)}:${w.marker === null ? '(absent)' : JSON.stringify(w.marker)}`).join(' | ')}`);
- }
- }
- for (const fact of this.acks.filter(a => a.touches === 'fact')) {
- const excused = pageWrites.some(w => w.slug === fact.slug && (w.touches as string[]).includes('db') && w.ack > fact.start);
- if (!excused && !facts.has(fact.marker!)) out.push(`fact on ${fact.slug}: lost acknowledged remember ${JSON.stringify(fact.marker)}`);
- }
- return out;
- }
-}
-
-// ---------------------------------------------------------------- the fixture
-
-function noteText(i: number, version: string): string {
- return `---\ntype: concept\ntitle: Note ${i} Example\n---\n\n${version} body for note ${i}, an observation about widget-co and acme-example.\n\n${'Supporting detail sentence. '.repeat(8 + (i % 5))}\n`;
-}
-const noteMarker = (i: number, version: string) => `${version} body for note ${i},`;
-const noteSlug = (i: number) => `notes/note-${String(i).padStart(4, '0')}`;
-function bigPage(facts: number, timeline: number, version: string): string {
- const day = (n: number) => new Date(Date.UTC(2018, 0, 1) + n * 86_400_000).toISOString().slice(0, 10);
- const paragraphs = Array.from({ length: 30 }, (_, i) => `Section ${i} (${version}): acme-example ${'expanded with a steady stream of synthetic operating detail. '.repeat(6)}`);
- const rows = Array.from({ length: facts }, (_, i) =>
- `| ${i + 1} | Synthetic claim ${i + 1} about acme-example | fact | 0.9 | world | medium | ${day(i)} | | fixture | |`);
- const bullets = Array.from({ length: timeline }, (_, i) => `- **${day(i)}** | Synthetic event ${i + 1} at acme-example`);
- return [
- '---', 'type: company', 'title: Acme Example', '---', '', '# Acme Example', '', ...paragraphs.flatMap(p => [p, '']),
- '## Facts', '', '<!--- gbrain:facts:begin -->',
- '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |',
- '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|',
- ...rows, '<!--- gbrain:facts:end -->', '', '<!-- timeline -->', '', ...bullets, '',
- ].join('\n');
-}
-const entityText = '---\ntype: person\ntitle: Alice Example\n---\n\nAlice Example works with acme-example.\n';
-
-// ---------------------------------------------------------------- one brain
-
-interface Fault { point: FaultPoint; nth: number; marker: string; operation?: string; path?: string; action?: 'kill' | 'edit' | 'stall' | 'throw';
- edit?: { file: string; content: string }; release?: string; ms?: number; sqlstate?: string;
- prime?: { point: FaultPoint; nth: number; path?: string; edit: { file: string; content: string } } }
-interface Run { code: number; out: string; err: string; ms: number }
-interface SyncRun { proc: ReturnType<typeof Bun.spawn>; result: Promise<Run> }
-
-async function until(check: () => boolean | Promise<boolean>, label: string, timeoutMs = 300_000): Promise<void> {
- const deadline = performance.now() + timeoutMs;
- while (performance.now() < deadline) {
- if (await check()) return;
- await Bun.sleep(100);
- }
- throw new Error(`timed out waiting for ${label}`);
-}
-function lastJson(out: string): Record<string, unknown> | null {
- try { return JSON.parse(out) as Record<string, unknown>; } catch { /* several documents */ }
- for (const line of out.trim().split('\n').reverse()) { try { return JSON.parse(line) as Record<string, unknown>; } catch { /* not the envelope */ } }
- return null;
-}
-/** The longest `stalled Ns` progress line a sync printed: how long it waited on a predecessor's leftovers. */
-function worstStall(err: string): number { return Math.max(0, ...[...err.matchAll(/stalled (\d+)s/g)].map(m => Number(m[1]))); }
-
-/**
- * One scenario's brain: its database, home, Git worktree, proxies, child processes, ledger and log. `context()` is
- * what every failure prints: seed, modes, the fault plan and the one-line repro.
- */
-class Brain {
- home = ''; repo = ''; database = '';
- admin: ReturnType<typeof postgres> | undefined;
- engine: PostgresEngine | undefined;
- pooledProxy: LatencyProxy | undefined; directProxy: LatencyProxy | undefined;
- pooledUrl = ''; directUrl = '';
- serves: StdioServe[] = [];
- ledger = new Ledger();
- /** Writes the brain accepted (exit 10) that had not settled when their writer stopped; `settlePending` reads their receipts after recovery. */
- pending: Array<{ write: Omit<Ack, 'ack'>; requestId: string }> = [];
- plan: string[] = [];
- fired: Record<string, string | null> = {};
- killedPids = new Set<number>();
- recoveryRuns: Array<Run & { envelope: Record<string, unknown> | null }> = [];
- elapsed: Record<string, number> = {};
- calibration: string[] = [];
- readonly rand: () => number;
- constructor(readonly name: string, readonly modes: Mode[]) { this.rand = prng(SEED ^ [...name].reduce((h, c) => Math.imul(h, 31) + c.charCodeAt(0), 7)); }
-
- between(lo: number, hi: number): number { return lo + Math.floor(this.rand() * (hi - lo + 1)); }
- note(line: string): void {
- console.error(`[convergence ${this.name}] ${line}`);
- if (LOG_DIR) { mkdirSync(LOG_DIR, { recursive: true }); appendFileSync(join(LOG_DIR, 'convergence.log'), `[${this.name}] ${line}\n`); }
- }
- keepLog(name: string, text: string): void { if (LOG_DIR) { mkdirSync(join(LOG_DIR, this.name), { recursive: true }); writeFileSync(join(LOG_DIR, this.name, name), text); } }
- context(): string {
- const env = [`GBRAIN_TEST_CONVERGENCE_SEED=${SEED}`, `GBRAIN_TEST_CONVERGENCE_MODES=${this.modes.join(',')}`, `GBRAIN_TEST_CONVERGENCE_NOTES=${NOTES}`,
- `GBRAIN_TEST_CONVERGENCE_KILLS=${KILLS}`, `GBRAIN_TEST_CONVERGENCE_LATENCY_MS=${LATENCY_MS}`];
- return [`seed ${SEED}; modes ${this.modes.join(',')}; notes ${NOTES}; latency ${LATENCY_MS} ms`,
- `fault plan: ${this.plan.join('; ') || 'none'}`,
- `fired: ${Object.entries(this.fired).map(([k, v]) => `${k} => ${v ?? 'did not fire'}`).join('; ') || 'none'}`,
- `repro (Postgres + PgBouncer from scripts/persistence/README.md): ${env.join(' ')} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun run test:convergence`].join('\n');
- }
- /** Throws with the problems and the failure context; a passing check prints nothing. */
- verdict(problems: string[]): void {
- if (problems.length) throw new Error(`${problems.length} problem(s):\n ${problems.slice(0, 40).join('\n ')}\n${this.context()}`);
- }
- marker(tag: string): string { return join(this.home, `fault-${tag}.json`); }
- recordFault(tag: string, label: string): string | null {
- const at = existsSync(this.marker(tag)) ? readFileSync(this.marker(tag), 'utf8') : null;
- this.fired[label] = at;
- if (at) { const fired = JSON.parse(at) as { pid: number; action: string }; if (fired.action === 'kill') this.killedPids.add(fired.pid); }
- return at;
- }
-
- git(...args: string[]): string { return execFileSync('git', ['-C', this.repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
- /**
- * Writes files and commits exactly those paths, as the shared-checkout rules require of every committer; each write
- * is acknowledged when the commit returns. Sweeping the worktree (`git add -A`) would commit a writer's in-flight
- * write-through bytes as an out-of-band Git change of a page that writer is still writing.
- */
- commit(message: string, files: Array<{ slug: string; content: string; marker: string }>): void {
- const start = performance.now();
- for (const f of files) { mkdirSync(dirname(join(this.repo, `${f.slug}.md`)), { recursive: true }); writeFileSync(join(this.repo, `${f.slug}.md`), f.content); }
- const paths = files.map(f => `${f.slug}.md`);
- this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);
- const ack = performance.now();
- for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });
- }
- writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {
- this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);
- }
- bigFile(facts: number, timeline: number, version: string) { return { slug: BIG, content: bigPage(facts, timeline, version), marker: `Section 0 (${version})` }; }
-
- childEnv(extra: Record<string, string | undefined> = {}): Record<string, string> {
- return keylessBrainEnv(process.env, this.home, {
- DATABASE_URL: this.pooledUrl, GBRAIN_DATABASE_URL: undefined, GBRAIN_DIRECT_DATABASE_URL: this.directUrl,
- GBRAIN_BRAIN_ID: 'host', GBRAIN_SOURCE: SOURCE, GBRAIN_SWEEP: '0', GBRAIN_HOOKS: undefined,
- GBRAIN_SKIP_STARTUP_HOOKS: '1', GBRAIN_NO_SKILL_NAG: '1', GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS: '300',
- GBRAIN_SYNC_NO_DELEGATE: undefined, GBRAIN_TEST_CONVERGENCE_FAULT: undefined,
- ...extra,
- });
- }
- faultArgs(fault?: Fault): { preload: string[]; env: Record<string, string | undefined> } {
- if (!fault) return { preload: [], env: {} };
- return { preload: ['--preload', FAULT_PRELOAD], env: { GBRAIN_TEST_CONVERGENCE_FAULT: JSON.stringify(fault) } };
- }
- spawnCli(args: string[], opts: { fault?: Fault; stdin?: string; timeoutMs?: number } = {}): SyncRun {
- const { preload, env } = this.faultArgs(opts.fault);
- const started = performance.now();
- const proc = Bun.spawn([process.execPath, '--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), ...args], {
- cwd: REPO_ROOT, env: this.childEnv(env), stdin: opts.stdin === undefined ? 'ignore' : new Blob([opts.stdin]), stdout: 'pipe', stderr: 'pipe',
- });
- const timer = setTimeout(() => proc.kill('SIGKILL'), opts.timeoutMs ?? SYNC_TIMEOUT_MS);
- const result = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])
- .then(([out, err, code]) => ({ code, out, err, ms: performance.now() - started }))
- .finally(() => clearTimeout(timer));
- return { proc, result };
- }
- sync(args: string[], fault?: Fault): SyncRun { return this.spawnCli(['sync', '--source', SOURCE, ...args], { fault }); }
- async committedSince(at: string): Promise<number> {
- return (await this.engine!.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM persistence_requests WHERE state='committed' AND completed_at > $1::timestamptz AND operation='submit_job'`, [at]))[0].n;
- }
- async now(): Promise<string> { return (await this.engine!.executeRaw<{ at: string }>('SELECT now()::text AS at'))[0].at; }
-
- async setup(seed: (b: Brain) => void, config: Record<string, string> = {}): Promise<void> {
- assertSafeE2eDatabaseUrl(directTarget!);
- this.home = mkdtempSync(join(tmpdir(), 'gb-conv-'));
- mkdirSync(join(this.home, '.gbrain'));
- this.database = `gbrain_test_convergence_${randomUUID().replaceAll('-', '')}`;
- this.admin = postgres(directTarget!, { max: 1, prepare: false });
- await this.admin.unsafe(`CREATE DATABASE ${this.database}`);
- const direct = new URL(directTarget!); direct.pathname = `/${this.database}`;
- const pooled = new URL(pooledTarget!); pooled.pathname = `/${this.database}`;
- this.pooledProxy = new LatencyProxy({ host: pooled.hostname, port: Number(pooled.port) || 5432 }, LATENCY_MS / 2);
- this.directProxy = new LatencyProxy({ host: direct.hostname, port: Number(direct.port) || 5432 }, LATENCY_MS / 2);
- await Promise.all([this.pooledProxy.listen(), this.directProxy.listen()]);
- const viaPooled = new URL(pooled); viaPooled.hostname = '127.0.0.1'; viaPooled.port = String(this.pooledProxy.port); viaPooled.searchParams.set('prepare', 'false');
- const viaDirect = new URL(direct); viaDirect.hostname = '127.0.0.1'; viaDirect.port = String(this.directProxy.port);
- this.pooledUrl = viaPooled.toString(); this.directUrl = viaDirect.toString();
- this.engine = new PostgresEngine();
- await withEnv({ GBRAIN_HOME: this.home, GBRAIN_DIRECT_DATABASE_URL: undefined }, async () => {
- await this.engine!.connect({ database_url: direct.toString(), poolSize: 4 });
- await this.engine!.initSchema();
- });
- writeFileSync(join(this.home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: this.pooledUrl, embedding_disabled: true }));
- this.repo = mkdtempSync(join(tmpdir(), 'gb-conv-repo-'));
- this.git('init', '-q'); this.git('config', 'user.email', '[email protected]'); this.git('config', 'user.name', 'Example');
- seed(this);
- await withEnv({ GBRAIN_HOME: this.home }, async () => {
- await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
- await this.engine!.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [SOURCE, this.repo]);
- await claimWorktree(this.engine!, SOURCE, this.repo);
- for (const [key, value] of Object.entries(config)) await this.engine!.setConfig(key, value);
- await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
- });
- this.note(`seed=${SEED} latency=${LATENCY_MS}ms round trip via proxies (pooled -> ${pooled.host}, direct -> ${direct.host}); modes ${this.modes.join(',')}`);
- // Phase 0: a fault-free import, so the oracle is calibrated before any chaos.
- const started = performance.now();
- const baseline = await this.sync(['--no-pull', '--no-delegate', '--json']).result;
- this.elapsed.baseline_import_ms = performance.now() - started;
- if (baseline.code !== 0) throw new Error(`baseline sync exited ${baseline.code}\n${baseline.out}\n${baseline.err}\n${this.context()}`);
- this.calibration = await this.diskDbMismatches(new Set());
- if (existsSync(join(this.repo, `${BIG}.md`))) {
- const rows = await this.childRowProblems();
- this.calibration.push(...rows.problems, ...(rows.timelineDuplicates || rows.timelineStrays ? [`timeline: ${rows.timelineDuplicates} duplicate and ${rows.timelineStrays} stray rows after a fault-free import`] : []));
- }
- this.note(`phase 0: fault-free import in ${(this.elapsed.baseline_import_ms / 1000).toFixed(1)}s; oracle calibration problems: ${this.calibration.length}`);
- }
-
- /** A fresh resident serve and at most MAX_RECOVERY_RUNS plain syncs, no operator step. */
- async recover(): Promise<void> {
- const started = performance.now();
- const serve = new StdioServe(this);
- await serve.start();
- for (let run = 1; run <= MAX_RECOVERY_RUNS; run++) {
- const r = await this.sync(['--no-pull', '--json']).result;
- const envelope = lastJson(r.out);
- this.recoveryRuns.push({ ...r, envelope });
- this.keepLog(`recovery-${run}-client.log`, `${r.out}\n${r.err}`);
- this.note(`recovery run ${run}: exit ${r.code} in ${(r.ms / 1000).toFixed(1)}s outcome=${String(envelope?.outcome ?? envelope?.sync_status ?? envelope?.error ?? 'none')}`);
- if (r.code === 0 && envelope?.outcome === 'synced') break;
- }
- this.elapsed.recovery_ms = performance.now() - started;
- this.keepLog('recovery-serve.log', serve.stderr);
- this.note(`elapsed: ${Object.entries(this.elapsed).map(([k, ms]) => `${k.replace(/_ms$/, '')} ${(ms / 1000).toFixed(1)}s`).join(', ')}; recovery took ${this.recoveryRuns.length} sync run(s)`);
- }
-
- /**
- * #6841: an accepted write still queued when its writer gave up may commit later, after a write the ledger holds.
- * Once the brain has converged, each one's receipt is read the way the write protocol tells a client to after exit
- * 10 (get_write_request). A committed write is acknowledged then: it may have landed any time since its start, so it
- * is concurrent with every write in that window. Any other state means it never landed and stays out of the ledger.
- */
- async settlePending(receiptState = async (requestId: string) =>
- lastJson((await this.spawnCli(['call', '--source', SOURCE, 'get_write_request', JSON.stringify({ request_id: requestId })]).result).out)?.state): Promise<void> {
- for (const { write, requestId } of this.pending) {
- const state = await receiptState(requestId);
- if (state === 'committed') this.ledger.record({ ...write, ack: performance.now() });
- this.note(`accepted ${write.kind} of ${write.slug} (request ${requestId}) unsettled when its writer stopped: receipt ${String(state ?? 'unreadable')}`);
- }
- }
-
- async teardown(): Promise<void> {
- for (const serve of this.serves) await serve.stop().catch(() => {});
- try { await this.engine?.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1'); } catch { /* best effort */ }
- await this.engine?.disconnect().catch(() => {});
- await this.pooledProxy?.close().catch(() => {});
- await this.directProxy?.close().catch(() => {});
- if (this.admin) {
- try { if (this.database) await this.admin.unsafe(`DROP DATABASE IF EXISTS ${this.database} WITH (FORCE)`); } finally { await this.admin.end(); }
- }
- for (const dir of [this.home, this.repo]) if (dir) rmSync(dir, { recursive: true, force: true });
- }
-
- // ------------------------------------------------------------ oracle reads
-
- diskPages(): Map<string, string> {
- const out = new Map<string, string>();
- const walk = (dir: string) => {
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
- if (entry.name.startsWith('.')) continue;
- const full = join(dir, entry.name);
- if (entry.isDirectory()) walk(full);
- else if (entry.name.endsWith('.md')) out.set(relative(this.repo, full).replace(/\.md$/, ''), readFileSync(full, 'utf8'));
- }
- };
- walk(this.repo);
- return out;
- }
- async dbPages(): Promise<Map<string, { hash: string | null; body: string }>> {
- const rows = await this.engine!.executeRaw<{ slug: string; content_hash: string | null; body: string }>(
- `SELECT slug, content_hash, coalesce(compiled_truth,'') || E'\\n' || coalesce(timeline,'') AS body FROM pages WHERE source_id=$1 AND deleted_at IS NULL`, [SOURCE]);
- return new Map(rows.map(r => [r.slug, { hash: r.content_hash, body: r.body }]));
- }
- async diskDbMismatches(exclude: Set<string>): Promise<string[]> {
- const disk = this.diskPages(), db = await this.dbPages(), out: string[] = [];
- for (const [slug, content] of disk) {
- if (exclude.has(slug)) continue;
- if (!db.has(slug)) out.push(`${slug}: file exists, no live page`);
- else if (db.get(slug)!.hash !== fileHash(slug, content)) out.push(`${slug}: page content_hash differs from the file`);
- }
- for (const slug of db.keys()) if (!disk.has(slug) && !exclude.has(slug)) out.push(`${slug}: live page, no file`);
- return out;
- }
- async ledgerProblems(): Promise<string[]> {
- const facts = new Set((await this.engine!.executeRaw<{ fact: string }>('SELECT fact FROM facts WHERE source_id=$1 AND expired_at IS NULL', [SOURCE])).map(r => r.fact));
- return this.ledger.problems(this.diskPages(), await this.dbPages(), facts);
- }
- async openRequests(): Promise<unknown[]> {
- const open = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; recovery: boolean; blocked_reason: string | null }>(
- `SELECT request_id::text, slug, state, recovery IS NOT NULL AS recovery, blocked_reason FROM persistence_requests
- WHERE state IN ('queued','running','recovering') OR recovery IS NOT NULL ORDER BY sequence`);
- const worktrees = await this.engine!.executeRaw<{ id: string; state: string }>(`SELECT id::text, state FROM persistence_worktrees WHERE state<>'active'`);
- return [...open.map(r => `open request ${JSON.stringify(r)}`), ...worktrees.map(w => `worktree ${w.id} is ${w.state}`)];
- }
- async unexplainedTerminal(exclude: Set<string>): Promise<string[]> {
- const rows = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; error_code: string | null; operation: string }>(
- `SELECT request_id::text, slug, state, error_code, operation FROM persistence_requests WHERE state IN ('conflict','failed','cancelled') ORDER BY sequence`);
- const disk = this.diskPages(), db = await this.dbPages();
- this.note(`terminal non-committed requests: ${rows.length} (${[...new Set(rows.map(r => `${r.state}:${r.error_code}`))].join(', ') || 'none'})`);
- return rows.filter(row => {
- const content = disk.get(row.slug);
- const superseded = exclude.has(row.slug) || (content !== undefined && db.get(row.slug)?.hash === fileHash(row.slug, content));
- return !superseded && !(row.error_code && row.error_code in CODES);
- }).map(row => `unexplained ${row.state} request ${row.request_id} (${row.slug}, ${row.error_code ?? 'no code'})`);
- }
- async cursorProblems(): Promise<string[]> {
- const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);
- const head = this.git('rev-parse', 'HEAD');
- return source.last_commit === head ? [] : [`sources.last_commit ${source.last_commit} is not HEAD ${head}`];
- }
- async status(detail = true): Promise<Record<string, unknown> | null> {
- const r = await this.spawnCli(['sync', 'status', '--source', SOURCE, '--json', ...(detail ? ['--detail'] : [])]).result;
- this.keepLog(detail ? 'status.json' : 'status-fast.json', `${r.out}\n${r.err}`);
- return lastJson(r.out);
- }
- recoveryProblems(): string[] {
- const last = this.recoveryRuns.at(-1);
- return last?.code === 0 && last.envelope?.outcome === 'synced' ? []
- : [`no plain sync succeeded within ${MAX_RECOVERY_RUNS} runs: ${this.recoveryRuns.map(r => `exit ${r.code} outcome ${String(r.envelope?.outcome ?? r.envelope?.error ?? 'none')}`).join('; ')}`];
- }
- /**
- * The large page's child rows against its file. Facts are keyed by fence row, so
- * loss, duplication and strays are all problems. Timeline rows are judged for
- * loss only; duplicate and stray timeline rows come back as a measured count,
- * because derived timeline rows are never pruned (a known defect outside #6650).
- */
- async childRowProblems(): Promise<{ problems: string[]; timelineDuplicates: number; timelineStrays: number }> {
- const content = this.diskPages().get(BIG);
- if (!content) return { problems: [`${BIG}: no file`], timelineDuplicates: 0, timelineStrays: 0 };
- const out: string[] = [];
- const expectedTimeline = [...content.matchAll(/^- \*\*(\d{4}-\d{2}-\d{2})\*\* \| (.+)$/gm)].map(m => `${m[1]}|${m[2]}`);
- const timeline = await this.engine!.executeRaw<{ key: string; n: number }>(
- `SELECT t.date::text || '|' || t.summary AS key, count(*)::int AS n FROM timeline_entries t JOIN pages p ON p.id=t.page_id
- WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL AND t.event_page_id IS NULL GROUP BY 1`, [SOURCE, BIG]);
- const have = new Map(timeline.map(r => [r.key, r.n]));
- const lostTimeline = expectedTimeline.filter(k => !have.has(k));
- const dupTimeline = timeline.filter(r => r.n > 1);
- const extraTimeline = timeline.filter(r => !expectedTimeline.includes(r.key));
- if (lostTimeline.length) out.push(`timeline: ${lostTimeline.length} of ${expectedTimeline.length} file entries missing (first: ${lostTimeline[0]})`);
- const expectedFacts = parseFactsFence(content).facts.map(f => `${f.rowNum}|${f.claim}`);
- const facts = await this.engine!.executeRaw<{ key: string; n: number }>(
- `SELECT row_num::text || '|' || fact AS key, count(*)::int AS n FROM facts
- WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL GROUP BY 1`, [SOURCE, BIG]);
- const haveFacts = new Map(facts.map(r => [r.key, r.n]));
- const lostFacts = expectedFacts.filter(k => !haveFacts.has(k));
- const dupFacts = facts.filter(r => r.n > 1);
- const extraFacts = facts.filter(r => !expectedFacts.includes(r.key));
- if (lostFacts.length) out.push(`facts: ${lostFacts.length} of ${expectedFacts.length} fence rows missing (first: ${lostFacts[0]})`);
- if (dupFacts.length) out.push(`facts: ${dupFacts.length} rows duplicated (first: ${dupFacts[0].key} x${dupFacts[0].n})`);
- if (extraFacts.length) out.push(`facts: ${extraFacts.length} rows not in the fence (first: ${extraFacts[0].key})`);
- return { problems: out, timelineDuplicates: dupTimeline.reduce((n, r) => n + r.n - 1, 0), timelineStrays: extraTimeline.length };
- }
-}
-
-function fileHash(slug: string, content: string): string {
- const parsed = parseMarkdown(content, `${slug}.md`);
- return contentHash({ title: parsed.title, type: parsed.type, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline, frontmatter: parsed.frontmatter, tags: [...parsed.tags].sort() });
-}
-
-/** One stdio MCP session against a spawned `gbrain serve`; the serve is the host's resident persistence owner. */
-class StdioServe {
- readonly child: ChildProcess;
- stderr = '';
- exited: Promise<number | null>;
- private nextId = 1;
- private pending = new Map<number, (msg: Record<string, unknown>) => void>();
- private buf = '';
- constructor(private brain: Brain, fault?: Fault) {
- const { preload, env } = brain.faultArgs(fault);
- this.child = spawn(process.execPath, ['--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), 'serve'], { cwd: REPO_ROOT, env: brain.childEnv(env), stdio: ['pipe', 'pipe', 'pipe'] });
- this.exited = new Promise(done => this.child.once('exit', code => done(code)));
- this.child.stderr!.on('data', (d: Buffer) => { this.stderr += d.toString('utf8'); });
- this.child.stdin!.on('error', () => {});
- this.child.stdout!.on('data', (d: Buffer) => {
- this.buf += d.toString('utf8');
- let idx: number;
- while ((idx = this.buf.indexOf('\n')) !== -1) {
- const line = this.buf.slice(0, idx).trim(); this.buf = this.buf.slice(idx + 1);
- if (!line) continue;
- try {
- const msg = JSON.parse(line) as Record<string, unknown>;
- const id = msg.id as number | undefined;
- if (id !== undefined && this.pending.has(id)) { this.pending.get(id)!(msg); this.pending.delete(id); }
- } catch { /* serve chatter */ }
- }
- });
- brain.serves.push(this);
- }
- get pid(): number { return this.child.pid!; }
- get alive(): boolean { return this.child.exitCode === null && this.child.signalCode === null; }
- request(method: string, params: Record<string, unknown>, timeoutMs = 240_000): Promise<Record<string, unknown>> {
- const id = this.nextId++;
- return new Promise((done, reject) => {
- const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`rpc ${method} timed out`)); }, timeoutMs);
- this.pending.set(id, msg => { clearTimeout(t); done(msg); });
- this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
- });
- }
- async start(): Promise<void> {
- await this.request('initialize', { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'convergence', version: '1' } });
- this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
- await until(async () => (await this.brain.engine!.executeRaw<{ n: number }>(
- `SELECT count(*)::int AS n FROM persistence_consumers WHERE pid=$1 AND kind='serve' AND renewed_at > now() - interval '30 seconds'`, [this.pid]))[0].n > 0,
- `serve ${this.pid} heartbeat`, 120_000);
- }
- async stop(): Promise<void> {
- if (!this.alive) return;
- this.child.kill('SIGTERM');
- const timer = setTimeout(() => this.child.kill('SIGKILL'), 10_000);
- await this.exited; clearTimeout(timer);
- }
-}
-
// ---------------------------------------------------------------- the writers of mode (c)
-/**
- * A writer call that retries an unsettled attempt (exit 10, a lost owner) with the same request id and a terminal refusal with a new one.
- * When the attempts or the chaos run out on an unsettled attempt, the request it names may still commit, so it comes back `pending`.
- */
-async function acknowledged(b: Brain, label: string, call: (requestId: string) => SyncRun, stop: () => boolean): Promise<{ start: number; ack: number } | { pending: string } | null> {
- let requestId = randomUUID();
- let unsettled = false;
- for (let attempt = 1; attempt <= 20 && !stop(); attempt++) {
- const start = performance.now();
- const r = await call(requestId).result;
- b.keepLog(`writer-${label.replaceAll('/', '_')}-${attempt}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
- if (r.code === 0) return { start, ack: performance.now() };
- const terminal = /\((conflict|failed|cancelled)\)/.test(r.err);
- b.note(`writer ${label} attempt ${attempt} exit ${r.code}${terminal ? ' (terminal; next attempt uses a new request id)' : ''}: ${r.err.trim().split('\n').slice(-1)[0] ?? ''}`);
- unsettled = !terminal;
- if (terminal) requestId = randomUUID();
- await Bun.sleep(3000);
- }
- return unsettled ? { pending: requestId } : null;
-}
-const outcome = (ok: { ack: number } | { pending: string } | null) => !ok ? 'unacknowledged' : 'pending' in ok ? `pending ${ok.pending}` : 'ok';
/** Rewrites notes through the coordinator (`gbrain put`), remembers facts on the entity page and runs the managed timeline maintenance job until `stop`. */
async function runWriters(b: Brain, putTargets: string[], stop: () => boolean): Promise<string[]> {
const results: string[] = [];
@@ -715,31 +196,6 @@ async function runWriters(b: Brain, putTargets: string[], stop: () => boolean):
return results;
}
-// ---------------------------------------------------------------- the shared oracle
-
-interface EndState { exclude?: Set<string>; status?: StatusExpectation; bigPage?: boolean }
-/** The oracle tests every scenario ends with; `end()` is read when the tests run (after the scenario's beforeAll). */
-function oracleTests(b: Brain, end: () => EndState): void {
- test('the oracle is calibrated: a fault-free import satisfies it', () => b.verdict(b.calibration));
- test('every planned fault fired, so the chaos actually happened', () => b.verdict(Object.entries(b.fired).filter(([, at]) => at === null).map(([fault]) => `${fault} did not fire`)));
- test(as('wedge', `a plain sync succeeds within ${MAX_RECOVERY_RUNS} runs after the chaos, with no operator step`), () => b.verdict(b.recoveryProblems()));
- test(as('projection_drift', 'disk == database for every page the end state does not name'), async () => b.verdict(await b.diskDbMismatches(end().exclude ?? new Set())));
- test(as('wedge', 'no request blocks the checkpoint or a root'), async () => b.verdict((await b.openRequests()).map(String)));
- test(as('untyped_error', 'every non-committed terminal request is superseded or carries a registered error code'), async () => b.verdict(await b.unexplainedTerminal(end().exclude ?? new Set())));
- test(as('wedge', 'the cursor reached HEAD'), async () => b.verdict(await b.cursorProblems()));
- test(as('lost_write', 'every acknowledged write survives or is causally superseded by a later acknowledged write'), async () => {
- b.note(`ledger: ${b.ledger.acks.length} acknowledged writes (${[...new Set(b.ledger.acks.map(a => a.kind))].map(kind => `${kind} ${b.ledger.acks.filter(a => a.kind === kind).length}`).join(', ')})`);
- b.verdict(await b.ledgerProblems());
- });
- test(as('wedge', 'sync status --json reports the converged state with every field present'), async () => b.verdict(statusProblems(await b.status(), end().status)));
- test(as('wedge', 'plain sync status --json names every heavy read it skipped instead of reading as all-clear'), async () => b.verdict(fastStatusProblems(await b.status(false))));
- if (end().bigPage !== false) test(as('lost_write', 'the large page keeps every fence fact exactly once and loses no timeline entry'), async () => {
- const { problems, timelineDuplicates, timelineStrays } = await b.childRowProblems();
- b.note(`large page timeline (measured, not asserted): ${timelineDuplicates} duplicate rows, ${timelineStrays} rows not in the file`);
- b.verdict(problems);
- });
-}
-
/** A describe that runs when any of its modes runs; skipped (with every skip reason in its title) otherwise. */
function scenario(modes: Mode[], title: string, body: () => void): void {
const active = enabled && modes.some(runs);