Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
test(convergence): busy-agent soak cell (modes h-m) on a shared convergence harness; informational nightly job (L5, GST-1502)
garrytan0dce824a3f44
diff --git a/.github/workflows/convergence-nightly.yml b/.github/workflows/convergence-nightly.ymlindex ef25799058..18f6aed4e6 100644--- a/.github/workflows/convergence-nightly.yml+++ b/.github/workflows/convergence-nightly.yml@@ -9,6 +9,11 @@ name: Convergence nightly# concurrency group, and a red lands in the nightly-red issue (nightly-watch.yml).# Every job name carries its seed, and a failure writes the seed, the modes, the# fault plan, the fired faults and the one-line repro to the step summary.+# The busy-agent job (L5, test/busy-agent-soak.serial.test.ts) runs the same harness+# under a busy agent's workload, from the committed generic profile+# test/fixtures/busy-agent-profile.json. It is informational (continue-on-error) for+# its first seven nights; after that a release touching persistence, sync or Git+# needs it green on the release's exact sha.on:schedule:@@ -115,3 +120,89 @@ jobs:path: .context/convergenceinclude-hidden-files: trueif-no-files-found: warn++ busy-agent:+ name: Busy-agent soak cell (informational) / Bun ${{ matrix.bun }} / seed ${{ inputs.seed || github.run_id }}+ runs-on: ubicloud-standard-4-ubuntu-2404+ timeout-minutes: 150+ continue-on-error: true+ strategy:+ fail-fast: false+ matrix:+ bun: ['1.4.4']+ services:+ postgres:+ image: pgvector/pgvector:pg16+ env:+ POSTGRES_USER: gbrain_test+ POSTGRES_PASSWORD: gbrain_test+ POSTGRES_DB: gbrain_test+ ports: ['5432:5432']+ options: >-+ --health-cmd "pg_isready -U gbrain_test -d gbrain_test"+ --health-interval 5s --health-timeout 5s --health-retries 20+ pgbouncer:+ image: edoburu/pgbouncer:latest+ env:+ DB_HOST: postgres+ DB_PORT: '5432'+ DB_USER: gbrain_test+ DB_PASSWORD: gbrain_test+ POOL_MODE: transaction+ AUTH_TYPE: plain+ MAX_CLIENT_CONN: '200'+ DEFAULT_POOL_SIZE: '10'+ IGNORE_STARTUP_PARAMETERS: extra_float_digits,statement_timeout,idle_in_transaction_session_timeout,search_path+ ports: ['55433:5432']+ env:+ GBRAIN_TEST_BUSY_AGENT_LOG_DIR: ${{ github.workspace }}/.context/busy-agent+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6+ with:+ bun-version: ${{ matrix.bun }}+ - name: Install dependencies (bounded, retried once)+ timeout-minutes: 8+ run: bun scripts/ci-bun-install.ts --frozen-lockfile --ignore-scripts+ - name: Run the busy-agent modes (h)-(m) from the generic profile (50 ms)+ timeout-minutes: 120+ env:+ GBRAIN_TEST_BUSY_AGENT: '1'+ GBRAIN_PGBOUNCER_URL: postgres://gbrain_test:[email protected]:55433/gbrain_test+ GBRAIN_PGBOUNCER_DIRECT_URL: postgres://gbrain_test:[email protected]:5432/gbrain_test+ GBRAIN_TEST_BUSY_AGENT_LATENCY_MS: '50'+ GBRAIN_TEST_BUSY_AGENT_SEED: ${{ inputs.seed || github.run_id }}+ GBRAIN_TEST_BUSY_AGENT_SYNC_TIMEOUT_MS: '1800000'+ GBRAIN_TEST_BUSY_AGENT_SCENARIO_TIMEOUT_MS: '3600000'+ run: |+ set -o pipefail+ mkdir -p "$GBRAIN_TEST_BUSY_AGENT_LOG_DIR"+ bun --no-env-file test --timeout="$GBRAIN_TEST_BUSY_AGENT_SCENARIO_TIMEOUT_MS" test/busy-agent-soak.serial.test.ts 2>&1 | tee "$GBRAIN_TEST_BUSY_AGENT_LOG_DIR/bun-test.log"+ - name: Write the seed-carrying failure summary+ if: failure() || cancelled()+ env:+ SEED: ${{ inputs.seed || github.run_id }}+ BUN_CELL: ${{ matrix.bun }}+ run: |+ log="$GBRAIN_TEST_BUSY_AGENT_LOG_DIR/bun-test.log"+ {+ echo "### Busy-agent soak cell red (informational): Bun $BUN_CELL, seed $SEED"+ echo+ echo '```'+ grep -E '^\(fail\)|^seed [0-9]|^fault plan:|^fired:|^repro ' "$log" 2>/dev/null | head -60 || true+ [ -s "$log" ] || echo "no test output: the step timed out or never started"+ echo '```'+ echo+ echo "Replay: gh workflow run convergence-nightly.yml -f seed=$SEED (or the repro line above, locally)."+ } >> "$GITHUB_STEP_SUMMARY"+ - name: Stop harness processes a timed-out run left behind+ if: always()+ run: pkill -9 -f 'src/cli\.ts (serve|sync|put|call|extract|sources|git|doctor)' || true+ - name: Upload the busy-agent evidence (profile, fault plan, every writer, serve and refresh log)+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02+ with:+ name: busy-agent-nightly-bun-${{ matrix.bun }}+ path: .context/busy-agent+ include-hidden-files: true+ if-no-files-found: warndiff --git a/package.json b/package.jsonindex ca841dff89..5e5b3828df 100644--- a/package.json+++ b/package.json@@ -102,6 +102,7 @@"test:profile": "bash scripts/profile-tests.sh","test:serial": "bash scripts/run-serial-tests.sh","test:convergence": "GBRAIN_TEST_CONVERGENCE=1 bun --no-env-file test --timeout=3600000 test/sync-convergence.serial.test.ts",+ "test:busy-agent": "GBRAIN_TEST_BUSY_AGENT=1 bun --no-env-file test --timeout=3600000 test/busy-agent-soak.serial.test.ts","test:tla": "bash scripts/spec/tlc.sh && bun scripts/spec/mutants.ts drain-lease && bun scripts/spec/mutants.ts refresh-barrier","ci:local": "bash scripts/ci-local.sh","ci:local:diff": "bash scripts/ci-local.sh --diff",diff --git a/scripts/ci-convergence-scope.sh b/scripts/ci-convergence-scope.shindex 64a9650efc..5dd99cb386 100755--- a/scripts/ci-convergence-scope.sh+++ b/scripts/ci-convergence-scope.sh@@ -11,7 +11,7 @@# runs never call this; they always run the gate.set -euo pipefail-CONVERGENCE_PATHS='^(bun\.lock$|vendor/postgres/|src/core/persistence/|src/commands/sync/|src/core/page-state/|src/core/engine-sql/|src/core/storage/|test/fixtures/convergence-fault-kill\.ts$|test/sync-convergence\.serial\.test\.ts$|scripts/ci-convergence-scope\.sh$|\.github/workflows/(persistence-validation|convergence-nightly)\.yml$)|^src/commands/sync[^/]*\.ts$|^src/core/(sync[^/]*|import-file|markdown|write-through|timeline-write-through|storage|storage-config|page-lock|ghost-pages|postgres-engine)\.ts$'+CONVERGENCE_PATHS='^(bun\.lock$|vendor/postgres/|src/core/persistence/|src/commands/sync/|src/core/page-state/|src/core/engine-sql/|src/core/storage/|test/fixtures/convergence-fault-kill\.ts$|test/helpers/convergence-brain\.ts$|test/sync-convergence\.serial\.test\.ts$|scripts/ci-convergence-scope\.sh$|\.github/workflows/(persistence-validation|convergence-nightly)\.yml$)|^src/commands/sync[^/]*\.ts$|^src/core/(sync[^/]*|import-file|markdown|write-through|timeline-write-through|storage|storage-config|page-lock|ghost-pages|postgres-engine)\.ts$'files=$(cat) || { echo run; exit 0; }# Consume the full input: grep -q closes an early match's pipe and printfdiff --git a/scripts/persistence/README.md b/scripts/persistence/README.mdindex 628942232e..c6643abb39 100644--- a/scripts/persistence/README.md+++ b/scripts/persistence/README.md@@ -659,6 +659,49 @@ summary (every failing test with its seed, fault plan, fired faults and reproline) and the upload of `LOG_DIR` still run. The nightly watcher(`nightly-watch.yml`) files its failures like every other scheduled workflow.+### Busy-agent soak cell++`test/busy-agent-soak.serial.test.ts` (L5) runs the convergence gate's+real-process brain (`test/helpers/convergence-brain.ts`, shared with+`test/sync-convergence.serial.test.ts`: Postgres behind transaction-mode+PgBouncer through latency proxies, real `src/cli.ts` processes, a resident+serve, the fault seam, the acknowledged-write ledger and the oracle) under the+workload a busy personal agent puts on one managed checkout. Its counts and+cadences come from a profile: the committed generic one,+`test/fixtures/busy-agent-profile.json`, unless+`GBRAIN_TEST_BUSY_AGENT_PROFILE` names another (a profile derived from a real+deployment stays private). A profile holds exactly the test's `PROFILE_SHAPE`+keys, each a positive integer, plus an optional `description`; it never holds+content.++| Mode | What happens | Expected end state |+| --- | --- | --- |+| (h) refresh ticks | With `persistence.refresh_barrier` on, scheduled `sources refresh` ticks pull upstream commits while foreground `put`s land inside the fence, some on notes the tick changes, and an agent writes across every tick | every tick's refresh completes and releases its fence; every write that reached the refreshing checkout is answered pending with a receipt naming the refresh, never refused, and commits after it, or ends `revision_conflict` with nothing applied when the refresh changed its file |+| (i) foreign writers | Plain-git committers and an uncommitted-file writer beside gbrain | `doctor` `foreign_writers` names every commit gbrain did not make and none it made, and the aged uncommitted files |+| (j) legacy rows | Rows with no import record in every content-evidence direction | `sources git-gap` classifies each by its direction evidence; `both_moved` and `no_evidence` rows end held `file_drift` |+| (k) strict conversion | `sources canonical --to git` over several adopt groups, stalled after the first (`conversion:group_committed`) while another clone pushes, door writes land and a refresh runs | the conversion ends `converted`, strict, with nothing left to plan; every door write and refresh is accepted or refused with a registered code; the other agent's commit reaches HEAD |+| (l) segmented staging | A segmented write whose staging expires or is aborted at `segmented:manifest_staged` | the write refuses `body_staging_expired`, the page keeps its body, a resend commits and the sweep leaves no live staging |+| (m) lost update | A file-writing cron (`git commit-push --base`) and an MCP `edit_page` writer race on one page | `edit_page` refuses `revision_conflict`, `commit-push --base` refuses `commit_base_moved`, and no acknowledged write is silently reverted |++Every scenario then gets the convergence gate's oracle (above). It skips+unless `GBRAIN_TEST_BUSY_AGENT=1`; with the Postgres and PgBouncer of the+gate above:++```bash+GBRAIN_TEST_BUSY_AGENT_LATENCY_MS=0 bun run test:busy-agent # every mode, fast local loop+GBRAIN_TEST_BUSY_AGENT_MODES=k bun run test:busy-agent # one mode+```++Knobs (`GBRAIN_TEST_BUSY_AGENT_*`): `SEED` (default 8800), `MODES`,+`PROFILE`, `LATENCY_MS` (default 50), `SYNC_TIMEOUT_MS` (default 900000),+`SCENARIO_TIMEOUT_MS` (default 3600000) and `LOG_DIR`.++CI: the `busy-agent` job of `.github/workflows/convergence-nightly.yml`, on+the same runner class, at 50 ms with the run id as the seed. It is+informational (`continue-on-error`) for its first seven nights; after that a+release touching persistence, sync or Git needs it green on the release's+exact sha.+## Engine graduation testsGraduation (`gbrain migrate --to postgres`) is tested against two fixtures.diff --git a/scripts/serial-files.tsv b/scripts/serial-files.tsvindex 610c3b4587..65fe3e411d 100644--- a/scripts/serial-files.tsv+++ b/scripts/serial-files.tsv@@ -398,6 +398,7 @@ test/spawn-seam-batch1-4992.serial.test.ts R1 R1: mutates process.env (GBRAIN_HOtest/stdio-stderr-redirect.serial.test.ts global-state global-state: patches process stdout/stderr/consoletest/subagent-put-page-rejection.serial.test.ts global-state global-state: process-wide test seams __setChatTransportForTests; configureGateway changes the process-wide AI gateway; spyOn putPage.handlertest/sync-convergence.serial.test.ts global-state global-state: binds local servers/ports (two latency proxies per scenario) and spawns serve/sync/put/call processes against each scenario's brain; its oracle tests read the chaos scenario their describe's beforeAll ran, across it() boundaries+test/busy-agent-soak.serial.test.ts global-state global-state: binds local servers/ports (two latency proxies per scenario) and spawns serve/sync/put/sources/git/doctor processes against each scenario's brain; its mode tests read the scenario their describe's beforeAll ran, across it() boundariestest/sync-cost-gate.serial.test.ts global-state global-state: process-wide test seams __setEmbedTransportForTests; configureGateway changes the process-wide AI gateway; patches process stdout/stderr/consoletest/sync-default-write-guard.serial.test.ts global-state global-state: spyOn console.error, console.log, process.exittest/sync-delegated-options-6696.serial.test.ts global-state global-state: process-wide test seams __resetDelegatedSyncForTests and the embedding transport; binds local servers/ports; patches process stdout/stderr/consolediff --git a/scripts/serial-weights.json b/scripts/serial-weights.jsonindex 965f2ff5f4..e15edaf29c 100644--- a/scripts/serial-weights.json+++ b/scripts/serial-weights.json@@ -415,6 +415,7 @@"test/stdio-stderr-redirect.serial.test.ts": 0,"test/subagent-put-page-rejection.serial.test.ts": 3,"test/sync-convergence.serial.test.ts": 3,+ "test/busy-agent-soak.serial.test.ts": 1,"test/sync-cost-gate.serial.test.ts": 18,"test/sync-default-write-guard.serial.test.ts": 4,"test/sync-delegated-options-6696.serial.test.ts": 11,diff --git a/test/busy-agent-soak.serial.test.ts b/test/busy-agent-soak.serial.test.tsnew file mode 100644index 0000000000..ed9c57e02b--- /dev/null+++ b/test/busy-agent-soak.serial.test.ts@@ -0,0 +1,772 @@+/**+ * L5 (GST-1502): the busy-agent soak cell, the convergence harness under the workload a busy personal agent puts on+ * one managed checkout. It shares the real-process brain of the managed-sync convergence gate+ * (test/helpers/convergence-brain.ts: Postgres behind transaction-mode PgBouncer through latency proxies, real+ * `src/cli.ts` processes, a resident serve, the acknowledged-write ledger and the oracle), and its counts and cadences+ * come from a committed generic profile (test/fixtures/busy-agent-profile.json; GBRAIN_TEST_BUSY_AGENT_PROFILE points+ * at another, such as a private one derived from a real deployment).+ *+ * Modes (MODES below; GBRAIN_TEST_BUSY_AGENT_MODES=h,m runs exactly those):+ * (h) scheduled `sources refresh` ticks with foreground writes inside the fence: with persistence.refresh_barrier on,+ * a write that reaches the checkout while a refresh holds it is answered pending and commits after the refresh,+ * or ends revision_conflict when the refresh changed its file;+ * (i) plain-git committers and uncommitted-file writers beside gbrain: doctor foreign_writers names every commit+ * gbrain did not make and none it made;+ * (j) legacy rows with no import record in every direction: `sources git-gap` classifies each by content evidence;+ * (k) a strict conversion with live door writers and a refresh between adopt groups;+ * (l) a segmented page whose staging expires between staging and publication: the write refuses+ * body_staging_expired, the page keeps its body, a resend commits and the sweep leaves no live staging;+ * (m) the lost-update race between a file-writing cron and an MCP write to the same page: edit_page refuses+ * revision_conflict and `git commit-push --base` refuses commit_base_moved, and no acknowledged write is+ * silently reverted.+ *+ * Every scenario ends with the convergence gate's oracle (a fresh serve, at most two plain syncs, disk == database,+ * no open request, registered terminal codes, the cursor at HEAD, the ledger and the status oracle). The cell is+ * informational in .github/workflows/convergence-nightly.yml for seven nights, then gates releases that touch+ * persistence, sync or Git. Setup: scripts/persistence/README.md ("Busy-agent soak cell"). It is skipped unless+ * GBRAIN_TEST_BUSY_AGENT=1.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { execFileSync } from 'node:child_process';+import { randomUUID } from 'node:crypto';+import { existsSync, mkdirSync, readFileSync, utimesSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { FOREIGN_DIRTY_WARN_MS } from '../src/commands/doctor/checks/foreign-writers.ts';+import { inlineExportMarkdown, readExportPage, readExportWithdrawals } from '../src/core/export-snapshot.ts';+import { GIT_HOLD_OP, SYNC_IMPORT_PROVENANCE_OP } from '../src/core/persistence/sync-holds.ts';+import { CODES } from '../src/core/error-registry.ts';+import { CONVERSION_GROUP_MAX } from '../src/core/persistence/git-conversion.ts';+import { sweepSegmentStorage } from '../src/core/persistence/journal.ts';+import { withEnv } from './helpers/with-env.ts';+import {+ Brain as HarnessBrain, SOURCE, StdioServe, as, fileHash, lastJson, noteMarker, noteSlug, noteText, oracleTests, until,+ type Fault, type HarnessKnobs, type SyncRun,+} from './helpers/convergence-brain.ts';++const REPO_ROOT = resolve(import.meta.dir, '..');+const knob = (name: string, fallback: number): number => {+ const raw = process.env[`GBRAIN_TEST_BUSY_AGENT_${name}`];+ const value = raw === undefined || raw === '' ? fallback : Number(raw);+ if (!Number.isInteger(value) || value < 0) throw new Error(`GBRAIN_TEST_BUSY_AGENT_${name} must be a non-negative integer, got '${raw}'.`);+ return value;+};+const SEED = knob('SEED', 8800);+const LATENCY_MS = knob('LATENCY_MS', 50);+const SYNC_TIMEOUT_MS = knob('SYNC_TIMEOUT_MS', 900_000);+const SCENARIO_TIMEOUT_MS = knob('SCENARIO_TIMEOUT_MS', 3_600_000);+const PROFILE_PATH = process.env.GBRAIN_TEST_BUSY_AGENT_PROFILE || join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json');+const pooledTarget = process.env.GBRAIN_PGBOUNCER_URL;+const directTarget = process.env.GBRAIN_PGBOUNCER_DIRECT_URL;+const enabled = process.env.GBRAIN_TEST_BUSY_AGENT === '1';+if (enabled && !(pooledTarget && directTarget)) {+ throw new Error('GBRAIN_TEST_BUSY_AGENT=1 needs GBRAIN_PGBOUNCER_URL (transaction-mode PgBouncer) and GBRAIN_PGBOUNCER_DIRECT_URL (the Postgres behind it).');+}+const LOG_DIR = process.env.GBRAIN_TEST_BUSY_AGENT_LOG_DIR;++// ---------------------------------------------------------------- the profile++/** Counts and cadences only: the shape of a busy agent's day, never its content. */+const PROFILE_SHAPE = {+ notes: 'pages in the checkout',+ refresh_ticks: '(h) scheduled refresh ticks',+ upstream_commits_per_tick: '(h) notes another clone changes and pushes before each tick',+ fence_writes_per_tick: '(h) foreground puts of unchanged notes while the tick holds the fence',+ fence_conflicts_per_tick: '(h) foreground puts of notes the tick changes, while it holds the fence',+ background_write_interval_ms: '(h) cadence of the agent puts that run across every tick',+ plain_git_committers: '(i) plain-git committers beside gbrain',+ plain_git_commits_each: '(i) commits per plain-git committer',+ uncommitted_files: '(i) page files an uncommitted-file writer leaves behind',+ legacy_rows_per_direction: '(j) legacy rows per evidence direction',+ conversion_groups: '(k) adopt groups of the strict conversion',+ conversion_door_writes: '(k) door writes during the conversion',+ segmented_page_kib: '(l) size of the segmented page',+ segmented_writes: '(l) writes to the segmented page after the expired one',+ lost_update_rounds: '(m) free-running rounds of the cron and the MCP writer',+ cron_interval_ms: '(m) cadence of the file-writing cron',+ agent_interval_ms: '(m) cadence of the MCP writer',+} as const;+type Profile = Record<keyof typeof PROFILE_SHAPE, number>;+/** The profile file: a JSON object of exactly PROFILE_SHAPE's keys, each a positive integer, plus an optional `description`. */+function readProfile(raw: string, where: string): Profile {+ const parsed = JSON.parse(raw) as Record<string, unknown>;+ const problems = [+ ...Object.keys(parsed).filter(key => key !== 'description' && !(key in PROFILE_SHAPE)).map(key => `unknown key ${key}`),+ ...Object.keys(PROFILE_SHAPE).filter(key => !Number.isInteger(parsed[key]) || (parsed[key] as number) < 1).map(key => `${key} must be a positive integer, got ${JSON.stringify(parsed[key])}`),+ ];+ if (problems.length) throw new Error(`busy-agent profile ${where}: ${problems.join('; ')}`);+ return Object.fromEntries(Object.keys(PROFILE_SHAPE).map(key => [key, parsed[key]])) as Profile;+}+const PROFILE = readProfile(readFileSync(PROFILE_PATH, 'utf8'), PROFILE_PATH);++// ---------------------------------------------------------------- modes++const MODES = {+ h: { title: 'scheduled sources refresh ticks with foreground writes inside the fence', skip: null },+ i: { title: 'plain-git committers and uncommitted-file writers beside gbrain', skip: null },+ j: { title: 'legacy rows with no import record in every direction', skip: null },+ k: { title: 'a strict conversion with live door writers and a refresh between adopt groups', skip: null },+ l: { title: 'a segmented page whose staging expires between staging and publication', skip: null },+ m: { title: 'the lost-update race between a file-writing cron and an MCP write to the same page', skip: null },+} as const satisfies Record<string, { title: string; skip: string | null }>;+type Mode = keyof typeof MODES;+const requestedModes = process.env.GBRAIN_TEST_BUSY_AGENT_MODES?.split(',').map(mode => mode.trim()).filter(Boolean);+for (const mode of requestedModes ?? []) if (!(mode in MODES)) throw new Error(`GBRAIN_TEST_BUSY_AGENT_MODES names unknown mode '${mode}' (known: ${Object.keys(MODES).join(', ')}).`);+const runs = (mode: Mode): boolean => requestedModes ? requestedModes.includes(mode) : MODES[mode].skip === null;++const KNOBS: HarnessKnobs = {+ tag: 'busy-agent', seed: SEED, latencyMs: LATENCY_MS, syncTimeoutMs: SYNC_TIMEOUT_MS, logDir: LOG_DIR,+ pooledTarget: pooledTarget ?? '', directTarget: directTarget ?? '', summary: `profile ${PROFILE_PATH}; latency ${LATENCY_MS} ms`,+ repro: modes => `GBRAIN_TEST_BUSY_AGENT=1 GBRAIN_TEST_BUSY_AGENT_SEED=${SEED} GBRAIN_TEST_BUSY_AGENT_MODES=${modes.join(',')} GBRAIN_TEST_BUSY_AGENT_LATENCY_MS=${LATENCY_MS} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun test test/busy-agent-soak.serial.test.ts`,+};+class Brain extends HarnessBrain { constructor(name: string, modes: Mode[]) { super(name, modes, KNOBS); } }++/** A describe that runs when its mode runs; skipped (with its skip reason in the title) otherwise. */+function scenario(mode: Mode, title: string, body: () => void): void {+ const skip = MODES[mode].skip;+ (enabled && runs(mode) ? describe : describe.skip)(`${title} [(${mode})]${!(enabled && runs(mode)) && skip ? ` (skipped until ${skip})` : ''}`, body);+}++// ---------------------------------------------------------------- shared steps++interface Receipt { state: string | null; error_code: string | null; deferred_refresh_id: string | null; completed_at: string | null }+/** A request's row by the caller's request id, read from the journal (null before it was journaled or when it was refused at admission). */+async function receipt(b: Brain, requestId: string): Promise<Receipt | null> {+ const [row] = await b.engine!.executeRaw<Receipt>(+ `SELECT state, error_code, deferred_refresh_id::text, completed_at::text FROM persistence_requests WHERE request_id=$1::uuid ORDER BY sequence DESC LIMIT 1`, [requestId]);+ return row ?? null;+}+const TERMINAL = new Set(['committed', 'conflict', 'failed', 'cancelled']);+/** The error code a CLI refusal printed (`(code)` or `"code":`), for reports. */+function printedCode(r: { out: string; err: string }): string {+ const json = lastJson(r.out) ?? lastJson(r.err);+ const code = (json?.error as { code?: string } | undefined)?.code ?? json?.code;+ if (typeof code === 'string') return code;+ return /\b([a-z]+(?:_[a-z]+)+)\b/.exec(r.err)?.[1] ?? 'unknown';+}+function put(b: Brain, slug: string, content: string, requestId: string) {+ return b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '2000'], { stdin: content });+}+/**+ * Waits for a stalled child to write its fault marker. A child that exits first, or never gets there, fails the+ * scenario with its output kept (`<tag>-unreached.log`) instead of leaving a stall nobody releases.+ */+async function reached(b: Brain, tag: string, run: SyncRun, label: string): Promise<void> {+ let exited = false;+ void run.result.then(() => { exited = true; });+ try {+ await until(() => exited || existsSync(b.marker(tag)), label, SYNC_TIMEOUT_MS);+ } catch { run.proc.kill('SIGKILL'); }+ if (existsSync(b.marker(tag))) return;+ const r = await run.result;+ b.keepLog(`${tag}-unreached.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ throw new Error(`${label}: the child ${exited ? `exited ${r.code}` : 'timed out'} first\n${r.out.slice(-2000)}\n${r.err.slice(-4000)}\n${b.context()}`);+}+/** Commits files in a clone that is not the managed checkout (the upstream another agent pushes to). */+function cloneGit(root: string, ...args: string[]): string {+ return execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args],+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();+}+function writeInto(root: string, slug: string, content: string): void {+ mkdirSync(dirname(join(root, `${slug}.md`)), { recursive: true });+ writeFileSync(join(root, `${slug}.md`), content);+}+/**+ * A busy agent's checkout is Git-durable: the owner commits every page write through its Git effect. The harness records+ * the setting the way `gbrain sources writer git-durability --enable` does, without its push probe (a checkout with no+ * remote commits locally). Each scenario's database holds this one checkout's binding.+ */+async function durable(b: Brain): Promise<void> {+ await b.engine!.executeRaw("UPDATE persistence_host_bindings SET git_durability='enabled'");+}+/** Seeds the notes, publishes them to a bare origin the checkout tracks, and clones that origin as the upstream writer. */+function seedTracked(b: Brain, notes: number): { origin: string; upstream: string } {+ const origin = join(b.home, 'origin.git'), upstream = join(b.home, 'upstream');+ b.writeNotes(Array.from({ length: notes }, (_, i) => i), 'Initial', 'fixture');+ execFileSync('git', ['init', '-q', '--bare', origin]);+ b.git('remote', 'add', 'origin', origin);+ b.git('push', '-q', '-u', 'origin', 'HEAD');+ execFileSync('git', ['clone', '-q', origin, upstream], { stdio: 'ignore' });+ return { origin, upstream };+}++// ---------------------------------------------------------------- always-on calibration++describe('busy-agent cell calibration (L5, GST-1502)', () => {+ test('the committed profile names every count and cadence, and nothing else', () => {+ const committed = readProfile(readFileSync(join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json'), 'utf8'), 'committed');+ expect(Object.keys(committed).sort()).toEqual(Object.keys(PROFILE_SHAPE).sort());+ });+ test('a profile with an unknown key, a missing count or a non-integer is refused', () => {+ const good = readFileSync(join(REPO_ROOT, 'test/fixtures/busy-agent-profile.json'), 'utf8');+ const edit = (patch: (p: Record<string, unknown>) => void) => { const p = JSON.parse(good) as Record<string, unknown>; patch(p); return JSON.stringify(p); };+ expect(() => readProfile(edit(p => { p.page_titles = 3; }), 't')).toThrow('unknown key page_titles');+ expect(() => readProfile(edit(p => { delete p.notes; }), 't')).toThrow('notes must be a positive integer');+ expect(() => readProfile(edit(p => { p.refresh_ticks = 1.5; }), 't')).toThrow('refresh_ticks must be a positive integer');+ });+ test('every mode runs: none is skipped', () => {+ expect(Object.entries(MODES).filter(([, mode]) => mode.skip !== null).map(([name]) => name)).toEqual([]);+ });+});++// ---------------------------------------------------------------- (h) refresh ticks with writes inside the fence++scenario('h', 'writes that reach a refreshing checkout wait for the refresh and then commit or conflict', () => {+ const b = new Brain('refresh-fence', ['h']);+ interface FenceWrite { tick: number; slug: string; requestId: string; expect: 'committed' | 'revision_conflict'; exit: number; printed: string; start: number; receipt: Receipt | null }+ const fenceWrites: FenceWrite[] = [];+ const refreshes: Array<{ tick: number; exit: number; outcome: string; state: string | null; released: string | null }> = [];+ beforeAll(async () => {+ let upstream = '';+ await b.setup(brain => { upstream = seedTracked(brain, PROFILE.notes).upstream; }, { 'persistence.refresh_barrier': 'true' });+ await durable(b);+ const serve = new StdioServe(b);+ await serve.start();+ // The agent writes the first half of the notes; the other clone changes the second half, so a refresh never has to+ // merge two edits of one file. Only the fence conflicts write a note the tick changes, and those apply nothing.+ const half = Math.floor(PROFILE.notes / 2);+ let done = false;+ const background = (async () => {+ for (let k = 0; !done; k++) {+ const slug = noteSlug(b.between(0, half - 1)), marker = `Agent put ${k} of ${slug} (${randomUUID().slice(0, 8)})`;+ const requestId = randomUUID(), start = performance.now();+ const r = await put(b, slug, `---\ntype: concept\ntitle: Agent ${k}\n---\n\n${marker}.\n`, requestId).result;+ b.keepLog(`agent-put-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });+ await Bun.sleep(PROFILE.background_write_interval_ms);+ }+ })();+ for (let tick = 1; tick <= PROFILE.refresh_ticks; tick++) {+ const pick = (lo: number, hi: number, n: number) => [...new Set(Array.from({ length: 4 * n }, () => b.between(lo, hi - 1)))].slice(0, n);+ const changed = pick(half, PROFILE.notes, PROFILE.upstream_commits_per_tick);+ const untouched = pick(0, half, PROFILE.fence_writes_per_tick);+ const start = performance.now();+ cloneGit(upstream, 'pull', '-q', '--no-rebase', '--no-edit');+ for (const i of changed) writeInto(upstream, noteSlug(i), noteText(i, `Upstream tick ${tick}`));+ cloneGit(upstream, 'add', '--', ...changed.map(i => `${noteSlug(i)}.md`));+ cloneGit(upstream, 'commit', '-qm', `tick ${tick}: another agent's notes`);+ cloneGit(upstream, 'push', '-q', 'origin', 'HEAD');+ const ack = performance.now();+ for (const i of changed) b.ledger.record({ kind: 'commit', slug: noteSlug(i), marker: noteMarker(i, `Upstream tick ${tick}`), touches: ['db', 'file'], start, ack });+ // The tick's member sync stalls at its first round, so the refresh holds the fence (state syncing) while the writes arrive.+ const tag = `tick-${tick}`, release = join(b.home, `release-${tag}`);+ const fault: Fault = { point: 'sync:round_start', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 };+ b.plan.push(`tick ${tick}: stall the refresh's member sync at sync:round_start #1`);+ const refresh = b.spawnCli(['sources', 'refresh', SOURCE, '--json'], { fault });+ await reached(b, tag, refresh, `tick ${tick}'s refresh to reach its member sync`);+ b.recordFault(tag, `tick ${tick} sync:round_start stall`);+ const writes = [+ ...untouched.map(i => ({ i, expect: 'committed' as const })),+ ...changed.slice(0, PROFILE.fence_conflicts_per_tick).map(i => ({ i, expect: 'revision_conflict' as const })),+ ];+ await Promise.all(writes.map(async ({ i, expect: want }) => {+ const slug = noteSlug(i), requestId = randomUUID(), started = performance.now();+ const marker = `Fence write tick ${tick} of ${slug}`;+ const r = await put(b, slug, `---\ntype: concept\ntitle: Fence ${tick}\n---\n\n${marker}.\n`, requestId).result;+ b.keepLog(`fence-${tick}-${slug.replaceAll('/', '_')}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ fenceWrites.push({ tick, slug, requestId, expect: want, exit: r.code, printed: r.code === 0 || r.code === 10 ? '' : printedCode(r), start: started, receipt: null });+ }));+ writeFileSync(release, '');+ const r = await refresh.result;+ b.keepLog(`refresh-${tick}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ const [row] = await b.engine!.executeRaw<{ state: string; released_at: string | null }>(+ `SELECT state, released_at::text FROM persistence_worktree_refreshes ORDER BY created_at DESC LIMIT 1`);+ refreshes.push({ tick, exit: r.code, outcome: String(lastJson(r.out)?.status ?? lastJson(r.out)?.error ?? 'none'), state: row?.state ?? null, released: row?.released_at ?? null });+ b.note(`tick ${tick}: refresh exit ${r.code} state ${row?.state}; ${writes.length} write(s) inside the fence`);+ for (const w of fenceWrites.filter(f => f.tick === tick)) {+ await until(async () => TERMINAL.has((await receipt(b, w.requestId))?.state ?? ''), `fence write ${w.slug} of tick ${tick} to settle`, 120_000).catch(() => {});+ w.receipt = await receipt(b, w.requestId);+ const marker = `Fence write tick ${tick} of ${w.slug}`;+ if (w.receipt?.state === 'committed') b.ledger.record({ kind: 'put', slug: w.slug, marker, touches: ['db', 'file'], start: w.start, ack: performance.now() });+ }+ }+ done = true;+ await background;+ await serve.stop();+ await b.recover();+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test('every tick\'s refresh completed and released its fence', () => b.verdict(refreshes.filter(r => r.exit !== 0 || r.state !== 'completed' || !r.released)+ .map(r => `tick ${r.tick}: refresh exit ${r.exit} outcome ${r.outcome} state ${r.state} released ${r.released}`)));+ test(as('wedge', 'every write that reached the refreshing checkout was answered pending with a receipt naming the refresh, never refused'), () =>+ b.verdict(fenceWrites.filter(w => w.exit !== 10 || !w.receipt?.deferred_refresh_id)+ .map(w => `tick ${w.tick} ${w.slug}: exit ${w.exit}${w.printed ? ` (${w.printed})` : ''}, deferred_refresh_id ${w.receipt?.deferred_refresh_id ?? 'none'}`)));+ test(as('lost_write', 'a fence write to a file the refresh left alone committed after the refresh'), () =>+ b.verdict(fenceWrites.filter(w => w.expect === 'committed' && w.receipt?.state !== 'committed')+ .map(w => `tick ${w.tick} ${w.slug}: ${w.receipt?.state ?? 'no receipt'} ${w.receipt?.error_code ?? ''}`)));+ test(as('lost_write', 'a fence write to a file the refresh changed ended revision_conflict with nothing applied'), () =>+ b.verdict(fenceWrites.filter(w => w.expect === 'revision_conflict' && !(w.receipt?.state === 'conflict' && w.receipt.error_code === 'revision_conflict'))+ .map(w => `tick ${w.tick} ${w.slug}: ${w.receipt?.state ?? 'no receipt'} ${w.receipt?.error_code ?? ''}`)));+ test('every tick put writes inside the fence, so the barrier was exercised', () =>+ b.verdict(fenceWrites.length >= PROFILE.refresh_ticks ? [] : [`${fenceWrites.length} fence write(s) over ${PROFILE.refresh_ticks} tick(s)`]));+ oracleTests(b, () => ({ bigPage: false }));+});++// ---------------------------------------------------------------- (i) plain-git and uncommitted-file writers++scenario('i', 'doctor foreign_writers names every commit made beside gbrain, none gbrain made, and every stale uncommitted page', () => {+ const b = new Brain('foreign-writers', ['i']);+ const plain: Array<{ sha: string; slug: string }> = [];+ const dropped: string[] = [];+ let gbrainShas: string[] = [];+ let check: { status?: string; details?: { foreign?: Array<{ sha: string }>; dirty?: Array<{ path: string; age_ms: number }>; dirty_count?: number; partial?: boolean } } | undefined;+ beforeAll(async () => {+ await b.setup(brain => brain.writeNotes(Array.from({ length: PROFILE.notes }, (_, i) => i), 'Initial', 'fixture'));+ await durable(b);+ const fixtureHead = b.git('rev-parse', 'HEAD');+ const serve = new StdioServe(b);+ await serve.start();+ // The agent writes the first half of the notes through gbrain (its door commits start the commit ledger); the+ // plain-git committers write the second half, so no plain commit races a gbrain write of the same file.+ const half = Math.floor(PROFILE.notes / 2);+ const agentPut = async (k: number) => {+ const slug = noteSlug(k % half), marker = `Agent put ${k} of ${slug} (${randomUUID().slice(0, 8)})`, start = performance.now();+ const requestId = randomUUID();+ const r = await b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '20000'],+ { stdin: `---\ntype: concept\ntitle: Agent ${k}\n---\n\n${marker}.\n` }).result;+ b.keepLog(`agent-put-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });+ return r.code;+ };+ if (await agentPut(0) !== 0) throw new Error(`the first agent put did not commit, so gbrain's commit ledger may not exist\n${b.context()}`);+ await until(() => b.git('rev-list', '--count', `${fixtureHead}..HEAD`) !== '0', 'the first door commit', 120_000);+ let done = false;+ const agent = (async () => { for (let k = 1; !done; k++) { await agentPut(k); await Bun.sleep(PROFILE.background_write_interval_ms); } })();+ const committer = async (c: number) => {+ for (let k = 0; k < PROFILE.plain_git_commits_each; k++) {+ const i = half + b.between(0, PROFILE.notes - half - 1), slug = noteSlug(i), version = `Plain git ${c}.${k}`;+ if (plain.some(p => p.slug === slug)) continue;+ const start = performance.now();+ writeInto(b.repo, slug, noteText(i, version));+ for (let attempt = 1; ; attempt++) {+ try { b.git('add', '--', `${slug}.md`); b.git('commit', '-qm', `committer ${c}: ${version}`, '--', `${slug}.md`); break; }+ catch (error) { if (attempt >= 10) throw error; await Bun.sleep(500); }+ }+ plain.push({ sha: b.git('log', '-1', '--format=%H', '--', `${slug}.md`), slug });+ b.ledger.record({ kind: 'commit', slug, marker: noteMarker(i, version), touches: ['db', 'file'], start, ack: performance.now() });+ await Bun.sleep(b.between(300, 1500));+ }+ };+ await Promise.all(Array.from({ length: PROFILE.plain_git_committers }, (_, c) => committer(c)));+ // The uncommitted-file writer leaves new page files behind, last touched past the warn window.+ const stale = (Date.now() - FOREIGN_DIRTY_WARN_MS - 5 * 60_000) / 1000;+ for (let k = 0; k < PROFILE.uncommitted_files; k++) {+ const slug = `inbox/dropped-${k}`;+ writeInto(b.repo, slug, `---\ntype: note\ntitle: Dropped ${k}\n---\n\nLeft uncommitted by another writer.\n`);+ utimesSync(join(b.repo, `${slug}.md`), stale, stale);+ dropped.push(slug);+ }+ done = true;+ await agent;+ await until(async () => (await b.openRequests()).length === 0, 'the agent writes and their door commits to settle', 300_000).catch(() => {});+ const r = await b.spawnCli(['doctor', '--only', 'foreign_writers', '--json']).result;+ b.keepLog('doctor-foreign-writers.json', `exit ${r.code}\n${r.out}\n${r.err}`);+ check = ((lastJson(r.out)?.checks ?? []) as Array<typeof check & { name: string }>).find(c => c?.name === 'foreign_writers');+ gbrainShas = b.git('rev-list', `${fixtureHead}..HEAD`).split('\n').filter(sha => sha && !plain.some(p => p.sha === sha));+ b.note(`${plain.length} plain commit(s), ${gbrainShas.length} gbrain commit(s), ${dropped.length} uncommitted file(s); foreign_writers ${check?.status}`);+ await serve.stop();+ await b.recover();+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test('foreign_writers fails and names every plain-git commit', () => {+ const named = new Set((check?.details?.foreign ?? []).map(c => c.sha));+ b.verdict([+ ...(check?.status === 'fail' ? [] : [`foreign_writers status ${check?.status ?? 'missing'}, expected fail`]),+ ...(check?.details?.partial ? ['foreign_writers read only part of the checkout (partial)'] : []),+ ...plain.filter(p => !named.has(p.sha)).map(p => `plain commit ${p.sha.slice(0, 12)} (${p.slug}) not named`),+ ]);+ });+ test('foreign_writers names no commit gbrain made', () => {+ const gbrain = new Set(gbrainShas);+ b.verdict((check?.details?.foreign ?? []).filter(c => gbrain.has(c.sha)).map(c => `gbrain commit ${c.sha.slice(0, 12)} named as foreign`));+ });+ test('every stale uncommitted page file is listed past the warn window', () => {+ const listed = new Map((check?.details?.dirty ?? []).map(d => [d.path.replace(/\.md$/, ''), d.age_ms]));+ b.verdict(dropped.filter(slug => ![...listed.keys()].some(path => path.endsWith(slug)) || [...listed.entries()].find(([path]) => path.endsWith(slug))![1] < FOREIGN_DIRTY_WARN_MS)+ .map(slug => `${slug}: not listed as a stale uncommitted page file`));+ });+ oracleTests(b, () => ({ exclude: new Set(dropped), bigPage: false }));+});++// ---------------------------------------------------------------- (j) legacy rows with no import record++scenario('j', 'legacy rows with no import record are classified by content evidence in every direction', () => {+ const b = new Brain('legacy-rows', ['j']);+ const DIRECTIONS = ['adopt_db_moved', 'git_wins_file_moved', 'both_moved', 'no_evidence'] as const;+ const targets = new Map<string, typeof DIRECTIONS[number]>();+ let census: Record<string, unknown> | null = null;+ const after = new Map<string, { file: string; db: string; held: string | null }>();+ beforeAll(async () => {+ await b.setup(brain => brain.writeNotes(Array.from({ length: PROFILE.notes }, (_, i) => i), 'Initial', 'fixture'));+ await durable(b);+ const pool = [...new Set(Array.from({ length: 8 * PROFILE.legacy_rows_per_direction }, () => b.between(0, PROFILE.notes - 1)))];+ DIRECTIONS.forEach((direction, d) => { for (let k = 0; k < PROFILE.legacy_rows_per_direction; k++) targets.set(noteSlug(pool[d * PROFILE.legacy_rows_per_direction + k]!), direction); });+ const slugs = [...targets.keys()];+ // Each file becomes the page's own export, imported once, so the page and its committed blob share a state.+ const rendered = async (slug: string) => b.engine!.transaction(async tx => {+ const [key] = await tx.executeRaw<{ id: string; source_id: string; slug: string }>('SELECT id::text, source_id, slug FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, slug]);+ return inlineExportMarkdown(await readExportPage(tx, key!, await readExportWithdrawals(tx, SOURCE)));+ });+ for (const slug of slugs) writeFileSync(join(b.repo, `${slug}.md`), await rendered(slug));+ b.git('add', '--', ...slugs.map(slug => `${slug}.md`)); b.git('commit', '-qm', 'files equal their pages', '--', ...slugs.map(slug => `${slug}.md`));+ const imported = await b.sync(['--no-pull', '--no-delegate', '--json']).result;+ if (imported.code !== 0) throw new Error(`the import of the rendered files exited ${imported.code}\n${imported.err}\n${b.context()}`);+ // The rows lose their import records (imported before records existed), then each side moves as its direction says.+ await withEnv({ GBRAIN_HOME: b.home }, async () => {+ await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ try {+ await b.engine!.executeRaw(`DELETE FROM op_checkpoints WHERE op=$1 AND completed_keys->0->>'page_id' IN (SELECT id::text FROM pages WHERE source_id=$2 AND slug = ANY($3::text[]))`,+ [SYNC_IMPORT_PROVENANCE_OP, SOURCE, slugs]);+ for (const [slug, direction] of targets) {+ if (direction === 'git_wins_file_moved') continue;+ if (direction !== 'no_evidence') await b.engine!.createVersion(slug, { sourceId: SOURCE });+ const start = performance.now();+ await b.engine!.putPage(slug, { type: 'concept', title: slug.split('/')[1]!, compiled_truth: `Edited in the database (${direction}).`, timeline: '' }, { sourceId: SOURCE });+ b.ledger.record({ kind: 'db_only', slug, marker: `Edited in the database (${direction}).`, touches: ['db'], start, ack: performance.now() });+ }+ } finally { await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1'); }+ });+ const fileMoved = [...targets].filter(([, d]) => d !== 'adopt_db_moved').map(([slug]) => slug);+ for (const slug of fileMoved) writeFileSync(join(b.repo, `${slug}.md`), noteText(Number(slug.slice(-4)), `Plain git edit (${targets.get(slug)})`));+ const editStart = performance.now();+ b.git('add', '--', ...fileMoved.map(slug => `${slug}.md`)); b.git('commit', '-qm', 'plain git edits', '--', ...fileMoved.map(slug => `${slug}.md`));+ // A held file never reaches its page, so a plain edit of a row whose page also moved lands in the file alone.+ for (const slug of fileMoved) b.ledger.record({ kind: 'commit', slug, marker: noteMarker(Number(slug.slice(-4)), `Plain git edit (${targets.get(slug)})`),+ touches: targets.get(slug) === 'git_wins_file_moved' ? ['db', 'file'] : ['file'], start: editStart, ack: performance.now() });+ const r = await b.spawnCli(['sources', 'git-gap', SOURCE, '--json', '--sample', String(4 * targets.size + 20)]).result;+ b.keepLog('git-gap.json', `exit ${r.code}\n${r.out}\n${r.err}`);+ census = lastJson(r.out);+ b.note(`git-gap: direction_evidence ${JSON.stringify(census?.direction_evidence)}`);+ await b.recover();+ const disk = b.diskPages(), db = await b.dbPages();+ const holds = await b.engine!.executeRaw<{ record: Record<string, unknown> }>(`SELECT completed_keys->0 AS record FROM op_checkpoints WHERE op=$1`, [GIT_HOLD_OP]);+ for (const slug of slugs) after.set(slug, { file: disk.get(slug) ?? '', db: db.get(slug)?.body ?? '', held: (holds.find(h => h.record.slug === slug)?.record.code as string | undefined) ?? null });+ for (const [slug, state] of after) b.note(`${targets.get(slug)} ${slug}: held ${state.held}; db ${JSON.stringify(state.db.slice(0, 60))}; file ${JSON.stringify(state.file.slice(-60))}`);+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test('sources git-gap classifies every legacy row by the side that moved', () => {+ const samples = new Map(((census?.samples ?? census?.sample ?? []) as Array<{ slug: string; evidence?: string }>).map(s => [s.slug, s.evidence]));+ b.verdict([+ ...(census?.complete === true ? [] : [`git-gap census incomplete or missing: ${JSON.stringify(census)?.slice(0, 300)}`]),+ ...[...targets].filter(([slug, want]) => samples.get(slug) !== want).map(([slug, want]) => `${slug}: evidence ${samples.get(slug) ?? 'none'}, expected ${want}`),+ ...DIRECTIONS.filter(d => (census?.direction_evidence as Record<string, number> | null)?.[d] !== PROFILE.legacy_rows_per_direction)+ .map(d => `direction_evidence.${d} is ${(census?.direction_evidence as Record<string, number> | null)?.[d]}, expected ${PROFILE.legacy_rows_per_direction}`),+ ]);+ });+ test(as('lost_write', 'the sync after the census overwrote no database move: a legacy row whose page moved keeps it'), () =>+ b.verdict([...targets].filter(([slug, d]) => d !== 'git_wins_file_moved' && !after.get(slug)?.db.includes(`Edited in the database (${d}).`))+ .map(([slug, d]) => `${d} ${slug}: the page lost its database edit (${JSON.stringify(after.get(slug)?.db.slice(0, 80))})`)));+ test('a row only the file moved imports the file; a row both sides moved is held file_drift for a person', () =>+ b.verdict([...targets].flatMap(([slug, d]) => {+ const state = after.get(slug)!;+ if (d === 'git_wins_file_moved') return state.db.includes('Plain git edit') && !state.held ? [] : [`${slug}: the file move was not imported (held ${state.held})`];+ if (d === 'adopt_db_moved') return state.held ? [`${slug}: held ${state.held} although its file never changed`] : [];+ return state.held === 'file_drift' ? [] : [`${d} ${slug}: held ${state.held}, expected file_drift`];+ })));+ oracleTests(b, () => {+ const held = [...targets].filter(([, d]) => d === 'both_moved' || d === 'no_evidence').map(([slug]) => slug);+ return { exclude: new Set([...targets].filter(([, d]) => d !== 'git_wins_file_moved').map(([slug]) => slug)), bigPage: false,+ status: { holds: held.map(slug => ({ code: 'file_drift', path: `${slug}.md` })), needsHuman: held.length > 0 } };+ });+});++// ---------------------------------------------------------------- (k) a strict conversion under live writers++scenario('k', 'a strict conversion with live door writers and a refresh between adopt groups converts and loses nothing', () => {+ const b = new Brain('strict-conversion', ['k']);+ const adoptCount = (PROFILE.conversion_groups - 1) * CONVERSION_GROUP_MAX + 10;+ const notes = Math.max(PROFILE.notes, adoptCount + 40);+ const upstreamNotes = [adoptCount, adoptCount + 1, adoptCount + 2], doorBase = adoptCount + 20;+ let preview: Record<string, unknown> | null = null;+ const applies: Array<{ exit: number; status: string; code: string | null; closing: unknown }> = [];+ const refreshes: Array<{ when: string; exit: number; status: string; code: string }> = [];+ const doorWrites: Array<{ slug: string; exit: number; code: string }> = [];+ let upstreamHead = '', strictAfter = false, remainingAfter: unknown = null;+ const canonical = (args: string[], fault?: Fault) => b.spawnCli(['sources', 'canonical', SOURCE, '--to', 'git', ...args, '--json'], { fault });+ const refresh = async (when: string) => {+ const r = await b.spawnCli(['sources', 'refresh', SOURCE, '--json']).result;+ b.keepLog(`refresh-${when}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ refreshes.push({ when, exit: r.code, status: String(lastJson(r.out)?.status ?? 'none'), code: r.code === 0 ? '' : String(lastJson(r.out)?.code ?? printedCode(r)) });+ };+ const apply = async (hash: string, fault?: Fault, tag?: string) => {+ const run = canonical(['--apply', '--expect', hash, '--no-extract'], fault);+ return { run, done: async () => {+ const r = await run.result;+ b.keepLog(`apply-${applies.length + 1}.log`, `exit ${r.code}\n${r.out.slice(-20000)}\n${r.err.slice(-20000)}`);+ const out = lastJson(r.out);+ applies.push({ exit: r.code, status: String(out?.status ?? 'none'), code: (out?.code as string | undefined) ?? null, closing: (out?.result as Record<string, unknown> | undefined)?.closing_passes ?? null });+ if (tag) b.recordFault(tag, 'apply: conversion:group_committed stall');+ } };+ };+ const previewHash = async () => {+ const r = await canonical(['--dry-run']).result;+ b.keepLog(`preview-${applies.length}.json`, `exit ${r.code}\n${r.out.slice(-20000)}\n${r.err.slice(-4000)}`);+ const printed = lastJson(r.out);+ const out = (printed?.preview as Record<string, unknown> | undefined) ?? printed;+ return { out, hash: String(out?.preview_hash ?? '') };+ };+ beforeAll(async () => {+ let upstream = '';+ await b.setup(brain => { upstream = seedTracked(brain, notes).upstream; }, { 'persistence.refresh_barrier': 'true' });+ await durable(b);+ // Rows the database moved since their import: the conversion adopts them into Git, CONVERSION_GROUP_MAX per commit.+ await withEnv({ GBRAIN_HOME: b.home }, async () => {+ await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ try {+ for (let i = 0; i < adoptCount; i++) {+ const start = performance.now(), marker = `Database edit of note ${i}`;+ await b.engine!.putPage(noteSlug(i), { type: 'concept', title: `Note ${i} Example`, compiled_truth: `${marker}, adopted by the conversion.`, timeline: '' }, { sourceId: SOURCE });+ b.ledger.record({ kind: 'db_only', slug: noteSlug(i), marker, touches: ['db', 'file'], start, ack: performance.now() });+ }+ } finally { await b.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1'); }+ });+ const serve = new StdioServe(b);+ await serve.start();+ const first = await previewHash();+ preview = first.out;+ // Another agent pushes while the conversion runs; the refresh between adopt groups brings it in.+ const pushStart = performance.now();+ cloneGit(upstream, 'pull', '-q', '--no-rebase', '--no-edit');+ for (const i of upstreamNotes) writeInto(upstream, noteSlug(i), noteText(i, 'Upstream during conversion'));+ cloneGit(upstream, 'add', '--', ...upstreamNotes.map(i => `${noteSlug(i)}.md`));+ cloneGit(upstream, 'commit', '-qm', 'another agent during the conversion');+ cloneGit(upstream, 'push', '-q', 'origin', 'HEAD');+ upstreamHead = cloneGit(upstream, 'rev-parse', 'HEAD');+ for (const i of upstreamNotes) b.ledger.record({ kind: 'commit', slug: noteSlug(i), marker: noteMarker(i, 'Upstream during conversion'), touches: ['db', 'file'], start: pushStart, ack: performance.now() });+ const tag = 'between-groups', release = join(b.home, `release-${tag}`);+ b.plan.push('stall the apply after its first adopt group commits (conversion:group_committed #1); refresh and door writes there');+ const stalled = await apply(first.hash, { point: 'conversion:group_committed', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 }, tag);+ await reached(b, tag, stalled.run, 'the apply to commit its first adopt group');+ const writers = Array.from({ length: PROFILE.conversion_door_writes }, async (_, k) => {+ const i = doorBase + k, slug = noteSlug(i), marker = `Door write ${k} during the conversion`, start = performance.now(), requestId = randomUUID();+ const r = await b.spawnCli(['put', slug, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '20000'],+ { stdin: `---\ntype: concept\ntitle: Door ${k}\n---\n\n${marker}.\n` }).result;+ b.keepLog(`door-${k}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ doorWrites.push({ slug, exit: r.code, code: r.code === 0 || r.code === 10 ? '' : printedCode(r) });+ if (r.code === 0) b.ledger.record({ kind: 'put', slug, marker, touches: ['db', 'file'], start, ack: performance.now() });+ else if (r.code === 10) b.pending.push({ write: { kind: 'put', slug, marker, touches: ['db', 'file'], start }, requestId });+ });+ await refresh('between adopt groups');+ writeFileSync(release, '');+ await Promise.all(writers);+ await stalled.done();+ // An apply that ends incomplete names its rerun (git_conversion_incomplete): one more preview and apply, no fault.+ if (applies.at(-1)?.status !== 'converted') { const again = await previewHash(); await (await apply(again.hash)).done(); }+ if (!refreshes.some(r => r.exit === 0)) await refresh('after the conversion');+ strictAfter = (await b.engine!.executeRaw("SELECT 1 FROM sources WHERE id=$1 AND config->>'canonical'='git'", [SOURCE])).length === 1;+ remainingAfter = (await previewHash()).out?.remaining ?? null;+ b.note(`adopt ${adoptCount} row(s) in ${JSON.stringify(preview?.adopt)}; applies ${JSON.stringify(applies)}; refreshes ${JSON.stringify(refreshes)}; door writes ${JSON.stringify(doorWrites)}; strict ${strictAfter}; remaining ${remainingAfter}`);+ await serve.stop();+ await b.recover();+ await b.settlePending(async requestId => (await receipt(b, requestId))?.state);+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test('the conversion had more adopt groups than one, so the refresh landed between two of them', () => {+ const groups = (preview?.adopt as { groups?: number } | undefined)?.groups ?? 0;+ b.verdict(groups >= PROFILE.conversion_groups ? [] : [`the preview planned ${groups} adopt group(s), expected at least ${PROFILE.conversion_groups}`]);+ });+ test(as('wedge', 'the conversion converts the source to strict Git-canonical with nothing left to plan'), () =>+ b.verdict([...(applies.at(-1)?.status === 'converted' ? [] : [`the last apply ended ${JSON.stringify(applies.at(-1))}`]),+ ...(strictAfter ? [] : ['the source is not git-canonical']), ...(remainingAfter === 0 ? [] : [`the preview after the conversion has ${remainingAfter} remaining`])]));+ test(as('wedge', 'the refresh between adopt groups, or the one after the conversion, brought in the other agent\'s commit'), () =>+ b.verdict(refreshes.some(r => r.exit === 0) && b.git('merge-base', '--is-ancestor', upstreamHead, 'HEAD') === '' ? [] : [`refreshes ${JSON.stringify(refreshes)}; HEAD does not contain ${upstreamHead.slice(0, 12)}`]));+ test(as('untyped_error', 'every door write and refresh during the conversion was accepted or refused with a registered code'), () =>+ b.verdict([...doorWrites.filter(w => w.exit !== 0 && w.exit !== 10 && !(w.code in CODES)).map(w => `${w.slug}: exit ${w.exit} ${w.code}`),+ ...refreshes.filter(r => r.exit !== 0 && !(r.code in CODES)).map(r => `refresh ${r.when}: exit ${r.exit} ${r.code}`)]));+ oracleTests(b, () => ({ bigPage: false }));+});++// ---------------------------------------------------------------- (l) segmented staging expiry++scenario('l', 'a segmented write whose staging expires or aborts before publication publishes nothing, and a resend commits', () => {+ const b = new Brain('segmented-staging', ['l']);+ const PAGE = 'archive/segmented-log';+ const archive = (version: string) => `---\ntype: note\ntitle: Segmented log\n---\n\n# Segmented log (${version})\n\n` ++ Array.from({ length: Math.ceil(PROFILE.segmented_page_kib * 1024 / 96) }, (_, i) => `Entry ${i} of the ${version} archive, a steady line of synthetic operating detail.\n`).join('');+ const injected: Array<{ how: 'expired' | 'aborted'; staged: string | null; exit: number; code: string; receipt: Receipt | null; kept: boolean }> = [];+ let storage: string | null = null;+ let liveAfterSweep: unknown[] = [];+ let tiling: { status?: string } | undefined;+ /** A segmented page keeps its body in segment rows, not compiled_truth, so the page is compared by content hash. */+ const pageIs = async (version: string) => (await b.dbPages()).get(PAGE)?.hash === fileHash(PAGE, archive(version));+ const putArchive = (version: string, requestId: string, fault?: Fault) =>+ b.spawnCli(['put', PAGE, '--source-id', SOURCE, '--force', '--request-id', requestId, '--wait-ms', '30000'], { stdin: archive(version), fault });+ const sweep = () => withEnv({ GBRAIN_HOME: b.home }, () => sweepSegmentStorage(b.engine!));+ beforeAll(async () => {+ await b.setup(brain => brain.commit('fixture', [+ ...Array.from({ length: Math.min(PROFILE.notes, 20) }, (_, i) => ({ slug: noteSlug(i), content: noteText(i, 'Initial'), marker: noteMarker(i, 'Initial') })),+ { slug: PAGE, content: archive('v0'), marker: 'Segmented log (v0)' }]), { 'pages.segmented_storage': 'on' });+ await durable(b);+ for (const ack of b.ledger.acks) if (ack.slug === PAGE) ack.touches = ['file'];+ storage = (await b.engine!.executeRaw<{ body_storage: string }>('SELECT body_storage FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, PAGE]))[0]?.body_storage ?? null;+ // Each write stalls once its manifest is staged; the harness then ends its staging the way a sweep (expired) or a+ // fact purge (aborted) does, and releases it into publication.+ for (const how of ['expired', 'aborted'] as const) {+ const tag = `staging-${how}`, release = join(b.home, `release-${tag}`), requestId = randomUUID();+ b.plan.push(`stall the ${how} write at segmented:manifest_staged #1`);+ const write = putArchive(`v-${how}`, requestId, { point: 'segmented:manifest_staged', nth: 1, marker: b.marker(tag), action: 'stall', release, ms: 300_000 });+ await reached(b, tag, write, `the ${how} write to stage its manifest`);+ b.recordFault(tag, `${how}: segmented:manifest_staged stall`);+ if (how === 'expired') {+ await b.engine!.executeRaw(`UPDATE page_body_stagings SET idle_expires_at=now() - interval '1 hour', hard_expires_at=now() - interval '1 minute'+ WHERE source_id=$1 AND slug=$2 AND state IN ('open','sealed','prepared')`, [SOURCE, PAGE]);+ await sweep();+ } else {+ await b.engine!.executeRaw(`UPDATE page_body_stagings SET state='aborted', updated_at=now() WHERE source_id=$1 AND slug=$2 AND state IN ('open','sealed','prepared')`, [SOURCE, PAGE]);+ }+ const [staged] = await b.engine!.executeRaw<{ state: string }>('SELECT state FROM page_body_stagings WHERE source_id=$1 AND slug=$2 ORDER BY created_at DESC LIMIT 1', [SOURCE, PAGE]);+ writeFileSync(release, '');+ const r = await write.result;+ b.keepLog(`${tag}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ await until(async () => TERMINAL.has((await receipt(b, requestId))?.state ?? ''), `the ${how} write to settle`, 120_000).catch(() => {});+ injected.push({ how, staged: staged?.state ?? null, exit: r.code, code: r.code === 0 ? '' : printedCode(r), receipt: await receipt(b, requestId), kept: await pageIs('v0') });+ }+ // The resend, then the profile's later writes, each a new version of the whole page.+ for (let k = 0; k <= PROFILE.segmented_writes; k++) {+ const version = `v${k + 1}`, start = performance.now();+ const r = await putArchive(version, randomUUID()).result;+ b.keepLog(`write-${version}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ // The ledger reads the file; disk == database (by content hash) carries the page side of a segmented write.+ if (r.code === 0) b.ledger.record({ kind: 'put', slug: PAGE, marker: `Segmented log (${version})`, touches: ['file'], start, ack: performance.now() });+ else b.note(`write ${version}: exit ${r.code} ${printedCode(r)}`);+ }+ await sweep();+ liveAfterSweep = await b.engine!.executeRaw(`SELECT id::text, kind, state FROM page_body_stagings WHERE state IN ('open','sealed','prepared')`);+ const doctor = await b.spawnCli(['doctor', '--only', 'segmented_tiling', '--json']).result;+ b.keepLog('doctor-segmented-tiling.json', `exit ${doctor.code}\n${doctor.out}\n${doctor.err}`);+ tiling = ((lastJson(doctor.out)?.checks ?? []) as Array<{ name: string; status?: string }>).find(c => c.name === 'segmented_tiling');+ b.note(`storage ${storage}; injected ${JSON.stringify(injected.map(i => ({ how: i.how, staged: i.staged, exit: i.exit, code: i.code, receipt: i.receipt?.error_code ?? i.receipt?.state })))}; tiling ${tiling?.status}`);+ await b.recover();+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test('the page is stored in segments, so the staging path is the one under test', () => b.verdict(storage === 'segmented' ? [] : [`body_storage is ${storage}`]));+ test(as('lost_write', 'a write whose staging expired or aborted before publication refuses body_staging_expired and the page keeps its body'), () =>+ b.verdict(injected.flatMap(i => [+ ...(i.staged === i.how ? [] : [`${i.how}: the staging was ${i.staged} when the write was released`]),+ ...((i.code === 'body_staging_expired' || i.receipt?.error_code === 'body_staging_expired') && i.receipt?.state !== 'committed' ? [] : [`${i.how}: exit ${i.exit} ${i.code}, receipt ${i.receipt?.state} ${i.receipt?.error_code}`]),+ ...(i.kept ? [] : [`${i.how}: the page lost its previous body`]),+ ])));+ test('the sweep leaves no live staging behind', () => b.verdict(liveAfterSweep.map(row => `live staging after the sweep: ${JSON.stringify(row)}`)));+ test(as('projection_drift', 'doctor segmented_tiling finds every segmented page tiled'), () => b.verdict(tiling?.status === 'ok' ? [] : [`segmented_tiling is ${tiling?.status ?? 'missing'}`]));+ oracleTests(b, () => ({ bigPage: false }));+});++// ---------------------------------------------------------------- (m) the lost-update race++scenario('m', 'a file-writing cron and an MCP writer racing on one page lose no acknowledged write', () => {+ const b = new Brain('lost-update', ['m']);+ const BOARD = 'ops/daily-board', PATH = `${BOARD}.md`;+ const board = (entries: string[]) => `---\ntype: note\ntitle: Daily board\n---\n\n# Daily board\n\n${entries.map(e => `- ${e}\n`).join('')}END\n`;+ /**+ * Every refusal either writer met, by code. The race may only produce refusals that apply nothing: commit_base_moved+ * for the cron; for the agent revision_conflict, or source_changed while the file holds the cron's commit before the+ * sync imports it (pending_import) or the cron's bytes before it commits them (an uncoordinated local edit).+ */+ const refusals: Array<{ writer: 'cron' | 'agent'; code: string }> = [];+ const acked: Array<{ writer: 'cron' | 'agent'; marker: string }> = [];+ const forced: Record<'A' | 'B', string | null> = { A: null, B: null };+ let serve!: StdioServe;+ const mcp = async (name: string, args: Record<string, unknown>) => {+ const msg = await serve.request('tools/call', { name, arguments: args });+ const result = msg.result as { isError?: boolean; content?: Array<{ text?: string }> } | undefined;+ const text = result?.content?.[0]?.text ?? JSON.stringify(msg.error ?? null);+ let body: Record<string, unknown> = {};+ try { body = JSON.parse(text) as Record<string, unknown>; } catch { body = { text }; }+ return { isError: result?.isError === true || !!msg.error, body };+ };+ const revision = async () => String((await mcp('get_page', { slug: BOARD, include_content: true, source_id: SOURCE })).body.revision ?? '');+ /** The agent's MCP edit at a revision it read: committed, or the code that refused it (a pending write is followed to its receipt). */+ const agentEdit = async (marker: string, rev: string): Promise<string> => {+ const requestId = randomUUID();+ const r = await mcp('edit_page', { slug: BOARD, source_id: SOURCE, expected_revision: rev, request_id: requestId, edits: [{ old_text: '\nEND\n', new_text: `\n- ${marker}\nEND\n` }] });+ if (r.isError) return String(r.body.code ?? r.body.error ?? 'unknown');+ if (r.body.status === 'pending') {+ await until(async () => TERMINAL.has((await receipt(b, requestId))?.state ?? ''), `the agent edit ${marker} to settle`, 300_000);+ const settled = await receipt(b, requestId);+ if (settled?.state !== 'committed') return settled?.error_code ?? settled?.state ?? 'unknown';+ }+ return 'committed';+ };+ /** The cron's commit of the file it read at HEAD (blob `base`), bound to that blob with --base; a refusal discards its stale bytes. */+ const cronCommit = async (marker: string, read: { base: string; content: string }): Promise<string> => {+ writeFileSync(join(b.repo, PATH), read.content.replace('\nEND\n', `\n- ${marker}\nEND\n`));+ const r = await b.spawnCli(['git', 'commit-push', '--path', b.repo, '--base', `${PATH}=${read.base}`, '-m', `cron: ${marker}`, '--json', '--', PATH]).result;+ b.keepLog(`cron-${marker.replaceAll(' ', '_')}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ if (r.code === 0) return 'committed';+ b.git('checkout', '--', PATH);+ return printedCode(r);+ };+ const cronRead = () => ({ base: b.git('rev-parse', `HEAD:${PATH}`), content: b.git('show', `HEAD:${PATH}`) + '\n' });+ const tick = () => b.sync(['--no-pull', '--json']).result;+ /**+ * One writer's acknowledged append, retried from a fresh read after each refusal, at most 20 times. An agent refused+ * because the file holds the cron's commit before the next sync imports it waits for that sync like any client.+ */+ const append = async (writer: 'cron' | 'agent', marker: string) => {+ for (let attempt = 1; attempt <= 20; attempt++) {+ const outcome = writer === 'agent' ? await agentEdit(marker, await revision()) : await cronCommit(marker, cronRead());+ if (outcome === 'committed') { acked.push({ writer, marker }); return; }+ refusals.push({ writer, code: outcome });+ await Bun.sleep(b.between(300, 1500));+ }+ };+ beforeAll(async () => {+ await b.setup(brain => brain.commit('fixture', [+ ...Array.from({ length: Math.min(PROFILE.notes, 20) }, (_, i) => ({ slug: noteSlug(i), content: noteText(i, 'Initial'), marker: noteMarker(i, 'Initial') })),+ { slug: BOARD, content: board(['seeded']), marker: '- seeded' }]), { 'persistence.refresh_barrier': 'true' });+ await durable(b);+ serve = new StdioServe(b);+ await serve.start();+ // Forced order A: the agent reads, the cron commits and the sync imports it, then the agent writes at its stale revision.+ const staleRev = await revision();+ if (await cronCommit('cron forced A', cronRead()) !== 'committed') throw new Error(`forced A: the cron commit was refused\n${b.context()}`);+ acked.push({ writer: 'cron', marker: 'cron forced A' });+ await tick();+ await until(async () => (await revision()) !== staleRev, 'the cron commit to be imported', 120_000);+ forced.A = await agentEdit('agent forced A', staleRev);+ // Forced order B: the cron reads the blob, the agent's edit commits through the door, then the cron commits on its stale read.+ const staleRead = cronRead();+ if (await agentEdit('agent forced B', await revision()) !== 'committed') throw new Error(`forced B: the agent edit was refused\n${b.context()}`);+ acked.push({ writer: 'agent', marker: 'agent forced B' });+ await until(() => b.git('rev-parse', `HEAD:${PATH}`) !== staleRead.base, 'the agent edit\'s door commit', 120_000);+ forced.B = await cronCommit('cron forced B', staleRead);+ // Free-running: both writers on their cadences, and the agent's scheduled sync importing the cron's commits.+ let done = false;+ const ticks = (async () => { while (!done) { await tick(); await Bun.sleep(PROFILE.cron_interval_ms); } })();+ await Promise.all([+ (async () => { for (let k = 0; k < PROFILE.lost_update_rounds; k++) { await append('cron', `cron ${k}`); await Bun.sleep(PROFILE.cron_interval_ms); } })(),+ (async () => { for (let k = 0; k < PROFILE.lost_update_rounds; k++) { await append('agent', `agent ${k}`); await Bun.sleep(PROFILE.agent_interval_ms); } })(),+ ]);+ done = true;+ await ticks;+ b.note(`forced A: ${forced.A}; forced B: ${forced.B}; ${acked.length} acknowledged append(s); refusals ${JSON.stringify(Object.fromEntries([...new Set(refusals.map(r => `${r.writer}:${r.code}`))].map(k => [k, refusals.filter(r => `${r.writer}:${r.code}` === k).length])))}`);+ await serve.stop();+ await b.recover();+ }, SCENARIO_TIMEOUT_MS);+ afterAll(() => b.teardown());++ test(as('lost_write', 'an MCP edit at a revision the cron\'s imported commit moved refuses revision_conflict'), () =>+ b.verdict(forced.A === 'revision_conflict' ? [] : [`forced order A ended ${forced.A}`]));+ test(as('lost_write', 'a commit-push bound to a blob the agent\'s commit moved refuses commit_base_moved'), () =>+ b.verdict(forced.B === 'commit_base_moved' ? [] : [`forced order B ended ${forced.B}`]));+ test(as('lost_write', 'no acknowledged append is silently reverted: every one is in the file and the page'), async () => {+ const file = b.diskPages().get(BOARD) ?? '', page = (await b.dbPages()).get(BOARD)?.body ?? '';+ b.verdict(acked.flatMap(a => [...(file.includes(`- ${a.marker}\n`) ? [] : [`${a.writer} append ${a.marker}: missing from the file`]),+ ...(page.includes(`- ${a.marker}`) ? [] : [`${a.writer} append ${a.marker}: missing from the page`])]));+ });+ test('every refusal the free-running writers met is the race\'s own and applied nothing', () =>+ b.verdict(refusals.filter(r => !(r.writer === 'agent' ? ['revision_conflict', 'source_changed'].includes(r.code) : r.code === 'commit_base_moved'))+ .map(r => `${r.writer} refused ${r.code}`)));+ test('both writers got every append in', () =>+ b.verdict((['cron', 'agent'] as const).filter(w => acked.filter(a => a.writer === w && /^\w+ \d+$/.test(a.marker)).length !== PROFILE.lost_update_rounds)+ .map(w => `${w}: ${acked.filter(a => a.writer === w && /^\w+ \d+$/.test(a.marker)).length} of ${PROFILE.lost_update_rounds} appends acknowledged`)));+ oracleTests(b, () => ({ bigPage: false }));+});diff --git a/test/fixtures/busy-agent-profile.json b/test/fixtures/busy-agent-profile.jsonnew file mode 100644index 0000000000..7bb8c6cd4b--- /dev/null+++ b/test/fixtures/busy-agent-profile.json@@ -0,0 +1,20 @@+{+ "description": "A generic busy-agent workload for the busy-agent soak cell (test/busy-agent-soak.serial.test.ts): counts and cadences only, no content. A profile derived from a real deployment stays private and is passed with GBRAIN_TEST_BUSY_AGENT_PROFILE.",+ "notes": 120,+ "refresh_ticks": 4,+ "upstream_commits_per_tick": 3,+ "fence_writes_per_tick": 4,+ "fence_conflicts_per_tick": 2,+ "background_write_interval_ms": 1500,+ "plain_git_committers": 2,+ "plain_git_commits_each": 3,+ "uncommitted_files": 2,+ "legacy_rows_per_direction": 2,+ "conversion_groups": 3,+ "conversion_door_writes": 4,+ "segmented_page_kib": 1536,+ "segmented_writes": 2,+ "lost_update_rounds": 12,+ "cron_interval_ms": 900,+ "agent_interval_ms": 700+}diff --git a/test/helpers/convergence-brain.ts b/test/helpers/convergence-brain.tsnew file mode 100644index 0000000000..e0f22c0dd2--- /dev/null+++ b/test/helpers/convergence-brain.ts@@ -0,0 +1,590 @@+/**+ * The real-process brain harness shared by the managed-sync convergence gate (test/sync-convergence.serial.test.ts)+ * and the busy-agent soak cell (test/busy-agent-soak.serial.test.ts): one isolated Postgres database behind+ * transaction-mode PgBouncer, reached by every child process through latency proxies, a Git worktree bound to one+ * managed source, real `src/cli.ts` child processes (optionally under the fault preload+ * test/fixtures/convergence-fault-kill.ts), a resident stdio serve, the acknowledged-write ledger and the oracle reads.+ * Each suite passes its own knobs (seed, latency, log directory, repro line). Setup: scripts/persistence/README.md.+ */+import { test } from 'bun:test';+import { execFileSync, spawn, type ChildProcess } from 'node:child_process';+import { randomUUID } from 'node:crypto';+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, relative, resolve } from 'node:path';+import postgres from '#postgres';+import { PostgresEngine } from '../../src/core/postgres-engine.ts';+import { claimWorktree } from '../../src/core/persistence/ownership.ts';+import { parseMarkdown } from '../../src/core/markdown.ts';+import { parseFactsFence } from '../../src/core/facts-fence.ts';+import { contentHash } from '../../src/core/utils.ts';+import { CODES } from '../../src/core/error-registry.ts';+import type { FaultPoint } from '../../src/core/persistence/fault-points.ts';+import { assertSafeE2eDatabaseUrl } from './db-guard.ts';+import { keylessBrainEnv } from './provider-env.ts';+import { withEnv } from './with-env.ts';+import { LatencyProxy } from './latency-proxy.ts';+import type { ViolationClass } from '../../scripts/persistence/model.ts';++const REPO_ROOT = resolve(import.meta.dir, '../..');+const FAULT_PRELOAD = join(REPO_ROOT, 'test/fixtures/convergence-fault-kill.ts');+export const SOURCE = 'workspace';+export const BIG = 'companies/acme-example';+export const ENTITY = 'people/alice-example';+export const MAX_RECOVERY_RUNS = 2;++/** What a suite passes to every brain: the knobs its failures print, where its logs go, and the targets behind the proxies. */+export interface HarnessKnobs {+ /** Prefixes every progress line (`[<tag> <scenario>]`) and names the log file `<tag>.log` in logDir. */+ tag: string;+ seed: number;+ latencyMs: number;+ /** Per child process (a sync or a writer call) unless a call passes its own. */+ syncTimeoutMs: number;+ logDir: string | undefined;+ pooledTarget: string;+ directTarget: string;+ /** The suite's other knobs, printed after the seed and modes in every failure. */+ summary: string;+ /** The local command that reruns these modes, printed in every failure. */+ repro: (modes: string[]) => string;+}++/** mulberry32: a fixed seed gives the same fault plan and writer targets on every run. */+export function prng(seed: number): () => number {+ let a = seed >>> 0;+ return () => {+ a = (a + 0x6d2b79f5) >>> 0;+ let t = a;+ t = Math.imul(t ^ (t >>> 15), t | 1);+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61);+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;+ };+}++// ---------------------------------------------------------------- the status oracle++/** Every `gbrain sync status --json` field the operator loop reads (sync-status.ts `SyncStatus`). */+export const STATUS_FIELDS = ['source_id', 'run_id', 'cursor', 'committed_last_10m', 'live_job_id', 'holds', 'last_error', 'needs_human',+ 'resume_argv', 'checkpoint_blockers', 'cursors', 'terminal_failures', 'next'] as const;+export interface StatusExpectation { holds?: Array<{ code: string; path: string }>; needsHuman?: boolean }+/**+ * What a converged source's status must say. Field-present assertions first: a renderer that drops a field fails+ * here, not in an agent's loop. `checkpoint_blockers` and `cursors` are arrays on every status (#6712 PR 0 made+ * `checkpoint_blockers` always present). The harness reads `sync status --json --detail`: since #6712 PR 2 the+ * terminal summary, blocker list and cursor join run only behind the flag.+ */+/** The default (no `--detail`) status after the chaos: every skipped heavy read says so and names the `--detail` command (#6712 PR 2, DX F1). */+export function fastStatusProblems(status: Record<string, unknown> | null): string[] {+ if (!status) return ['sync status --json printed no JSON document'];+ return (['checkpoint_blockers_read', 'cursors_read', 'terminal_failures_read'] as const).flatMap(field => {+ const read = status[field] as { status?: string; fix?: { argv?: string[] } } | undefined;+ return read?.status === 'not_read' && read.fix?.argv?.includes('--detail') ? [] : [`sync status --json: ${field} is ${JSON.stringify(read)}, expected not_read with a --detail fix`];+ });+}++export function statusProblems(status: Record<string, unknown> | null, expected: StatusExpectation = {}): string[] {+ if (!status) return ['sync status --json printed no JSON document'];+ const out = STATUS_FIELDS.filter(field => !(field in status)).map(field => `sync status --json: field ${field} missing`);+ if (out.length) return out;+ const cursor = status.cursor as { done?: boolean } | null;+ if (!Array.isArray(status.cursors)) out.push('sync status --json: cursors is not an array');+ if (!Array.isArray(status.checkpoint_blockers)) out.push('sync status --json: checkpoint_blockers is not an array');+ else if (status.checkpoint_blockers.length) out.push(`sync status --json: ${status.checkpoint_blockers.length} checkpoint blocker(s)`);+ // A finished run's cursor is retired, so a converged source shows none (null) or a done one.+ if (cursor && cursor.done !== true) out.push(`sync status --json: cursor not done (${JSON.stringify(cursor)})`);+ const terminal = status.terminal_failures as { status?: string; active_blockers?: unknown[] } | null;+ if (terminal?.status !== 'ok') out.push(`sync status --json: terminal_failures.status is ${terminal?.status}`);+ else if (!Array.isArray(terminal.active_blockers)) out.push('sync status --json: terminal_failures.active_blockers is not an array');+ else if (terminal.active_blockers.length) out.push(`sync status --json: ${terminal.active_blockers.length} active terminal blocker(s)`);+ if (status.needs_human !== (expected.needsHuman ?? false)) out.push(`sync status --json: needs_human is ${status.needs_human}, expected ${expected.needsHuman ?? false}`);+ const holds = Array.isArray(status.holds) ? (status.holds as Array<{ code: string; path: string }>).map(h => `${h.code}:${h.path}`).sort() : ['holds is not an array'];+ const want = (expected.holds ?? []).map(h => `${h.code}:${h.path}`).sort();+ if (JSON.stringify(holds) !== JSON.stringify(want)) out.push(`sync status --json: holds [${holds.join(', ')}], expected [${want.join(', ')}]`);+ if (status.last_error !== null) out.push(`sync status --json: last_error ${JSON.stringify(status.last_error)}`);+ if (!expected.holds?.length && status.next !== null) out.push(`sync status --json: next ${JSON.stringify(status.next)}`);+ return out;+}++// ---------------------------------------------------------------- the acknowledged-write ledger++/**+ * One writer operation that returned success, recorded by the harness at the moment it saw the success (never read+ * back from the brain), or a write the brain accepted (exit 10) that had not settled when its writer stopped, recorded+ * when its receipt, read after recovery, said committed (`Brain.settlePending`). `touches` is where the write landed: the page in the database, its file, or (for `remember`)+ * an additive fact row. `marker` is text that only this write put there; `null` on a page location is a deletion.+ */+export interface Ack { kind: 'commit' | 'put' | 'remember' | 'maintenance' | 'db_only' | 'git_mv' | 'git_rm' | 'unlink'; slug: string;+ marker: string | null; touches: Array<'db' | 'file'> | 'fact'; start: number; ack: number }+export class Ledger {+ acks: Ack[] = [];+ record(ack: Ack): void { this.acks.push(ack); }+ /**+ * For every slug and location, the writes no later acknowledged write causally supersedes (none started after it+ * was acknowledged); the final state must show one of them. An additive fact survives unless a page write on its+ * slug may have been ordered after it.+ */+ problems(disk: Map<string, string>, db: Map<string, { body: string } | null>, facts: Set<string>): string[] {+ const out: string[] = [];+ const pageWrites = this.acks.filter(a => a.touches !== 'fact' && a.kind !== 'maintenance');+ for (const slug of new Set(pageWrites.map(a => a.slug))) {+ for (const where of ['db', 'file'] as const) {+ const writes = pageWrites.filter(a => a.slug === slug && (a.touches as string[]).includes(where));+ if (!writes.length) continue;+ const maximal = writes.filter(w => !writes.some(later => later.start > w.ack));+ const seen = where === 'file' ? disk.get(slug) ?? null : db.get(slug)?.body ?? null;+ if (maximal.some(w => w.marker === null ? seen === null : seen?.includes(w.marker))) continue;+ out.push(`${where} ${slug}: lost acknowledged write; has ${seen === null ? 'nothing' : JSON.stringify(seen.slice(0, 80))}, expected one of ${maximal.map(w => `${w.kind}@${Math.round(w.ack)}:${w.marker === null ? '(absent)' : JSON.stringify(w.marker)}`).join(' | ')}`);+ }+ }+ for (const fact of this.acks.filter(a => a.touches === 'fact')) {+ const excused = pageWrites.some(w => w.slug === fact.slug && (w.touches as string[]).includes('db') && w.ack > fact.start);+ if (!excused && !facts.has(fact.marker!)) out.push(`fact on ${fact.slug}: lost acknowledged remember ${JSON.stringify(fact.marker)}`);+ }+ return out;+ }+}++// ---------------------------------------------------------------- the fixture++export function noteText(i: number, version: string): string {+ return `---\ntype: concept\ntitle: Note ${i} Example\n---\n\n${version} body for note ${i}, an observation about widget-co and acme-example.\n\n${'Supporting detail sentence. '.repeat(8 + (i % 5))}\n`;+}+export const noteMarker = (i: number, version: string) => `${version} body for note ${i},`;+export const noteSlug = (i: number) => `notes/note-${String(i).padStart(4, '0')}`;+export function bigPage(facts: number, timeline: number, version: string): string {+ const day = (n: number) => new Date(Date.UTC(2018, 0, 1) + n * 86_400_000).toISOString().slice(0, 10);+ const paragraphs = Array.from({ length: 30 }, (_, i) => `Section ${i} (${version}): acme-example ${'expanded with a steady stream of synthetic operating detail. '.repeat(6)}`);+ const rows = Array.from({ length: facts }, (_, i) =>+ `| ${i + 1} | Synthetic claim ${i + 1} about acme-example | fact | 0.9 | world | medium | ${day(i)} | | fixture | |`);+ const bullets = Array.from({ length: timeline }, (_, i) => `- **${day(i)}** | Synthetic event ${i + 1} at acme-example`);+ return [+ '---', 'type: company', 'title: Acme Example', '---', '', '# Acme Example', '', ...paragraphs.flatMap(p => [p, '']),+ '## Facts', '', '<!--- gbrain:facts:begin -->',+ '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |',+ '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|',+ ...rows, '<!--- gbrain:facts:end -->', '', '<!-- timeline -->', '', ...bullets, '',+ ].join('\n');+}+export const entityText = '---\ntype: person\ntitle: Alice Example\n---\n\nAlice Example works with acme-example.\n';++// ---------------------------------------------------------------- one brain++export interface Fault { point: FaultPoint; nth: number; marker: string; operation?: string; path?: string; action?: 'kill' | 'edit' | 'stall' | 'throw';+ edit?: { file: string; content: string }; release?: string; ms?: number; sqlstate?: string;+ prime?: { point: FaultPoint; nth: number; path?: string; edit: { file: string; content: string } } }+export interface Run { code: number; out: string; err: string; ms: number }+export interface SyncRun { proc: ReturnType<typeof Bun.spawn>; result: Promise<Run> }++export async function until(check: () => boolean | Promise<boolean>, label: string, timeoutMs = 300_000): Promise<void> {+ const deadline = performance.now() + timeoutMs;+ while (performance.now() < deadline) {+ if (await check()) return;+ await Bun.sleep(100);+ }+ throw new Error(`timed out waiting for ${label}`);+}+export function lastJson(out: string): Record<string, unknown> | null {+ try { return JSON.parse(out) as Record<string, unknown>; } catch { /* several documents */ }+ for (const line of out.trim().split('\n').reverse()) { try { return JSON.parse(line) as Record<string, unknown>; } catch { /* not the envelope */ } }+ return null;+}+/** The longest `stalled Ns` progress line a sync printed: how long it waited on a predecessor's leftovers. */+export function worstStall(err: string): number { return Math.max(0, ...[...err.matchAll(/stalled (\d+)s/g)].map(m => Number(m[1]))); }++/**+ * One scenario's brain: its database, home, Git worktree, proxies, child processes, ledger and log. `context()` is+ * what every failure prints: seed, modes, the fault plan and the one-line repro.+ */+export class Brain {+ home = ''; repo = ''; database = '';+ admin: ReturnType<typeof postgres> | undefined;+ engine: PostgresEngine | undefined;+ pooledProxy: LatencyProxy | undefined; directProxy: LatencyProxy | undefined;+ pooledUrl = ''; directUrl = '';+ serves: StdioServe[] = [];+ ledger = new Ledger();+ /** Writes the brain accepted (exit 10) that had not settled when their writer stopped; `settlePending` reads their receipts after recovery. */+ pending: Array<{ write: Omit<Ack, 'ack'>; requestId: string }> = [];+ plan: string[] = [];+ fired: Record<string, string | null> = {};+ killedPids = new Set<number>();+ recoveryRuns: Array<Run & { envelope: Record<string, unknown> | null }> = [];+ elapsed: Record<string, number> = {};+ calibration: string[] = [];+ readonly rand: () => number;+ constructor(readonly name: string, readonly modes: string[], readonly knobs: HarnessKnobs) { this.rand = prng(knobs.seed ^ [...name].reduce((h, c) => Math.imul(h, 31) + c.charCodeAt(0), 7)); }++ between(lo: number, hi: number): number { return lo + Math.floor(this.rand() * (hi - lo + 1)); }+ note(line: string): void {+ console.error(`[${this.knobs.tag} ${this.name}] ${line}`);+ if (this.knobs.logDir) { mkdirSync(this.knobs.logDir, { recursive: true }); appendFileSync(join(this.knobs.logDir, `${this.knobs.tag}.log`), `[${this.name}] ${line}\n`); }+ }+ keepLog(name: string, text: string): void { if (this.knobs.logDir) { mkdirSync(join(this.knobs.logDir, this.name), { recursive: true }); writeFileSync(join(this.knobs.logDir, this.name, name), text); } }+ context(): string {+ return [`seed ${this.knobs.seed}; modes ${this.modes.join(',')}; ${this.knobs.summary}`,+ `fault plan: ${this.plan.join('; ') || 'none'}`,+ `fired: ${Object.entries(this.fired).map(([k, v]) => `${k} => ${v ?? 'did not fire'}`).join('; ') || 'none'}`,+ `repro (Postgres + PgBouncer from scripts/persistence/README.md): ${this.knobs.repro(this.modes)}`].join('\n');+ }+ /** Throws with the problems and the failure context; a passing check prints nothing. */+ verdict(problems: string[]): void {+ if (problems.length) throw new Error(`${problems.length} problem(s):\n ${problems.slice(0, 40).join('\n ')}\n${this.context()}`);+ }+ marker(tag: string): string { return join(this.home, `fault-${tag}.json`); }+ recordFault(tag: string, label: string): string | null {+ const at = existsSync(this.marker(tag)) ? readFileSync(this.marker(tag), 'utf8') : null;+ this.fired[label] = at;+ if (at) { const fired = JSON.parse(at) as { pid: number; action: string }; if (fired.action === 'kill') this.killedPids.add(fired.pid); }+ return at;+ }++ git(...args: string[]): string { return execFileSync('git', ['-C', this.repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }+ /**+ * Writes files and commits exactly those paths, as the shared-checkout rules require of every committer; each write+ * is acknowledged when the commit returns. Sweeping the worktree (`git add -A`) would commit a writer's in-flight+ * write-through bytes as an out-of-band Git change of a page that writer is still writing.+ */+ commit(message: string, files: Array<{ slug: string; content: string; marker: string }>): void {+ const start = performance.now();+ for (const f of files) { mkdirSync(dirname(join(this.repo, `${f.slug}.md`)), { recursive: true }); writeFileSync(join(this.repo, `${f.slug}.md`), f.content); }+ const paths = files.map(f => `${f.slug}.md`);+ this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);+ const ack = performance.now();+ for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });+ }+ writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {+ this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);+ }+ bigFile(facts: number, timeline: number, version: string) { return { slug: BIG, content: bigPage(facts, timeline, version), marker: `Section 0 (${version})` }; }++ childEnv(extra: Record<string, string | undefined> = {}): Record<string, string> {+ return keylessBrainEnv(process.env, this.home, {+ DATABASE_URL: this.pooledUrl, GBRAIN_DATABASE_URL: undefined, GBRAIN_DIRECT_DATABASE_URL: this.directUrl,+ GBRAIN_BRAIN_ID: 'host', GBRAIN_SOURCE: SOURCE, GBRAIN_SWEEP: '0', GBRAIN_HOOKS: undefined,+ GBRAIN_SKIP_STARTUP_HOOKS: '1', GBRAIN_NO_SKILL_NAG: '1', GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS: '300',+ GBRAIN_SYNC_NO_DELEGATE: undefined, GBRAIN_TEST_CONVERGENCE_FAULT: undefined,+ ...extra,+ });+ }+ faultArgs(fault?: Fault): { preload: string[]; env: Record<string, string | undefined> } {+ if (!fault) return { preload: [], env: {} };+ return { preload: ['--preload', FAULT_PRELOAD], env: { GBRAIN_TEST_CONVERGENCE_FAULT: JSON.stringify(fault) } };+ }+ spawnCli(args: string[], opts: { fault?: Fault; stdin?: string; timeoutMs?: number } = {}): SyncRun {+ const { preload, env } = this.faultArgs(opts.fault);+ const started = performance.now();+ const proc = Bun.spawn([process.execPath, '--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), ...args], {+ cwd: REPO_ROOT, env: this.childEnv(env), stdin: opts.stdin === undefined ? 'ignore' : new Blob([opts.stdin]), stdout: 'pipe', stderr: 'pipe',+ });+ const timer = setTimeout(() => proc.kill('SIGKILL'), opts.timeoutMs ?? this.knobs.syncTimeoutMs);+ const result = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])+ .then(([out, err, code]) => ({ code, out, err, ms: performance.now() - started }))+ .finally(() => clearTimeout(timer));+ return { proc, result };+ }+ sync(args: string[], fault?: Fault): SyncRun { return this.spawnCli(['sync', '--source', SOURCE, ...args], { fault }); }+ async committedSince(at: string): Promise<number> {+ return (await this.engine!.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM persistence_requests WHERE state='committed' AND completed_at > $1::timestamptz AND operation='submit_job'`, [at]))[0].n;+ }+ async now(): Promise<string> { return (await this.engine!.executeRaw<{ at: string }>('SELECT now()::text AS at'))[0].at; }++ async setup(seed: (b: Brain) => void, config: Record<string, string> = {}): Promise<void> {+ assertSafeE2eDatabaseUrl(this.knobs.directTarget);+ this.home = mkdtempSync(join(tmpdir(), 'gb-conv-'));+ mkdirSync(join(this.home, '.gbrain'));+ this.database = `gbrain_test_${this.knobs.tag.replaceAll('-', '_')}_${randomUUID().replaceAll('-', '')}`;+ this.admin = postgres(this.knobs.directTarget, { max: 1, prepare: false });+ await this.admin.unsafe(`CREATE DATABASE ${this.database}`);+ const direct = new URL(this.knobs.directTarget); direct.pathname = `/${this.database}`;+ const pooled = new URL(this.knobs.pooledTarget); pooled.pathname = `/${this.database}`;+ this.pooledProxy = new LatencyProxy({ host: pooled.hostname, port: Number(pooled.port) || 5432 }, this.knobs.latencyMs / 2);+ this.directProxy = new LatencyProxy({ host: direct.hostname, port: Number(direct.port) || 5432 }, this.knobs.latencyMs / 2);+ await Promise.all([this.pooledProxy.listen(), this.directProxy.listen()]);+ const viaPooled = new URL(pooled); viaPooled.hostname = '127.0.0.1'; viaPooled.port = String(this.pooledProxy.port); viaPooled.searchParams.set('prepare', 'false');+ const viaDirect = new URL(direct); viaDirect.hostname = '127.0.0.1'; viaDirect.port = String(this.directProxy.port);+ this.pooledUrl = viaPooled.toString(); this.directUrl = viaDirect.toString();+ this.engine = new PostgresEngine();+ await withEnv({ GBRAIN_HOME: this.home, GBRAIN_DIRECT_DATABASE_URL: undefined }, async () => {+ await this.engine!.connect({ database_url: direct.toString(), poolSize: 4 });+ await this.engine!.initSchema();+ });+ writeFileSync(join(this.home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: this.pooledUrl, embedding_disabled: true }));+ this.repo = mkdtempSync(join(tmpdir(), 'gb-conv-repo-'));+ this.git('init', '-q'); this.git('config', 'user.email', '[email protected]'); this.git('config', 'user.name', 'Example');+ seed(this);+ await withEnv({ GBRAIN_HOME: this.home }, async () => {+ await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ await this.engine!.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [SOURCE, this.repo]);+ await claimWorktree(this.engine!, SOURCE, this.repo);+ for (const [key, value] of Object.entries(config)) await this.engine!.setConfig(key, value);+ await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');+ });+ this.note(`seed=${this.knobs.seed} latency=${this.knobs.latencyMs}ms round trip via proxies (pooled -> ${pooled.host}, direct -> ${direct.host}); modes ${this.modes.join(',')}`);+ // Phase 0: a fault-free import, so the oracle is calibrated before any chaos.+ const started = performance.now();+ const baseline = await this.sync(['--no-pull', '--no-delegate', '--json']).result;+ this.elapsed.baseline_import_ms = performance.now() - started;+ if (baseline.code !== 0) throw new Error(`baseline sync exited ${baseline.code}\n${baseline.out}\n${baseline.err}\n${this.context()}`);+ this.calibration = await this.diskDbMismatches(new Set());+ if (existsSync(join(this.repo, `${BIG}.md`))) {+ const rows = await this.childRowProblems();+ this.calibration.push(...rows.problems, ...(rows.timelineDuplicates || rows.timelineStrays ? [`timeline: ${rows.timelineDuplicates} duplicate and ${rows.timelineStrays} stray rows after a fault-free import`] : []));+ }+ this.note(`phase 0: fault-free import in ${(this.elapsed.baseline_import_ms / 1000).toFixed(1)}s; oracle calibration problems: ${this.calibration.length}`);+ }++ /** A fresh resident serve and at most MAX_RECOVERY_RUNS plain syncs, no operator step. */+ async recover(): Promise<void> {+ const started = performance.now();+ const serve = new StdioServe(this);+ await serve.start();+ for (let run = 1; run <= MAX_RECOVERY_RUNS; run++) {+ const r = await this.sync(['--no-pull', '--json']).result;+ const envelope = lastJson(r.out);+ this.recoveryRuns.push({ ...r, envelope });+ this.keepLog(`recovery-${run}-client.log`, `${r.out}\n${r.err}`);+ this.note(`recovery run ${run}: exit ${r.code} in ${(r.ms / 1000).toFixed(1)}s outcome=${String(envelope?.outcome ?? envelope?.sync_status ?? envelope?.error ?? 'none')}`);+ if (r.code === 0 && envelope?.outcome === 'synced') break;+ }+ this.elapsed.recovery_ms = performance.now() - started;+ this.keepLog('recovery-serve.log', serve.stderr);+ this.note(`elapsed: ${Object.entries(this.elapsed).map(([k, ms]) => `${k.replace(/_ms$/, '')} ${(ms / 1000).toFixed(1)}s`).join(', ')}; recovery took ${this.recoveryRuns.length} sync run(s)`);+ }++ /**+ * #6841: an accepted write still queued when its writer gave up may commit later, after a write the ledger holds.+ * Once the brain has converged, each one's receipt is read the way the write protocol tells a client to after exit+ * 10 (get_write_request). A committed write is acknowledged then: it may have landed any time since its start, so it+ * is concurrent with every write in that window. Any other state means it never landed and stays out of the ledger.+ */+ async settlePending(receiptState = async (requestId: string) =>+ lastJson((await this.spawnCli(['call', '--source', SOURCE, 'get_write_request', JSON.stringify({ request_id: requestId })]).result).out)?.state): Promise<void> {+ for (const { write, requestId } of this.pending) {+ const state = await receiptState(requestId);+ if (state === 'committed') this.ledger.record({ ...write, ack: performance.now() });+ this.note(`accepted ${write.kind} of ${write.slug} (request ${requestId}) unsettled when its writer stopped: receipt ${String(state ?? 'unreadable')}`);+ }+ }++ async teardown(): Promise<void> {+ for (const serve of this.serves) await serve.stop().catch(() => {});+ try { await this.engine?.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1'); } catch { /* best effort */ }+ await this.engine?.disconnect().catch(() => {});+ await this.pooledProxy?.close().catch(() => {});+ await this.directProxy?.close().catch(() => {});+ if (this.admin) {+ try { if (this.database) await this.admin.unsafe(`DROP DATABASE IF EXISTS ${this.database} WITH (FORCE)`); } finally { await this.admin.end(); }+ }+ for (const dir of [this.home, this.repo]) if (dir) rmSync(dir, { recursive: true, force: true });+ }++ // ------------------------------------------------------------ oracle reads++ diskPages(): Map<string, string> {+ const out = new Map<string, string>();+ const walk = (dir: string) => {+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ if (entry.name.startsWith('.')) continue;+ const full = join(dir, entry.name);+ if (entry.isDirectory()) walk(full);+ else if (entry.name.endsWith('.md')) out.set(relative(this.repo, full).replace(/\.md$/, ''), readFileSync(full, 'utf8'));+ }+ };+ walk(this.repo);+ return out;+ }+ async dbPages(): Promise<Map<string, { hash: string | null; body: string }>> {+ const rows = await this.engine!.executeRaw<{ slug: string; content_hash: string | null; body: string }>(+ `SELECT slug, content_hash, coalesce(compiled_truth,'') || E'\\n' || coalesce(timeline,'') AS body FROM pages WHERE source_id=$1 AND deleted_at IS NULL`, [SOURCE]);+ return new Map(rows.map(r => [r.slug, { hash: r.content_hash, body: r.body }]));+ }+ async diskDbMismatches(exclude: Set<string>): Promise<string[]> {+ const disk = this.diskPages(), db = await this.dbPages(), out: string[] = [];+ for (const [slug, content] of disk) {+ if (exclude.has(slug)) continue;+ if (!db.has(slug)) out.push(`${slug}: file exists, no live page`);+ else if (db.get(slug)!.hash !== fileHash(slug, content)) out.push(`${slug}: page content_hash differs from the file`);+ }+ for (const slug of db.keys()) if (!disk.has(slug) && !exclude.has(slug)) out.push(`${slug}: live page, no file`);+ return out;+ }+ async ledgerProblems(): Promise<string[]> {+ const facts = new Set((await this.engine!.executeRaw<{ fact: string }>('SELECT fact FROM facts WHERE source_id=$1 AND expired_at IS NULL', [SOURCE])).map(r => r.fact));+ return this.ledger.problems(this.diskPages(), await this.dbPages(), facts);+ }+ async openRequests(): Promise<unknown[]> {+ const open = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; recovery: boolean; blocked_reason: string | null }>(+ `SELECT request_id::text, slug, state, recovery IS NOT NULL AS recovery, blocked_reason FROM persistence_requests+ WHERE state IN ('queued','deferred','running','recovering') OR recovery IS NOT NULL ORDER BY sequence`);+ const worktrees = await this.engine!.executeRaw<{ id: string; state: string }>(`SELECT id::text, state FROM persistence_worktrees WHERE state<>'active'`);+ return [...open.map(r => `open request ${JSON.stringify(r)}`), ...worktrees.map(w => `worktree ${w.id} is ${w.state}`)];+ }+ async unexplainedTerminal(exclude: Set<string>): Promise<string[]> {+ const rows = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; error_code: string | null; operation: string }>(+ `SELECT request_id::text, slug, state, error_code, operation FROM persistence_requests WHERE state IN ('conflict','failed','cancelled') ORDER BY sequence`);+ const disk = this.diskPages(), db = await this.dbPages();+ this.note(`terminal non-committed requests: ${rows.length} (${[...new Set(rows.map(r => `${r.state}:${r.error_code}`))].join(', ') || 'none'})`);+ return rows.filter(row => {+ const content = disk.get(row.slug);+ const superseded = exclude.has(row.slug) || (content !== undefined && db.get(row.slug)?.hash === fileHash(row.slug, content));+ return !superseded && !(row.error_code && row.error_code in CODES);+ }).map(row => `unexplained ${row.state} request ${row.request_id} (${row.slug}, ${row.error_code ?? 'no code'})`);+ }+ async cursorProblems(): Promise<string[]> {+ const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);+ const head = this.git('rev-parse', 'HEAD');+ return source.last_commit === head ? [] : [`sources.last_commit ${source.last_commit} is not HEAD ${head}`];+ }+ async status(detail = true): Promise<Record<string, unknown> | null> {+ const r = await this.spawnCli(['sync', 'status', '--source', SOURCE, '--json', ...(detail ? ['--detail'] : [])]).result;+ this.keepLog(detail ? 'status.json' : 'status-fast.json', `${r.out}\n${r.err}`);+ return lastJson(r.out);+ }+ recoveryProblems(): string[] {+ const last = this.recoveryRuns.at(-1);+ return last?.code === 0 && last.envelope?.outcome === 'synced' ? []+ : [`no plain sync succeeded within ${MAX_RECOVERY_RUNS} runs: ${this.recoveryRuns.map(r => `exit ${r.code} outcome ${String(r.envelope?.outcome ?? r.envelope?.error ?? 'none')}`).join('; ')}`];+ }+ /**+ * The large page's child rows against its file. Facts are keyed by fence row, so+ * loss, duplication and strays are all problems. Timeline rows are judged for+ * loss only; duplicate and stray timeline rows come back as a measured count,+ * because derived timeline rows are never pruned (a known defect outside #6650).+ */+ async childRowProblems(): Promise<{ problems: string[]; timelineDuplicates: number; timelineStrays: number }> {+ const content = this.diskPages().get(BIG);+ if (!content) return { problems: [`${BIG}: no file`], timelineDuplicates: 0, timelineStrays: 0 };+ const out: string[] = [];+ const expectedTimeline = [...content.matchAll(/^- \*\*(\d{4}-\d{2}-\d{2})\*\* \| (.+)$/gm)].map(m => `${m[1]}|${m[2]}`);+ const timeline = await this.engine!.executeRaw<{ key: string; n: number }>(+ `SELECT t.date::text || '|' || t.summary AS key, count(*)::int AS n FROM timeline_entries t JOIN pages p ON p.id=t.page_id+ WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL AND t.event_page_id IS NULL GROUP BY 1`, [SOURCE, BIG]);+ const have = new Map(timeline.map(r => [r.key, r.n]));+ const lostTimeline = expectedTimeline.filter(k => !have.has(k));+ const dupTimeline = timeline.filter(r => r.n > 1);+ const extraTimeline = timeline.filter(r => !expectedTimeline.includes(r.key));+ if (lostTimeline.length) out.push(`timeline: ${lostTimeline.length} of ${expectedTimeline.length} file entries missing (first: ${lostTimeline[0]})`);+ const expectedFacts = parseFactsFence(content).facts.map(f => `${f.rowNum}|${f.claim}`);+ const facts = await this.engine!.executeRaw<{ key: string; n: number }>(+ `SELECT row_num::text || '|' || fact AS key, count(*)::int AS n FROM facts+ WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL GROUP BY 1`, [SOURCE, BIG]);+ const haveFacts = new Map(facts.map(r => [r.key, r.n]));+ const lostFacts = expectedFacts.filter(k => !haveFacts.has(k));+ const dupFacts = facts.filter(r => r.n > 1);+ const extraFacts = facts.filter(r => !expectedFacts.includes(r.key));+ if (lostFacts.length) out.push(`facts: ${lostFacts.length} of ${expectedFacts.length} fence rows missing (first: ${lostFacts[0]})`);+ if (dupFacts.length) out.push(`facts: ${dupFacts.length} rows duplicated (first: ${dupFacts[0].key} x${dupFacts[0].n})`);+ if (extraFacts.length) out.push(`facts: ${extraFacts.length} rows not in the fence (first: ${extraFacts[0].key})`);+ return { problems: out, timelineDuplicates: dupTimeline.reduce((n, r) => n + r.n - 1, 0), timelineStrays: extraTimeline.length };+ }+}++export function fileHash(slug: string, content: string): string {+ const parsed = parseMarkdown(content, `${slug}.md`);+ return contentHash({ title: parsed.title, type: parsed.type, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline, frontmatter: parsed.frontmatter, tags: [...parsed.tags].sort() });+}++/** One stdio MCP session against a spawned `gbrain serve`; the serve is the host's resident persistence owner. */+export class StdioServe {+ readonly child: ChildProcess;+ stderr = '';+ exited: Promise<number | null>;+ private nextId = 1;+ private pending = new Map<number, (msg: Record<string, unknown>) => void>();+ private buf = '';+ constructor(private brain: Brain, fault?: Fault) {+ const { preload, env } = brain.faultArgs(fault);+ this.child = spawn(process.execPath, ['--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), 'serve'], { cwd: REPO_ROOT, env: brain.childEnv(env), stdio: ['pipe', 'pipe', 'pipe'] });+ this.exited = new Promise(done => this.child.once('exit', code => done(code)));+ this.child.stderr!.on('data', (d: Buffer) => { this.stderr += d.toString('utf8'); });+ this.child.stdin!.on('error', () => {});+ this.child.stdout!.on('data', (d: Buffer) => {+ this.buf += d.toString('utf8');+ let idx: number;+ while ((idx = this.buf.indexOf('\n')) !== -1) {+ const line = this.buf.slice(0, idx).trim(); this.buf = this.buf.slice(idx + 1);+ if (!line) continue;+ try {+ const msg = JSON.parse(line) as Record<string, unknown>;+ const id = msg.id as number | undefined;+ if (id !== undefined && this.pending.has(id)) { this.pending.get(id)!(msg); this.pending.delete(id); }+ } catch { /* serve chatter */ }+ }+ });+ brain.serves.push(this);+ }+ get pid(): number { return this.child.pid!; }+ get alive(): boolean { return this.child.exitCode === null && this.child.signalCode === null; }+ request(method: string, params: Record<string, unknown>, timeoutMs = 240_000): Promise<Record<string, unknown>> {+ const id = this.nextId++;+ return new Promise((done, reject) => {+ const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`rpc ${method} timed out`)); }, timeoutMs);+ this.pending.set(id, msg => { clearTimeout(t); done(msg); });+ this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');+ });+ }+ async start(): Promise<void> {+ await this.request('initialize', { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'convergence', version: '1' } });+ this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');+ await until(async () => (await this.brain.engine!.executeRaw<{ n: number }>(+ `SELECT count(*)::int AS n FROM persistence_consumers WHERE pid=$1 AND kind='serve' AND renewed_at > now() - interval '30 seconds'`, [this.pid]))[0].n > 0,+ `serve ${this.pid} heartbeat`, 120_000);+ }+ async stop(): Promise<void> {+ if (!this.alive) return;+ this.child.kill('SIGTERM');+ const timer = setTimeout(() => this.child.kill('SIGKILL'), 10_000);+ await this.exited; clearTimeout(timer);+ }+}++/**+ * A writer call that retries an unsettled attempt (exit 10, a lost owner) with the same request id and a terminal refusal with a new one.+ * When the attempts or the chaos run out on an unsettled attempt, the request it names may still commit, so it comes back `pending`.+ */+export async function acknowledged(b: Brain, label: string, call: (requestId: string) => SyncRun, stop: () => boolean): Promise<{ start: number; ack: number } | { pending: string } | null> {+ let requestId = randomUUID();+ let unsettled = false;+ for (let attempt = 1; attempt <= 20 && !stop(); attempt++) {+ const start = performance.now();+ const r = await call(requestId).result;+ b.keepLog(`writer-${label.replaceAll('/', '_')}-${attempt}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);+ if (r.code === 0) return { start, ack: performance.now() };+ const terminal = /\((conflict|failed|cancelled)\)/.test(r.err);+ b.note(`writer ${label} attempt ${attempt} exit ${r.code}${terminal ? ' (terminal; next attempt uses a new request id)' : ''}: ${r.err.trim().split('\n').slice(-1)[0] ?? ''}`);+ unsettled = !terminal;+ if (terminal) requestId = randomUUID();+ await Bun.sleep(3000);+ }+ return unsettled ? { pending: requestId } : null;+}+export const outcome = (ok: { ack: number } | { pending: string } | null) => !ok ? 'unacknowledged' : 'pending' in ok ? `pending ${ok.pending}` : 'ok';++/** Oracle tests are named by the persistence gate's violation class (scripts/persistence/model.ts) they would report. */+export const as = (violation: ViolationClass, title: string): string => `[${violation}] ${title}`;++// ---------------------------------------------------------------- the shared oracle++export interface EndState { exclude?: Set<string>; status?: StatusExpectation; bigPage?: boolean }+/** The oracle tests every scenario ends with; `end()` is read when the tests run (after the scenario's beforeAll). */+export function oracleTests(b: Brain, end: () => EndState): void {+ test('the oracle is calibrated: a fault-free import satisfies it', () => b.verdict(b.calibration));+ test('every planned fault fired, so the chaos actually happened', () => b.verdict(Object.entries(b.fired).filter(([, at]) => at === null).map(([fault]) => `${fault} did not fire`)));+ test(as('wedge', `a plain sync succeeds within ${MAX_RECOVERY_RUNS} runs after the chaos, with no operator step`), () => b.verdict(b.recoveryProblems()));+ test(as('projection_drift', 'disk == database for every page the end state does not name'), async () => b.verdict(await b.diskDbMismatches(end().exclude ?? new Set())));+ test(as('wedge', 'no request blocks the checkpoint or a root'), async () => b.verdict((await b.openRequests()).map(String)));+ test(as('untyped_error', 'every non-committed terminal request is superseded or carries a registered error code'), async () => b.verdict(await b.unexplainedTerminal(end().exclude ?? new Set())));+ test(as('wedge', 'the cursor reached HEAD'), async () => b.verdict(await b.cursorProblems()));+ test(as('lost_write', 'every acknowledged write survives or is causally superseded by a later acknowledged write'), async () => {+ b.note(`ledger: ${b.ledger.acks.length} acknowledged writes (${[...new Set(b.ledger.acks.map(a => a.kind))].map(kind => `${kind} ${b.ledger.acks.filter(a => a.kind === kind).length}`).join(', ')})`);+ b.verdict(await b.ledgerProblems());+ });+ test(as('wedge', 'sync status --json reports the converged state with every field present'), async () => b.verdict(statusProblems(await b.status(), end().status)));+ test(as('wedge', 'plain sync status --json names every heavy read it skipped instead of reading as all-clear'), async () => b.verdict(fastStatusProblems(await b.status(false))));+ if (end().bigPage !== false) test(as('lost_write', 'the large page keeps every fence fact exactly once and loses no timeline entry'), async () => {+ const { problems, timelineDuplicates, timelineStrays } = await b.childRowProblems();+ b.note(`large page timeline (measured, not asserted): ${timelineDuplicates} duplicate rows, ${timelineStrays} rows not in the file`);+ b.verdict(problems);+ });+}diff --git a/test/scripts/ci-pr-scope.test.ts b/test/scripts/ci-pr-scope.test.tsindex 71870fa31d..6c336b7e2e 100644--- a/test/scripts/ci-pr-scope.test.ts+++ b/test/scripts/ci-pr-scope.test.ts@@ -225,7 +225,7 @@ esacfor (const path of ['src/core/persistence/sync-run.ts', 'src/core/persistence/fault-points.ts', 'src/commands/sync/run.ts', 'src/commands/sync.ts','src/core/sync-holds.ts', 'src/core/page-state/projections.ts', 'src/core/engine-sql/links.ts', 'src/core/storage/local.ts', 'src/core/storage-config.ts','src/core/import-file.ts', 'src/core/markdown.ts', 'src/core/write-through.ts', 'src/core/postgres-engine.ts',- 'test/sync-convergence.serial.test.ts', 'test/fixtures/convergence-fault-kill.ts', 'scripts/ci-convergence-scope.sh', '.github/workflows/persistence-validation.yml',+ 'test/sync-convergence.serial.test.ts', 'test/fixtures/convergence-fault-kill.ts', 'test/helpers/convergence-brain.ts', 'scripts/ci-convergence-scope.sh', '.github/workflows/persistence-validation.yml','.github/workflows/convergence-nightly.yml', 'bun.lock', 'vendor/postgres/src/connection.js']) {expect(convergenceScope(`README.md\n${path}\n`), path).toBe('run');}diff --git a/test/sync-convergence.serial.test.ts b/test/sync-convergence.serial.test.tsindex 46764984a3..998c72dd5b 100644--- a/test/sync-convergence.serial.test.ts+++ b/test/sync-convergence.serial.test.ts@@ -56,29 +56,19 @@* skipped unless GBRAIN_TEST_CONVERGENCE=1.*/import { afterAll, beforeAll, describe, expect, test } from 'bun:test';-import { execFileSync, spawn, type ChildProcess } from 'node:child_process';+import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';-import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';-import { dirname, join, relative, resolve } from 'node:path';-import postgres from '#postgres';-import { PostgresEngine } from '../src/core/postgres-engine.ts';-import { claimWorktree } from '../src/core/persistence/ownership.ts';-import { parseMarkdown } from '../src/core/markdown.ts';-import { parseFactsFence } from '../src/core/facts-fence.ts';-import { contentHash } from '../src/core/utils.ts';-import { CODES } from '../src/core/error-registry.ts';+import { join } from 'node:path';import { CONSUMER_LIVE_MS, CONSUMER_RENEWAL_MS } from '../src/core/persistence/consumer-heartbeat.ts';import type { FaultPoint } from '../src/core/persistence/fault-points.ts';import { PUBLICATION_CEILING_DEFAULT_MS, publicationLeaseMs } from '../src/core/persistence/publication-deadline.ts';-import { assertSafeE2eDatabaseUrl } from './helpers/db-guard.ts';-import { keylessBrainEnv } from './helpers/provider-env.ts';-import { withEnv } from './helpers/with-env.ts';-import { LatencyProxy } from './helpers/latency-proxy.ts';-import type { ViolationClass } from '../scripts/persistence/model.ts';+import {+ Brain as HarnessBrain, ENTITY, Ledger, SOURCE, STATUS_FIELDS, StdioServe, acknowledged, as, entityText, fastStatusProblems, lastJson,+ noteMarker, noteSlug, noteText, oracleTests, outcome, prng, statusProblems, until, worstStall, type Ack, type Fault, type HarnessKnobs,+} from './helpers/convergence-brain.ts';-const REPO_ROOT = resolve(import.meta.dir, '..');-const FAULT_PRELOAD = join(REPO_ROOT, 'test/fixtures/convergence-fault-kill.ts');const knob = (name: string, fallback: number): number => {const raw = process.env[`GBRAIN_TEST_CONVERGENCE_${name}`];const value = raw === undefined || raw === '' ? fallback : Number(raw);@@ -120,9 +110,15 @@ const requestedModes = process.env.GBRAIN_TEST_CONVERGENCE_MODES?.split(',').mapfor (const mode of requestedModes ?? []) if (!(mode in MODES)) throw new Error(`GBRAIN_TEST_CONVERGENCE_MODES names unknown mode '${mode}' (known: ${Object.keys(MODES).join(', ')}).`);const runs = (mode: Mode): boolean => requestedModes ? requestedModes.includes(mode) : MODES[mode].skip === null;-const SOURCE = 'workspace';-const BIG = 'companies/acme-example';-const ENTITY = 'people/alice-example';+const KNOBS: HarnessKnobs = {+ tag: 'convergence', seed: SEED, latencyMs: LATENCY_MS, syncTimeoutMs: SYNC_TIMEOUT_MS, logDir: LOG_DIR,+ pooledTarget: pooledTarget ?? '', directTarget: directTarget ?? '', summary: `notes ${NOTES}; latency ${LATENCY_MS} ms`,+ repro: modes => `${[`GBRAIN_TEST_CONVERGENCE_SEED=${SEED}`, `GBRAIN_TEST_CONVERGENCE_MODES=${modes.join(',')}`, `GBRAIN_TEST_CONVERGENCE_NOTES=${NOTES}`,+ `GBRAIN_TEST_CONVERGENCE_KILLS=${KILLS}`, `GBRAIN_TEST_CONVERGENCE_LATENCY_MS=${LATENCY_MS}`].join(' ')} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun run test:convergence`,+};+/** One scenario's brain: the shared harness brain (test/helpers/convergence-brain.ts) with this gate's knobs. */+class Brain extends HarnessBrain { constructor(name: string, modes: Mode[]) { super(name, modes, KNOBS); } }+/*** The kill loop's fault points: the seams a managed sync import crosses. Its pages are already in Git, so the* file-publication pair (`before_publication`/`after_publication`) never fires for it. Per-page seams are counted@@ -145,9 +141,6 @@ const KILL_POINTS = [// #6712 PR 2 (L5): after the checkpoint committed and the lease was released, before the owed tail runs (the next sync runs it).{ point: 'sync:tail_started', per: 'group' },] as const satisfies ReadonlyArray<{ point: FaultPoint; per: 'page' | 'group' | 'save' | 'held' | 'deferred' }>;-const MAX_RECOVERY_RUNS = 2;-/** Oracle tests are named by the persistence gate's violation class (scripts/persistence/model.ts) they would report. */-const as = (violation: ViolationClass, title: string): string => `[${violation}] ${title}`;/*** The longest a sync may sit with no progress after an owner was killed: the* window in which a dead owner's heartbeat still reads live, one renewal@@ -156,520 +149,8 @@ const as = (violation: ViolationClass, title: string): string => `[${violation}]*/const OWNER_KILL_STALL_BOUND_S = (CONSUMER_LIVE_MS + CONSUMER_RENEWAL_MS + 30_000) / 1000;-/** mulberry32: a fixed seed gives the same fault plan and writer targets on every run. */-function prng(seed: number): () => number {- let a = seed >>> 0;- return () => {- a = (a + 0x6d2b79f5) >>> 0;- let t = a;- t = Math.imul(t ^ (t >>> 15), t | 1);- t ^= t + Math.imul(t ^ (t >>> 7), t | 61);- return ((t ^ (t >>> 14)) >>> 0) / 4294967296;- };-}--// ---------------------------------------------------------------- the status oracle--/** Every `gbrain sync status --json` field the operator loop reads (sync-status.ts `SyncStatus`). */-const STATUS_FIELDS = ['source_id', 'run_id', 'cursor', 'committed_last_10m', 'live_job_id', 'holds', 'last_error', 'needs_human',- 'resume_argv', 'checkpoint_blockers', 'cursors', 'terminal_failures', 'next'] as const;-interface StatusExpectation { holds?: Array<{ code: string; path: string }>; needsHuman?: boolean }-/**- * What a converged source's status must say. Field-present assertions first: a renderer that drops a field fails- * here, not in an agent's loop. `checkpoint_blockers` and `cursors` are arrays on every status (#6712 PR 0 made- * `checkpoint_blockers` always present). The harness reads `sync status --json --detail`: since #6712 PR 2 the- * terminal summary, blocker list and cursor join run only behind the flag.- */-/** The default (no `--detail`) status after the chaos: every skipped heavy read says so and names the `--detail` command (#6712 PR 2, DX F1). */-function fastStatusProblems(status: Record<string, unknown> | null): string[] {- if (!status) return ['sync status --json printed no JSON document'];- return (['checkpoint_blockers_read', 'cursors_read', 'terminal_failures_read'] as const).flatMap(field => {- const read = status[field] as { status?: string; fix?: { argv?: string[] } } | undefined;- return read?.status === 'not_read' && read.fix?.argv?.includes('--detail') ? [] : [`sync status --json: ${field} is ${JSON.stringify(read)}, expected not_read with a --detail fix`];- });-}--function statusProblems(status: Record<string, unknown> | null, expected: StatusExpectation = {}): string[] {- if (!status) return ['sync status --json printed no JSON document'];- const out = STATUS_FIELDS.filter(field => !(field in status)).map(field => `sync status --json: field ${field} missing`);- if (out.length) return out;- const cursor = status.cursor as { done?: boolean } | null;- if (!Array.isArray(status.cursors)) out.push('sync status --json: cursors is not an array');- if (!Array.isArray(status.checkpoint_blockers)) out.push('sync status --json: checkpoint_blockers is not an array');- else if (status.checkpoint_blockers.length) out.push(`sync status --json: ${status.checkpoint_blockers.length} checkpoint blocker(s)`);- // A finished run's cursor is retired, so a converged source shows none (null) or a done one.- if (cursor && cursor.done !== true) out.push(`sync status --json: cursor not done (${JSON.stringify(cursor)})`);- const terminal = status.terminal_failures as { status?: string; active_blockers?: unknown[] } | null;- if (terminal?.status !== 'ok') out.push(`sync status --json: terminal_failures.status is ${terminal?.status}`);- else if (!Array.isArray(terminal.active_blockers)) out.push('sync status --json: terminal_failures.active_blockers is not an array');- else if (terminal.active_blockers.length) out.push(`sync status --json: ${terminal.active_blockers.length} active terminal blocker(s)`);- if (status.needs_human !== (expected.needsHuman ?? false)) out.push(`sync status --json: needs_human is ${status.needs_human}, expected ${expected.needsHuman ?? false}`);- const holds = Array.isArray(status.holds) ? (status.holds as Array<{ code: string; path: string }>).map(h => `${h.code}:${h.path}`).sort() : ['holds is not an array'];- const want = (expected.holds ?? []).map(h => `${h.code}:${h.path}`).sort();- if (JSON.stringify(holds) !== JSON.stringify(want)) out.push(`sync status --json: holds [${holds.join(', ')}], expected [${want.join(', ')}]`);- if (status.last_error !== null) out.push(`sync status --json: last_error ${JSON.stringify(status.last_error)}`);- if (!expected.holds?.length && status.next !== null) out.push(`sync status --json: next ${JSON.stringify(status.next)}`);- return out;-}--// ---------------------------------------------------------------- the acknowledged-write ledger--/**- * One writer operation that returned success, recorded by the harness at the moment it saw the success (never read- * back from the brain), or a write the brain accepted (exit 10) that had not settled when its writer stopped, recorded- * when its receipt, read after recovery, said committed (`Brain.settlePending`). `touches` is where the write landed: the page in the database, its file, or (for `remember`)- * an additive fact row. `marker` is text that only this write put there; `null` on a page location is a deletion.- */-interface Ack { kind: 'commit' | 'put' | 'remember' | 'maintenance' | 'db_only' | 'git_mv' | 'git_rm' | 'unlink'; slug: string;- marker: string | null; touches: Array<'db' | 'file'> | 'fact'; start: number; ack: number }-class Ledger {- acks: Ack[] = [];- record(ack: Ack): void { this.acks.push(ack); }- /**- * For every slug and location, the writes no later acknowledged write causally supersedes (none started after it- * was acknowledged); the final state must show one of them. An additive fact survives unless a page write on its- * slug may have been ordered after it.- */- problems(disk: Map<string, string>, db: Map<string, { body: string } | null>, facts: Set<string>): string[] {- const out: string[] = [];- const pageWrites = this.acks.filter(a => a.touches !== 'fact' && a.kind !== 'maintenance');- for (const slug of new Set(pageWrites.map(a => a.slug))) {- for (const where of ['db', 'file'] as const) {- const writes = pageWrites.filter(a => a.slug === slug && (a.touches as string[]).includes(where));- if (!writes.length) continue;- const maximal = writes.filter(w => !writes.some(later => later.start > w.ack));- const seen = where === 'file' ? disk.get(slug) ?? null : db.get(slug)?.body ?? null;- if (maximal.some(w => w.marker === null ? seen === null : seen?.includes(w.marker))) continue;- out.push(`${where} ${slug}: lost acknowledged write; has ${seen === null ? 'nothing' : JSON.stringify(seen.slice(0, 80))}, expected one of ${maximal.map(w => `${w.kind}@${Math.round(w.ack)}:${w.marker === null ? '(absent)' : JSON.stringify(w.marker)}`).join(' | ')}`);- }- }- for (const fact of this.acks.filter(a => a.touches === 'fact')) {- const excused = pageWrites.some(w => w.slug === fact.slug && (w.touches as string[]).includes('db') && w.ack > fact.start);- if (!excused && !facts.has(fact.marker!)) out.push(`fact on ${fact.slug}: lost acknowledged remember ${JSON.stringify(fact.marker)}`);- }- return out;- }-}--// ---------------------------------------------------------------- the fixture--function noteText(i: number, version: string): string {- return `---\ntype: concept\ntitle: Note ${i} Example\n---\n\n${version} body for note ${i}, an observation about widget-co and acme-example.\n\n${'Supporting detail sentence. '.repeat(8 + (i % 5))}\n`;-}-const noteMarker = (i: number, version: string) => `${version} body for note ${i},`;-const noteSlug = (i: number) => `notes/note-${String(i).padStart(4, '0')}`;-function bigPage(facts: number, timeline: number, version: string): string {- const day = (n: number) => new Date(Date.UTC(2018, 0, 1) + n * 86_400_000).toISOString().slice(0, 10);- const paragraphs = Array.from({ length: 30 }, (_, i) => `Section ${i} (${version}): acme-example ${'expanded with a steady stream of synthetic operating detail. '.repeat(6)}`);- const rows = Array.from({ length: facts }, (_, i) =>- `| ${i + 1} | Synthetic claim ${i + 1} about acme-example | fact | 0.9 | world | medium | ${day(i)} | | fixture | |`);- const bullets = Array.from({ length: timeline }, (_, i) => `- **${day(i)}** | Synthetic event ${i + 1} at acme-example`);- return [- '---', 'type: company', 'title: Acme Example', '---', '', '# Acme Example', '', ...paragraphs.flatMap(p => [p, '']),- '## Facts', '', '<!--- gbrain:facts:begin -->',- '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |',- '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|',- ...rows, '<!--- gbrain:facts:end -->', '', '<!-- timeline -->', '', ...bullets, '',- ].join('\n');-}-const entityText = '---\ntype: person\ntitle: Alice Example\n---\n\nAlice Example works with acme-example.\n';--// ---------------------------------------------------------------- one brain--interface Fault { point: FaultPoint; nth: number; marker: string; operation?: string; path?: string; action?: 'kill' | 'edit' | 'stall' | 'throw';- edit?: { file: string; content: string }; release?: string; ms?: number; sqlstate?: string;- prime?: { point: FaultPoint; nth: number; path?: string; edit: { file: string; content: string } } }-interface Run { code: number; out: string; err: string; ms: number }-interface SyncRun { proc: ReturnType<typeof Bun.spawn>; result: Promise<Run> }--async function until(check: () => boolean | Promise<boolean>, label: string, timeoutMs = 300_000): Promise<void> {- const deadline = performance.now() + timeoutMs;- while (performance.now() < deadline) {- if (await check()) return;- await Bun.sleep(100);- }- throw new Error(`timed out waiting for ${label}`);-}-function lastJson(out: string): Record<string, unknown> | null {- try { return JSON.parse(out) as Record<string, unknown>; } catch { /* several documents */ }- for (const line of out.trim().split('\n').reverse()) { try { return JSON.parse(line) as Record<string, unknown>; } catch { /* not the envelope */ } }- return null;-}-/** The longest `stalled Ns` progress line a sync printed: how long it waited on a predecessor's leftovers. */-function worstStall(err: string): number { return Math.max(0, ...[...err.matchAll(/stalled (\d+)s/g)].map(m => Number(m[1]))); }--/**- * One scenario's brain: its database, home, Git worktree, proxies, child processes, ledger and log. `context()` is- * what every failure prints: seed, modes, the fault plan and the one-line repro.- */-class Brain {- home = ''; repo = ''; database = '';- admin: ReturnType<typeof postgres> | undefined;- engine: PostgresEngine | undefined;- pooledProxy: LatencyProxy | undefined; directProxy: LatencyProxy | undefined;- pooledUrl = ''; directUrl = '';- serves: StdioServe[] = [];- ledger = new Ledger();- /** Writes the brain accepted (exit 10) that had not settled when their writer stopped; `settlePending` reads their receipts after recovery. */- pending: Array<{ write: Omit<Ack, 'ack'>; requestId: string }> = [];- plan: string[] = [];- fired: Record<string, string | null> = {};- killedPids = new Set<number>();- recoveryRuns: Array<Run & { envelope: Record<string, unknown> | null }> = [];- elapsed: Record<string, number> = {};- calibration: string[] = [];- readonly rand: () => number;- constructor(readonly name: string, readonly modes: Mode[]) { this.rand = prng(SEED ^ [...name].reduce((h, c) => Math.imul(h, 31) + c.charCodeAt(0), 7)); }-- between(lo: number, hi: number): number { return lo + Math.floor(this.rand() * (hi - lo + 1)); }- note(line: string): void {- console.error(`[convergence ${this.name}] ${line}`);- if (LOG_DIR) { mkdirSync(LOG_DIR, { recursive: true }); appendFileSync(join(LOG_DIR, 'convergence.log'), `[${this.name}] ${line}\n`); }- }- keepLog(name: string, text: string): void { if (LOG_DIR) { mkdirSync(join(LOG_DIR, this.name), { recursive: true }); writeFileSync(join(LOG_DIR, this.name, name), text); } }- context(): string {- const env = [`GBRAIN_TEST_CONVERGENCE_SEED=${SEED}`, `GBRAIN_TEST_CONVERGENCE_MODES=${this.modes.join(',')}`, `GBRAIN_TEST_CONVERGENCE_NOTES=${NOTES}`,- `GBRAIN_TEST_CONVERGENCE_KILLS=${KILLS}`, `GBRAIN_TEST_CONVERGENCE_LATENCY_MS=${LATENCY_MS}`];- return [`seed ${SEED}; modes ${this.modes.join(',')}; notes ${NOTES}; latency ${LATENCY_MS} ms`,- `fault plan: ${this.plan.join('; ') || 'none'}`,- `fired: ${Object.entries(this.fired).map(([k, v]) => `${k} => ${v ?? 'did not fire'}`).join('; ') || 'none'}`,- `repro (Postgres + PgBouncer from scripts/persistence/README.md): ${env.join(' ')} GBRAIN_PGBOUNCER_URL="$GBRAIN_PGBOUNCER_URL" GBRAIN_PGBOUNCER_DIRECT_URL="$GBRAIN_PGBOUNCER_DIRECT_URL" bun run test:convergence`].join('\n');- }- /** Throws with the problems and the failure context; a passing check prints nothing. */- verdict(problems: string[]): void {- if (problems.length) throw new Error(`${problems.length} problem(s):\n ${problems.slice(0, 40).join('\n ')}\n${this.context()}`);- }- marker(tag: string): string { return join(this.home, `fault-${tag}.json`); }- recordFault(tag: string, label: string): string | null {- const at = existsSync(this.marker(tag)) ? readFileSync(this.marker(tag), 'utf8') : null;- this.fired[label] = at;- if (at) { const fired = JSON.parse(at) as { pid: number; action: string }; if (fired.action === 'kill') this.killedPids.add(fired.pid); }- return at;- }-- git(...args: string[]): string { return execFileSync('git', ['-C', this.repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }- /**- * Writes files and commits exactly those paths, as the shared-checkout rules require of every committer; each write- * is acknowledged when the commit returns. Sweeping the worktree (`git add -A`) would commit a writer's in-flight- * write-through bytes as an out-of-band Git change of a page that writer is still writing.- */- commit(message: string, files: Array<{ slug: string; content: string; marker: string }>): void {- const start = performance.now();- for (const f of files) { mkdirSync(dirname(join(this.repo, `${f.slug}.md`)), { recursive: true }); writeFileSync(join(this.repo, `${f.slug}.md`), f.content); }- const paths = files.map(f => `${f.slug}.md`);- this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);- const ack = performance.now();- for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });- }- writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {- this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);- }- bigFile(facts: number, timeline: number, version: string) { return { slug: BIG, content: bigPage(facts, timeline, version), marker: `Section 0 (${version})` }; }-- childEnv(extra: Record<string, string | undefined> = {}): Record<string, string> {- return keylessBrainEnv(process.env, this.home, {- DATABASE_URL: this.pooledUrl, GBRAIN_DATABASE_URL: undefined, GBRAIN_DIRECT_DATABASE_URL: this.directUrl,- GBRAIN_BRAIN_ID: 'host', GBRAIN_SOURCE: SOURCE, GBRAIN_SWEEP: '0', GBRAIN_HOOKS: undefined,- GBRAIN_SKIP_STARTUP_HOOKS: '1', GBRAIN_NO_SKILL_NAG: '1', GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS: '300',- GBRAIN_SYNC_NO_DELEGATE: undefined, GBRAIN_TEST_CONVERGENCE_FAULT: undefined,- ...extra,- });- }- faultArgs(fault?: Fault): { preload: string[]; env: Record<string, string | undefined> } {- if (!fault) return { preload: [], env: {} };- return { preload: ['--preload', FAULT_PRELOAD], env: { GBRAIN_TEST_CONVERGENCE_FAULT: JSON.stringify(fault) } };- }- spawnCli(args: string[], opts: { fault?: Fault; stdin?: string; timeoutMs?: number } = {}): SyncRun {- const { preload, env } = this.faultArgs(opts.fault);- const started = performance.now();- const proc = Bun.spawn([process.execPath, '--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), ...args], {- cwd: REPO_ROOT, env: this.childEnv(env), stdin: opts.stdin === undefined ? 'ignore' : new Blob([opts.stdin]), stdout: 'pipe', stderr: 'pipe',- });- const timer = setTimeout(() => proc.kill('SIGKILL'), opts.timeoutMs ?? SYNC_TIMEOUT_MS);- const result = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])- .then(([out, err, code]) => ({ code, out, err, ms: performance.now() - started }))- .finally(() => clearTimeout(timer));- return { proc, result };- }- sync(args: string[], fault?: Fault): SyncRun { return this.spawnCli(['sync', '--source', SOURCE, ...args], { fault }); }- async committedSince(at: string): Promise<number> {- return (await this.engine!.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM persistence_requests WHERE state='committed' AND completed_at > $1::timestamptz AND operation='submit_job'`, [at]))[0].n;- }- async now(): Promise<string> { return (await this.engine!.executeRaw<{ at: string }>('SELECT now()::text AS at'))[0].at; }-- async setup(seed: (b: Brain) => void, config: Record<string, string> = {}): Promise<void> {- assertSafeE2eDatabaseUrl(directTarget!);- this.home = mkdtempSync(join(tmpdir(), 'gb-conv-'));- mkdirSync(join(this.home, '.gbrain'));- this.database = `gbrain_test_convergence_${randomUUID().replaceAll('-', '')}`;- this.admin = postgres(directTarget!, { max: 1, prepare: false });- await this.admin.unsafe(`CREATE DATABASE ${this.database}`);- const direct = new URL(directTarget!); direct.pathname = `/${this.database}`;- const pooled = new URL(pooledTarget!); pooled.pathname = `/${this.database}`;- this.pooledProxy = new LatencyProxy({ host: pooled.hostname, port: Number(pooled.port) || 5432 }, LATENCY_MS / 2);- this.directProxy = new LatencyProxy({ host: direct.hostname, port: Number(direct.port) || 5432 }, LATENCY_MS / 2);- await Promise.all([this.pooledProxy.listen(), this.directProxy.listen()]);- const viaPooled = new URL(pooled); viaPooled.hostname = '127.0.0.1'; viaPooled.port = String(this.pooledProxy.port); viaPooled.searchParams.set('prepare', 'false');- const viaDirect = new URL(direct); viaDirect.hostname = '127.0.0.1'; viaDirect.port = String(this.directProxy.port);- this.pooledUrl = viaPooled.toString(); this.directUrl = viaDirect.toString();- this.engine = new PostgresEngine();- await withEnv({ GBRAIN_HOME: this.home, GBRAIN_DIRECT_DATABASE_URL: undefined }, async () => {- await this.engine!.connect({ database_url: direct.toString(), poolSize: 4 });- await this.engine!.initSchema();- });- writeFileSync(join(this.home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: this.pooledUrl, embedding_disabled: true }));- this.repo = mkdtempSync(join(tmpdir(), 'gb-conv-repo-'));- this.git('init', '-q'); this.git('config', 'user.email', '[email protected]'); this.git('config', 'user.name', 'Example');- seed(this);- await withEnv({ GBRAIN_HOME: this.home }, async () => {- await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');- await this.engine!.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [SOURCE, this.repo]);- await claimWorktree(this.engine!, SOURCE, this.repo);- for (const [key, value] of Object.entries(config)) await this.engine!.setConfig(key, value);- await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');- });- this.note(`seed=${SEED} latency=${LATENCY_MS}ms round trip via proxies (pooled -> ${pooled.host}, direct -> ${direct.host}); modes ${this.modes.join(',')}`);- // Phase 0: a fault-free import, so the oracle is calibrated before any chaos.- const started = performance.now();- const baseline = await this.sync(['--no-pull', '--no-delegate', '--json']).result;- this.elapsed.baseline_import_ms = performance.now() - started;- if (baseline.code !== 0) throw new Error(`baseline sync exited ${baseline.code}\n${baseline.out}\n${baseline.err}\n${this.context()}`);- this.calibration = await this.diskDbMismatches(new Set());- if (existsSync(join(this.repo, `${BIG}.md`))) {- const rows = await this.childRowProblems();- this.calibration.push(...rows.problems, ...(rows.timelineDuplicates || rows.timelineStrays ? [`timeline: ${rows.timelineDuplicates} duplicate and ${rows.timelineStrays} stray rows after a fault-free import`] : []));- }- this.note(`phase 0: fault-free import in ${(this.elapsed.baseline_import_ms / 1000).toFixed(1)}s; oracle calibration problems: ${this.calibration.length}`);- }-- /** A fresh resident serve and at most MAX_RECOVERY_RUNS plain syncs, no operator step. */- async recover(): Promise<void> {- const started = performance.now();- const serve = new StdioServe(this);- await serve.start();- for (let run = 1; run <= MAX_RECOVERY_RUNS; run++) {- const r = await this.sync(['--no-pull', '--json']).result;- const envelope = lastJson(r.out);- this.recoveryRuns.push({ ...r, envelope });- this.keepLog(`recovery-${run}-client.log`, `${r.out}\n${r.err}`);- this.note(`recovery run ${run}: exit ${r.code} in ${(r.ms / 1000).toFixed(1)}s outcome=${String(envelope?.outcome ?? envelope?.sync_status ?? envelope?.error ?? 'none')}`);- if (r.code === 0 && envelope?.outcome === 'synced') break;- }- this.elapsed.recovery_ms = performance.now() - started;- this.keepLog('recovery-serve.log', serve.stderr);- this.note(`elapsed: ${Object.entries(this.elapsed).map(([k, ms]) => `${k.replace(/_ms$/, '')} ${(ms / 1000).toFixed(1)}s`).join(', ')}; recovery took ${this.recoveryRuns.length} sync run(s)`);- }-- /**- * #6841: an accepted write still queued when its writer gave up may commit later, after a write the ledger holds.- * Once the brain has converged, each one's receipt is read the way the write protocol tells a client to after exit- * 10 (get_write_request). A committed write is acknowledged then: it may have landed any time since its start, so it- * is concurrent with every write in that window. Any other state means it never landed and stays out of the ledger.- */- async settlePending(receiptState = async (requestId: string) =>- lastJson((await this.spawnCli(['call', '--source', SOURCE, 'get_write_request', JSON.stringify({ request_id: requestId })]).result).out)?.state): Promise<void> {- for (const { write, requestId } of this.pending) {- const state = await receiptState(requestId);- if (state === 'committed') this.ledger.record({ ...write, ack: performance.now() });- this.note(`accepted ${write.kind} of ${write.slug} (request ${requestId}) unsettled when its writer stopped: receipt ${String(state ?? 'unreadable')}`);- }- }-- async teardown(): Promise<void> {- for (const serve of this.serves) await serve.stop().catch(() => {});- try { await this.engine?.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1'); } catch { /* best effort */ }- await this.engine?.disconnect().catch(() => {});- await this.pooledProxy?.close().catch(() => {});- await this.directProxy?.close().catch(() => {});- if (this.admin) {- try { if (this.database) await this.admin.unsafe(`DROP DATABASE IF EXISTS ${this.database} WITH (FORCE)`); } finally { await this.admin.end(); }- }- for (const dir of [this.home, this.repo]) if (dir) rmSync(dir, { recursive: true, force: true });- }-- // ------------------------------------------------------------ oracle reads-- diskPages(): Map<string, string> {- const out = new Map<string, string>();- const walk = (dir: string) => {- for (const entry of readdirSync(dir, { withFileTypes: true })) {- if (entry.name.startsWith('.')) continue;- const full = join(dir, entry.name);- if (entry.isDirectory()) walk(full);- else if (entry.name.endsWith('.md')) out.set(relative(this.repo, full).replace(/\.md$/, ''), readFileSync(full, 'utf8'));- }- };- walk(this.repo);- return out;- }- async dbPages(): Promise<Map<string, { hash: string | null; body: string }>> {- const rows = await this.engine!.executeRaw<{ slug: string; content_hash: string | null; body: string }>(- `SELECT slug, content_hash, coalesce(compiled_truth,'') || E'\\n' || coalesce(timeline,'') AS body FROM pages WHERE source_id=$1 AND deleted_at IS NULL`, [SOURCE]);- return new Map(rows.map(r => [r.slug, { hash: r.content_hash, body: r.body }]));- }- async diskDbMismatches(exclude: Set<string>): Promise<string[]> {- const disk = this.diskPages(), db = await this.dbPages(), out: string[] = [];- for (const [slug, content] of disk) {- if (exclude.has(slug)) continue;- if (!db.has(slug)) out.push(`${slug}: file exists, no live page`);- else if (db.get(slug)!.hash !== fileHash(slug, content)) out.push(`${slug}: page content_hash differs from the file`);- }- for (const slug of db.keys()) if (!disk.has(slug) && !exclude.has(slug)) out.push(`${slug}: live page, no file`);- return out;- }- async ledgerProblems(): Promise<string[]> {- const facts = new Set((await this.engine!.executeRaw<{ fact: string }>('SELECT fact FROM facts WHERE source_id=$1 AND expired_at IS NULL', [SOURCE])).map(r => r.fact));- return this.ledger.problems(this.diskPages(), await this.dbPages(), facts);- }- async openRequests(): Promise<unknown[]> {- const open = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; recovery: boolean; blocked_reason: string | null }>(- `SELECT request_id::text, slug, state, recovery IS NOT NULL AS recovery, blocked_reason FROM persistence_requests- WHERE state IN ('queued','running','recovering') OR recovery IS NOT NULL ORDER BY sequence`);- const worktrees = await this.engine!.executeRaw<{ id: string; state: string }>(`SELECT id::text, state FROM persistence_worktrees WHERE state<>'active'`);- return [...open.map(r => `open request ${JSON.stringify(r)}`), ...worktrees.map(w => `worktree ${w.id} is ${w.state}`)];- }- async unexplainedTerminal(exclude: Set<string>): Promise<string[]> {- const rows = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; error_code: string | null; operation: string }>(- `SELECT request_id::text, slug, state, error_code, operation FROM persistence_requests WHERE state IN ('conflict','failed','cancelled') ORDER BY sequence`);- const disk = this.diskPages(), db = await this.dbPages();- this.note(`terminal non-committed requests: ${rows.length} (${[...new Set(rows.map(r => `${r.state}:${r.error_code}`))].join(', ') || 'none'})`);- return rows.filter(row => {- const content = disk.get(row.slug);- const superseded = exclude.has(row.slug) || (content !== undefined && db.get(row.slug)?.hash === fileHash(row.slug, content));- return !superseded && !(row.error_code && row.error_code in CODES);- }).map(row => `unexplained ${row.state} request ${row.request_id} (${row.slug}, ${row.error_code ?? 'no code'})`);- }- async cursorProblems(): Promise<string[]> {- const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);- const head = this.git('rev-parse', 'HEAD');- return source.last_commit === head ? [] : [`sources.last_commit ${source.last_commit} is not HEAD ${head}`];- }- async status(detail = true): Promise<Record<string, unknown> | null> {- const r = await this.spawnCli(['sync', 'status', '--source', SOURCE, '--json', ...(detail ? ['--detail'] : [])]).result;- this.keepLog(detail ? 'status.json' : 'status-fast.json', `${r.out}\n${r.err}`);- return lastJson(r.out);- }- recoveryProblems(): string[] {- const last = this.recoveryRuns.at(-1);- return last?.code === 0 && last.envelope?.outcome === 'synced' ? []- : [`no plain sync succeeded within ${MAX_RECOVERY_RUNS} runs: ${this.recoveryRuns.map(r => `exit ${r.code} outcome ${String(r.envelope?.outcome ?? r.envelope?.error ?? 'none')}`).join('; ')}`];- }- /**- * The large page's child rows against its file. Facts are keyed by fence row, so- * loss, duplication and strays are all problems. Timeline rows are judged for- * loss only; duplicate and stray timeline rows come back as a measured count,- * because derived timeline rows are never pruned (a known defect outside #6650).- */- async childRowProblems(): Promise<{ problems: string[]; timelineDuplicates: number; timelineStrays: number }> {- const content = this.diskPages().get(BIG);- if (!content) return { problems: [`${BIG}: no file`], timelineDuplicates: 0, timelineStrays: 0 };- const out: string[] = [];- const expectedTimeline = [...content.matchAll(/^- \*\*(\d{4}-\d{2}-\d{2})\*\* \| (.+)$/gm)].map(m => `${m[1]}|${m[2]}`);- const timeline = await this.engine!.executeRaw<{ key: string; n: number }>(- `SELECT t.date::text || '|' || t.summary AS key, count(*)::int AS n FROM timeline_entries t JOIN pages p ON p.id=t.page_id- WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL AND t.event_page_id IS NULL GROUP BY 1`, [SOURCE, BIG]);- const have = new Map(timeline.map(r => [r.key, r.n]));- const lostTimeline = expectedTimeline.filter(k => !have.has(k));- const dupTimeline = timeline.filter(r => r.n > 1);- const extraTimeline = timeline.filter(r => !expectedTimeline.includes(r.key));- if (lostTimeline.length) out.push(`timeline: ${lostTimeline.length} of ${expectedTimeline.length} file entries missing (first: ${lostTimeline[0]})`);- const expectedFacts = parseFactsFence(content).facts.map(f => `${f.rowNum}|${f.claim}`);- const facts = await this.engine!.executeRaw<{ key: string; n: number }>(- `SELECT row_num::text || '|' || fact AS key, count(*)::int AS n FROM facts- WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL GROUP BY 1`, [SOURCE, BIG]);- const haveFacts = new Map(facts.map(r => [r.key, r.n]));- const lostFacts = expectedFacts.filter(k => !haveFacts.has(k));- const dupFacts = facts.filter(r => r.n > 1);- const extraFacts = facts.filter(r => !expectedFacts.includes(r.key));- if (lostFacts.length) out.push(`facts: ${lostFacts.length} of ${expectedFacts.length} fence rows missing (first: ${lostFacts[0]})`);- if (dupFacts.length) out.push(`facts: ${dupFacts.length} rows duplicated (first: ${dupFacts[0].key} x${dupFacts[0].n})`);- if (extraFacts.length) out.push(`facts: ${extraFacts.length} rows not in the fence (first: ${extraFacts[0].key})`);- return { problems: out, timelineDuplicates: dupTimeline.reduce((n, r) => n + r.n - 1, 0), timelineStrays: extraTimeline.length };- }-}--function fileHash(slug: string, content: string): string {- const parsed = parseMarkdown(content, `${slug}.md`);- return contentHash({ title: parsed.title, type: parsed.type, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline, frontmatter: parsed.frontmatter, tags: [...parsed.tags].sort() });-}--/** One stdio MCP session against a spawned `gbrain serve`; the serve is the host's resident persistence owner. */-class StdioServe {- readonly child: ChildProcess;- stderr = '';- exited: Promise<number | null>;- private nextId = 1;- private pending = new Map<number, (msg: Record<string, unknown>) => void>();- private buf = '';- constructor(private brain: Brain, fault?: Fault) {- const { preload, env } = brain.faultArgs(fault);- this.child = spawn(process.execPath, ['--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), 'serve'], { cwd: REPO_ROOT, env: brain.childEnv(env), stdio: ['pipe', 'pipe', 'pipe'] });- this.exited = new Promise(done => this.child.once('exit', code => done(code)));- this.child.stderr!.on('data', (d: Buffer) => { this.stderr += d.toString('utf8'); });- this.child.stdin!.on('error', () => {});- this.child.stdout!.on('data', (d: Buffer) => {- this.buf += d.toString('utf8');- let idx: number;- while ((idx = this.buf.indexOf('\n')) !== -1) {- const line = this.buf.slice(0, idx).trim(); this.buf = this.buf.slice(idx + 1);- if (!line) continue;- try {- const msg = JSON.parse(line) as Record<string, unknown>;- const id = msg.id as number | undefined;- if (id !== undefined && this.pending.has(id)) { this.pending.get(id)!(msg); this.pending.delete(id); }- } catch { /* serve chatter */ }- }- });- brain.serves.push(this);- }- get pid(): number { return this.child.pid!; }- get alive(): boolean { return this.child.exitCode === null && this.child.signalCode === null; }- request(method: string, params: Record<string, unknown>, timeoutMs = 240_000): Promise<Record<string, unknown>> {- const id = this.nextId++;- return new Promise((done, reject) => {- const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`rpc ${method} timed out`)); }, timeoutMs);- this.pending.set(id, msg => { clearTimeout(t); done(msg); });- this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');- });- }- async start(): Promise<void> {- await this.request('initialize', { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'convergence', version: '1' } });- this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');- await until(async () => (await this.brain.engine!.executeRaw<{ n: number }>(- `SELECT count(*)::int AS n FROM persistence_consumers WHERE pid=$1 AND kind='serve' AND renewed_at > now() - interval '30 seconds'`, [this.pid]))[0].n > 0,- `serve ${this.pid} heartbeat`, 120_000);- }- async stop(): Promise<void> {- if (!this.alive) return;- this.child.kill('SIGTERM');- const timer = setTimeout(() => this.child.kill('SIGKILL'), 10_000);- await this.exited; clearTimeout(timer);- }-}-// ---------------------------------------------------------------- the writers of mode (c)-/**- * A writer call that retries an unsettled attempt (exit 10, a lost owner) with the same request id and a terminal refusal with a new one.- * When the attempts or the chaos run out on an unsettled attempt, the request it names may still commit, so it comes back `pending`.- */-async function acknowledged(b: Brain, label: string, call: (requestId: string) => SyncRun, stop: () => boolean): Promise<{ start: number; ack: number } | { pending: string } | null> {- let requestId = randomUUID();- let unsettled = false;- for (let attempt = 1; attempt <= 20 && !stop(); attempt++) {- const start = performance.now();- const r = await call(requestId).result;- b.keepLog(`writer-${label.replaceAll('/', '_')}-${attempt}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);- if (r.code === 0) return { start, ack: performance.now() };- const terminal = /\((conflict|failed|cancelled)\)/.test(r.err);- b.note(`writer ${label} attempt ${attempt} exit ${r.code}${terminal ? ' (terminal; next attempt uses a new request id)' : ''}: ${r.err.trim().split('\n').slice(-1)[0] ?? ''}`);- unsettled = !terminal;- if (terminal) requestId = randomUUID();- await Bun.sleep(3000);- }- return unsettled ? { pending: requestId } : null;-}-const outcome = (ok: { ack: number } | { pending: string } | null) => !ok ? 'unacknowledged' : 'pending' in ok ? `pending ${ok.pending}` : 'ok';/** Rewrites notes through the coordinator (`gbrain put`), remembers facts on the entity page and runs the managed timeline maintenance job until `stop`. */async function runWriters(b: Brain, putTargets: string[], stop: () => boolean): Promise<string[]> {const results: string[] = [];@@ -715,31 +196,6 @@ async function runWriters(b: Brain, putTargets: string[], stop: () => boolean):return results;}-// ---------------------------------------------------------------- the shared oracle--interface EndState { exclude?: Set<string>; status?: StatusExpectation; bigPage?: boolean }-/** The oracle tests every scenario ends with; `end()` is read when the tests run (after the scenario's beforeAll). */-function oracleTests(b: Brain, end: () => EndState): void {- test('the oracle is calibrated: a fault-free import satisfies it', () => b.verdict(b.calibration));- test('every planned fault fired, so the chaos actually happened', () => b.verdict(Object.entries(b.fired).filter(([, at]) => at === null).map(([fault]) => `${fault} did not fire`)));- test(as('wedge', `a plain sync succeeds within ${MAX_RECOVERY_RUNS} runs after the chaos, with no operator step`), () => b.verdict(b.recoveryProblems()));- test(as('projection_drift', 'disk == database for every page the end state does not name'), async () => b.verdict(await b.diskDbMismatches(end().exclude ?? new Set())));- test(as('wedge', 'no request blocks the checkpoint or a root'), async () => b.verdict((await b.openRequests()).map(String)));- test(as('untyped_error', 'every non-committed terminal request is superseded or carries a registered error code'), async () => b.verdict(await b.unexplainedTerminal(end().exclude ?? new Set())));- test(as('wedge', 'the cursor reached HEAD'), async () => b.verdict(await b.cursorProblems()));- test(as('lost_write', 'every acknowledged write survives or is causally superseded by a later acknowledged write'), async () => {- b.note(`ledger: ${b.ledger.acks.length} acknowledged writes (${[...new Set(b.ledger.acks.map(a => a.kind))].map(kind => `${kind} ${b.ledger.acks.filter(a => a.kind === kind).length}`).join(', ')})`);- b.verdict(await b.ledgerProblems());- });- test(as('wedge', 'sync status --json reports the converged state with every field present'), async () => b.verdict(statusProblems(await b.status(), end().status)));- test(as('wedge', 'plain sync status --json names every heavy read it skipped instead of reading as all-clear'), async () => b.verdict(fastStatusProblems(await b.status(false))));- if (end().bigPage !== false) test(as('lost_write', 'the large page keeps every fence fact exactly once and loses no timeline entry'), async () => {- const { problems, timelineDuplicates, timelineStrays } = await b.childRowProblems();- b.note(`large page timeline (measured, not asserted): ${timelineDuplicates} duplicate rows, ${timelineStrays} rows not in the file`);- b.verdict(problems);- });-}-/** A describe that runs when any of its modes runs; skipped (with every skip reason in its title) otherwise. */function scenario(modes: Mode[], title: string, body: () => void): void {const active = enabled && modes.some(runs);