Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { assertLegacyEngineMigration, assertUnmanagedCanonicalWriter } from '../core/persistence/maintenance.ts';
/**
* Engine migration: transfer brain data between PGLite and Postgres.
*
* Usage:
* gbrain migrate --to supabase [--url <connection_string>] (PGLite -> Postgres: graduation, src/commands/migrate-graduation.ts)
* gbrain migrate --to pglite [--path <db_path>]
* gbrain migrate --to <engine> --force (overwrite non-empty target)
*/
import { opError } from '../core/ops/contract.ts';
import { sqlEngineKind } from '../core/sql-engine-kind.ts';
import { ItemRun } from '../core/item-failure.ts';
import { createEngine } from '../core/engine-factory.ts';
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
import type { BrainEngine } from '../core/engine.ts';
import type { EffectiveDateSource, EngineConfig, Page, PageKind } from '../core/types.ts';
import { writeFileSync, readFileSync, existsSync, unlinkSync, statSync, mkdirSync, renameSync } from 'fs';
import { createHash } from 'crypto';
import { resolve, dirname } from 'path';
import { createProgress, type ProgressReporter } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import { registerCleanup } from '../core/process-cleanup.ts';
// LEAF module, deliberately NOT './autopilot.ts': the flag-registry generator
// follows a command's dynamic imports and would fold autopilot's entire flag
// surface into `migrate`'s accepted-flag allowlist.
import { autopilotPausedMarkerPath, autopilotLockPath, markerHolderAlive, MIGRATE_PAUSE_MARKER_PREFIX } from '../core/autopilot-paths.ts';
export { MIGRATE_PAUSE_MARKER_PREFIX };
import { listLiveLocks } from '../core/db-lock.ts';
import { drainProjections, queuePageProjection } from '../core/page-state/projections.ts';
import { carryChunkMetadata } from '../core/embed-stale.ts';
import { normalizeAliasList } from '../core/search/alias-normalize.ts';
interface MigrateOpts {
targetEngine: 'postgres' | 'pglite';
targetUrl?: string;
targetPath?: string;
force: boolean;
}
function parseArgs(args: string[]): MigrateOpts {
const toIdx = args.indexOf('--to');
if (toIdx === -1 || !args[toIdx + 1]) {
throw opError('invalid_params', 'gbrain migrate needs --to <postgres|supabase|pglite>.',
'Usage: gbrain migrate --to <postgres|supabase|pglite> [--url <url>] [--path <path>] [--force]. Example: gbrain migrate --to pglite --path ~/.gbrain/brain.pglite');
}
const targetRaw = args[toIdx + 1];
const targetEngine = targetRaw === 'supabase' ? 'postgres' : targetRaw as 'postgres' | 'pglite';
if (targetEngine !== 'postgres' && targetEngine !== 'pglite') {
throw opError('invalid_params', `Unknown target engine: "${targetRaw}". Use postgres (alias supabase) or pglite.`,
'Usage: gbrain migrate --to <postgres|supabase|pglite> [--url <url>] [--path <path>] [--force].',
{ why: `gbrain migrate moves a brain between its two engines, postgres and pglite; "${targetRaw}" is neither.`,
fix: { argv: ['gbrain', 'migrate', '--to', '<engine>'], inputs: [{ name: 'engine', how: 'postgres (alias supabase) or pglite, whichever engine the user wants to move to.' }],
consent: [], actor: 'agent', requires_exclusive: true, verify: { argv: ['gbrain', 'engine', 'status', '--json'] },
why: 'Names an engine gbrain supports.' } });
}
const urlIdx = args.indexOf('--url');
const pathIdx = args.indexOf('--path');
return {
targetEngine,
targetUrl: urlIdx !== -1 ? args[urlIdx + 1] : undefined,
targetPath: pathIdx !== -1 ? args[pathIdx + 1] : undefined,
force: args.includes('--force'),
};
}
function getManifestPath(): string {
return gbrainPath('migrate-manifest.json');
}
export interface MigrateManifest {
completed_slugs: string[];
target_engine: string;
target_id?: string;
schema_version?: number;
started_at: string;
}
export function migrationTargetId(config: EngineConfig): string {
const locator = config.engine === 'postgres'
? config.database_url ?? ''
: resolve(config.database_path ?? gbrainPath('brain.pglite'));
return createHash('sha256')
.update(JSON.stringify([config.engine, locator]))
.digest('hex');
}
export function manifestMatchesTarget(manifest: MigrateManifest, targetId: string): boolean {
return manifest.schema_version === 2 && manifest.target_id === targetId;
}
function makeManifestKey(sourceId: string, slug: string): string {
return sourceId === 'default' ? slug : `${sourceId}::${slug}`;
}
function loadManifest(): MigrateManifest | null {
const path = getManifestPath();
if (!existsSync(path)) return null;
try {
return JSON.parse(readFileSync(path, 'utf-8'));
} catch {
return null;
}
}
function saveManifest(manifest: MigrateManifest): void {
writeFileSync(getManifestPath(), JSON.stringify(manifest, null, 2));
}
function clearManifest(): void {
const path = getManifestPath();
if (existsSync(path)) unlinkSync(path);
}
interface MigratedSourceRow {
id: string;
name: string;
local_path: string | null;
last_commit: string | null;
last_sync_at: Date | string | null;
config_json: string;
archived: boolean;
archived_at: Date | string | null;
archive_expires_at: Date | string | null;
contextual_retrieval_mode: string | null;
trust_frontmatter_overrides: boolean;
newest_content_at: Date | string | null;
created_at: Date | string;
}
export async function copyMigrationSources(source: BrainEngine, target: BrainEngine): Promise<number> {
const sources = await source.executeRaw<MigratedSourceRow>(`
SELECT id, name, local_path, last_commit, last_sync_at, config::text AS config_json, archived,
archived_at, archive_expires_at, contextual_retrieval_mode,
trust_frontmatter_overrides, newest_content_at, created_at
FROM sources
ORDER BY (id = 'default') DESC, id`);
for (const row of sources) {
await target.executeRaw(`
INSERT INTO sources
(id, name, local_path, last_commit, last_sync_at, config, archived,
archived_at, archive_expires_at, contextual_retrieval_mode,
trust_frontmatter_overrides, newest_content_at, created_at)
VALUES ($1, $2, $3, $4, $5, $6::text::jsonb, $7, $8, $9, $10, $11, $12, $13)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
local_path = EXCLUDED.local_path,
last_commit = EXCLUDED.last_commit,
last_sync_at = EXCLUDED.last_sync_at,
config = EXCLUDED.config,
archived = EXCLUDED.archived,
archived_at = EXCLUDED.archived_at,
archive_expires_at = EXCLUDED.archive_expires_at,
contextual_retrieval_mode = EXCLUDED.contextual_retrieval_mode,
trust_frontmatter_overrides = EXCLUDED.trust_frontmatter_overrides,
newest_content_at = EXCLUDED.newest_content_at,
created_at = EXCLUDED.created_at`, [
row.id, row.name, row.local_path, row.last_commit, row.last_sync_at,
row.config_json, row.archived, row.archived_at, row.archive_expires_at,
row.contextual_retrieval_mode, row.trust_frontmatter_overrides,
row.newest_content_at, row.created_at,
]);
}
return sources.length;
}
/** Result of the facts-table copy — surfaced in the per-table summary (#4350). */
export interface MigrateFactsResult {
copied: number;
failed: Array<{ id: string; reason: string }>;
/** Rows whose vector could not land on the target (e.g. a dimension
* mismatch between the two schemas' embedding columns) and were copied
* with embedding NULL instead of being lost. */
embeddings_dropped: number;
/** True when the source schema predates the facts table entirely. */
table_missing: boolean;
}
/**
* Non-empty-target guard, facts leg: `getStats().page_count` can't see a
* facts-only brain (a DB-only remember corpus — conversation facts with zero
* pages), so the page guard alone would let `copyMigrationFacts`'s
* delete-and-recopy destroy the target's only copy of its hot memory.
* Bounded single-row probe; a target schema that predates the facts table
* has nothing to protect, so probe failure returns false.
*/
async function targetHasFactRows(target: BrainEngine): Promise<boolean> {
try {
const rows = await target.executeRaw<{ one: number }>('SELECT 1 AS one FROM facts LIMIT 1');
return rows.length > 0;
} catch {
return false;
}
}
async function factsColumns(engine: BrainEngine): Promise<string[]> {
const rows = await engine.executeRaw<{ column_name: string }>(`
SELECT column_name FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = 'facts'
ORDER BY ordinal_position`);
return rows.map(r => r.column_name);
}
/**
* #4350: copy the facts table (hot memory) verbatim. Facts are the one data
* table whose rows can exist ONLY in the DB — conversation facts (`source`
* LIKE 'cli:%') have no markdown fence to re-sync from, so a migration that
* skips them loses them permanently (`recall` comes back empty on the new
* engine, the original report).
*
* Shape decisions:
* - Column list is the source∩target intersection read from
* information_schema, so an older source schema (missing newer columns
* like event_type/dimension) still copies; absent columns land as the
* target's defaults.
* - Delete-and-recopy: facts on the target can only have come from this
* same migration (the non-empty guard blocks foreign targets), so wiping
* and re-copying converges every re-run to source truth — including after
* --force, whose pages wipe does NOT cascade into facts.
* - Two passes for `superseded_by`: it's a self-FK where the OLD row points
* at the NEWER (higher) id, so id-ordered inserts would violate the FK;
* insert with NULL, then restore the pointers.
* - Embeddings round-trip as text (`'[...]'` parses into vector AND halfvec,
* whichever the target column is). A row whose vector can't land is
* retried with embedding NULL and counted — losing a re-buildable vector
* beats losing the fact.
* - The id sequence is bumped past MAX(id) so post-migration inserts don't
* collide with copied rows.
*
* Takes are deliberately NOT copied here: they're fence-canonical (markdown-
* mirrored) and keyed on page_id, which the target reassigns — the next
* extract cycle rebuilds them from the migrated pages. Facts'
* `consolidated_into` take-pointers therefore dangle until that rebuild;
* `consolidated_at` still marks the rows as promoted, so the consolidate
* phase won't double-promote them.
*/
/**
* v0.47 open-loop engine: open_loops + loop_suppressions carry
* NON-derivable state (manual mutes, manual done/dropped decisions) — a
* Gmail re-sync on the target cannot reconstruct them, so the engine
* migration copies them like facts. Column-intersection + table_missing
* tolerance mirrors copyMigrationFacts; ids are preserved (fact_id
* references survive because facts ids are preserved too).
*/
export interface MigrateSimpleTableResult {
copied: number;
/** Rows the target refused, with the classified code and the database's reason; any of them blocks the engine switch. */
failed: Array<{ id: string; code: string; reason: string }>;
table_missing: boolean;
}
export async function copyMigrationSimpleTable(
source: BrainEngine,
target: BrainEngine,
table: 'open_loops' | 'loop_suppressions',
onRow?: () => void,
): Promise<MigrateSimpleTableResult> {
const colsOf = async (engine: BrainEngine): Promise<string[]> => {
try {
const rows = await engine.executeRaw<{ column_name: string }>(
`SELECT column_name FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = $1`,
[table],
);
return rows.map((r) => r.column_name);
} catch {
return [];
}
};
const sourceCols = await colsOf(source);
if (sourceCols.length === 0) return { copied: 0, failed: [], table_missing: true };
const targetColSet = new Set(await colsOf(target));
if (targetColSet.size === 0) return { copied: 0, failed: [], table_missing: true };
const cols = sourceCols.filter((c) => targetColSet.has(c));
const rows = await source.executeRaw<Record<string, unknown>>(
`SELECT ${cols.map((c) => (c === 'evidence' ? 'evidence::text AS evidence' : `"${c}"`)).join(', ')} FROM ${table} ORDER BY id`,
);
await target.executeRaw(`DELETE FROM ${table}`);
const insertSql = `INSERT INTO ${table} (${cols.map((c) => `"${c}"`).join(', ')})
VALUES (${cols.map((c, i) => (c === 'evidence' ? `$${i + 1}::text::jsonb` : `$${i + 1}`)).join(', ')})`;
const result: MigrateSimpleTableResult = { copied: 0, failed: [], table_missing: false };
// GBRA-130 A-31: a refused row is recorded with its reason (it blocks the switch); a run-scoped failure (lost
// connection, failed transaction) or a systemic run (most rows refused) stops the copy instead.
const run = new ItemRun('migrate_engine.loop_rows');
for (const raw of rows) {
const row = nullifyUndefinedColumns(raw);
try {
await target.executeRaw(insertSql, cols.map((c) => row[c]));
result.copied++;
run.succeed();
} catch (error) {
const failure = run.fail(`${table}:${String(row.id)}`, error);
result.failed.push({ id: String(row.id), code: failure.code, reason: failure.message });
}
onRow?.();
}
// Explicit-id copy leaves the BIGSERIAL sequence at 1 — the first new
// insert would collide with a copied row's PRIMARY KEY (and the ON
// CONFLICT dedup arbiter does NOT absorb a pkey 23505). Mirror the facts
// copy's setval.
try {
await target.executeRaw(
`SELECT setval(pg_get_serial_sequence('${table}', 'id'), (SELECT COALESCE(MAX(id), 0) + 1 FROM ${table}), false)`,
);
} catch { /* sequence bump is best-effort on exotic schemas */ }
return result;
}
export async function copyMigrationFacts(
source: BrainEngine,
target: BrainEngine,
onRow?: () => void,
): Promise<MigrateFactsResult> {
const sourceCols = await factsColumns(source);
if (sourceCols.length === 0) {
return { copied: 0, failed: [], embeddings_dropped: 0, table_missing: true };
}
const targetColSet = new Set(await factsColumns(target));
const cols = sourceCols.filter(c => targetColSet.has(c));
const selectList = cols
.map(c => (c === 'embedding' ? 'embedding::text AS embedding' : `"${c}"`))
.join(', ');
const rows = await source.executeRaw<Record<string, unknown>>(
`SELECT ${selectList} FROM facts ORDER BY id`);
// The write-side cast must name the target column's physical type
// (halfvec on pgvector >= 0.7, vector otherwise); text parses into either.
const udt = await target.executeRaw<{ udt_name: string }>(`
SELECT udt_name FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = 'facts' AND column_name = 'embedding'`);
const embType = udt[0]?.udt_name === 'halfvec' ? 'halfvec' : 'vector';
await target.executeRaw('DELETE FROM facts');
const embIdx = cols.indexOf('embedding');
const insertSql = `INSERT INTO facts (${cols.map(c => `"${c}"`).join(', ')})
VALUES (${cols.map((c, i) => (c === 'embedding' ? `$${i + 1}::text::${embType}` : `$${i + 1}`)).join(', ')})`;
const result: MigrateFactsResult = { copied: 0, failed: [], embeddings_dropped: 0, table_missing: false };
const chainPairs: Array<{ id: unknown; superseded_by: unknown }> = [];
for (const raw of rows) {
const row = nullifyUndefinedColumns(raw);
if (row.superseded_by != null) chainPairs.push({ id: row.id, superseded_by: row.superseded_by });
// Pass 1 inserts with superseded_by NULL (see the two-pass note above).
const values = cols.map(c => (c === 'superseded_by' ? null : row[c]));
try {
await target.executeRaw(insertSql, values);
result.copied++;
} catch (e) {
if (embIdx !== -1 && values[embIdx] != null) {
try {
const withoutEmbedding = [...values];
withoutEmbedding[embIdx] = null;
await target.executeRaw(insertSql, withoutEmbedding);
result.copied++;
result.embeddings_dropped++;
onRow?.();
continue;
} catch { /* fall through to the failure record below */ }
}
result.failed.push({
id: String(row.id),
reason: e instanceof Error ? e.message : String(e),
});
}
onRow?.();
}
const failedIds = new Set(result.failed.map(f => f.id));
for (const pair of chainPairs) {
if (failedIds.has(String(pair.id)) || failedIds.has(String(pair.superseded_by))) continue;
await target.executeRaw('UPDATE facts SET superseded_by = $1 WHERE id = $2', [pair.superseded_by, pair.id]);
}
// BIGSERIAL doesn't advance on explicit-id inserts; without this the first
// post-migration insertFact would collide with a copied row's id.
await target.executeRaw(
`SELECT setval(pg_get_serial_sequence('facts', 'id'), (SELECT COALESCE(MAX(id), 0) + 1 FROM facts), false)`);
return result;
}
export const MIGRATE_CONFIG_ENGINE_LOCAL_KEYS: ReadonlySet<string> = new Set([
'engine',
'version',
'embedding_columns',
'search_embedding_column',
]);
/**
* Copy every DB-plane config row except the engine-local denylist above.
* Subsumes the old 3-key allowlist (embedding_model / embedding_dimensions /
* chunk_strategy — schema metadata recording what the schema was sized
* with); those still copy, now alongside everything else.
*/
export async function copyMigrationConfig(
source: BrainEngine,
target: BrainEngine,
): Promise<{ copied: number; skipped: string[] }> {
const rows = await source.executeRaw<{ key: string; value: string }>(
'SELECT key, value FROM config ORDER BY key');
const skipped: string[] = [];
let copied = 0;
for (const row of rows) {
if (MIGRATE_CONFIG_ENGINE_LOCAL_KEYS.has(row.key)) {
skipped.push(row.key);
continue;
}
await target.setConfig(row.key, row.value);
copied++;
}
return { copied, skipped };
}
export async function copyPageLinksToTarget(
source: BrainEngine,
target: BrainEngine,
page: MigratePageRef,
failedKeys: ReadonlySet<string> = new Set(),
): Promise<number> {
const links = await source.getLinks(page.slug, { sourceId: page.source_id });
let copied = 0;
for (const link of links) {
const toSourceId = link.to_source_id ?? page.source_id;
if (failedKeys.has(makeManifestKey(toSourceId, link.to_slug))) continue;
await target.addLink(
link.from_slug, link.to_slug,
link.context, link.link_type,
undefined, undefined, undefined,
{ fromSourceId: page.source_id, toSourceId },
);
copied++;
}
// Temporal typed edges: manual dated statements (add_link valid_from /
// valid_until) exist only in the database, so they travel with the page.
// Statements derived from page content re-derive on the target's next
// extract pass.
const manual = await source.executeRaw<{ to_slug: string; to_source_id: string; link_type: string; kind: string; occurred_on: string }>(
`SELECT t.slug AS to_slug, t.source_id AS to_source_id, lt.link_type, lt.kind, lt.occurred_on::text AS occurred_on
FROM link_transitions lt JOIN pages f ON f.id = lt.from_page_id JOIN pages t ON t.id = lt.to_page_id
WHERE f.slug = $1 AND f.source_id = $2 AND lt.producer = 'manual'`, [page.slug, page.source_id]).catch(() => []);
if (manual.length) {
const { writeManualTransitions } = await import('../core/link-temporal-apply.ts');
for (const m of manual) {
if (failedKeys.has(makeManifestKey(m.to_source_id, m.to_slug))) continue;
await writeManualTransitions(target, { from: page.slug, to: m.to_slug, linkType: m.link_type, sourceId: page.source_id },
m.kind === 'start' ? { validFrom: m.occurred_on.slice(0, 10) } : { validUntil: m.occurred_on.slice(0, 10) });
}
}
return copied;
}
/**
* postgres.js's UNDEFINED_VALUE guard rejects any bound parameter that is JS
* `undefined` — unlike PGLite, it will not silently treat it as SQL NULL.
* A page read back from a PGLite source can carry `undefined` for a column
* that is legitimately empty/NULL (a read-side driver-shape difference, not
* a data problem), and passing that value straight into a Postgres
* `putPage` throws mid-insert (#3194). Normalizing at this migrate-only
* boundary — rather than inside `putPage` itself, which many non-migrate
* callers also use — turns that driver-shape difference into an explicit
* SQL NULL, so only a genuine NOT-NULL constraint violation (an actual data
* problem) still surfaces as a page-copy failure.
*/
function nullifyUndefinedColumns<T extends Record<string, unknown>>(row: T): T {
const normalized = { ...row };
for (const key of Object.keys(normalized) as (keyof T)[]) {
if (normalized[key] === undefined) normalized[key] = null as T[typeof key];
}
return normalized;
}
/**
* Copy one page's full row (page body, chunks, tags, timeline, raw data)
* from source to target. Throws on any failure — the caller (the per-page
* loop in runMigrateEngine) decides how to account for that: track it as a
* failed page and keep going, rather than letting one bad row silently
* disappear from the progress count (#3194). Exported so unit tests can
* inject fake engines and exercise the failure path without a live
* DATABASE_URL.
*/
/** Per-page sub-row copy counts, accumulated into the per-table summary (#4350). */
export interface PageCopyCounts {
chunks: number;
tags: number;
timeline_entries: number;
raw_data: number;
}
/** Queue reasons the migration rebuilds before its flip: pages it copied sealed, and earlier attempts that failed. */
const MIGRATION_PROJECTION_REASONS = ['engine_migration', 'rebuild_failed'] as const;
export async function copyPageToTarget(
source: BrainEngine,
target: BrainEngine,
page: Page,
): Promise<PageCopyCounts> {
const sourceOpts = { sourceId: page.source_id };
// Copy page (preserve source_id). v0.32.8 F8: thread source_id end-to-end
// so multi-source pages migrate intact.
// Verbatim copy — the source engine's row is authoritative, so a
// legitimately blank body (image page, deliberate clear) must land even
// when a re-run's target row already holds an older non-empty body.
// #6286: the full row travels: origin, kind and provenance columns `Page` does not carry are read from the source row.
const [row] = await source.executeRaw<{ page_kind: PageKind | null; source_path: string | null; effective_date: Date | string | null; effective_date_source: EffectiveDateSource | null;
import_filename: string | null; source_kind: string | null; source_uri: string | null; ingested_via: string | null; ingested_at: Date | string | null; sealed: boolean }>(
`SELECT page_kind,source_path,effective_date,effective_date_source,import_filename,source_kind,source_uri,ingested_via,ingested_at,
text_projection_revision IS NOT DISTINCT FROM knowledge_revision AS sealed
FROM pages WHERE slug=$1 AND source_id=$2`, [page.slug, page.source_id ?? 'default']);
const asDate = (value: Date | string | null | undefined) => value == null ? null : value instanceof Date ? value : new Date(value);
await target.putPage(page.slug, nullifyUndefinedColumns({
type: page.type,
title: page.title,
compiled_truth: page.compiled_truth,
timeline: page.timeline,
frontmatter: page.frontmatter,
content_hash: page.content_hash,
...(row ? { page_kind: row.page_kind ?? undefined, source_path: row.source_path, effective_date: asDate(row.effective_date), effective_date_source: row.effective_date_source,
import_filename: row.import_filename, source_kind: row.source_kind, source_uri: row.source_uri, ingested_via: row.ingested_via } : {}),
}), { ...sourceOpts, allowEmptyOverwrite: true });
await target.setPageAliases(page.slug, page.source_id ?? 'default', normalizeAliasList(page.frontmatter?.aliases));
// #4527: putPage stamps created_at/updated_at with now() (PageInput has no
// timestamp fields), so without this every migrated page loses its
// chronology — recency ranking, `--since` filters, and timeline ordering
// all silently reset to the migration date. Restore the source row's
// timestamps directly; COALESCE keeps the putPage-stamped value if the
// source engine ever hands back a NULL (never expected, but a copy must
// not null out a NOT NULL column).
if (page.created_at != null || page.updated_at != null || row?.ingested_at != null) {
await target.executeRaw(
`UPDATE pages
SET created_at = COALESCE($1, created_at),
updated_at = COALESCE($2, updated_at),
ingested_at = COALESCE($5::timestamptz, ingested_at)
WHERE slug = $3 AND source_id = $4`,
[page.created_at ?? null, page.updated_at ?? null, page.slug, page.source_id ?? 'default', asDate(row?.ingested_at)],
);
}
// Migration preserves stored data even when it is not a verified search
// projection. The target rebuilds sanitized text under its new revision.
const chunks = await source.getChunksWithEmbeddings(page.slug, { ...sourceOpts, includeUnsealed: true });
if (chunks.length > 0) {
await target.upsertChunks(page.slug, chunks.map(c => carryChunkMetadata(c, {
chunk_index: c.chunk_index,
chunk_text: c.chunk_text,
chunk_source: c.chunk_source,
embedding: c.embedding || undefined,
embedding_image: c.embedding_image || undefined,
model: c.model,
token_count: c.token_count || undefined,
})), sourceOpts);
}
// Copy tags
const tags = await source.getTags(page.slug, sourceOpts);
for (const tag of tags) {
await target.addTag(page.slug, tag, sourceOpts);
}
// Copy timeline. #4485: getTimeline defaults to LIMIT 100 (newest-first)
// in both engines, so a page with a longer history silently lost everything
// past its newest 100 entries — pass an explicit unbounded limit (max int4,
// safely above any real per-page timeline).
const timeline = await source.getTimeline(page.slug, { ...sourceOpts, limit: 2_147_483_647 });
for (const entry of timeline) {
await target.addTimelineEntry(page.slug, {
date: entry.date,
source: entry.source,
summary: entry.summary,
detail: entry.detail,
}, sourceOpts);
}
// Copy raw data (includeDeleted: a migration copies whatever the page row
// carries — the page list already decided which rows travel).
const rawData = await source.getRawData(page.slug, undefined, { ...sourceOpts, includeDeleted: true });
for (const rd of rawData) {
await target.putRawData(page.slug, rd.source, rd.data, sourceOpts);
}
// #6286: a page sealed on the source is rebuilt on the target before the flip (MIGRATION_PROJECTION_REASONS); one
// the source itself had not sealed stays queued as it was, so the copy keeps its verbatim chunks and vectors.
await queuePageProjection(target, page.source_id ?? 'default', page.slug, row?.sealed === false ? 'engine_migration_source_unsealed' : 'engine_migration');
return {
chunks: chunks.length,
tags: tags.length,
timeline_entries: timeline.length,
raw_data: rawData.length,
};
}
/** A page's identity: what the copy lists before it reads each page body. */
export type MigratePageRef = Pick<Page, 'slug' | 'source_id'>;
/** A page that failed to copy during migrate — tracked so the run's final
* summary reports it honestly instead of letting the "N copied" counter
* imply every page landed (#3194). */
export interface MigratePageFailure {
source_id: string;
slug: string;
reason: string;
}
/**
* Copy each listed page, reading its body by (source_id, slug) at copy time so
* memory holds refs, not the whole brain. v0.32.8 F8: source_id is threaded
* end-to-end so non-default-source tags / timeline / raw / links land on the
* right row. A page soft-deleted after the listing reads null and is returned
* in `vanished` (not copied, not a failure). #3194: a per-page write failure is
* never swallowed into the success count; it stays OUT of completed_slugs (a
* resume retries it; every write is an upsert) and is returned in `failures`.
*/
export async function copyMigrationPages(
sourceEngine: BrainEngine,
targetEngine: BrainEngine,
refs: MigratePageRef[],
manifest: MigrateManifest,
rowCounts: PageCopyCounts,
progress: ProgressReporter,
): Promise<{ migrated: number; failures: MigratePageFailure[]; vanished: MigratePageRef[] }> {
let migrated = 0;
const failures: MigratePageFailure[] = [];
const vanished: MigratePageRef[] = [];
for (const ref of refs) {
try {
const page = await sourceEngine.getPage(ref.slug, { sourceId: ref.source_id });
if (!page) {
vanished.push(ref);
} else {
const counts = await copyPageToTarget(sourceEngine, targetEngine, page);
rowCounts.chunks += counts.chunks;
rowCounts.tags += counts.tags;
rowCounts.timeline_entries += counts.timeline_entries;
rowCounts.raw_data += counts.raw_data;
manifest.completed_slugs.push(makeManifestKey(page.source_id, page.slug));
saveManifest(manifest);
migrated++;
}
} catch (e) {
failures.push({ source_id: ref.source_id, slug: ref.slug, reason: e instanceof Error ? e.message : String(e) });
}
progress.tick(1, ref.slug);
}
return { migrated, failures, vanished };
}
/**
* Quiesce a running autopilot daemon for the duration of the copy.
*
* Why this brackets the copy instead of trailing it: `saveConfig` runs at the
* very END of the migration, gated on zero failures. A live daemon therefore
* keeps syncing and embedding into the SOURCE engine for the entire copy window,
* and everything it writes after page enumeration is silently lost the moment
* the config flips. `migrate-engine.ts` previously contained ZERO references to
* autopilot, which is the root cause of the observed incident: an engine
* migration rewrote `config.json` while a daemon built on the old config kept
* running, crashed on the stale object, and nothing linked the two events.
*
* Cooperative marker, not a supervisor stop — see `autopilotPausedMarkerPath`.
* Returns a resume function that is safe to call twice.
*/
/**
* A twice-safe, ownership-conditional unlinker for the pause marker: it
* deletes the marker ONLY while the content is still the exact body this
* process wrote. If the marker changed hands (an adoption race resolved
* against us), deleting by path would un-pause the daemon in the middle of
* the new owner's copy window — the precise failure the fence exists to
* stop. (The read-then-unlink pair has a microscopic window of its own;
* every layer here shrinks the race multiplicatively rather than claiming
* to erase it.)
*/
function markerRelease(marker: string, ownBody: string): () => void {
let released = false;
return () => {
if (released) return;
released = true;
try {
if (readFileSync(marker, 'utf8') !== ownBody) {
console.log('[migrate] pause marker changed hands; leaving it for its new owner.');
return;
}
} catch { return; /* already gone */ }
try { unlinkSync(marker); console.log('[migrate] autopilot resumed.'); }
catch { /* already gone */ }
};
}
/** Our marker body: prefix + ownership pid + timestamp. */
function markerBody(): string {
return `${MIGRATE_PAUSE_MARKER_PREFIX} (pid ${process.pid}) at ${new Date().toISOString()}\n`;
}
/**
* Drain cap in seconds. Empty/unset → default; `0` disables the wait
* entirely; garbage warns and falls back (an env typo must not silently
* remove the drain).
*/
function resolveDrainCapSeconds(): number {
const raw = process.env.GBRAIN_MIGRATE_QUIESCE_SECONDS;
if (raw === undefined || raw === '') return 300;
if (raw === '0') return 0;
const n = Number(raw);
if (!Number.isFinite(n) || n < 0) {
console.warn(`[migrate] ignoring invalid GBRAIN_MIGRATE_QUIESCE_SECONDS=${JSON.stringify(raw)}; using 300.`);
return 300;
}
return n;
}
/**
* Returns a resume closure, or NULL when the pause is owned by someone else —
* in which case the caller must ABORT: proceeding without owning the pause
* means the real owner's cleanup un-pauses the daemon in the middle of OUR
* copy window, and two migrations racing one manifest/target corrupt both.
*/
export async function quiesceAutopilot(engine?: BrainEngine): Promise<(() => void) | null> {
const marker = autopilotPausedMarkerPath();
// The marker is written UNCONDITIONALLY — not gated on the daemon's lock
// file existing. The lock is unlinked on every clean daemon exit (self
// upgrade swap, supervisor restart, SIGTERM) and the supervisor relaunches
// within a minute, so "no lock right now" recurs routinely on healthy
// installs; cron installs have no lock BETWEEN runs, which is most of the
// time. A migrate that skipped the marker in such a window would let the
// (re)started daemon run full cycles into the source engine for the whole
// copy — all silently lost at the config flip. A marker on a machine with
// no autopilot at all costs one tiny file, removed by the finally.
mkdirSync(dirname(marker), { recursive: true });
// Claim it atomically ('wx' = fail if it exists) — an existsSync pre-check
// would race a concurrent migrate/pauser between check and write. The pid
// in the body is the ownership handle for orphan adoption below.
let claimed = false;
const ownBody = markerBody();
try {
writeFileSync(marker, ownBody, { flag: 'wx' });
claimed = true;
} catch (e) {
if ((e as NodeJS.ErrnoException)?.code === 'EEXIST') {
// A marker already exists. Adopt it ONLY when it was written by a
// migrate whose process is provably dead — refusing to adopt a dead
// run's orphan would make it permanent (every retry defers to it and
// the daemon sits paused forever), while adopting a LIVE run's marker
// would be worse: our resume would un-pause the daemon in the middle
// of the other migrate's copy window. Content prefix alone cannot tell
// those apart; the pid can.
try {
const body = readFileSync(marker, 'utf8');
if (body.startsWith(MIGRATE_PAUSE_MARKER_PREFIX) && markerHolderAlive(body) === 'dead') {
// Take OWNERSHIP atomically. Two migrates racing the same orphan
// both read the dead pid; a plain rewrite would let BOTH claim it
// and one's cleanup would un-pause the daemon mid-copy for the
// other. rename(2) is the compare-and-claim: exactly one racer
// moves the orphan aside; the loser gets ENOENT and aborts. The
// fresh write then goes through the same exclusive 'wx' gate, so a
// third claimant sneaking into the gap loses cleanly too.
const tomb = `${marker}.reclaim.${process.pid}`;
renameSync(marker, tomb);
// Verify identity AFTER the rename: between our read above and the
// rename, a racing adopter may have completed the whole
// adopt-and-rewrite — in which case we just renamed THEIR live
// marker, not the orphan. Restore it no-clobber and bow out.
let tombBody = '';
try { tombBody = readFileSync(tomb, 'utf8'); } catch { /* vanished */ }
if (tombBody.startsWith(MIGRATE_PAUSE_MARKER_PREFIX) && markerHolderAlive(tombBody) === 'dead') {
try { unlinkSync(tomb); } catch { /* best-effort */ }
writeFileSync(marker, ownBody, { flag: 'wx' });
console.log('[migrate] adopting a pause marker left by a dead migrate run.');
claimed = true;
} else {
try { writeFileSync(marker, tombBody, { flag: 'wx' }); } catch { /* a racer claimed the gap; theirs now */ }
try { unlinkSync(tomb); } catch { /* best-effort */ }
}
}
} catch { /* lost an adoption race, or unreadable: treat as foreign */ }
if (!claimed) {
// Live migrate, operator hold, or unreadable: NOT ours. Abort rather
// than run unowned — see the null contract above.
return null;
}
} else {
// EACCES / EROFS / ENOSPC: fail CLOSED, same as the foreign-marker
// case. Proceeding unquiesced here would let a running (or supervisor
// relaunched) daemon write into the source for the entire copy — all
// silently lost at the flip, the exact incident this marker prevents.
console.error(`[migrate] cannot write the autopilot pause marker at ${marker}: ${e instanceof Error ? e.message : String(e)}`);
return null;
}
}
// Cleanup registered IMMEDIATELY after the claim — a SIGTERM during the
// drain below must not orphan the marker. (The daemon also self-clears a
// migrate-prefixed marker whose recorded pid is dead, so even a SIGKILL
// here heals within one daemon poll.)
const release = markerRelease(marker, ownBody);
const deregister = registerCleanup('migrate-autopilot-resume', async () => release());
const resume = () => { release(); deregister(); };
let heartbeatAgeMs = Infinity;
try { heartbeatAgeMs = Date.now() - statSync(autopilotLockPath()).mtimeMs; }
catch { /* no lock: no live daemon */ }
console.log(heartbeatAgeMs < 600_000
? '[migrate] autopilot detected and paused for the duration of the copy.'
: '[migrate] autopilot pause marker parked for the copy window (no live daemon detected; covers one starting mid-copy).');
// Drain in-flight writers instead of sleeping a blind grace: cycle runs,
// sync imports, and embed backfills all hold live rows in the DB lock
// table while they work, so "no live locks" is the actual quiesced signal.
// An idle daemon holds nothing and the drain returns immediately (faster
// than any fixed sleep); a mid-cycle daemon is waited on up to the cap.
// Fail-open: a probe error must never block a migration — the marker still
// parks the NEXT cycle, which was the whole pre-drain guarantee.
const capMs = resolveDrainCapSeconds() * 1000;
if (engine && capMs > 0) {
const started = Date.now();
let lastLog = 0;
for (;;) {
let live: Awaited<ReturnType<typeof listLiveLocks>>;
try {
live = (await listLiveLocks(engine))
// The supervisor lock is PRESENCE, not work: a healthy minion
// supervisor holds it continuously, so counting it would park
// every drain at the cap and cry wolf. Actual work (cycle, sync,
// embed backfill, unify, ...) holds its own rows.
.filter((l) => !l.id.startsWith('gbrain-supervisor:'));
} catch (e) {
console.warn(`[migrate] could not probe the lock table (${e instanceof Error ? e.message : String(e)}); relying on the pause marker alone.`);
break;
}
// A claimed minion job writes through the worker process; count healthy
// active jobs as writers too. Fail-open — a missing queue table (older
// schema) must not block the migration.
try {
const rows = await engine.executeRaw(
`SELECT count(*)::text AS count FROM minion_jobs WHERE status = 'active' AND lock_until > now()`,
) as Array<{ count?: string }>;
const activeJobs = Number(rows?.[0]?.count ?? 0);
if (Number.isFinite(activeJobs) && activeJobs > 0) {
live = [...live, { id: `minion-jobs-active:${activeJobs}` } as (typeof live)[number]];
}
} catch { /* no queue table: nothing to count */ }
if (live.length === 0) {
if (started !== lastLog && lastLog !== 0) console.log('[migrate] in-flight work drained; starting the copy.');
break;
}
const waited = Date.now() - started;
if (waited >= capMs) {
console.warn(
`[migrate] ${live.length} writer lock(s) still live after ${Math.round(waited / 1000)}s (${live.map((l) => l.id).join(', ')}). ` +
'Proceeding — anything they write during the copy will NOT be carried to the target. ' +
'For large migrations, stop autopilot and idle the job queue first.',
);
break;
}
if (waited - lastLog >= 30_000) {
lastLog = waited;
console.log(`[migrate] waiting for in-flight work to drain: ${live.map((l) => l.id).join(', ')} (${Math.round((capMs - waited) / 1000)}s left)...`);
}
await new Promise((r) => setTimeout(r, 3_000));
}
}
return resume;
}
/**
* PGLite -> Postgres reaches this legacy copier only when graduation is not
* routed (src/commands/migrate-graduation.ts routesToGraduation): opted out
* with migrate.graduation=false, or Windows. On Windows a brain with write
* history gets graduation's platform refusal; a history-free brain copies here.
*/
export async function assertWindowsGraduationPlatform(sourceEngine: BrainEngine, opts: Pick<MigrateOpts, 'targetEngine'>, platform: NodeJS.Platform = process.platform): Promise<void> {
if (platform !== 'win32' || sourceEngine.kind !== 'pglite' || opts.targetEngine !== 'postgres') return;
const { graduationOptedOut } = await import('./migrate-graduation.ts');
if (graduationOptedOut(loadConfig())) return;
const tables = ['persistence_requests', 'fact_withdrawals', 'persistence_worktrees'];
const present = await sourceEngine.executeRaw<{ name: string }>(
`SELECT table_name AS name FROM information_schema.tables WHERE table_schema = 'public' AND table_name = ANY($1::text[])`, [tables]);
for (const { name } of present) {
const [row] = await sourceEngine.executeRaw<{ present: boolean }>(`SELECT EXISTS(SELECT 1 FROM ${name}) AS present`);
if (row?.present) {
const { unsupportedPlatformError } = await import('../core/persistence/graduation-errors.ts');
throw unsupportedPlatformError({ platform: 'Windows' });
}
}
}
/** GBRA-130 A-31: prints the open-loop rows the target refused (each blocks the engine switch), sets the exit verdict, and returns how many. */
function reportLoopRowFailures(tables: Record<string, MigrateSimpleTableResult>, opts: Pick<MigrateOpts, 'targetEngine' | 'targetPath' | 'targetUrl'>): number {
const failures = Object.entries(tables).flatMap(([table, r]) => r.failed.map(f => ({ table, ...f })));
if (failures.length === 0) return 0;
console.error(`\n${failures.length} open-loop row(s) FAILED to copy:`);
for (const f of failures.slice(0, 20)) console.error(` - ${f.table} id ${f.id} (${f.code}): ${f.reason}`);
if (failures.length > 20) console.error(` ... and ${failures.length - 20} more`);
const retry = ['gbrain', 'migrate', '--to', opts.targetEngine, ...(opts.targetPath ? ['--path', opts.targetPath] : []), ...(opts.targetUrl ? ['--url', '<the same connection string>'] : [])];
console.error(`The active engine was NOT switched. Fix the cause each reason names on the target, then re-run: ${retry.join(' ')}`);
setCliExitVerdict(1);
return failures.length;
}
export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]): Promise<void> {
const opts = parseArgs(args);
const sourceKind = sqlEngineKind(sourceEngine.kind, 'gbrain migrate'); // SQL on both sides: a Rust source refuses before any read.
await assertWindowsGraduationPlatform(sourceEngine, opts);
const migration = { from: sourceKind, to: opts.targetEngine };
await assertUnmanagedCanonicalWriter(sourceEngine, 'engine migration', { migration: { side: 'source', ...migration } });
await assertLegacyEngineMigration(sourceEngine, { side: 'source', ...migration });
const config = loadConfig();
if (!config) {
console.error('No brain configured. Run: gbrain init');
process.exit(1);
}
// Check source != target
if (config.engine === opts.targetEngine) {
console.error(`Already using ${opts.targetEngine} engine. Nothing to migrate.`);
process.exit(1);
}
// Quiesce FIRST — before the target is even connected. The pause marker
// doubles as the migrate mutex: a second migrate aborts here instead of
// racing this one's manifest and target (a concurrent force run could
// delete pages the first run already copied). It also means everything
// below (target initSchema, manifest handling, the copy) runs with the
// daemon parked. Every process.exit in this function is above this line;
// the paths below it return through the finally instead.
const resumeAutopilot = await quiesceAutopilot(sourceEngine);
if (!resumeAutopilot) {
console.error('Could not take ownership of the autopilot pause (a concurrent migrate, an operator hold, or an unwritable marker — see above).');
console.error(`Refusing to migrate without it: a daemon this migration cannot pause keeps writing into the source engine, and those writes are lost at the flip. Inspect ${autopilotPausedMarkerPath()} if you are sure nothing is running.`);
process.exit(1);
}
// Build target config
const targetConfig: EngineConfig = { engine: opts.targetEngine };
if (opts.targetEngine === 'postgres') {
// #427 guard: don't let a cwd-.env DATABASE_URL become a migration target.
targetConfig.database_url = opts.targetUrl || effectiveEnvDatabaseUrl();
if (!targetConfig.database_url) {
console.error('Target is Supabase but no connection string provided. Use: --url <connection_string>');
// Not process.exit: that would skip the resume in the finally below and
// leave the daemon parked on the pause marker.
setCliExitVerdict(1);
resumeAutopilot();
return;
}
} else {
targetConfig.database_path = opts.targetPath || gbrainPath('brain.pglite');
}
const targetId = migrationTargetId(targetConfig);
// Connect to target
console.log(`Connecting to target (${opts.targetEngine})...`);
const targetEngine = await createEngine(targetConfig);
await targetEngine.connect(targetConfig);
await targetEngine.initSchema();
try {
await assertUnmanagedCanonicalWriter(targetEngine, 'engine migration', { migration: { side: 'target', ...migration } });
await assertLegacyEngineMigration(targetEngine, { side: 'target', ...migration });
} catch (error) {
try { await targetEngine.disconnect(); } finally { resumeAutopilot(); }
throw error;
}
// Load or create manifest for resume. Checked BEFORE the non-empty-target
// guard below: a manifest matching this exact target means the target's
// existing rows came from OUR OWN in-progress migration (#3194's per-page
// failures now leave the target non-empty by design instead of crashing),
// so a resume must not be treated as "attempting to migrate into a
// foreign non-empty brain".
let manifest = loadManifest();
if (manifest && !manifestMatchesTarget(manifest, targetId)) {
console.log('Previous migration was to a different target. Starting fresh.');
manifest = null;
}
const resumingMatchingManifest = manifest !== null;
// Check if target has data
const targetStats = await targetEngine.getStats();
if (opts.force) {
if (targetStats.page_count > 0) {
console.log('--force: wiping target brain...');
// v0.18.0+ multi-source: deletePage(slug) is now source-scoped (defaults
// to 'default'), so per-page iteration would skip non-default-source
// rows. migrate-engine --force is a destructive wipe across the entire
// brain — all sources, all pages — so we issue a raw DELETE that matches
// the original semantic. Cascades through content_chunks / page_links /
// tags / timeline_entries / page_versions via existing FKs.
await targetEngine.executeRaw('DELETE FROM pages');
}
// --force always starts this exact migration fresh against this target:
// a manifest tracking a previous attempt must not be trusted to skip
// pages, regardless of whether the target LOOKED non-empty just now
// (e.g. the target DB file was recreated out-of-band but
// ~/.gbrain/migrate-manifest.json survived) — round 2 of #3194.
manifest = null;
} else if (targetStats.page_count > 0 && !resumingMatchingManifest) {
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
console.error('Run with --force to overwrite, or migrate to an empty brain.');
await targetEngine.disconnect();
// Not process.exit: the resume must run (see the quiesce block above).
setCliExitVerdict(1);
resumeAutopilot();
return;
} else if (targetStats.page_count > 0 && resumingMatchingManifest) {
console.log(`Resuming previous migration: ${manifest!.completed_slugs.length} page(s) already copied.`);
} else if (!resumingMatchingManifest && await targetHasFactRows(targetEngine)) {
// Facts leg of the non-empty guard: a facts-only foreign target (zero
// pages, so the page_count check above passed) would lose its hot memory
// to copyMigrationFacts's DELETE FROM facts. A matching manifest means
// those rows came from OUR OWN in-progress run (delete-and-recopy
// converges them), so only a manifest-less target refuses. Same refusal
// shape as the page guard.
console.error('Target brain is not empty (its facts table has rows, though no pages).');
console.error('Run with --force to overwrite, or migrate to an empty brain.');
await targetEngine.disconnect();
// Not process.exit: the resume must run (see the quiesce block above).
setCliExitVerdict(1);
resumeAutopilot();
return;
}
// v0.32.8 F8: manifest keys are now `${source_id}::${slug}` so multi-source
// migrations don't collide on same-slug-different-source pages. Pre-v0.32.8
// entries were bare slugs; we keep treating those as default-source for
// back-compat resume.
const completedSet = new Set(manifest?.completed_slugs || []);
if (!manifest) {
manifest = {
completed_slugs: [],
target_engine: opts.targetEngine,
target_id: targetId,
schema_version: 2,
started_at: new Date().toISOString(),
};
}
// Persist immediately, before any page copy runs. Otherwise a run where
// EVERY page fails after its putPage lands (but before completed_slugs
// ever gets a successful entry) leaves the target non-empty with no
// manifest file on disk at all — the next invocation can't tell this
// was a resumable in-progress migration and hits the non-empty guard
// above requiring --force (round 2 of #3194).
saveManifest(manifest);
// Pages.source_id is a foreign key. Copy the complete source catalog first,
// including archived rows and sync/routing metadata, so every page write has
// a valid parent and the target preserves source behavior.
//
// The pause marker must not outlive this process. `registerCleanup` covers
// signals and UNhandled rejections, but a caught-and-rethrown error (or any
// handled failure path) exits through normal unwinding — without this
// finally, that path leaks the marker and the daemon idles forever while
// the status command keeps reporting fresh. On success, finally still runs
// AFTER the config flip above, preserving the resume-after-flip ordering:
// resuming any earlier would let the daemon write into the source engine
// again, and those writes would be the exact ones the migration cannot
// carry over.
//
// Known residual: the finally fires before verifyTarget below, so a fast
// supervisor relaunch can touch the target during verification — verify is
// read-only, so the cost is at most a noisy count, not corruption.
//
// NOTE the daemon still holds the PRE-migration config in memory, and its
// health probe keeps succeeding against the old engine (the backup stays
// alive), so it would never reconnect on its own. Its tick loop compares
// the file-plane engine identity against what it booted with
// (`autopilotEngineIdentity`) and exits cleanly for supervisor relaunch on
// the new config — that check, not reconnect(), is what converges it.
let sourceStats!: Awaited<ReturnType<BrainEngine['getStats']>>;
let pagesToMigrate: MigratePageRef[] = [];
let migrated = 0;
let failures: MigratePageFailure[] = [];
// Pages soft-deleted on the source after the listing: not copied, not failures.
let vanished: MigratePageRef[] = [];
// Per-table copy counts for the end-of-run summary (#4350): every table
// the migration touches reports what actually landed, so an omitted table
// is visible instead of silent.
let sourcesCopied = 0;
const rowCounts: PageCopyCounts = { chunks: 0, tags: 0, timeline_entries: 0, raw_data: 0 };
let linksCopied = 0;
let factsResult: MigrateFactsResult = { copied: 0, failed: [], embeddings_dropped: 0, table_missing: false };
let openLoopsResult: MigrateSimpleTableResult = { copied: 0, failed: [], table_missing: true };
let loopSuppressionsResult: MigrateSimpleTableResult = { copied: 0, failed: [], table_missing: true };
let loopRowsFailed = 0;
let configResult: { copied: number; skipped: string[] } = { copied: 0, skipped: [] };
try {
sourcesCopied = await copyMigrationSources(sourceEngine, targetEngine);
// Every live page's (source_id, slug) from one uncapped query; each body
// is read in the copy loop, so memory holds refs, not the whole brain.
sourceStats = await sourceEngine.getStats();
const allPages = await sourceEngine.listAllPageRefs();
pagesToMigrate = allPages.filter(p => !completedSet.has(makeManifestKey(p.source_id, p.slug)));
console.log(`Migrating ${pagesToMigrate.length} pages (${allPages.length} total, ${completedSet.size} already done)...`);
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
progress.start('migrate.copy_pages', pagesToMigrate.length);
({ migrated, failures, vanished } = await copyMigrationPages(sourceEngine, targetEngine, pagesToMigrate, manifest!, rowCounts, progress));
progress.finish();
if (vanished.length > 0) {
console.log(`${vanished.length} page(s) were deleted on the source during the copy and were not migrated.`);
}
if (failures.length > 0) {
console.error(`\n${failures.length} of ${pagesToMigrate.length} page(s) FAILED to copy and were NOT migrated:`);
for (const f of failures) {
const key = f.source_id === 'default' ? f.slug : `${f.source_id}::${f.slug}`;
console.error(` - ${key}: ${f.reason}`);
}
console.error('Re-run `gbrain migrate` to retry the failed pages (already-copied pages resume via the manifest).');
// Non-fatal so the run still copies links + config for everything that
// DID land, but the process must exit non-zero — a partial migration
// must never look identical to a clean one.
setCliExitVerdict(1);
}
// Copy links (after all pages exist in target).
// v0.32.8 F8: thread source_id so cross-source links migrate correctly.
// #3194: a page that failed to copy above does NOT exist on the target,
// so any link touching it would violate the target's FK and abort this
// whole phase (the exact "addLink failed: page ... not found" crash from
// the original report). Skip links on either end of a known-failed page —
// a retry that successfully copies the page also re-copies its links.
// A page deleted mid-copy is absent on the target the same way.
const failedKeys = new Set([...failures, ...vanished].map(f => makeManifestKey(f.source_id, f.slug)));
console.log('Copying links...');
progress.start('migrate.copy_links', allPages.length);
for (const page of allPages) {
if (failedKeys.has(makeManifestKey(page.source_id, page.slug))) {
progress.tick(1);
continue;
}
linksCopied += await copyPageLinksToTarget(sourceEngine, targetEngine, page, failedKeys);
progress.tick(1);
}
progress.finish();
// Copy facts (#4350): hot memory has no markdown mirror for
// conversation facts, so skipping this table loses them permanently.
console.log('Copying facts...');
progress.start('migrate.copy_facts');
factsResult = await copyMigrationFacts(sourceEngine, targetEngine, () => progress.tick(1));
openLoopsResult = await copyMigrationSimpleTable(sourceEngine, targetEngine, 'open_loops', () => progress.tick(1));
loopSuppressionsResult = await copyMigrationSimpleTable(sourceEngine, targetEngine, 'loop_suppressions', () => progress.tick(1));
progress.finish();
loopRowsFailed = reportLoopRowFailures({ open_loops: openLoopsResult, loop_suppressions: loopSuppressionsResult }, opts);
if (factsResult.failed.length > 0) {
console.error(`\n${factsResult.failed.length} fact row(s) FAILED to copy:`);
for (const f of factsResult.failed.slice(0, 10)) {
console.error(` - fact id ${f.id}: ${f.reason}`);
}
if (factsResult.failed.length > 10) {
console.error(` ... and ${factsResult.failed.length - 10} more`);
}
console.error('Re-run `gbrain migrate` to retry (facts re-copy from scratch each run).');
// Same contract as page failures: the run keeps going so everything
// else lands, but it must exit non-zero and must not flip the config.
setCliExitVerdict(1);
}
// Copy config (#4350): every DB-plane row except the engine-local
// denylist (see MIGRATE_CONFIG_ENGINE_LOCAL_KEYS). The old 3-key
// allowlist (embedding_model / embedding_dimensions / chunk_strategy —
// the v0.37 Lane C.4 schema metadata) silently dropped everything else:
// sync anchors, search settings, feature toggles. Those three still copy
// as part of copy-all. The file-plane config (`~/.gbrain/config.json`)
// is untouched by this — the newConfig flip below preserves it.
console.log('Copying config rows...');
configResult = await copyMigrationConfig(sourceEngine, targetEngine);
// Update local config. v0.37 fix wave: preserve existing file-plane
// embedding/expansion/chat config across the engine migration; only
// the engine + connection target should change.
//
// #3194: only flip the ACTIVE config when the migration is fully clean
// (no page failures AND no fact-row failures). A partial migration
// leaves the target's data incomplete; auto-switching
// every subsequent `gbrain` invocation onto that incomplete target would
// (a) make the failure invisible behind otherwise-normal usage and (b)
// break the natural retry — `gbrain migrate --to X` again would hit the
// "Already using X engine" guard even though the migration never actually
// finished. Leaving the file-plane config untouched keeps the source the
// active engine, so a retry (which resumes via the still-intact manifest)
// is a same-shaped command, not a special case.
// #6286: every copied page was queued for a projection rebuild; rebuild them on the target now (keyless, no provider
// calls), before the flip and while autopilot is paused, so the new active brain starts sealed and its first sync
// does not replace chunks. A page whose rebuild fails blocks the flip like a page that failed to copy.
let projectionsFailed = 0;
if (failures.length === 0 && factsResult.failed.length === 0 && loopRowsFailed === 0) {
console.log('Rebuilding text projections on the target...');
const drained = await drainProjections(targetEngine, { reasons: MIGRATION_PROJECTION_REASONS });
projectionsFailed = drained.failed.length;
console.log(`Projections: ${drained.rebuilt} rebuilt, ${drained.superseded} superseded, ${drained.failed.length} failed, ${drained.remaining} still queued.`);
if (projectionsFailed) {
for (const f of drained.failed.slice(0, 10)) console.error(` - ${f.source_id}/${f.slug}: ${f.reason}`);
console.error('The active engine was NOT switched: the target has pages whose text projection did not rebuild. Fix them (`gbrain projections drain` on the target names each next step), then re-run `gbrain migrate` to resume.');
setCliExitVerdict(1);
}
}
if (failures.length === 0 && factsResult.failed.length === 0 && loopRowsFailed === 0 && projectionsFailed === 0) {
const existingFile = (await import('../core/config.ts')).loadConfigFileOnly() ?? ({} as GBrainConfig);
const newConfig: GBrainConfig = {
...existingFile,
engine: opts.targetEngine,
...(opts.targetEngine === 'postgres'
? { database_url: targetConfig.database_url, database_path: undefined }
: { database_path: targetConfig.database_path, database_url: undefined }),
};
saveConfig(newConfig);
// An exported connection-string env var overrides the file config for
// every gbrain process (including the relaunched daemon), which would
// silently keep the whole brain on the OLD engine after this flip.
const envUrl = effectiveEnvDatabaseUrl();
if (envUrl && envUrl !== targetConfig.database_url) {
console.warn('WARNING: a DATABASE_URL-style env var is exported and does not match the migration target.');
console.warn('It overrides config.json for every gbrain process — unset it (shell profile, launchd plist, systemd unit) or the brain stays on the old engine.');
}
// Clean up the resume manifest — only safe once nothing is left pending.
clearManifest();
}
} finally {
resumeAutopilot();
}
// Per-table copied-count summary (#4350): every table the migration
// touches reports what landed THIS run, so a dropped table is visible
// instead of silent. Facts/config re-copy fully on every run; page-plane
// counts cover only pages not already banked in the resume manifest.
console.log('\nCopied to target (this run):');
const summaryRow = (label: string, detail: string) => console.log(` ${label.padEnd(18)} ${detail}`);
summaryRow('sources', String(sourcesCopied));
summaryRow('pages', failures.length > 0 ? `${migrated} (${failures.length} FAILED)` : String(migrated));
summaryRow('chunks', String(rowCounts.chunks));
summaryRow('tags', String(rowCounts.tags));
summaryRow('timeline entries', String(rowCounts.timeline_entries));
summaryRow('raw data rows', String(rowCounts.raw_data));
summaryRow('links', String(linksCopied));
if (factsResult.table_missing) {
summaryRow('facts', '0 (source schema predates the facts table)');
} else {
const factsNotes = [
factsResult.failed.length > 0 ? `${factsResult.failed.length} FAILED` : '',
factsResult.embeddings_dropped > 0
? `${factsResult.embeddings_dropped} embedding(s) dropped — run a re-embed on the target`
: '',
].filter(Boolean).join('; ');
summaryRow('facts', factsNotes ? `${factsResult.copied} (${factsNotes})` : String(factsResult.copied));
}
const loopsRow = (label: string, r: MigrateSimpleTableResult) =>
summaryRow(label, r.table_missing
? '0 (source schema predates the open-loop tables)'
: r.failed.length > 0 ? `${r.copied} (${r.failed.length} FAILED)` : String(r.copied));
loopsRow('open loops', openLoopsResult);
loopsRow('loop suppressions', loopSuppressionsResult);
summaryRow('config rows', configResult.skipped.length > 0
? `${configResult.copied} (skipped engine-local: ${configResult.skipped.join(', ')})`
: String(configResult.copied));
const cleanRun = failures.length === 0 && factsResult.failed.length === 0 && loopRowsFailed === 0;
if (!cleanRun) {
const parts = [
failures.length > 0 ? `${migrated} of ${pagesToMigrate.length} pages copied, ${failures.length} failed (${completedSet.size} already done from a prior run)` : '',
factsResult.failed.length > 0 ? `${factsResult.failed.length} fact row(s) failed` : '',
loopRowsFailed > 0 ? `${loopRowsFailed} open-loop row(s) failed` : '',
].filter(Boolean).join('; ');
console.log(`\nMigration completed with errors. ${parts}. See failure list above.`);
console.log(`Config NOT switched — still using engine: ${config.engine}. Re-run \`gbrain migrate --to ${opts.targetEngine}\` to retry; already-copied pages resume via the manifest.`);
} else {
console.log(`\nMigration complete. ${migrated} pages transferred.`);
console.log(`Config updated to engine: ${opts.targetEngine}`);
}
if (cleanRun && config.engine === 'pglite' && config.database_path) {
console.log(`Original PGLite brain preserved at ${config.database_path} (backup).`);
}
// Post-migrate verification: confirm the target is healthy before we
// leave the user. Catches incomplete copies, schema drift, and missing
// embeddings immediately instead of on next CLI use. Non-fatal — prints
// warnings and keeps going so the user sees the full picture.
console.log('\nVerifying target...');
try {
// #4485: compare timeline counts too — the LIMIT-100 truncation class of
// bug (a bounded read inside the copy) is invisible to the page-count
// check, so verify pins the row-level tables it can count cheaply on
// both sides. wave-g: counts are scoped to the COPIED page set (live
// pages), not the whole table — the copy iterates listPages() output,
// which excludes soft-deleted pages, so a soft-deleted source page that
// still owns timeline rows produced a false WARN under whole-table
// counts. Both sides use the identical predicate.
let sourceTimelineCount: number | undefined;
try {
const rows = await sourceEngine.executeRaw<{ count: string }>(COPIED_TIMELINE_COUNT_SQL);
sourceTimelineCount = Number(rows[0]?.count ?? NaN);
if (!Number.isFinite(sourceTimelineCount)) sourceTimelineCount = undefined;
} catch { /* older schema: skip the timeline parity check */ }
await verifyTarget(targetEngine, sourceStats.page_count, sourceTimelineCount);
} catch (e) {
console.warn(` Verification could not complete: ${e instanceof Error ? e.message : String(e)}`);
}
await targetEngine.disconnect();
}
/**
* wave-g (#4485 follow-up): timeline parity counts rows over the COPIED page
* set — timeline entries whose page is live (`deleted_at IS NULL`). The
* migration copies listPages() output, which excludes soft-deleted pages, so
* whole-table counts false-WARNed whenever a soft-deleted page still owned
* timeline rows on the source. Shared by the source read and verifyTarget so
* the two sides can never drift; throws on pre-deleted_at schemas, which the
* callers' try/catch already fail-opens to "skip the parity check".
* Exported for the test surface only.
*/
export const COPIED_TIMELINE_COUNT_SQL =
`SELECT count(*)::text AS count
FROM timeline_entries te
JOIN pages p ON p.id = te.page_id
WHERE p.deleted_at IS NULL`;
/**
* Lightweight doctor-style verify run against the migrated target.
* Prints a small table of signals; does not exit. Callers own engine
* lifecycle.
*/
async function verifyTarget(engine: BrainEngine, expectedPages: number, expectedTimelineEntries?: number): Promise<void> {
const stats = await engine.getStats();
if (stats.page_count === expectedPages) {
console.log(` ok pages: ${stats.page_count} (matches source)`);
} else {
console.warn(` WARN pages: ${stats.page_count} (source had ${expectedPages})`);
}
// #4485: row-level parity for the timeline table — a bounded read inside
// the copy (the old LIMIT-100 truncation) passes the page-count check just
// fine, so the verify step counts the rows themselves. wave-g: scoped to
// the copied (live) page set on BOTH sides — see COPIED_TIMELINE_COUNT_SQL.
if (expectedTimelineEntries !== undefined) {
try {
const rows = await engine.executeRaw<{ count: string }>(COPIED_TIMELINE_COUNT_SQL);
const targetTimeline = Number(rows[0]?.count ?? NaN);
if (targetTimeline === expectedTimelineEntries) {
console.log(` ok timeline entries: ${targetTimeline} (matches source)`);
} else {
console.warn(` WARN timeline entries: ${targetTimeline} (source had ${expectedTimelineEntries})`);
}
} catch (e) {
console.warn(` WARN timeline entries: could not count (${e instanceof Error ? e.message : String(e)})`);
}
}
try {
const health = await engine.getHealth();
const pct = (health.embed_coverage * 100).toFixed(0);
if (health.embed_coverage >= 0.9) {
console.log(` ok embeddings: ${pct}% coverage, ${health.missing_embeddings} missing`);
} else {
console.warn(` WARN embeddings: ${pct}% coverage, ${health.missing_embeddings} missing. Run: gbrain embed --stale`);
}
} catch (e) {
console.warn(` WARN embeddings: could not measure (${e instanceof Error ? e.message : String(e)})`);
}
try {
const version = await engine.getConfig('version');
const { LATEST_VERSION } = await import('../core/migrate.ts');
const schemaVersion = parseInt(version || '0', 10);
if (schemaVersion >= LATEST_VERSION) {
console.log(` ok schema: version ${schemaVersion}`);
} else {
console.warn(` WARN schema: version ${schemaVersion} (latest: ${LATEST_VERSION}). Run: gbrain apply-migrations --yes`);
}
} catch {
console.warn(' WARN schema: version could not be read');
}
console.log(' Full health check: gbrain doctor');
}