Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
71.8 KB
Raw Blame
/**
 * archive.ts — the built-in conversation archive (GST-2768): on by default on a personal brain, offered on a shared one
 * (archive-default.ts); the maintenance pass runs it (archive-maintenance.ts).
 *
 * gbrain transcripts archive run | backfill --from --to | status | adopt-legacy | on | off.
 *
 * Each run reads every readable harness (the file adapters through harnessRoots(), OpenClaw archived session files,
 * and the OpenClaw Gateway), redacts each session as a whole, cuts it into local days in the source's pinned
 * timezone, and writes one page per (harness, conversation, day) plus a day index page (archive-day.ts). It never
 * touches a per-session page: day pages have their own slug and id namespace.
 *
 * Ledger: one config row per (source, harness, conversation_key_hash, day) under transcripts.archive.ledger.*, holding
 * the content hash, the complete flag, the slug and part count, and a held error code. The config table never
 * expires rows (op_checkpoint does, after 7 days). A day is rewritten only when its rendered content changed, so a
 * rerun, an overlapping backfill and a crash retry all converge on the same pages; a closed day whose redaction
 * changed is rewritten because its hash changed. A day whose source now has fewer messages than the ledger recorded
 * (an OpenClaw reset in place drops the earlier messages from chat.history) is kept as written.
 *
 * Timezone: pinned per source at the first run (transcripts.archive.pinned_timezone.<source>) from
 * transcripts.archive.timezone or the machine's zone, and stamped into every page. A later change to the setting
 * never re-slices.
 *
 * Privacy: a personal brain writes no readable_by (every caller is the owner). A shared brain (brain.audience
 * shared, unknown, or two or more enabled members) gives every page readable_by [owner]; a verified group or room
 * adds group:chat-<hash8> only when that group principal already exists. With no owner member set, every day is
 * held with archive_owner_unset: no archive page is ever written without readable_by on a shared brain. A shared
 * brain whose archive source sits in a repository archives into <source>-owner, a database-only source, so nothing
 * is committed where repository readers could read it.
 *
 * Write paths: managed persistence goes through the coordinator; an unmanaged repository source gets the redacted
 * files written, committed with gbrain git commit-push -- <paths>, then imported by sync (git stays the parent); any
 * other brain writes the database directly. Nothing refuses for lack of a repository.
 *
 * Embedding: pages are written keyword-only. Backfilled pages are held in page_embedding_budget (reason
 * archive_backfill) until the owner approves the previewed cost with gbrain embed --held. The open day is held
 * (archive_open_day) until it closes or has been quiet for transcripts.archive.quiet_minutes. No fact extraction.
 */

import { createHash } from 'node:crypto';
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { dirname, join, relative, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { Action } from '../agent-output.ts';
import type { OperationContext } from '../ops/contract.ts';
import { importFromContent } from '../import-file.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { acceptedPendingReceipt } from '../persistence/accepted-pending.ts';
import { isTerminalWriteState, type WriteRequestState } from '../persistence/types.ts';
import { CLI_WRITE_WAIT_MS } from '../persistence/write-wait.ts';
import { classifyItemFailure } from '../item-failure.ts';
import { classifyBrainAudience } from '../facts/writeback-audience.ts';
import { ARCHIVE_ENABLED_KEY, resolveArchiveEnabled } from './archive-default.ts';
import { listGroups, listMembers } from '../access/members.ts';
import { ownerView } from '../access/owner.ts';
import { holdPageEmbedding } from '../page-embedding-budget.ts';
import { PAGE_MAX_BYTES } from '../page-size-policy.ts';
import type { ParsedSession, TranscriptFormat } from './types.ts';
import { transcriptFullId } from './types.ts';
import { loadImportRedactionPatterns, PAGE_CEILING_FILL, redactSession, type RedactedSession, type RenderedPart } from './render.ts';
import { deleteStaleTranscriptPart, submitTranscriptPart, transcriptPageCeiling } from './ingest.ts';
import {
  addDays, assertTimeZone, localDay, renderArchiveDay, renderDayIndex, sliceSessionByDay, conversationHash8,
  type ArchiveChannel, type DayIndexEntry,
} from './archive-day.ts';

export type { ArchiveChannel } from './archive-day.ts';

export { ARCHIVE_ENABLED_KEY } from './archive-default.ts';
export const ARCHIVE_TIMEZONE_KEY = 'transcripts.archive.timezone';
export const ARCHIVE_SOURCE_KEY = 'transcripts.archive.source';
export const ARCHIVE_BACKFILL_CAP_KEY = 'transcripts.archive.backfill_days_per_run';
export const ARCHIVE_RUN_WINDOW_KEY = 'transcripts.archive.run_window_days';
export const ARCHIVE_QUIET_MINUTES_KEY = 'transcripts.archive.quiet_minutes';
export const ARCHIVE_BACKFILL_HOLD_REASON = 'archive_backfill';
export const ARCHIVE_OPEN_DAY_HOLD_REASON = 'archive_open_day';
export const DEFAULT_ARCHIVE_SOURCE = 'conversations';
export const DEFAULT_BACKFILL_DAYS_PER_RUN = 30;
export const DEFAULT_RUN_WINDOW_DAYS = 2;
export const DEFAULT_QUIET_MINUTES = 180;
const LEDGER_PREFIX = 'transcripts.archive.ledger.';
const PIN_PREFIX = 'transcripts.archive.pinned_timezone.';
const LEGACY_PREFIX = 'transcripts.archive.legacy.';
/** The audience (personal or shared) the archive last ran under, per configured source. */
const AUDIENCE_PREFIX = 'transcripts.archive.audience.';
const RESCOPE_CAPPED_PREFIX = 'transcripts.archive.rescope_capped.';
/** Days whose index page still has to be written (a failed or interrupted index write), per source. */
const INDEX_PENDING_PREFIX = 'transcripts.archive.index_pending.';
/** What the last run that read each reader found, per source: status and doctor report it without reading again. */
const LAST_READ_PREFIX = 'transcripts.archive.last_read.';
const INGESTED_VIA = 'cli:transcripts-archive';
const DAY_RE = /^\d{4}-\d{2}-\d{2}$/;

export interface ArchiveConversation {
  harness: TranscriptFormat;
  /** The conversation's identity, the same whichever path read it: the session id. */
  conversationKey: string;
  /** The harness's own key for the conversation when it differs (the OpenClaw session key): it names the group principal and label, never the page. */
  scopeKey?: string;
  session: ParsedSession;
  channel?: ArchiveChannel;
  /** Where it was read; a Gateway conversation wins over an archived file of the same OpenClaw session. */
  via?: 'gateway' | 'file';
}

/** One conversation found by a scan: its days and how to read it again; no message is held between the two. */
export interface ArchiveScanEntry {
  harness: TranscriptFormat;
  conversationKey: string;
  sessionId: string;
  via?: 'gateway' | 'file';
  /** Local days (dayOf) that hold at least one message. */
  days: string[];
  /** Per day, the identities (archiveMessageId) of its messages, so backfill can tell a finished day from a changed one. */
  dayIds?: Record<string, string[]>;
  newest?: string;
  load(): Promise<ArchiveConversation | null>;
}

export interface ArchiveScanOpts { sinceMs?: number; minMtimeMs?: number; dayOf: (ts: string) => string }

export interface ArchiveReader {
  harness: TranscriptFormat;
  name: string;
  /** Every readable conversation (sinceMs narrows by activity). Throws a coded error; yields nothing partial. */
  list?(opts: { sinceMs?: number }): Promise<ArchiveConversation[]>;
  /** Streaming form for large stores: one entry per conversation, parsed one file at a time and loaded again only for the days a run writes. */
  scan?(opts: ArchiveScanOpts): AsyncIterable<ArchiveScanEntry>;
  /** Set when the harness has nothing to read on this machine: an informational line, never an error. */
  unavailable?: string;
}

export interface ArchiveHeld { code: string; message: string }

export interface ArchiveLedgerEntry {
  source_id: string;
  harness: string;
  conversation_key_hash: string;
  day: string;
  hash?: string;
  complete?: boolean;
  slug?: string;
  parts?: number;
  messages?: number;
  /** Identities (archiveMessageId) of the messages the page holds; a rewrite that would drop one is kept instead. */
  ids?: string[];
  /** Signature of the source's message identities for the day when it was written; backfill skips a finished day whose source still matches. */
  source?: string;
  /** Version of the redaction rules (built-in detectors and the user's patterns) the day was written under; backfill re-checks a day written under other rules. */
  redaction?: string;
  held?: ArchiveHeld;
  /** A write the coordinator accepted and has not published: its request ids and the content hash it carries. */
  pending?: { requests: string[]; hash: string; complete: boolean };
  updated_at?: string;
}

/** accepted: the write coordinator accepted the day's write and publishes it later; the ledger records its requests
 * without a content hash, so a later run waits while they are queued, records the day once they commit, and writes it
 * again when one was cancelled or failed. */
export type ArchiveDayStatus = 'written' | 'accepted' | 'unchanged' | 'held' | 'planned' | 'skipped_legacy' | 'kept' | 'error';

export interface ArchiveDayOutcome {
  harness: string;
  conversation_key_hash: string;
  day: string;
  status: ArchiveDayStatus;
  slug?: string;
  parts?: number;
  messages?: number;
  readable_by?: string[];
  redactions?: number;
  code?: string;
  message?: string;
}

export interface ArchiveReaderError { harness: string; reader: string; code: string; message: string; fix?: Action; /** When the run that met it read (status only). */ at?: string }

/**
 * One thing a run could not do: a reader that failed (reader set), a conversation day that failed (day set), the day
 * index, or the owner-unset hold on a shared brain. A run returns exit 1 exactly when this list is not empty.
 */
export interface ArchiveRunError { harness: string; reader?: string; day?: string; conversation_key_hash?: string; code: string; message: string; fix?: Action }

/** One reader as the last run that read it found it: the newest message any run saw, and the last run's error until a later read succeeds. */
interface ArchiveReadRecord {
  harness: string;
  at?: string;
  mode?: 'run' | 'backfill';
  conversations?: number;
  newest?: string;
  error?: { at: string; code: string; message: string; fix?: Action };
}

async function readArchiveReads(engine: BrainEngine, sourceId: string): Promise<Record<string, ArchiveReadRecord>> {
  const raw = await engine.getConfig(`${LAST_READ_PREFIX}${sourceId}`);
  try {
    const doc = raw ? JSON.parse(raw) as unknown : null;
    return doc && typeof doc === 'object' && !Array.isArray(doc) ? doc as Record<string, ArchiveReadRecord> : {};
  } catch {
    return {};
  }
}

export interface ArchiveRunResult {
  mode: 'run' | 'backfill';
  enabled: boolean;
  dry_run: boolean;
  source_id: string;
  timezone: string;
  write_mode: 'database' | 'repo' | 'coordinator';
  privacy: 'personal' | 'shared';
  from: string;
  to: string;
  days: ArchiveDayOutcome[];
  written: number;
  accepted: number;
  unchanged: number;
  held: number;
  planned: number;
  skipped_legacy: number;
  index_pages: string[];
  deferred_days: string[];
  info: string[];
  /** Every failure of this run (ArchiveRunError); empty exactly when the run exits 0. */
  errors: ArchiveRunError[];
  message?: string;
  fix?: Action;
  /** The run reached deadlineAtMs and left conversations for the next run. */
  stopped_at_deadline?: boolean;
}

export interface ArchiveRunOpts {
  mode: 'run' | 'backfill';
  from?: string;
  to?: string;
  dryRun?: boolean;
  sourceId?: string;
  readers?: ArchiveReader[];
  now?: Date;
  /** The CLI's trusted-local context: managed brains write through the coordinator with it. */
  context?: OperationContext;
  /** Builds that context for the archive source once it exists (the CLI passes its makeContext); called only on a managed brain. */
  makeContext?: (sourceId: string) => Promise<OperationContext>;
  /** Test seam: the redaction pass (default redactSession over the whole session). */
  redact?: (session: ParsedSession) => RedactedSession;
  /** Repository path: commit the written files (default gbrain git commit-push -- <paths>). */
  commitPaths?: (root: string, paths: string[], message: string) => Promise<void>;
  /** Stop starting new readers and conversations at this instant (epoch ms); what was not reached waits for the next run. */
  deadlineAtMs?: number;
  /** Repository path: import the committed files (default an in-process sync of the source). */
  syncSource?: (engine: BrainEngine, sourceId: string, root: string) => Promise<void>;
}

const sha256 = (s: string) => createHash('sha256').update(s).digest('hex');

/** The group principal a group or room conversation may be shared with, once the operator creates it: group:chat-<hash8>. */
export function archiveGroupPrincipal(harness: TranscriptFormat, conversationKey: string): string {
  return 'chat-' + conversationHash8(harness, conversationKey);
}

/** The resolved switch (archive-default.ts): an explicit on or off wins; unset is on for a personal brain only. */
export async function archiveEnabled(engine: Pick<BrainEngine, 'getConfig'>): Promise<boolean> {
  return (await resolveArchiveEnabled(engine)).enabled;
}

export async function setArchiveEnabled(engine: BrainEngine, on: boolean): Promise<void> {
  await engine.setConfig(ARCHIVE_ENABLED_KEY, on ? 'true' : 'false');
}

async function configuredSource(engine: BrainEngine, sourceId?: string): Promise<string> {
  return sourceId ?? ((await engine.getConfig(ARCHIVE_SOURCE_KEY))?.trim() || DEFAULT_ARCHIVE_SOURCE);
}

/** The source run, status and adopt-legacy all use: on a shared brain whose archive source is a checkout, its owner-only twin. */
async function effectiveSource(engine: BrainEngine, sourceId?: string): Promise<{ sourceId: string; privacy: Privacy }> {
  const configured = await configuredSource(engine, sourceId);
  const privacy = await resolvePrivacy(engine);
  const [existing] = await engine.executeRaw<{ local_path: string | null }>('SELECT local_path FROM sources WHERE id = $1', [configured]);
  return { sourceId: privacy.shared && existing?.local_path ? configured + '-owner' : configured, privacy };
}

/** One message's identity across readers and runs: role, timestamp and text, before redaction. */
export function archiveMessageId(m: { role: string; timestamp: string; text: string }): string {
  return createHash('sha256').update(m.role + '\u0000' + m.timestamp + '\u0000' + m.text).digest('hex').slice(0, 16);
}

const daySignature = (ids: Iterable<string>) => sha256([...new Set(ids)].sort().join(','));

function positiveInt(value: string | null, fallback: number): number {
  const n = Number(value?.trim());
  return Number.isSafeInteger(n) && n > 0 ? n : fallback;
}

function ledgerKey(sourceId: string, harness: string, hash: string, day: string): string {
  return LEDGER_PREFIX + sourceId + '.' + harness + '.' + hash + '.' + day;
}

export async function readArchiveLedger(engine: BrainEngine, sourceId: string): Promise<ArchiveLedgerEntry[]> {
  const prefix = LEDGER_PREFIX + sourceId + '.';
  const rows = await engine.executeRaw<{ key: string; value: string }>('SELECT key, value FROM config WHERE key LIKE $1 ORDER BY key', [prefix + '%']);
  const out: ArchiveLedgerEntry[] = [];
  for (const row of rows) {
    const [harness, hash, day] = row.key.slice(prefix.length).split('.');
    if (!harness || !hash || !day) continue;
    let value: Record<string, unknown> = {};
    try { value = JSON.parse(row.value) as Record<string, unknown>; } catch { /* an unreadable row reads as empty and is rewritten */ }
    out.push({ ...value, source_id: sourceId, harness, conversation_key_hash: hash, day } as ArchiveLedgerEntry);
  }
  return out;
}

async function writeLedger(engine: BrainEngine, entry: ArchiveLedgerEntry): Promise<void> {
  const { source_id, harness, conversation_key_hash, day, ...value } = entry;
  await engine.setConfig(ledgerKey(source_id, harness, conversation_key_hash, day), JSON.stringify({ ...value, updated_at: new Date().toISOString() }));
}

/**
 * Days an old archive folder covers, as adopt-legacy recorded them. adopt-legacy records them under the source it
 * resolved then (the configured archive source, or its <source>-owner twin on a shared brain whose source is a
 * checkout), so they are read under both ids: a brain that turned shared or personal in between still skips them.
 */
async function legacyCovered(engine: BrainEngine, configured: string): Promise<Set<string>> {
  const out = new Set<string>();
  for (const id of [configured, configured + '-owner']) {
    const prefix = LEGACY_PREFIX + id + '.';
    const rows = await engine.executeRaw<{ key: string }>('SELECT key FROM config WHERE key LIKE $1', [prefix + '%']);
    for (const r of rows) out.add(r.key.slice(prefix.length));
  }
  return out;
}

function machineTimeZone(): string {
  return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
}

/** The pinned timezone, or the one the first run will pin. */
async function archiveTimeZone(engine: BrainEngine, sourceId: string): Promise<{ timezone: string; pinned: boolean; configured: string }> {
  const configured = (await engine.getConfig(ARCHIVE_TIMEZONE_KEY))?.trim() || machineTimeZone();
  const pinned = (await engine.getConfig(`${PIN_PREFIX}${sourceId}`))?.trim();
  return pinned ? { timezone: pinned, pinned: true, configured } : { timezone: assertTimeZone(configured), pinned: false, configured };
}

/** unproven: the brain reads as personal only because its client-usage check failed, so it may still be shared. */
interface Privacy { shared: boolean; owner: string | null; unproven?: boolean }

async function resolvePrivacy(engine: BrainEngine): Promise<Privacy> {
  const { loadConfigFileOnly } = await import('../config.ts');
  const audience = await classifyBrainAudience(engine, loadConfigFileOnly());
  const enabled = (await listMembers(engine)).filter(m => m.status === 'enabled').length;
  const shared = audience.audience !== 'personal' || enabled >= 2;
  return { shared, owner: shared ? (await ownerView(engine)).owner : null, ...(!shared && audience.unproven ? { unproven: true } : {}) };
}

async function readersFor(engine: BrainEngine, privacy: Privacy, conv: ArchiveConversation, groups: Map<string, boolean>): Promise<string[] | undefined> {
  if (!privacy.shared) return undefined;
  const readers = [privacy.owner!];
  const ch = conv.channel;
  if (ch?.verified && (ch.chatType === 'group' || ch.chatType === 'room')) {
    const principal = archiveGroupPrincipal(conv.harness, conv.scopeKey ?? conv.conversationKey);
    if (!groups.has(principal)) groups.set(principal, (await listGroups(engine, principal).catch(() => [])).length > 0);
    if (groups.get(principal)) readers.push('group:' + principal);
  }
  return readers;
}

/**
 * Pages written while the brain was personal name no readers. When the brain has become shared, every folder under
 * conversations/ that holds an archive page at that moment, or a day the ledger recorded (a write the coordinator accepted
 * and has not published yet), is kept to the owner by a rule (a narrowing, so it applies and
 * restamps at once; a rule already at one of those folders is replaced, a wider one included) before the run writes
 * anything, then the new audience is recorded. Folders written later are not covered, so a group conversation archived
 * on the shared brain keeps its group. When those folders and the source's other rules would pass MAX_RULES, one owner-only rule on
 * conversations/ replaces them, so later group conversations stay owner-only too (archive_rescope_folder_cap). No other rule is
 * dropped, so a source that already held MAX_RULES other rules ends one rule over (rescopeCapped reports it). A source whose rules live in git is left to its access.yml.
 */
async function reconcileAudience(engine: BrainEngine, sourceId: string, privacy: Privacy, record = true): Promise<void> {
  const now = privacy.shared ? 'shared' : 'personal';
  const recorded = await engine.getConfig(`${AUDIENCE_PREFIX}${sourceId}`);
  if (recorded === now) return;
  if (privacy.shared && !privacy.owner) return;
  if (privacy.shared && recorded === 'personal' && (await engine.executeRaw('SELECT 1 FROM sources WHERE id = $1', [sourceId])).length) {
    const policy = await import('../access/policy.ts');
    const { MAX_RULES, normalizeRule } = await import('../access/folder-rules.ts');
    if (await policy.readAccessStore(engine, sourceId) === 'git') return;
    const folderOf = (slug: string) => slug.includes('/') ? slug.replace(/[^/]+$/, '') : slug;
    const pageFolders = await engine.executeRaw<{ folder: string }>(
      "SELECT DISTINCT regexp_replace(slug, '[^/]+$', '') AS folder FROM pages WHERE source_id = $1 AND slug LIKE 'conversations/%'", [sourceId]);
    const ledger = await readArchiveLedger(engine, sourceId);
    const folders = [...new Set([...pageFolders.map(r => r.folder), ...ledger.flatMap(e => e.slug ? [folderOf(e.slug)] : []), ...(ledger.length ? ['conversations/days/'] : [])])].sort();
    const current = await policy.appliedRules(engine, sourceId);
    const capped = current.filter(r => !folders.includes(r.prefix)).length + folders.length > MAX_RULES;
    const rules = (capped ? ['conversations/'] : folders).map(folder => {
      const normalized = normalizeRule(folder.endsWith('/') ? { prefix: folder, readable_by: [privacy.owner] } : { slug: folder, readable_by: [privacy.owner] });
      if ('problem' in normalized) throw new Error('the owner-only archive rule for ' + folder + ' is invalid: ' + normalized.problem);
      return normalized.rule;
    });
    const replaced = new Set(rules.map(r => r.prefix));
    if (rules.some(rule => current.find(r => r.prefix === rule.prefix)?.readable_by.join() !== rule.readable_by.join())) {
      await policy.proposeRevision(engine, sourceId, [...current.filter(r => !replaced.has(r.prefix)), ...rules], { origin: 'cli', store: 'database', proposedBy: 'transcripts-archive' });
    }
    if (capped) await engine.setConfig(`${RESCOPE_CAPPED_PREFIX}${sourceId}`, String(folders.length));
  }
  if (record) await engine.setConfig(`${AUDIENCE_PREFIX}${sourceId}`, now);
}

export interface ArchiveRescopeSkip { code: 'archive_rescope_owner_unset' | 'archive_rescope_git_rules' | 'archive_rescope_pending' | 'archive_rescope_folder_cap'; message: string }

/**
 * Runs the owner-only re-scope for a brain archived while personal that is now shared, whether or not the archive is
 * still on (an unset switch resolves off once the brain is shared). Returns whether one was due.
 */
export async function reconcileArchiveAudience(engine: BrainEngine, sourceId?: string): Promise<boolean> {
  const configured = await configuredSource(engine, sourceId);
  if (await engine.getConfig(`${AUDIENCE_PREFIX}${configured}`) !== 'personal') return false;
  const privacy = await resolvePrivacy(engine);
  if (!privacy.shared) return false;
  await reconcileAudience(engine, configured, privacy);
  return true;
}

/**
 * The owner-only re-scope for every archive source written while the brain was personal, run by the commands that can
 * turn a brain shared (gbrain access member add and member enable, gbrain access owner set, gbrain config set
 * brain.audience) so earlier pages are kept to the owner before a second member reads them, not at the next archive
 * run or maintenance pass. Returns each source whose re-scope was due, with why it could not apply (null when it did).
 * It holds the source's archive run lock. While a run that started on the personal brain holds it, that run keeps
 * writing pages with no readers into folders no rule covers yet, so the pages archived so far are kept to the owner now
 * but the audience stays recorded personal and the next run re-scopes again (archive_rescope_pending).
 */
export async function rescopeArchiveOnShared(engine: BrainEngine): Promise<Array<{ source_id: string; skipped: ArchiveRescopeSkip | null }>> {
  const rows = await engine.executeRaw<{ key: string }>("SELECT key FROM config WHERE key LIKE $1 AND value = 'personal' ORDER BY key", [AUDIENCE_PREFIX + '%']);
  if (!rows.length) return [];
  const privacy = await resolvePrivacy(engine);
  if (!privacy.shared) return [];
  const { withRefreshingLock, LockUnavailableError } = await import('../db-lock.ts');
  const out: Array<{ source_id: string; skipped: ArchiveRescopeSkip | null }> = [];
  for (const { key } of rows) {
    const sourceId = key.slice(AUDIENCE_PREFIX.length);
    try {
      if (!await withRefreshingLock(engine, ARCHIVE_LOCK_PREFIX + sourceId, () => reconcileArchiveAudience(engine, sourceId))) continue;
    } catch (err) {
      if (!(err instanceof LockUnavailableError)) throw err;
      await reconcileAudience(engine, sourceId, privacy, false);
      const skipped = await archiveRescopeSkipped(engine, sourceId);
      out.push({ source_id: sourceId, skipped: skipped?.code !== 'archive_rescope_pending' ? skipped : { code: 'archive_rescope_pending', message: 'Conversation archive pages archived so far are kept to the owner, but an archive run that started while this brain was personal is writing source ' + sourceId + ' now and its new pages are not (gbrain status --section locks --json shows the holder); once it finishes, run gbrain transcripts archive run.' } });
      continue;
    }
    out.push({ source_id: sourceId, skipped: await archiveRescopeSkipped(engine, sourceId) });
  }
  return out;
}

/**
 * Before a member change that adds an enabled member (gbrain access member add, or member enable of a disabled one)
 * and so may make the brain shared: keeps the archive pages written while personal to the owner first, so the member's
 * credentials never read them, even for the moment between the change and rescopeArchiveOnShared. The audience is not
 * recorded here; rescopeArchiveOnShared does that once the change has committed. Throws when the re-scope fails, so the
 * caller makes no change.
 */
export async function rescopeArchiveBeforeJoin(engine: BrainEngine, memberId: string): Promise<void> {
  const rows = await engine.executeRaw<{ key: string }>("SELECT key FROM config WHERE key LIKE $1 AND value = 'personal' ORDER BY key", [AUDIENCE_PREFIX + '%']);
  if (!rows.length) return;
  let privacy = await resolvePrivacy(engine);
  if (!privacy.shared) {
    const enabled = (await listMembers(engine)).filter(m => m.status === 'enabled').map(m => m.id);
    if (enabled.includes(memberId) || enabled.length + 1 < 2) return;
    const view = await ownerView(engine);
    privacy = { shared: true, owner: view.owner ?? (view.inactive?.value === memberId ? memberId : null) };
  }
  for (const { key } of rows) await reconcileAudience(engine, key.slice(AUDIENCE_PREFIX.length), privacy, false);
}

/** Why the owner-only re-scope (reconcileAudience) has not run on a brain that became shared, or null. */
export async function archiveRescopeSkipped(engine: BrainEngine, sourceId?: string): Promise<ArchiveRescopeSkip | null> {
  const configured = await configuredSource(engine, sourceId);
  if (await engine.getConfig(`${AUDIENCE_PREFIX}${configured}`) !== 'personal') return rescopeCapped(engine, configured);
  const privacy = await resolvePrivacy(engine);
  if (!privacy.shared) return null;
  if (!privacy.owner) {
    return { code: 'archive_rescope_owner_unset', message: 'Conversation archive pages written while this brain was personal are not yet kept to the owner, because the shared brain names no owner; run gbrain access owner set <member>, which applies the owner-only rules at once.' };
  }
  const policy = await import('../access/policy.ts');
  const pending: ArchiveRescopeSkip = { code: 'archive_rescope_pending', message: 'Conversation archive pages written while this brain was personal are not yet kept to the owner; the next maintenance pass applies the owner-only rules, or run gbrain transcripts archive run now.' };
  if ((await engine.executeRaw('SELECT 1 FROM sources WHERE id = $1', [configured])).length === 0) return pending;
  if (await policy.readAccessStore(engine, configured) !== 'git') return pending;
  if ((await policy.appliedRules(engine, configured)).some(r => r.prefix === 'conversations/')) return null;
  return { code: 'archive_rescope_git_rules', message: 'Conversation archive pages written while this brain was personal are not yet kept to the owner, because source ' + configured + ' keeps its access rules in git; add a rule with prefix conversations/ and readable_by [' + privacy.owner + '] to its .gbrain/access.yml and sync.' };
}

/**
 * When one owner-only rule per archive folder, with the source's other rules, would have passed the rules a source holds
 * (MAX_RULES), the re-scope kept the whole conversations/ folder to the owner instead; reported while that rule's readers
 * are the owner alone. A source that already held MAX_RULES other rules is left one rule over the limit, so a rule export
 * no longer imports; reported while it stays over.
 */
async function rescopeCapped(engine: BrainEngine, sourceId: string): Promise<ArchiveRescopeSkip | null> {
  const folders = await engine.getConfig(`${RESCOPE_CAPPED_PREFIX}${sourceId}`);
  if (!folders) return null;
  const [{ appliedRules }, { MAX_RULES, normalizeRule }] = await Promise.all([import('../access/policy.ts'), import('../access/folder-rules.ts')]);
  const rules = await appliedRules(engine, sourceId);
  const rule = rules.find(r => r.prefix === 'conversations/');
  if (!rule) return null;
  const owner = (await ownerView(engine)).owner;
  const ownerOnly = owner ? normalizeRule({ prefix: 'conversations/', readable_by: [owner] }) : null;
  const kept = !!ownerOnly && 'rule' in ownerOnly && rule.readable_by.join() === ownerOnly.rule.readable_by.join();
  const over = rules.length > MAX_RULES
    ? 'Source ' + sourceId + ' now holds ' + rules.length + ' rules, more than the ' + MAX_RULES + ' a rule file accepts, so gbrain access rule export then import fails validation until you remove one of its other rules.'
    : '';
  if (!kept) return over ? { code: 'archive_rescope_folder_cap', message: over } : null;
  return { code: 'archive_rescope_folder_cap', message: 'Conversation archive pages written while this brain was personal fill ' + folders + ' folders, and one owner-only rule per folder with the source\'s other rules would pass the ' + MAX_RULES + ' rules a source holds, so one rule keeps the whole conversations/ folder of source ' + sourceId + ' to the owner: group conversations archived since stay owner-only too, and widening that conversations/ rule opens every earlier personal day to the same readers, not just one chat.' + (over ? ' ' + over : '') };
}

function isGitRepo(path: string | null): boolean {
  return !!path && existsSync(join(path, '.git'));
}

async function ensureSource(engine: BrainEngine, sourceId: string): Promise<{ local_path: string | null }> {
  const [row] = await engine.executeRaw<{ local_path: string | null }>('SELECT local_path FROM sources WHERE id = $1', [sourceId]);
  if (row) return row;
  const { addSource } = await import('../sources-ops.ts');
  await addSource(engine, { id: sourceId, name: 'Conversation archive' });
  return { local_path: null };
}

const offFix: Action = { argv: ['gbrain', 'transcripts', 'archive', 'on'], consent: [], actor: 'user', why: 'Turns the conversation archive on for this brain: it is off here because it was turned off, or because a shared brain is only offered it.', requires_exclusive: false };
const ownerFix: Action = {
  argv: ['gbrain', 'access', 'owner', 'set', '<MEMBER>'], consent: [], actor: 'user', ask_user: true, requires_exclusive: false,
  inputs: [{ name: 'MEMBER', how: 'Ask the user which enabled member is the brain owner (gbrain access member list).' }],
  why: 'On a shared brain every archive page is kept to the owner member; with none named the archive holds every day and writes nothing.',
} as Action;

function emptyResult(mode: 'run' | 'backfill', base: Partial<ArchiveRunResult>): ArchiveRunResult {
  return {
    mode, enabled: false, dry_run: false, source_id: DEFAULT_ARCHIVE_SOURCE, timezone: 'UTC', write_mode: 'database', privacy: 'personal',
    from: '', to: '', days: [], written: 0, accepted: 0, unchanged: 0, held: 0, planned: 0, skipped_legacy: 0, index_pages: [], deferred_days: [], info: [], errors: [],
    ...base,
  };
}

/** Readers this machine has: file adapters for every harness root, OpenClaw archived files, and the OpenClaw Gateway. */
export async function defaultArchiveReaders(): Promise<ArchiveReader[]> {
  const { fileArchiveReaders } = await import('./archive-readers.ts');
  return fileArchiveReaders();
}

/** A list reader's conversations as scan entries (in memory already); a scan reader as it is. A list reader that throws yields nothing. */
export async function* scanReader(reader: ArchiveReader, opts: ArchiveScanOpts): AsyncGenerator<ArchiveScanEntry> {
  if (reader.scan) { yield* reader.scan(opts); return; }
  const convs = reader.list ? await reader.list({ sinceMs: opts.sinceMs }) : [];
  for (const conv of convs) {
    if (conv.session.messages.length === 0) continue;
    const stamps = conv.session.messages.map(m => m.timestamp).filter(Boolean);
    const dayIds: Record<string, string[]> = {};
    for (const m of conv.session.messages) if (m.timestamp) (dayIds[opts.dayOf(m.timestamp)] ??= []).push(archiveMessageId(m));
    yield {
      harness: conv.harness, conversationKey: conv.conversationKey, sessionId: conv.session.meta.sessionId, ...(conv.via ? { via: conv.via } : {}),
      days: Object.keys(dayIds), dayIds, newest: stamps.sort().pop(), load: async () => conv,
    };
  }
}

/** A Gateway conversation with the session files of the same session id: its key and channel, every message once, oldest first. */
function mergeScanEntries(gateway: ArchiveScanEntry, files: ArchiveScanEntry[]): ArchiveScanEntry {
  const dayIds: Record<string, string[]> = {};
  for (const e of [gateway, ...files]) for (const [day, ids] of Object.entries(e.dayIds ?? {})) dayIds[day] = [...new Set([...(dayIds[day] ?? []), ...ids])];
  const days = [...new Set([gateway, ...files].flatMap(e => e.days))];
  const newest = [gateway, ...files].map(e => e.newest).filter((s): s is string => !!s).sort().pop();
  return {
    ...gateway, days, dayIds, ...(newest ? { newest } : {}),
    async load() {
      const base = await gateway.load();
      if (!base) return null;
      const seen = new Set(base.session.messages.map(archiveMessageId));
      const messages = [...base.session.messages];
      for (const f of files) {
        const conv = await f.load();
        for (const m of conv?.session.messages ?? []) {
          const id = archiveMessageId(m);
          if (seen.has(id)) continue;
          seen.add(id);
          messages.push(m);
        }
      }
      messages.sort((a, b) => (a.timestamp < b.timestamp ? -1 : a.timestamp > b.timestamp ? 1 : 0));
      return { ...base, session: { ...base.session, messages } };
    },
  };
}

/** A run writes its pages in batches of this many conversation-days or bytes, so memory stays bounded on a large store. */
const FLUSH_DAYS = 200;
/** Bump when the built-in redaction detectors change, so backfill re-redacts finished days written under the old ones. */
const ARCHIVE_REDACTION_RULES = 'archive-redaction-1';
const ARCHIVE_LOCK_PREFIX = 'transcripts-archive:';
const FLUSH_BYTES = 8 * 1024 * 1024;

interface DayWrite {
  outcome: ArchiveDayOutcome;
  parts: RenderedPart[];
  harness: string;
  raw?: Record<string, unknown>;
  ledger?: ArchiveLedgerEntry;
  hold: 'backfill' | 'open' | 'release' | null;
  staleSlugs: string[];
  /** Request ids of the coordinator writes accepted and not yet published. */
  pending?: string[];
}

/** Reads every reader once for a run, and unless it is a dry run records what each one found (transcripts.archive.last_read.<source>) for status and doctor. */
async function readReaders(engine: BrainEngine, readers: ArchiveReader[], scanOpts: ArchiveScanOpts,
  ctx: { sourceId: string; mode: 'run' | 'backfill'; dryRun: boolean; at: string; result: ArchiveRunResult; pastDeadline: () => boolean }): Promise<ArchiveScanEntry[]> {
  const entries: ArchiveScanEntry[] = [];
  const { result, pastDeadline } = ctx;
  const reads = new Map<string, { harness: string; conversations: number; newest?: string; error?: ArchiveReaderError }>();
  for (const reader of readers) {
    if (entries.length && pastDeadline()) break;
    if (reader.unavailable) { result.info.push(reader.unavailable); continue; }
    try {
      let found = 0;
      let newest: string | undefined;
      for await (const entry of scanReader(reader, scanOpts)) {
        entries.push(entry);
        found++;
        if (entry.newest && (!newest || entry.newest > newest)) newest = entry.newest;
      }
      reads.set(reader.name, { harness: reader.harness, conversations: found, ...(newest ? { newest } : {}) });
      if (found === 0) result.info.push('no readable sessions for ' + reader.harness + '; use gbrain connectors or an export with gbrain transcripts ingest');
    } catch (err) {
      if (classifyItemFailure(err).scope === 'run') throw err;
      const e = err as { code?: string; message?: string; fix?: Action };
      const error = { harness: reader.harness, reader: reader.name, code: e.code ?? 'archive_reader_failed', message: e.message ?? String(err), ...(e.fix ? { fix: e.fix } : {}) };
      result.errors.push(error);
      reads.set(reader.name, { harness: reader.harness, conversations: 0, error });
    }
  }
  if (ctx.dryRun || !reads.size) return entries;
  const { at } = ctx;
  const record = await readArchiveReads(engine, ctx.sourceId);
  for (const [name, read] of reads) {
    const prior = record[name];
    if (read.error) {
      record[name] = { ...prior, harness: read.harness, error: { at, code: read.error.code, message: read.error.message, ...(read.error.fix ? { fix: read.error.fix } : {}) } };
      continue;
    }
    const newest = [prior?.newest, read.newest].filter((v): v is string => !!v).sort().pop();
    record[name] = { harness: read.harness, at, mode: ctx.mode, conversations: read.conversations, ...(newest ? { newest } : {}) };
  }
  await engine.setConfig(LAST_READ_PREFIX + ctx.sourceId, JSON.stringify(record));
  return entries;
}

export async function runTranscriptArchive(engine: BrainEngine, opts: ArchiveRunOpts): Promise<ArchiveRunResult> {
  const dryRun = opts.dryRun === true;
  const enabled = await archiveEnabled(engine);
  if (!enabled && !dryRun) {
    await reconcileArchiveAudience(engine, opts.sourceId);
    return emptyResult(opts.mode, {
      source_id: await configuredSource(engine, opts.sourceId),
      message: 'The conversation archive is off on this brain, so nothing was read or written. It stays off until the owner turns it on (a shared brain is only offered it).',
      fix: offFix,
    });
  }
  const { sourceId, privacy } = await effectiveSource(engine, opts.sourceId);
  if (dryRun) return archiveSource(engine, opts, { enabled, sourceId, privacy });
  if (privacy.unproven) {
    const { opError } = await import('../ops/contract.ts');
    throw opError('archive_audience_unproven', 'The brain\'s audience could not be checked (its client-usage read failed), so the conversation archive wrote nothing: a page written as personal could be open to other members.',
      'Rerun gbrain transcripts archive run; if it repeats, declare the audience with gbrain config set brain.audience personal|shared.');
  }
  // One run per source at a time: a run decides from the ledger it read at its start, so an overlapping run could publish
  // older history over what the other run just wrote.
  const { withRefreshingLock, LockUnavailableError } = await import('../db-lock.ts');
  try {
    return await withRefreshingLock(engine, ARCHIVE_LOCK_PREFIX + sourceId, signal => archiveSource(engine, opts, { enabled, sourceId, privacy, signal }));
  } catch (err) {
    if (!(err instanceof LockUnavailableError)) throw err;
    const { opError } = await import('../ops/contract.ts');
    throw opError('archive_run_busy', 'Another conversation archive run holds source ' + sourceId + ', so this run read and wrote nothing.',
      'Wait for the other run to finish, then rerun; gbrain status --section locks --json shows the holder.');
  }
}

async function archiveSource(engine: BrainEngine, opts: ArchiveRunOpts, ctx: { enabled: boolean; sourceId: string; privacy: Privacy; signal?: AbortSignal }): Promise<ArchiveRunResult> {
  const dryRun = opts.dryRun === true;
  const now = opts.now ?? new Date();
  const { enabled, sourceId, privacy } = ctx;
  const source = dryRun
    ? (await engine.executeRaw<{ local_path: string | null }>('SELECT local_path FROM sources WHERE id = $1', [sourceId]))[0] ?? { local_path: null }
    : await ensureSource(engine, sourceId);
  const context = dryRun ? undefined
    : opts.context ?? (opts.makeContext && await managedPersistenceEnabled(engine) ? await opts.makeContext(sourceId) : undefined);
  // A dry run builds no write context, but reports the write path the real run would take with the same options.
  const managed = (dryRun ? !!(opts.context ?? opts.makeContext) : !!context) && await managedPersistenceEnabled(engine);
  const writeMode: ArchiveRunResult['write_mode'] = managed ? 'coordinator' : (!privacy.shared && isGitRepo(source.local_path) ? 'repo' : 'database');
  if (!dryRun) await reconcileAudience(engine, await configuredSource(engine, opts.sourceId), privacy);
  const tz = await archiveTimeZone(engine, sourceId);
  if (!tz.pinned && !dryRun) await engine.setConfig(PIN_PREFIX + sourceId, tz.timezone);
  const today = localDay(now.toISOString(), tz.timezone);

  let from: string;
  let to: string;
  if (opts.mode === 'run') {
    to = today;
    from = addDays(today, -(positiveInt(await engine.getConfig(ARCHIVE_RUN_WINDOW_KEY), DEFAULT_RUN_WINDOW_DAYS)));
  } else {
    if (!opts.from || !DAY_RE.test(opts.from) || (opts.to && !DAY_RE.test(opts.to))) {
      const { opError } = await import('../ops/contract.ts');
      throw opError('invalid_params', 'backfill needs --from YYYY-MM-DD (and an optional --to YYYY-MM-DD); nothing was read.',
        'Run gbrain transcripts archive backfill --from 2026-01-01 --to 2026-01-31.');
    }
    from = opts.from;
    to = opts.to ?? today;
  }

  const result = emptyResult(opts.mode, {
    enabled, dry_run: dryRun, source_id: sourceId, timezone: tz.timezone, write_mode: writeMode,
    privacy: privacy.shared ? 'shared' : 'personal', from, to,
  });
  if (privacy.shared && !privacy.owner) {
    result.message = 'This brain is shared and names no owner member, so every archive day is held (archive_owner_unset) and nothing was written.';
    result.fix = ownerFix;
    result.errors.push({ harness: '*', code: 'archive_owner_unset', message: result.message, fix: ownerFix });
  }

  const readers = opts.readers ?? await defaultArchiveReaders();
  const sinceMs = opts.mode === 'run' ? Date.parse(from + 'T00:00:00.000Z') - 2 * 86_400_000 : undefined;
  const minMtimeMs = opts.mode === 'backfill' ? Date.parse(from + 'T00:00:00.000Z') - 2 * 86_400_000 : sinceMs;
  const dayOf = (ts: string) => localDay(ts, tz.timezone);
  const pastDeadline = () => opts.deadlineAtMs !== undefined && Date.now() >= opts.deadlineAtMs && (result.stopped_at_deadline = true);
  const entries = await readReaders(engine, readers, { sinceMs, minMtimeMs, dayOf }, { sourceId, mode: opts.mode, dryRun, at: now.toISOString(), result, pastDeadline });
  // A Gateway conversation and the session files of the same session id are one conversation: a reset in place leaves
  // the earlier messages only in the .jsonl.reset.* file, so the file messages merge in, each message once.
  const filesBySession = new Map<string, ArchiveScanEntry[]>();
  for (const e of entries) if (e.harness === 'openclaw' && e.via === 'file') filesBySession.set(e.sessionId, [...(filesBySession.get(e.sessionId) ?? []), e]);
  const merged = new Set<ArchiveScanEntry>();
  const scanned: ArchiveScanEntry[] = [];
  for (const e of entries) {
    if (e.harness !== 'openclaw' || e.via !== 'gateway' || !filesBySession.has(e.sessionId)) continue;
    const files = filesBySession.get(e.sessionId)!;
    files.forEach(f => merged.add(f));
    merged.add(e);
    scanned.push(mergeScanEntries(e, files));
  }
  scanned.push(...entries.filter(e => !merged.has(e)));

  const patterns = loadImportRedactionPatterns();
  const redactionVersion = sha256(ARCHIVE_REDACTION_RULES + '\u0000' + patterns.map(p => p.source).join('\u0000'));
  const ledger = new Map((await readArchiveLedger(engine, sourceId)).map(e => [e.harness + '.' + e.conversation_key_hash + '.' + e.day, e]));
  const legacy = await legacyCovered(engine, await configuredSource(engine, opts.sourceId));
  const covered = (harness: string, day: string) => legacy.has(harness + '.' + day) || legacy.has('*.' + day);

  interface Item { entry: ArchiveScanEntry; hash: string; days: string[] }
  const items: Item[] = [];
  const allDays = new Set<string>();
  for (const entry of scanned) {
    const hash = transcriptFullId(entry.conversationKey);
    const live: string[] = [];
    for (const day of entry.days.filter(d => d >= from && d <= to)) {
      if (covered(entry.harness, day)) {
        result.days.push({ harness: entry.harness, conversation_key_hash: hash, day, status: 'skipped_legacy' });
        result.skipped_legacy++;
        continue;
      }
      const done = ledger.get(entry.harness + '.' + hash + '.' + day);
      const sig = entry.dayIds?.[day] ? daySignature(entry.dayIds[day]!) : undefined;
      if (opts.mode === 'backfill' && done?.hash && !done.held && done.complete && day < today && sig && done.source === sig && done.redaction === redactionVersion) { result.unchanged++; continue; }
      live.push(day);
      allDays.add(day);
    }
    if (live.length) items.push({ entry, hash, days: live });
  }
  // A day held in the ledger for every conversation on it is retried on every backfill run without counting against
  // the cap, and a day that fails during this run hands its slot to the next older day.
  const heldDays = new Set([...allDays].filter(day => items.every(it => !it.days.includes(day) || !!ledger.get(it.entry.harness + '.' + it.hash + '.' + day)?.held)));
  const cap = opts.mode === 'backfill' ? positiveInt(await engine.getConfig(ARCHIVE_BACKFILL_CAP_KEY), DEFAULT_BACKFILL_DAYS_PER_RUN) : 0;
  const ordered = [...allDays].filter(d => !heldDays.has(d)).sort().reverse();

  const redact = opts.redact ?? ((s: ParsedSession) => redactSession(s, { patterns }));
  const pageCeiling = dryRun ? Math.floor(PAGE_MAX_BYTES * PAGE_CEILING_FILL) : await transcriptPageCeiling(engine);
  const quietMs = positiveInt(await engine.getConfig(ARCHIVE_QUIET_MINUTES_KEY), DEFAULT_QUIET_MINUTES) * 60_000;
  const groups = new Map<string, boolean>();
  const pending: ArchiveDayOutcome[] = [];
  const pendingKey = INDEX_PENDING_PREFIX + sourceId;
  const touched = new Set<string>(dryRun ? [] : readDays(await engine.getConfig(`${INDEX_PENDING_PREFIX}${sourceId}`)));
  let batch: DayWrite[] = [];
  let batchBytes = 0;
  const flushBatch = async () => {
    if (!batch.length) return;
    if (ctx.signal?.aborted) throw ctx.signal.reason;
    await flushWrites(engine, opts, { sourceId, writeMode, root: source.local_path, context }, batch);
    for (const w of batch) {
      if ((w.outcome.status !== 'written' && w.outcome.status !== 'accepted') || !w.ledger) continue;
      ledger.set(w.ledger.harness + '.' + w.ledger.conversation_key_hash + '.' + w.ledger.day, w.ledger);
      touched.add(w.outcome.day);
    }
    if (!dryRun && touched.size) await engine.setConfig(pendingKey, JSON.stringify([...touched].sort()));
    batch = [];
    batchBytes = 0;
  };

  const hold = async (outcome: ArchiveDayOutcome, held: ArchiveHeld, prior: ArchiveLedgerEntry | undefined) => {
    Object.assign(outcome, { status: 'held', code: held.code, message: held.message });
    if (!dryRun) await writeLedger(engine, { ...(prior ?? {}), source_id: sourceId, harness: outcome.harness, conversation_key_hash: outcome.conversation_key_hash, day: outcome.day, held });
  };

  let queued = 0;
  const processRound = async (allowed: Set<string> | null) => { for (const { entry, hash, days } of items) {
    if (queued && pastDeadline()) break;
    const todo = [...(allowed ? days.filter(d => allowed!.has(d)) : days)].sort().reverse();
    if (todo.length === 0) continue;
    const outcomes = todo.map(day => ({ harness: entry.harness, conversation_key_hash: hash, day, status: 'planned' as ArchiveDayStatus } as ArchiveDayOutcome));
    pending.push(...outcomes);
    const prior = (day: string) => ledger.get(entry.harness + '.' + hash + '.' + day);
    if (privacy.shared && !privacy.owner) {
      for (const o of outcomes) await hold(o, { code: 'archive_owner_unset', message: 'The brain is shared and names no owner member.' }, prior(o.day));
      continue;
    }
    let conv: ArchiveConversation | null;
    try {
      conv = await entry.load();
    } catch (err) {
      conv = null;
      for (const o of outcomes) Object.assign(o, { status: 'error', code: 'archive_reader_failed', message: (err instanceof Error ? err.message : String(err)).slice(0, 300) });
      continue;
    }
    if (!conv) {
      for (const o of outcomes) Object.assign(o, { status: 'error', code: 'archive_day_missing', message: 'the conversation could not be read again for this run' });
      continue;
    }
    let redacted: RedactedSession;
    try {
      redacted = redact(conv.session);
    } catch (err) {
      const message = 'Redaction failed for this session, so its days were held, not written: ' + (err instanceof Error ? err.message : String(err)).slice(0, 200);
      for (const o of outcomes) await hold(o, { code: 'archive_redaction_failed', message }, prior(o.day));
      continue;
    }
    const readableBy = await readersFor(engine, privacy, conv, groups);
    const slices = new Map(sliceSessionByDay(redacted.session.messages, tz.timezone).map(s => [s.day, s]));
    const raw = conv.session.messages.length === redacted.session.messages.length ? conv.session.messages : redacted.session.messages;
    for (const outcome of outcomes) {
      const slice = slices.get(outcome.day);
      if (!slice) { Object.assign(outcome, { status: 'error', code: 'archive_day_missing', message: 'the redacted session has no messages on this day' }); continue; }
      const complete = outcome.day < today;
      const ids = [...new Set(slice.indexes.map(i => archiveMessageId(raw[i]!)))];
      const lastTs = slice.messages[slice.messages.length - 1]?.timestamp;
      const quiet = !!lastTs && now.getTime() - Date.parse(lastTs) >= quietMs;
      const hold: DayWrite['hold'] = opts.mode === 'backfill' ? 'backfill' : (!complete && !quiet ? 'open' : 'release');
      const dayInput = {
        harness: conv.harness, conversationKey: conv.conversationKey, ...(conv.scopeKey ? { scopeKey: conv.scopeKey } : {}), channel: conv.channel, day: outcome.day, timezone: tz.timezone,
        redacted, slice, complete, partTargetBytes: pageCeiling, readableBy,
      };
      let rendered = renderArchiveDay(dayInput);
      const contentHash = sha256(rendered.parts.map(p => p.content).join('\u0000'));
      Object.assign(outcome, {
        slug: rendered.baseSlug, parts: rendered.parts.length, messages: rendered.messages,
        redactions: Object.values((rendered.parts[0]!.frontmatter.transcript_archive as { redactions: Record<string, number> }).redactions).reduce((a, b) => a + b, 0),
        ...(readableBy ? { readable_by: readableBy } : {}),
      });
      const before = prior(outcome.day);
      const lost = before?.ids ? before.ids.some(id => !ids.includes(id)) : (before?.messages ?? 0) > rendered.messages;
      if (before?.hash && lost) {
        outcome.status = 'kept';
        outcome.message = 'the source no longer holds every message the archive wrote for this day (a reset or compaction); the archived page is kept';
        continue;
      }
      if (dryRun) { outcome.status = 'planned'; continue; }
      if (before?.pending && !before.hash && before.pending.hash === contentHash) {
        const states = (await engine.executeRaw<{ state: string }>('SELECT state FROM persistence_requests WHERE request_id::text = ANY($1::text[])', [before.pending.requests])).map(r => r.state);
        if (states.length === before.pending.requests.length && states.every(s => s === 'committed')) {
          const { pending, ...rest } = before;
          await writeLedger(engine, { ...rest, hash: pending.hash, complete: pending.complete });
          outcome.status = 'unchanged';
          continue;
        }
        if (states.length === before.pending.requests.length && states.every(s => s === 'committed' || !isTerminalWriteState(s as WriteRequestState))) { outcome.status = 'accepted'; continue; }
      }
      if (before?.hash === contentHash && !before.held && await pageExists(engine, sourceId, rendered.baseSlug)) {
        outcome.status = 'unchanged';
        if (before.redaction !== redactionVersion) await writeLedger(engine, { ...before, redaction: redactionVersion });
        if (hold === 'release') await releaseOpenDayHold(engine, sourceId, rendered.parts.map(p => p.slug));
        continue;
      }
      const embeddingHold = hold === 'backfill' ? ARCHIVE_BACKFILL_HOLD_REASON : hold === 'open' ? ARCHIVE_OPEN_DAY_HOLD_REASON : undefined;
      if (embeddingHold) rendered = renderArchiveDay({ ...dayInput, embeddingHold });
      const renamed = before?.slug && before.slug !== rendered.baseSlug
        ? [before.slug, ...Array.from({ length: Math.max(0, (before.parts ?? 1) - 1) }, (_, i) => before.slug + '-p' + (i + 2))]
        : [];
      batch.push({
        outcome,
        parts: rendered.parts,
        harness: conv.harness,
        ...(redacted.session.meta.raw ? { raw: redacted.session.meta.raw } : {}),
        ledger: { source_id: sourceId, harness: conv.harness, conversation_key_hash: hash, day: outcome.day, hash: contentHash, complete, slug: rendered.baseSlug, parts: rendered.parts.length, messages: rendered.messages, ids, source: daySignature(ids), redaction: redactionVersion },
        hold,
        staleSlugs: renamed.length ? renamed : Array.from({ length: Math.max(0, (before?.parts ?? 0) - rendered.parts.length) }, (_, i) => rendered.baseSlug + '-p' + (rendered.parts.length + i + 1)),
      });
      batchBytes += rendered.parts.reduce((n, p) => n + p.content.length, 0);
      queued++;
    }
    if (batch.length >= FLUSH_DAYS || batchBytes >= FLUSH_BYTES) await flushBatch();
  } };
  if (opts.mode !== 'backfill') await processRound(null);
  else {
    let next = 0;
    let good = 0;
    let round = new Set([...heldDays, ...ordered.slice(0, cap)]);
    next = Math.min(cap, ordered.length);
    while (round.size && (!queued || !pastDeadline())) {
      const start = pending.length;
      await processRound(round);
      const ok = new Set(pending.slice(start).filter(o => o.status !== 'held' && o.status !== 'error').map(o => o.day));
      good += [...round].filter(d => !heldDays.has(d) && ok.has(d)).length;
      round = new Set(ordered.slice(next, next + Math.max(0, cap - good)));
      next += round.size;
    }
    result.deferred_days = ordered.slice(next).sort();
  }
  const indexFor = (days: Set<string>, view: Map<string, ArchiveLedgerEntry>): DayWrite[] => [...days].sort().map(day => {
    const dayEntries: DayIndexEntry[] = [];
    for (const e of view.values()) {
      if (e.day !== day || !e.slug || e.held) continue;
      dayEntries.push({ slug: e.slug, harness: e.harness, messages: e.messages ?? 0 });
    }
    const page = renderDayIndex(day, tz.timezone, dayEntries, privacy.shared ? [privacy.owner!] : undefined);
    return { outcome: { harness: 'index', conversation_key_hash: '', day, status: 'planned', slug: page.slug }, parts: [page], harness: 'index', hold: null, staleSlugs: [] };
  });
  let indexWrites: DayWrite[] = [];
  if (writeMode === 'repo') {
    // A repo commit carries the last batch's days with their index pages, so a small run is one commit.
    const view = new Map(ledger);
    const days = new Set(touched);
    for (const w of batch) if (w.ledger) { view.set(w.ledger.harness + '.' + w.ledger.conversation_key_hash + '.' + w.ledger.day, w.ledger); days.add(w.outcome.day); }
    indexWrites = indexFor(days, view);
    batch.push(...indexWrites);
    await flushBatch();
  } else {
    await flushBatch();
    if (touched.size) {
      indexWrites = indexFor(touched, ledger);
      await flushWrites(engine, opts, { sourceId, writeMode, root: source.local_path, context }, indexWrites);
    }
  }
  if (!dryRun && touched.size) await engine.setConfig(pendingKey, JSON.stringify(indexWrites.filter(w => w.outcome.status !== 'written' && w.outcome.status !== 'accepted' && w.outcome.status !== 'unchanged').map(w => w.outcome.day).sort()));
  for (const w of indexWrites) {
    if (w.outcome.status === 'written' || w.outcome.status === 'accepted' || w.outcome.status === 'unchanged') result.index_pages.push(w.outcome.slug!);
    else if (w.outcome.status === 'error') result.errors.push({ harness: 'index', reader: 'day-index', code: w.outcome.code ?? 'archive_commit_failed', message: 'the day index ' + w.outcome.slug + ' was not written: ' + (w.outcome.message ?? 'no detail') });
  }

  pending.sort((a, b) => b.day.localeCompare(a.day));
  result.days.push(...pending);
  for (const d of pending) {
    if (d.status === 'error') result.errors.push({ harness: d.harness, day: d.day, conversation_key_hash: d.conversation_key_hash, code: d.code ?? 'archive_day_failed', message: d.message ?? 'no detail' });
  }
  for (const d of result.days) {
    if (d.status === 'written') result.written++;
    else if (d.status === 'accepted') result.accepted++;
    else if (d.status === 'unchanged') result.unchanged++;
    else if (d.status === 'held') result.held++;
    else if (d.status === 'planned') result.planned++;
  }
  pastDeadline();
  if (result.stopped_at_deadline) result.info.push('the run reached its time limit; the conversations it did not reach wait for the next run');
  if (result.deferred_days.length) result.info.push(result.deferred_days.length + ' older day(s) wait for the next backfill run (' + ARCHIVE_BACKFILL_CAP_KEY + ' caps one run)');
  return result;
}

function readDays(value: string | null): string[] {
  try {
    const days = JSON.parse(value ?? '[]') as unknown;
    return Array.isArray(days) ? days.filter((d): d is string => typeof d === 'string' && DAY_RE.test(d)) : [];
  } catch {
    return [];
  }
}

async function pageExists(engine: BrainEngine, sourceId: string, slug: string): Promise<boolean> {
  return (await engine.executeRaw('SELECT 1 FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL', [sourceId, slug])).length > 0;
}

async function defaultCommit(root: string, paths: string[], message: string): Promise<void> {
  const { commitPushChild } = await import('../git/reconcile.ts');
  await commitPushChild(root, ['-m', message], undefined, paths);
}

async function defaultSync(engine: BrainEngine, sourceId: string, root: string): Promise<void> {
  const { performSync } = await import('../../commands/sync/perform.ts');
  await performSync(engine, { sourceId, repoPath: root, noPull: true, noEmbed: true, noExtract: true });
}

/** Writes every rendered page through the brain's one write path, then records raw data, embedding holds and the ledger. */
/** A closed or quiet day's open-day hold is opened, not deleted, so the page's frontmatter marker never holds it again. */
async function releaseOpenDayHold(engine: BrainEngine, sourceId: string | null, slugs: string[], pageId?: number): Promise<void> {
  const ids = pageId !== undefined ? [pageId]
    : (await engine.executeRaw<{ id: number }>('SELECT id FROM pages WHERE source_id = $1 AND slug = ANY($2::text[]) AND deleted_at IS NULL', [sourceId, slugs])).map(r => r.id);
  if (!ids.length) return;
  await engine.executeRaw("UPDATE page_embedding_budget SET state = 'open', reason = NULL, updated_at = now() WHERE page_id = ANY($1::int[]) AND state = 'held' AND reason = $2", [ids, ARCHIVE_OPEN_DAY_HOLD_REASON]);
}

async function flushWrites(engine: BrainEngine, opts: ArchiveRunOpts, target: { sourceId: string; writeMode: ArchiveRunResult['write_mode']; root: string | null; context?: OperationContext }, writes: DayWrite[]): Promise<void> {
  const { sourceId, writeMode } = target;
  const provenance = (harness: string) => ({ source_kind: 'transcript:' + harness, source_uri: 'archive:' + harness, ingested_via: INGESTED_VIA });
  if (writeMode === 'repo') {
    const root = target.root!;
    const paths: string[] = [];
    for (const w of writes) {
      for (const part of w.parts) {
        const file = join(root, part.slug + '.md');
        mkdirSync(dirname(file), { recursive: true });
        writeFileSync(file, part.content);
        paths.push(relative(root, file).split(sep).join('/'));
      }
      for (const stale of w.staleSlugs) {
        const file = join(root, stale + '.md');
        if (existsSync(file)) { rmSync(file); paths.push(relative(root, file).split(sep).join('/')); }
      }
    }
    try {
      await (opts.commitPaths ?? defaultCommit)(root, paths, 'gbrain: archive ' + writes.filter(w => w.ledger).length + ' conversation day(s)');
      await (opts.syncSource ?? defaultSync)(engine, sourceId, root);
    } catch (err) {
      const e = err as { code?: string; message?: string };
      for (const w of writes) Object.assign(w.outcome, { status: 'error', code: e.code ?? 'archive_commit_failed', message: (e.message ?? String(err)).slice(0, 300) });
      return;
    }
    for (const w of writes) w.outcome.status = 'written';
  } else {
    const deadline = opts.deadlineAtMs;
    const coordinated = async (submit: (ctx: OperationContext) => Promise<unknown>): Promise<string[]> => {
      const ctx = target.context!;
      try {
        await submit(deadline === undefined ? ctx : { ...ctx, writeWaitMs: Math.max(0, Math.min(ctx.writeWaitMs ?? CLI_WRITE_WAIT_MS, deadline - Date.now())) });
        return [];
      } catch (err) {
        const receipt = acceptedPendingReceipt(err);
        if (receipt) return [receipt.request_id];
        throw err;
      }
    };
    let submitted = 0;
    for (const w of writes) {
      if (writeMode === 'coordinator' && deadline !== undefined && w.harness !== 'index' && submitted && Date.now() >= deadline) break;
      if (w.harness !== 'index') submitted++;
      try {
        const accepted: string[] = [];
        for (const part of w.parts) {
          if (writeMode === 'coordinator') {
            accepted.push(...await coordinated(async ctx => {
              const r = await submitTranscriptPart(ctx, engine, sourceId, part, provenance(w.harness));
              if (r.status === 'error') throw new Error('the page write for ' + part.slug + ' returned an error');
            }));
            continue;
          }
          const r = await importFromContent(engine, part.slug, part.content, { sourceId, noEmbed: true, ...provenance(w.harness) });
          if (r.status === 'error') throw new Error('the page write for ' + part.slug + ' returned an error');
        }
        for (const stale of w.staleSlugs) {
          if (writeMode === 'coordinator') accepted.push(...await coordinated(ctx => deleteStaleTranscriptPart(ctx, engine, sourceId, stale)));
          else await engine.deletePages([stale], { sourceId });
        }
        w.outcome.status = accepted.length ? 'accepted' : 'written';
        if (accepted.length && w.ledger) w.pending = accepted;
      } catch (err) {
        if (classifyItemFailure(err).scope === 'run') throw err;
        const e = err as { code?: string; message?: string };
        Object.assign(w.outcome, { status: 'error', code: e.code ?? classifyItemFailure(err).code, message: (e.message ?? String(err)).slice(0, 300) });
      }
    }
  }
  for (const w of writes) {
    if (w.outcome.status === 'accepted' && w.ledger) {
      const { hash, complete, ...queued } = w.ledger;
      await writeLedger(engine, { ...queued, pending: { requests: w.pending ?? [], hash: hash ?? '', complete: complete === true } });
    }
    if (w.outcome.status !== 'written') continue;
    if (w.raw) await engine.putRawData(w.parts[0]!.slug, 'transcript:' + w.harness, w.raw, { sourceId });
    if (w.hold) {
      const ids = await engine.executeRaw<{ id: number }>('SELECT id FROM pages WHERE source_id = $1 AND slug = ANY($2::text[]) AND deleted_at IS NULL', [sourceId, w.parts.map(p => p.slug)]);
      for (const { id } of ids) {
        if (w.hold === 'backfill') await holdPageEmbedding(engine, id, ARCHIVE_BACKFILL_HOLD_REASON);
        else if (w.hold === 'open') await holdPageEmbedding(engine, id, ARCHIVE_OPEN_DAY_HOLD_REASON);
        else await releaseOpenDayHold(engine, null, [], id);
      }
    }
    if (w.ledger) await writeLedger(engine, w.ledger);
  }
}

// ── status ──────────────────────────────────────────────────────────────────

export interface ArchiveStatus {
  enabled: boolean;
  source_id: string;
  timezone: string | null;
  configured_timezone: string;
  /**
   * One row per reader on this machine. conversations and newest_session_day are as of the last run that read it
   * (read_at, read_mode; null when no run has read it yet): status never reads a reader itself. In run mode a run
   * reads only recently active conversations, so conversations counts those; newest_session_day is the newest any run saw.
   */
  harnesses: Array<{ harness: string; reader: string; conversations: number; newest_session_day: string | null; newest_archived_day: string | null; read_at: string | null; read_mode?: 'run' | 'backfill'; info?: string }>;
  archived_days: number;
  held: Array<{ harness: string; day: string; code: string }>;
  legacy_days: number;
  /** Readers whose last run failed, with when (at); a later successful read clears one. */
  errors: ArchiveReaderError[];
  /** The brain became shared but the pages archived while personal could not be kept to the owner yet. */
  rescope?: ArchiveRescopeSkip;
}

/**
 * The archive's own records: config, ledger and what the last run read. It runs no reader (no Gateway call, no session
 * file parsed); building the reader list only checks which harness folders exist and whether openclaw is on PATH.
 */
export async function archiveStatus(engine: BrainEngine, opts: { sourceId?: string; readers?: ArchiveReader[] } = {}): Promise<ArchiveStatus> {
  const { sourceId } = await effectiveSource(engine, opts.sourceId);
  const pinned = (await engine.getConfig(`${PIN_PREFIX}${sourceId}`))?.trim() || null;
  const configured = (await engine.getConfig(ARCHIVE_TIMEZONE_KEY))?.trim() || machineTimeZone();
  const entries = await readArchiveLedger(engine, sourceId);
  const out: ArchiveStatus = {
    enabled: await archiveEnabled(engine), source_id: sourceId, timezone: pinned, configured_timezone: configured, harnesses: [],
    archived_days: entries.filter(e => e.hash && !e.held).length,
    held: entries.filter(e => e.held).map(e => ({ harness: e.harness, day: e.day, code: e.held!.code })),
    legacy_days: (await legacyCovered(engine, await configuredSource(engine, opts.sourceId))).size, errors: [],
  };
  const reads = await readArchiveReads(engine, sourceId);
  const zone = pinned ?? configured;
  for (const reader of opts.readers ?? await defaultArchiveReaders()) {
    const archived = entries.filter(e => e.harness === reader.harness && e.hash).map(e => e.day).sort();
    const row = { harness: reader.harness, reader: reader.name, conversations: 0, newest_session_day: null as string | null, newest_archived_day: archived[archived.length - 1] ?? null, read_at: null as string | null };
    if (reader.unavailable) { out.harnesses.push({ ...row, info: reader.unavailable }); continue; }
    const read = reads[reader.name];
    if (read?.error) out.errors.push({ harness: reader.harness, reader: reader.name, code: read.error.code, message: read.error.message, ...(read.error.fix ? { fix: read.error.fix } : {}), at: read.error.at });
    const newest = read?.newest ? localDay(read.newest, zone) : null;
    if (!read?.at) { out.harnesses.push({ ...row, newest_session_day: newest }); continue; }
    out.harnesses.push({ ...row, conversations: read.conversations ?? 0, newest_session_day: newest, read_at: read.at, ...(read.mode ? { read_mode: read.mode } : {}),
      ...(newest ? {} : { info: 'no readable sessions for ' + reader.harness + '; use gbrain connectors or an export with gbrain transcripts ingest' }) });
  }
  const rescope = await archiveRescopeSkipped(engine, opts.sourceId);
  if (rescope) out.rescope = rescope;
  return out;
}

// ── adopt-legacy ────────────────────────────────────────────────────────────

const HARNESSES: readonly string[] = ['claude-code', 'codex', 'openclaw', 'hermes', 'grok', 'chatgpt', 'claude-export'];

export interface LegacyDay { harness: string; day: string; file: string }
export interface AdoptLegacyResult { dir: string; source_id: string; applied: boolean; covered: LegacyDay[]; by_harness: Record<string, number> }

function legacyFiles(dir: string, base = dir, depth = 0, out: string[] = []): string[] {
  if (depth > 6) return out;
  let names: string[];
  try { names = readdirSync(dir).sort(); } catch { return out; }
  for (const name of names) {
    if (name.startsWith('.')) continue;
    const path = join(dir, name);
    const st = lstatSync(path);
    if (st.isSymbolicLink()) continue;
    if (st.isDirectory()) legacyFiles(path, base, depth + 1, out);
    else if (st.isFile()) out.push(path);
  }
  return out;
}

function frontmatterField(head: string, field: string): string | null {
  const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(head);
  if (!m) return null;
  for (const line of m[1]!.split(/\r?\n/)) {
    if (!line.startsWith(field + ':')) continue;
    const value = line.slice(field.length + 1).trim().replace(/^["']/, '');
    const end = value.search(/["']/);
    const out = (end < 0 ? value : value.slice(0, end)).trim();
    return out || null;
  }
  return null;
}

/**
 * Reads an old archive folder's day files (a YYYY-MM-DD in the file name, else a date: in the frontmatter) and lists
 * the days they cover per harness (from a harness: field, a folder or file name naming one, else every harness). With
 * apply it records each day as covered-by-legacy, so run and backfill skip it. Never edits, moves or deletes a file.
 */
export async function adoptLegacyArchive(engine: BrainEngine, opts: { dir: string; apply: boolean; sourceId?: string }): Promise<AdoptLegacyResult> {
  const { sourceId } = await effectiveSource(engine, opts.sourceId);
  if (!existsSync(opts.dir) || !statSync(opts.dir).isDirectory()) {
    const { opError } = await import('../ops/contract.ts');
    throw opError('invalid_params', opts.dir + ' is not a folder, so nothing was adopted.', 'Pass the old archive folder with --dir <folder>.');
  }
  const covered = new Map<string, LegacyDay>();
  for (const path of legacyFiles(opts.dir)) {
    const rel = relative(opts.dir, path).split(sep).join('/');
    let day = /(\d{4}-\d{2}-\d{2})/.exec(rel.split('/').pop()!)?.[1] ?? null;
    let head = '';
    if (/\.(md|markdown|txt)$/i.test(path)) {
      try { head = readFileSync(path, 'utf8').slice(0, 4096); } catch { head = ''; }
    }
    if (!day) day = frontmatterField(head, 'date')?.slice(0, 10) ?? null;
    if (!day || !DAY_RE.test(day) || Number.isNaN(Date.parse(day))) continue;
    const named = frontmatterField(head, 'harness');
    const harness = (named && HARNESSES.includes(named) ? named : null)
      ?? HARNESSES.find(h => rel.split('/').slice(0, -1).includes(h))
      ?? HARNESSES.find(h => rel.split('/').pop()!.includes(h))
      ?? '*';
    const key = harness + '.' + day;
    if (!covered.has(key)) covered.set(key, { harness, day, file: rel });
  }
  const list = [...covered.values()].sort((a, b) => a.day.localeCompare(b.day) || a.harness.localeCompare(b.harness));
  if (opts.apply) {
    for (const d of list) await engine.setConfig(LEGACY_PREFIX + sourceId + '.' + d.harness + '.' + d.day, JSON.stringify({ file: d.file, adopted_at: new Date().toISOString() }));
  }
  const by_harness: Record<string, number> = {};
  for (const d of list) by_harness[d.harness] = (by_harness[d.harness] ?? 0) + 1;
  return { dir: opts.dir, source_id: sourceId, applied: opts.apply, covered: list, by_harness };
}