Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.1 KB
Raw Blame
#!/usr/bin/env python3
"""Debug-only instrumentation for the #6278 stall reproduction. Never commit its output.

Applies, to the gbrain checkout given as argv[1] (default: the current one), a
SIGUSR2 dump of every preparation in flight: which `prepareManagedSyncMutation`
step each request is in, for how long, plus the consumer's `status()`
(phase observation with first_conn_ms / conn_wait_ms, preparations). The
repro script sends the signal with `--stall-signal SIGUSR2` once a stall is
detected, and the dump lands in `pass-<n>.stderr` as `[stall-debug] ...` lines.

It edits three files in place (idempotent: a second run is a no-op) and adds
`src/core/persistence/stall-debug.ts`. Undo with `git checkout -- src` in that checkout.
Anchors that do not exist in that checkout are reported and skipped.
"""
import os
import re
import sys

repo = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else '.')
P = lambda *parts: os.path.join(repo, 'src', 'core', 'persistence', *parts)

MODULE = '''/** Debug-only (#6278 Phase 0): SIGUSR2 dumps every preparation in flight with its current step and age. Installed by scripts/bench/stall-debug-instrument.py. */
const inflight = new Map<string, { step: string; since: number; started: number; steps: string[] }>();
const extras = new Map<string, () => unknown>();
let installed = false;
function install(): void {
  if (installed) return;
  installed = true;
  process.on('SIGUSR2', () => {
    const now = Date.now();
    const rows = [...inflight.entries()].map(([id, s]) => ({ id, step: s.step, step_age_ms: now - s.since, total_age_ms: now - s.started, steps: s.steps.slice(-20) }));
    const extra: Record<string, unknown> = {};
    for (const [name, fn] of extras) { try { extra[name] = fn(); } catch (error) { extra[name] = String(error); } }
    process.stderr.write(`[stall-debug] ${JSON.stringify({ at: new Date(now).toISOString(), pid: process.pid, inflight: rows, ...extra, memory: process.memoryUsage() })}\\n`);
  });
}
export function stallMark(id: string, step: string): void {
  install();
  const now = Date.now();
  const s = inflight.get(id);
  if (s) { s.step = step; s.since = now; s.steps.push(`${step}@${now - s.started}`); }
  else inflight.set(id, { step, since: now, started: now, steps: [`${step}@0`] });
}
export function stallDone(id: string): void { inflight.delete(id); }
export function stallRegister(name: string, fn: () => unknown): void { install(); extras.set(name, fn); }
/** #6317: every raw statement between its send in JS and its settle; one that is here but not in pg_stat_activity is waiting for a connection. */
const sqlInflight = new Map<number, { sql: string; since: number; pool: string; reserved: boolean }>();
let sqlSeq = 0;
export function stallSql(sql: string, pool: string, reserved = false): () => void {
  install();
  const id = ++sqlSeq;
  sqlInflight.set(id, { sql: sql.replace(/\\s+/g, ' ').slice(0, 160), since: Date.now(), pool, reserved });
  return () => { sqlInflight.delete(id); };
}
/** #6317: postgres.js pool queues (vendor/postgres/src/index.js exposes `queues` and registers each pool here when instrumented). */
export function stallPools(): unknown {
  const pools = (globalThis as { __gbrainPools?: Array<{ options: { max: number; host?: string[]; port?: number[]; database?: string }; queues?: Record<string, { length: number }> }> }).__gbrainPools ?? [];
  return pools.map((pool, i) => ({ i, max: pool.options?.max, database: pool.options?.database, port: pool.options?.port,
    ...Object.fromEntries(Object.entries(pool.queues ?? {}).map(([name, queue]) => [name, queue.length])) }));
}
/**
 * #6317 forced wedge (debug only): with STALL_DEBUG_HANG_KIND=<intent kind> (e.g. managed_sync_import), the preparation of
 * the STALL_DEBUG_HANG_AFTER-th (default 40th) request of that kind in this process parks on a promise that never settles and
 * ignores its signal: the reporter's never-settling await with nothing in flight at the database. STALL_DEBUG_HANG_ROLE limits
 * it to processes whose GBRAIN_SQL_TRACE_LABEL starts with that prefix (e.g. serve, cli-sync). Hung ids appear in the dump.
 */
const hung: Array<{ id: string; since: number }> = [];
let seenOfKind = 0;
const hungIds = new Set<string>();
export async function stallMaybeHang(id: string, kind: string | undefined, attempts = 0): Promise<void> {
  const wanted = process.env.STALL_DEBUG_HANG_KIND;
  if (!wanted || kind !== wanted) return;
  const role = process.env.STALL_DEBUG_HANG_ROLE;
  const mine = !role || (process.env.GBRAIN_SQL_TRACE_LABEL ?? '').startsWith(role);
  if (mine) seenOfKind++;
  // Sticky: the chosen request hangs again whenever this process prepares it, and (budget builds) any request already cut once
  // hangs in every process, so a re-claim after a cut parks again the way the reporter's did.
  if (!(mine && seenOfKind === Number(process.env.STALL_DEBUG_HANG_AFTER ?? '40')) && !hungIds.has(id) && !(attempts > 0)) return;
  hungIds.add(id);
  hung.push({ id, since: Date.now() });
  process.stderr.write(`[stall-debug] HANG request ${id} (${kind}) parked forever in pid ${process.pid}\n`);
  await new Promise<never>(() => {});
}
stallRegister('hung', () => { const now = Date.now(); return hung.map(h => ({ ...h, age_ms: now - h.since })); });
stallRegister('sql_inflight', () => { const now = Date.now(); return [...sqlInflight.values()].map(s => ({ ...s, age_ms: now - s.since })); });
stallRegister('pools', stallPools);
'''

def edit(path, fn):
    with open(path) as f: text = f.read()
    new = fn(text)
    if new != text:
        with open(path, 'w') as f: f.write(new)
    return new != text

def ensure_import(text, line):
    if line in text: return text
    m = re.search(r"^import .*?;\n(?!import)", text, re.S | re.M)
    at = m.end() if m else 0
    return text[:at] + line + '\n' + text[at:]

report = []

# 1. the module
mod = P('stall-debug.ts')
if not os.path.exists(mod):
    with open(mod, 'w') as f: f.write(MODULE)
    report.append('added stall-debug.ts')

# 2. service.ts: wrap preparePersistedMutation; register the consumer status
def service(text):
    if 'preparePersistedMutationInner' in text: return text
    text = ensure_import(text, "import { stallDone, stallMark, stallMaybeHang, stallRegister } from './stall-debug.ts';")
    text = text.replace('export async function preparePersistedMutation(e: BrainEngine, row: WriteRequest, cfg: GBrainConfig, signal?: AbortSignal) {',
        'export async function preparePersistedMutation(e: BrainEngine, row: WriteRequest, cfg: GBrainConfig, signal?: AbortSignal) {\n'
        "  stallMark(row.request_id, `dispatch:${row.operation}:${row.intent?.kind ?? ''}`);\n"
        "  stallMark(row.request_id, 'debug_hang_check'); await stallMaybeHang(row.request_id, row.intent?.kind as string | undefined, Number((row as { preparation_attempts?: number }).preparation_attempts ?? 0));\n"
        '  try { return await preparePersistedMutationInner(e, row, cfg, signal); } finally { stallDone(row.request_id); }\n'
        '}\n'
        'async function preparePersistedMutationInner(e: BrainEngine, row: WriteRequest, cfg: GBrainConfig, signal?: AbortSignal) {', 1)
    m = re.search(r"(  const consumer = new PersistenceConsumer\(engine, config, preparePersistedMutation,.*?\);\n)", text, re.S)
    if m:
        text = text[:m.end()] + "  stallRegister('consumer', () => consumer.status());\n" + text[m.end():]
    else: report.append('service.ts: consumer registration anchor missing')
    return text
report.append(f"service.ts edited={edit(P('service.ts'), service)}")

# 3. sync-prepare.ts: a mark before each major await of prepareManagedSyncMutation
ANCHORS = [
    ('  await assertManagedSyncActive(engine);', 'sync_active'),
    ('  await validateSyncAuthority(engine, p.syncAuthority, row.slug);', 'authority'),
    ('  const binding = await getWorktreeBinding(engine, row.source_id);', 'binding'),
    ("  const [configuredSource] = await engine.executeRaw<{ local_path: string | null }>('SELECT local_path FROM sources WHERE id=$1', [row.source_id]);", 'source_path'),
    ("  if (p.kind !== 'managed_sync_checkpoint') await assertKnowledgePublicationAllowed(engine, row,", 'knowledge_guard'),
    ("    originContext = { root, gitRoot: realpathSync(syncGit(root, ['rev-parse', '--show-toplevel']).trim()), target: p.target, slugMode: p.slugMode };", 'git_rev_parse'),
    ("    await assertSyncPageOrigin(engine, row.source_id, p.sourcePath, originPageId, p.kind === 'managed_sync_delete', originScope);", 'page_origin'),
    ('  const snapshot = await engine.readPageSnapshot(row.slug, { ...source, includeDeleted: true });', 'snapshot'),
    ('  const activePack = p.processingOptions?.noSchemaPack ? undefined', 'active_pack'),
    ('  const { screen, parsedInput, newerWorkingTree } = await settledSyncScreen(engine,', 'screen'),
    ('  const result = await importFromContent(engine, renamed?.slug ?? row.slug, importContent, { ...importOptions,', 'import_prepare'),
    ('  const project = await prepareCanonicalProjections(engine, ready.parsedPage, row.slug, row.source_id, base,', 'projections'),
    ('  const preparedImport: PreparedMutation = {', 'prepared'),
]
def sync_prepare(text):
    if "from './stall-debug.ts'" in text: return text
    text = ensure_import(text, "import { stallMark } from './stall-debug.ts';")
    for anchor, step in ANCHORS:
        lines = text.split('\n')
        hits = [i for i, l in enumerate(lines) if l.startswith(anchor)]
        if len(hits) != 1: report.append(f'sync-prepare.ts: anchor {step} matched {len(hits)}x, skipped'); continue
        indent = re.match(r'\s*', lines[hits[0]]).group(0)
        lines.insert(hits[0], f"{indent}stallMark(row.request_id, '{step}');")
        text = '\n'.join(lines)
    return text
report.append(f"sync-prepare.ts edited={edit(P('sync-prepare.ts'), sync_prepare)}")

# 4. prepared-maintenance.ts and page-prepare.ts: entry marks only (the dispatch wrapper already covers start/end)
def maintenance(text):
    if "from './stall-debug.ts'" in text: return text
    text = ensure_import(text, "import { stallMark } from './stall-debug.ts';")
    lines = text.split('\n')
    hits = [i for i, l in enumerate(lines) if l.startswith('export async function prepareMaintenanceMutation(')]
    if len(hits) == 1: lines.insert(hits[0] + 1, "  stallMark(row.request_id, `maintenance:${String(row.intent?.kind ?? '')}`);")
    else: report.append('prepared-maintenance.ts: entry anchor missing')
    return '\n'.join(lines)
if os.path.exists(P('prepared-maintenance.ts')): report.append(f"prepared-maintenance.ts edited={edit(P('prepared-maintenance.ts'), maintenance)}")


# 5. consumer.ts: the group claim path before executeClaimedGroup (#6317: head order, follower claim, group start)
def consumer(text):
    if "from './stall-debug.ts'" in text: return text
    text = ensure_import(text, "import { stallMark, stallDone } from './stall-debug.ts';")
    text = text.replace("    const order = await claimedHeadOrder(this.engine, row, lane !== null);",
        "    stallMark(row.request_id, 'consumer:head_order');\n    const order = await claimedHeadOrder(this.engine, row, lane !== null);\n    stallMark(row.request_id, `consumer:head_order_done:${order === 'wait' ? 'wait' : order ? 'cancel' : 'go'}`);", 1)
    text = text.replace("    const followers = await claimGroupFollowers(this.engine, row, group,",
        "    stallMark(row.request_id, 'consumer:claim_followers');\n    const followers = await claimGroupFollowers(this.engine, row, group,", 1)
    text = text.replace("      return await executeClaimedGroup(this.engine, rows, {",
        "      stallMark(row.request_id, `consumer:group_start:${rows.length}:${lane ? 'lane' : 'fifo'}`);\n      return await executeClaimedGroup(this.engine, rows, {", 1)
    text = text.replace("    } finally { for (const member of rows) this.executing.delete(member.id); }\n  }\n  /** Mandatory barrier",
        "    } finally { for (const member of rows) { this.executing.delete(member.id); stallDone(member.request_id); } stallDone(row.request_id); }\n  }\n  /** Mandatory barrier", 1)
    return text
report.append(f"consumer.ts edited={edit(P('consumer.ts'), consumer)}")

# 6. group-publish.ts: wave dispatch, lease outcome, publish entry (both the 0.60.110 and the budget shape)
def group_publish(text):
    if "from './stall-debug.ts'" in text: return text
    text = ensure_import(text, "import { stallMark } from './stall-debug.ts';")
    text = text.replace("          try { prepared[start + offset] = { ok: await run.prepare(row) }; }",
        "          stallMark(row.request_id, `group:dispatch:${start + offset}`);\n          try { prepared[start + offset] = { ok: await run.prepare(row) }; stallMark(row.request_id, 'group:prepared'); }", 1)
    text = text.replace("            return run.prepare(row, reads, undefined, clocks[i]);",
        "            stallMark(row.request_id, `group:dispatch:${i}`);\n            return run.prepare(row, reads, undefined, clocks[i]).then(value => { stallMark(row.request_id, 'group:prepared'); return value; });", 1)
    text = text.replace("    const preparing = (async () => {",
        "    for (const row of rows) stallMark(row.request_id, 'group:queued_in_group');\n    const preparing = (async () => {", 1)
    text = text.replace("  const lock = lane ? await acquireWorktreeShared(binding, engine)",
        "  for (const row of rows) stallMark(row.request_id, lane ? 'publish:acquire_worktree_shared' : 'publish:acquire_worktree');\n  const lock = lane ? await acquireWorktreeShared(binding, engine)", 1)
    text = text.replace("    if (lane) await awaitLaneBegin(lane, rows);\n    const done = await transaction(",
        "    if (lane) { for (const row of rows) stallMark(row.request_id, 'publish:await_lane_begin'); await awaitLaneBegin(lane, rows); }\n    for (const row of rows) stallMark(row.request_id, 'publish:transaction');\n    const done = await transaction(", 1)
    text = text.replace("    if (lane) await awaitLaneBegin(lane, rows);\n    const done = await engine.transaction(",
        "    if (lane) { for (const row of rows) stallMark(row.request_id, 'publish:await_lane_begin'); await awaitLaneBegin(lane, rows); }\n    for (const row of rows) stallMark(row.request_id, 'publish:transaction');\n    const done = await engine.transaction(", 1)
    return text
report.append(f"group-publish.ts edited={edit(P('group-publish.ts'), group_publish)}")

# 7. postgres-engine.ts: every raw statement and reserve while it is in flight in JS
PE = os.path.join(repo, 'src', 'core', 'postgres-engine.ts')
def postgres_engine(text):
    if "persistence/stall-debug.ts" in text: return text
    text = ensure_import(text, "import { stallSql } from './persistence/stall-debug.ts';")
    text = text.replace("        reserved = signal && typeof conn.reserve === 'function' ? await reserveWithCancellation(opts => conn.reserve(opts), signal) : undefined;",
        "        const __reserveDone = signal && typeof conn.reserve === 'function' ? stallSql(`RESERVE for ${sql}`, 'raw', true) : null;\n        try { reserved = signal && typeof conn.reserve === 'function' ? await reserveWithCancellation(opts => conn.reserve(opts), signal) : undefined; } finally { __reserveDone?.(); }", 1)
    text = text.replace("        pending = conn.unsafe(sql, params as Parameters<typeof conn.unsafe>[1], driverOpts);\n        return await pending as unknown as T[];",
        "        const __sqlDone = stallSql(sql, reserved ? 'raw:reserved' : this._pageTransaction ? 'tx' : 'raw', !!reserved);\n        try {\n          pending = conn.unsafe(sql, params as Parameters<typeof conn.unsafe>[1], driverOpts);\n          return await pending as unknown as T[];\n        } finally { __sqlDone(); }", 1)
    text = text.replace("      reserved = await pool.reserve();\n    } catch (e) {",
        "      const __reserveDone = stallSql('RESERVE withReservedConnection', 'reserved', true);\n      try { reserved = await pool.reserve(); } finally { __reserveDone(); }\n    } catch (e) {", 1)
    text = text.replace("      return await withHeldPageKeys(this._pageTransaction ? this._heldPageKeys : null, held => conn.begin(async (handle) => {",
        "      const __beginDone = this._pageTransaction ? null : stallSql('BEGIN (transaction checkout)', 'tx', true);\n      try { return await withHeldPageKeys(this._pageTransaction ? this._heldPageKeys : null, held => conn.begin(async (handle) => {\n        __beginDone?.();", 1)
    text = text.replace("        return fn(txEngine);\n      }) as Promise<T>);\n    } finally {\n      if (!this._pageTransaction) this.checkoutGauge.release('tx');",
        "        return fn(txEngine);\n      }) as Promise<T>); } finally { __beginDone?.(); }\n    } finally {\n      if (!this._pageTransaction) this.checkoutGauge.release('tx');", 1)
    return text
report.append(f"postgres-engine.ts edited={edit(PE, postgres_engine)}")

# 8. vendor/postgres/src/index.js: expose the pool queues and register every pool (debug only)
VP = os.path.join(repo, 'vendor', 'postgres', 'src', 'index.js')
def vendor_pool(text):
    if '__gbrainPools' in text: return text
    return text.replace("  Object.assign(sql, {\n    get parameters() { return options.parameters },",
        "  ;(globalThis.__gbrainPools ??= []).push(sql)\n  Object.assign(sql, {\n    queues,\n    get parameters() { return options.parameters },", 1)
report.append(f"vendor/postgres edited={edit(VP, vendor_pool)}")

print('\n'.join(report))