Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.0 KB
Raw Blame
/**
 * Synthesis completion keys: which transcripts a prior dream run already
 * synthesized (the legacy `dream:synth:` path-keyed family and the synth-v2
 * source-identity family), read from completed subagent jobs and the durable
 * dream_synthesis_completions archive. Peeled from synthesize.ts with #5145,
 * which moved the completion check ahead of the paid triage pass.
 */
import { basename } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { MaintenanceAuthority } from '../persistence/prepared-maintenance.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import type { DiscoveredTranscript } from './transcript-discovery.ts';
import type { PhaseResult } from '../cycle.ts';

/**
 * #5145: split transcripts into already-synthesized and candidates BEFORE the
 * paid triage pass. Classic brains (no maintenance) and dry runs (maintenance
 * forced null) used to triage completed transcripts first and only skip them
 * afterwards, re-buying triage whenever the verdict cache expired. A dry run
 * on a managed brain reads the source incarnation read-only so its synth-v2
 * keys match the identity a real run would use.
 *
 * `retainedSkips` maps each transcript the fan-out must skip without
 * re-attaching children to its skip reason: legacy `dream:synth:` completions
 * and, on a managed brain, synth-v2 completions under the bare source id.
 * Completions keyed to this run's own identity stay out of the map; the
 * fan-out re-checks them against `successfulV2Keys` (a managed run re-attaches
 * their children for postprocessing).
 *
 * #6924: on a managed brain a completion without an incarnation (legacy keys,
 * bare-source synth-v2 keys) was recorded before the brain went managed, so
 * its identity cannot say which row of this source id it belongs to. It
 * counts only when this source still holds that synthesis's output: a live
 * `dream_generated` page whose `raw_source` names the transcript (matched by
 * basename, like the keys). Pages are deleted with their source row, so a
 * completion of a removed and re-added source, or of another source, never
 * suppresses synthesis.
 */
export async function partitionCompletedSynthesis(
  engine: BrainEngine,
  transcripts: DiscoveredTranscript[],
  sourceId: string,
  maintenance: MaintenanceAuthority | null,
  dryRun: boolean,
): Promise<{ completed: Set<string>; retainedSkips: Map<string, string>; successfulV2Keys: string[]; synthesisIdentity: string }> {
  const incarnation = maintenance
    ? maintenance.writer.sourceIncarnation
    : dryRun ? await readManagedSourceIncarnation(engine, sourceId) : null;
  const synthesisIdentity = incarnation !== null ? `${sourceId}/${incarnation}` : sourceId;
  const successfulV2Keys = await loadSuccessfulSynthesisKeys(engine, sourceId, 'dream:synth-v2:');
  const successfulLegacyKeys = await loadSuccessfulSynthesisKeys(engine, sourceId, 'dream:synth:');
  const retainedOutputs = incarnation !== null ? await loadRetainedOutputSources(engine, sourceId) : null;
  const completed = new Set<string>();
  const retainedSkips = new Map<string, string>();
  for (const t of transcripts) {
    const hash16 = t.contentHash.slice(0, 16);
    const current = findSynthV2Completion(successfulV2Keys, t.filePath, hash16, synthesisIdentity);
    if (current) completed.add(t.filePath);
    if (retainedOutputs && (current || !retainedOutputs.has(basename(t.filePath)))) continue;
    const legacy = findLegacyCompletion(successfulLegacyKeys, t.filePath, hash16);
    const premanaged = retainedOutputs ? findSynthV2Completion(successfulV2Keys, t.filePath, hash16, sourceId) : null;
    const reason = legacy ? `already_synthesized_legacy_${legacy === 'chunked' ? 'chunked' : 'single_chunk'}`
      : premanaged ? `already_synthesized_v2_${premanaged === 'chunked' ? 'chunked' : 'single_chunk'}` : null;
    if (!reason) continue;
    completed.add(t.filePath);
    retainedSkips.set(t.filePath, reason);
  }
  return { completed, retainedSkips, successfulV2Keys, synthesisIdentity };
}

/**
 * #6924: basenames of the transcripts this source's live synthesis output
 * names (`raw_source`, stamped on every classic synthesis page by
 * stampDreamProvenance).
 */
async function loadRetainedOutputSources(engine: BrainEngine, sourceId: string): Promise<Set<string>> {
  const rows = await engine.executeRaw<{ raw_source: string }>(
    `SELECT DISTINCT frontmatter->>'raw_source' AS raw_source
       FROM pages
      WHERE source_id = $1
        AND deleted_at IS NULL
        AND frontmatter->>'dream_generated' = 'true'
        AND COALESCE(frontmatter->>'raw_source', '') <> ''`,
    [sourceId],
  );
  return new Set(rows.map(row => basename(row.raw_source)));
}

/**
 * #5145: the source incarnation a managed real run would key its synth-v2
 * completions under, read without the write preflight (dry runs never
 * register a writer). Null on a classic brain or when the read fails.
 */
async function readManagedSourceIncarnation(engine: BrainEngine, sourceId: string): Promise<string | null> {
  try {
    if (!await managedPersistenceEnabled(engine)) return null;
    const [row] = await engine.executeRaw<{ incarnation: string | null }>('SELECT incarnation FROM sources WHERE id = $1', [sourceId]);
    return row?.incarnation ?? null;
  } catch {
    return null;
  }
}

/**
 * Load every `completed` subagent job key in one synthesis key family for
 * one source. Called once per phase per family so the submit loop can skip
 * transcripts already synthesized BEFORE building their link manifest:
 *  - D8 legacy `dream:synth:<filePath>:<hash16>[:c<i>of<n>]` (pre-v2 shape;
 *    must not be re-submitted under v2 keys);
 *  - current `dream:synth-v2:<source>:filename:<basename>:<hash16>[:c<i>of<n>]`
 *    (the queue's idempotency dedupe would coalesce these too, but only after
 *    the manifest build, and the coalesced children would re-enter writtenRefs).
 * `dream:synth:%` does not match `dream:synth-v2:` keys.
 *
 * Plain `status = 'completed'` deliberately mirrors the queue-level
 * idempotency semantics the legacy keys relied on: a completed job blocks
 * re-submission regardless of `result.stop_reason` (pinned in
 * test/minions.test.ts). Filtering on stop_reason here would re-pay for
 * transcripts the old code path never re-ran, and reading `result` at all
 * would need the `(result #>> '{}')` double-encoded-jsonb defense.
 *
 * Completed rows that `jobs prune` removed live on in
 * `dream_synthesis_completions` (the prune archives them), so pruning never
 * makes a synthesized transcript eligible again. Loads source-scoped
 * completions once per phase; no repeated history scan per transcript.
 */
async function loadSuccessfulSynthesisKeys(
  engine: BrainEngine,
  sourceId: string,
  keyPrefix: 'dream:synth:' | 'dream:synth-v2:',
): Promise<string[]> {
  const rows = await engine.executeRaw<{ idempotency_key: string }>(
    `SELECT idempotency_key
       FROM minion_jobs
      WHERE name = 'subagent'
        AND status = 'completed'
        AND COALESCE(NULLIF(data->>'source_id', ''), 'default') = $1
        AND idempotency_key LIKE $2
     UNION
     SELECT idempotency_key FROM dream_synthesis_completions
      WHERE source_id = $1 AND idempotency_key LIKE $2`,
    [sourceId, `${keyPrefix}%`],
  );
  return rows.map(row => row.idempotency_key);
}

/**
 * #6753: the synth-v2 completions of the source recorded within the last `hours`, with when each finished: completed
 * subagent jobs and the durable completion archive. Failed, cancelled and still-running jobs are not completions.
 */
export async function recentSynthesisCompletions(engine: BrainEngine, sourceId: string, hours: number): Promise<Array<{ key: string; at: number }>> {
  if (hours <= 0) return [];
  const rows = await engine.executeRaw<{ idempotency_key: string; at: string | Date }>(
    `SELECT idempotency_key, finished_at AS at FROM minion_jobs
      WHERE name = 'subagent' AND status = 'completed' AND finished_at > now() - ($3 || ' hours')::interval
        AND COALESCE(NULLIF(data->>'source_id', ''), 'default') = $1 AND idempotency_key LIKE $2
     UNION ALL
     SELECT idempotency_key, completed_at FROM dream_synthesis_completions
      WHERE source_id = $1 AND idempotency_key LIKE $2 AND completed_at > now() - ($3 || ' hours')::interval`,
    [sourceId, 'dream:synth-v2:%', String(hours)],
  );
  return rows.map(row => ({ key: row.idempotency_key, at: new Date(row.at).getTime() }));
}

/**
 * #6753: a growing transcript (a live session) has a new content hash on every pass, so its completion key never
 * matches and it was synthesized again in full each time. When another version of the same file completed in full
 * (single or a whole chunk set, as findSynthV2Completion counts it) within `hours`, the current version waits: the
 * ISO time the hold ends, else null. The hold only delays; once it ends the current version synthesizes once, whole.
 */
export function resynthesisHoldUntil(recent: Array<{ key: string; at: number }>, filePath: string, hash16: string,
  sourceId: string, hours: number): string | null {
  const prefix = `dream:synth-v2:${encodeURIComponent(sourceId)}:filename:${encodeURIComponent(basename(filePath))}:`;
  const mine = recent.filter(r => r.key.startsWith(prefix));
  let latest = 0;
  for (const other of new Set(mine.map(r => r.key.slice(prefix.length, prefix.length + 16)))) {
    if (other === hash16 || !findSynthV2Completion(mine.map(r => r.key), filePath, other, sourceId)) continue;
    for (const r of mine) if (r.key.startsWith(prefix + other)) latest = Math.max(latest, r.at);
  }
  return latest > 0 ? new Date(latest + hours * 3_600_000).toISOString() : null;
}

/**
 * Mirror of findLegacyCompletion for the synth-v2 key family (grammar as
 * produced by the submit loop / parsed by `parseSynthV2Key`): `'single'` when
 * the unchunked key completed, `'chunked'` when a FULL `:c0of<n>`..`:c<n-1>of<n>`
 * set completed, null otherwise (a cancelled row never counts).
 */
export function findSynthV2Completion(
  successfulKeys: string[],
  filePath: string,
  hash16: string,
  sourceId: string,
): 'single' | 'chunked' | null {
  const prefix =
    `dream:synth-v2:${encodeURIComponent(sourceId)}` +
    `:filename:${encodeURIComponent(basename(filePath))}:${hash16}`;
  const chunkSets = new Map<number, Set<number>>();
  for (const key of successfulKeys) {
    if (key === prefix) return 'single';
    if (!key.startsWith(prefix + ':c')) continue;
    const chunk = /:c(\d+)of(\d+)$/.exec(key);
    if (!chunk) continue;
    const i = Number(chunk[1]);
    const n = Number(chunk[2]);
    if (n < 1 || i < 0 || i >= n) continue;
    let seen = chunkSets.get(n);
    if (!seen) chunkSets.set(n, seen = new Set());
    seen.add(i);
  }
  for (const [n, seen] of chunkSets) {
    if (seen.size === n) return 'chunked';
  }
  return null;
}

/**
 * Match a transcript (by filename + content hash) against completed legacy
 * keys. `'single'` when a `dream:synth:<path>:<hash16>` completion exists;
 * `'chunked'` when a FULL chunk set `:c0of<n>`..`:c<n-1>of<n>` completed
 * (chunk indices are 0-based). Partial chunk sets return null so the
 * transcript gets a fresh v2 synthesis instead of shipping with holes.
 */
export function findLegacyCompletion(
  successfulKeys: string[],
  filePath: string,
  hash16: string,
): 'single' | 'chunked' | null {
  const filename = basename(filePath);
  const hashSuffix = `:${hash16}`;
  /** total chunk count n → completed 0-based chunk indices */
  const chunkSets = new Map<number, Set<number>>();
  for (const key of successfulKeys) {
    const chunk = /:c(\d+)of(\d+)$/.exec(key);
    const base = chunk ? key.slice(0, -chunk[0].length) : key;
    if (!base.endsWith(hashSuffix)) continue;
    const historicalPath = base.slice('dream:synth:'.length, -hashSuffix.length);
    if (basename(historicalPath) !== filename) continue;
    if (!chunk) return 'single';
    const i = Number(chunk[1]);
    const n = Number(chunk[2]);
    if (n < 1 || i < 0 || i >= n) continue;
    let seen = chunkSets.get(n);
    if (!seen) chunkSets.set(n, seen = new Set());
    seen.add(i);
  }
  for (const [n, seen] of chunkSets) {
    if (seen.size === n) return 'chunked';
  }
  return null;
}

/**
 * #5848: the synthesize result for a window with zero candidates: no model call, no queue, no budget spend. Status
 * stays ok (not skipped) so finishDeferredPostprocess still finishes outputs an earlier pass deferred (#6757).
 */
export function noCandidates(summary: string, details: Record<string, unknown>): PhaseResult {
  return { phase: 'synthesize', status: 'ok', duration_ms: 0, summary, details: { reason: 'no_candidates', transcripts_processed: 0, pages_written: 0, ...details } };
}

/**
 * #5848: a classic real run whose every discovered transcript is already synthesized returns here, before triage,
 * the manifest context and the fan-out; it stamps the cooldown as a fully completed run would and reports each
 * transcript's skip reason. A managed run never comes here: it re-attaches completed children for publication.
 */
export async function noNewTranscripts(engine: BrainEngine, transcripts: DiscoveredTranscript[],
  partition: Pick<Awaited<ReturnType<typeof partitionCompletedSynthesis>>, 'retainedSkips' | 'successfulV2Keys' | 'synthesisIdentity'>): Promise<PhaseResult> {
  await engine.setConfig('dream.synthesize.last_completion_ts', new Date().toISOString());
  const skips = transcripts.map(t => ({ filePath: t.filePath, reason: partition.retainedSkips.get(t.filePath)
    ?? (findSynthV2Completion(partition.successfulV2Keys, t.filePath, t.contentHash.slice(0, 16), partition.synthesisIdentity) === 'chunked'
      ? 'already_synthesized_v2_chunked' : 'already_synthesized_v2_single_chunk') }));
  return noCandidates(`no new transcripts to synthesize (${transcripts.length} already synthesized)`, { transcripts_discovered: transcripts.length,
    synthesis_state: { candidates: 0, already_synthesized: transcripts.length }, children_submitted: 0, skips });
}

/**
 * #6753: the per-transcript re-synthesis hold for one run, read once: a function giving the `resynthesis_cooldown`
 * skip reason for a transcript an earlier version of which completed within `hours`, else null. Zero hours (an explicit
 * --input/--date run, or a cooldown of 0) holds nothing.
 */
export async function resynthesisHold(engine: BrainEngine, sourceId: string, hours: number): Promise<(filePath: string, hash16: string, identity: string) => string | null> {
  const recent = await recentSynthesisCompletions(engine, sourceId, hours);
  return (filePath, hash16, identity) => {
    const until = resynthesisHoldUntil(recent, filePath, hash16, identity, hours);
    return until && `resynthesis_cooldown: an earlier version of this transcript was synthesized within ${hours}h; the current version synthesizes after ${until}`;
  };
}

/** #6753: the summary's note of transcripts held by the per-transcript re-synthesis hold, empty when none. */
export function resynthesisHoldSuffix(skips: Array<{ reason: string }>): string {
  const held = skips.filter(s => s.reason.startsWith('resynthesis_cooldown')).length;
  return held ? ` (${held} grown transcript(s) held by resynthesis_cooldown)` : '';
}