Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
37.2 KB
Raw Blame
// v0.41 T6 — synthesize_concepts cycle phase (minimal-viable implementation).
//
// v0.41 ships a working concept synthesis path: group atoms by simple
// frontmatter tag/concept references, tier by count (T1 ≥10, T2 ≥5,
// T3 ≥2, T4 ≥1), Sonnet-synthesize T1/T2 narratives. Voice gate
// integration + dedup-by-embedding-similarity ship in v0.42+.
//
// Sequencing:
//   1. Query all atom-typed pages from DB (excluding imported_from
//      marker → atoms already extracted by your OpenClaw don't get
//      re-synthesized as concepts here; their original concept pages
//      come through greenfield import already).
//   2. Group by `concepts:` frontmatter field on each atom (when the
//      Haiku 3-check from extract_atoms decides "this atom is about
//      concept X", it stamps the field).
//   3. For each group with count ≥2: assign tier (T1/T2/T3/T4 by count).
//   4. Sort by tier, evidence count, and slug so the bounded LLM budget goes
//      to the strongest groups deterministically.
//   5. For T1/T2 groups: Sonnet call to produce a 1-paragraph narrative.
//      For T3/T4: deterministic stub narrative.
//   6. Write concept-typed pages with the synthesis mode made explicit.

import type { BrainEngine } from '../engine.ts';
import { resolveModel } from '../model-config.ts';
import type { PhaseResult } from '../cycle.ts';
import type { ProgressReporter } from '../progress.ts';
import { writeReceipt } from '../extract/receipt-writer.ts';
import { upsertExtractRollup } from '../extract/rollup-writer.ts';
import { chat as gatewayChat, isThinkingModel, THINKING_MODEL_MAX_OUTPUT_TOKENS } from '../ai/gateway.ts';
import { createGlobalLlmHaltTracker, haltedClassOf, type GlobalLlmErrorClass } from '../ai/errors.ts';
import { canonicalLookup, type ModelPricing } from '../model-pricing.ts';
import { priceFor, type PricingOverrides } from '../budget/reservation-cost.ts';
import { loadPricingOverrides } from '../budget/budget-tracker.ts';
import { pricingSetCommand } from '../budget/no-pricing.ts';
import { createHash } from 'node:crypto';
import { slugifySegment } from '../sync.ts';
import { validatePageSlug } from '../ops/context.ts';
import { privatePagesFilterFragment, strictestVisibility, type Visibility } from '../search/private-visibility.ts';
import { maintenancePreflight } from '../persistence/prepared-maintenance.ts';
import { derivedWriteThrough, reportDerivedWriteThrough } from './derived-write-through.ts';
import { addSpellingRedirects, canonicalConceptStem, loadConceptRedirects, loadLiveConceptStems } from './concept-redirects.ts';
import { conceptProvenanceCovers, conceptPublicationHold, stripFenceSections } from './concept-publication.ts';
import { conceptWriter, type ConceptWrite } from './concept-writes.ts';
import { readPhaseConfigNumber, SYNTHESIZE_CONCEPTS_BUDGET_KEY, SYNTHESIZE_CONCEPTS_DEFAULT_BUDGET_USD } from './phase-config-values.ts';
import { ConceptRetryBound, MAX_CONCEPT_ATTEMPTS } from './concept-retry-bound.ts';
import { ConceptDriftSkips } from '../persistence/drift-conflicts.ts';
import { throwIfAborted } from '../abort-check.ts';

// Miss policy for the shared resolver (`priceFor`: operator overrides, the
// claude-cli → Anthropic sibling, canonical rows): assume Sonnet-tier
// pricing for a model nothing prices, and name it in the phase details
// (`pricing_fallback_models`) plus one stderr line. Conservative and
// non-throwing; keeps this default budget effective instead of letting an
// unpriced model run unmetered. The rates are DERIVED from the canonical
// table (never hand-copied — CLAUDE.md invariant); the literal pair only
// fires if the Sonnet key itself ever leaves the table.
const FALLBACK_PRICING: ModelPricing = canonicalLookup('anthropic:claude-sonnet-4-6') ?? {
  input: 3.0,
  output: 15.0,
};
/** The rate a narrative call meters at: the shared resolver, else the Sonnet fallback, naming each fallback model once. */
function narrativePricing(model: string, overrides: PricingOverrides | undefined, fallbackModels: Set<string>, budgetUsd: number): ModelPricing {
  const priced = priceFor(model, 'chat', overrides);
  if (priced) return priced.pricing;
  if (!fallbackModels.has(model)) {
    fallbackModels.add(model);
    console.error(`[synthesize_concepts] ${model} has no pricing; metering it at Sonnet-tier rates against the $${budgetUsd.toFixed(2)} phase budget. To meter its real price, look it up and register it: ${pricingSetCommand(model, 'chat')}`);
  }
  return FALLBACK_PRICING;
}

/** #4907: the run's named-key budget warnings (invalid stored budget, groups refused at the cap), each echoed once to stderr. */
function budgetWarnings(configWarning: string | undefined, capped: number, budgetUsd: number): string[] {
  const warnings = configWarning ? [configWarning] : [];
  if (capped > 0) {
    warnings.push(`${capped} LLM-eligible concept(s) got no model narrative because the $${budgetUsd.toFixed(2)} phase budget was reached; they retry next run. ` +
      `Raise the cap: gbrain config set ${SYNTHESIZE_CONCEPTS_BUDGET_KEY} <usd>. Verify: gbrain config get ${SYNTHESIZE_CONCEPTS_BUDGET_KEY}`);
  }
  for (const warning of warnings) console.error(`[synthesize_concepts] ${warning}`);
  return warnings;
}

const TIER_T1_MIN = 10;
const TIER_T2_MIN = 5;
const TIER_T3_MIN = 2;
/**
 * Output cap for the per-concept narrative. 500 sizes the *answer* (a
 * 1-paragraph summary) and is ample for a non-reasoning model. It is NOT ample
 * for a thinking-by-default model: reasoning bills as output and counts against
 * max_tokens, so the budget is spent before any answer text is emitted — hosted
 * DeepSeek returns empty content (→ `deterministicNarrative` ships a template
 * stub as 'error_fallback'), while the native deepseek: recipe promotes the
 * truncated reasoning_content into content (→ chain-of-thought persisted as
 * the narrative with synthesis_mode 'llm'). This phase resolves at
 * `tier: 'reasoning'`, so a thinking model here is the expected case.
 */
const DEFAULT_SYNTH_MAX_OUTPUT_TOKENS = 500;

/**
 * Narrative output cap for the resolved model. `isThinkingModel` is the
 * gateway's shared predicate (name-matched Claude 5 OR recipe-declared
 * `thinking_by_default`; unknown providers count as non-thinking), the same
 * check think's maxOutputTokensFor and the subagent handler use. Thinking
 * models get the gateway's verified THINKING_MODEL_MAX_OUTPUT_TOKENS rather
 * than a phase-private number: a local 8000 contradicted it, and DeepSeek v4
 * truncates at 8192-class caps — the reasoning budget was gone before any
 * answer text.
 */
export function resolveSynthMaxOutputTokens(modelStr: string): number {
  return isThinkingModel(modelStr) ? THINKING_MODEL_MAX_OUTPUT_TOKENS : DEFAULT_SYNTH_MAX_OUTPUT_TOKENS;
}

export interface SynthesizeConceptsOpts {
  brainDir?: string;
  /**
   * #4416: the cycle's resolved source scope (cycleSourceId in cycle.ts).
   * Without it every write below falls through to the engine's `?? 'default'`
   * literal, which misfiles (or, on the createVersion update path, kills the
   * cycle) on any brain whose sole source is not named `default`: getPage's
   * undefined-source path is source-agnostic, so the existence probe passes,
   * then createVersion throws "page ... (source=default) not found".
   */
  sourceId?: string;
  dryRun?: boolean;
  yieldDuringPhase?: (() => Promise<void>) | undefined;
  /**
   * v0.41.19.0 (T4): progress reporter for in-phase ticks. Cycle.ts
   * passes the SAME reporter (not a child — see extract-atoms.ts for
   * the path-collision bug codex caught). Phases only call `tick()` /
   * `heartbeat()`; cycle.ts owns start/finish.
   */
  progress?: ProgressReporter;
  /**
   * #7986: the cycle's abort signal (job timeout or cancel, lock loss). Handed to the model call and checked
   * before each concept's spend and publication, so an aborted run publishes no further concept page.
   */
  signal?: AbortSignal;
  /** Test seam: alternative chat function. */
  _chat?: typeof gatewayChat;
  /** Test seam: skip DB query; cluster these atoms directly. */
  _atoms?: Array<{ slug: string; concept_refs: string[]; body: string; title: string; visibility?: Visibility }>;
}

interface AtomGroup {
  conceptSlug: string;
  /** #4589: member atom slugs — the provenance edges are written from these. */
  atomSlugs: string[];
  atomTitles: string[];
  atomBodies: string[];
  /** #5525: strictest effective visibility of the member atoms. */
  visibility: Visibility;
  tier: 'T1' | 'T2' | 'T3' | 'T4';
}

type ConceptSynthesisMode =
  | 'llm'
  | 'deterministic_tier'
  | 'budget_fallback'
  | 'error_fallback';

const SYNTH_PROMPT = `You write a 1-paragraph executive summary of a concept
based on multiple atom-shaped insights that reference it.

Output ONLY the summary paragraph (3-5 sentences). No headers, no JSON,
no preamble. Write in plain English, present-tense voice. Synthesize what
the atoms collectively SAY about the concept; don't enumerate the atoms.`;

export async function runPhaseSynthesizeConcepts(
  engine: BrainEngine,
  opts: SynthesizeConceptsOpts = {},
): Promise<PhaseResult> {
  const chat = opts._chat ?? gatewayChat;

  // 1. Get atom pages (test seam OR DB query)
  let atoms = opts._atoms ?? [];
  if (atoms.length === 0 && opts._atoms === undefined) {
    try {
      const rows = await engine.executeRaw<{
        slug: string;
        title: string;
        compiled_truth: string;
        frontmatter: { concepts?: string[]; imported_from?: string };
        private: boolean;
      }>(
        // Codex P2: scoped to the cycle source — the provenance edges below
        // are pinned to it, so a brain-global scan grouped same-slug atoms
        // from OTHER sources into this source's concepts.
        `SELECT slug, title, compiled_truth, frontmatter, NOT (${privatePagesFilterFragment('pages')}) AS private
           FROM pages
          WHERE type = 'atom'
            AND source_id = $1
            AND deleted_at IS NULL
            AND (frontmatter->>'imported_from') IS NULL`,
        [opts.sourceId ?? 'default'],
      );
      atoms = rows
        .filter((r) => Array.isArray(r.frontmatter?.concepts) && r.frontmatter.concepts.length > 0)
        .map((r) => ({
          slug: r.slug,
          title: r.title,
          body: r.compiled_truth,
          concept_refs: r.frontmatter!.concepts!,
          visibility: r.private ? 'private' as const : 'world' as const,
        }));
    } catch {
      // No atoms table or query failed — phase no-ops cleanly.
    }
  }

  if (atoms.length === 0) {
    return {
      phase: 'synthesize_concepts',
      status: 'skipped',
      duration_ms: 0,
      summary: 'synthesize_concepts: no atoms with concept refs',
      details: { reason: 'no_atoms' },
    };
  }

  // 2. Group atoms by concept stem (#6161 human redirects, then #5965 spelling folds; see resolveAtomConceptStems); one atom counts once per concept.
  const { atomStems, foldedSpellings } = await resolveAtomConceptStems(engine, opts.sourceId ?? 'default', atoms);
  const groups = new Map<string, { slugs: string[]; titles: string[]; bodies: string[]; visibilities: Visibility[] }>();
  for (const [i, atom] of atoms.entries()) {
    for (const conceptSlug of atomStems[i]!) {
      const existing = groups.get(conceptSlug) ?? { slugs: [], titles: [], bodies: [], visibilities: [] };
      existing.slugs.push(atom.slug);
      existing.titles.push(atom.title);
      existing.bodies.push(atom.body);
      // An atom with no recorded visibility is private (derived pages fail closed).
      existing.visibilities.push(atom.visibility ?? 'private');
      groups.set(conceptSlug, existing);
    }
  }

  // 3. Filter to count ≥2, assign tier
  const atomGroups: AtomGroup[] = [];
  for (const [conceptSlug, data] of groups) {
    const count = data.titles.length;
    if (count < TIER_T3_MIN) continue;
    const tier: AtomGroup['tier'] =
      count >= TIER_T1_MIN ? 'T1' : count >= TIER_T2_MIN ? 'T2' : 'T3';
    atomGroups.push({
      conceptSlug,
      atomSlugs: data.slugs,
      atomTitles: data.titles,
      atomBodies: data.bodies,
      visibility: strictestVisibility(data.visibilities),
      tier,
    });
  }

  if (atomGroups.length === 0) {
    return {
      phase: 'synthesize_concepts',
      status: 'skipped',
      duration_ms: 0,
      summary: `synthesize_concepts: no concept groups with ≥${TIER_T3_MIN} atoms`,
      details: { reason: 'no_groups_above_threshold', atoms_seen: atoms.length },
    };
  }

  // Spend the bounded LLM budget on the strongest concepts first, independent
  // of Postgres/PGLite row encounter order. Stable slug ordering makes equal
  // groups deterministic across engines and repeated runs.
  const tierRank: Record<AtomGroup['tier'], number> = { T1: 0, T2: 1, T3: 2, T4: 3 };
  atomGroups.sort((a, b) =>
    tierRank[a.tier] - tierRank[b.tier] ||
    b.atomTitles.length - a.atomTitles.length ||
    a.conceptSlug.localeCompare(b.conceptSlug));

  // 4. Per group: synthesize narrative (LLM for T1/T2, deterministic for T3+)
  let conceptsWritten = 0;
  let estimatedSpendUsd = 0;
  const budget = await readPhaseConfigNumber(engine, SYNTHESIZE_CONCEPTS_BUDGET_KEY);
  const budgetCap = budget.value ?? SYNTHESIZE_CONCEPTS_DEFAULT_BUDGET_USD;
  let budgetCapped = 0;
  const failures: Array<{ concept: string; error: string }> = [];
  // #4589 provenance-link problems. Kept OUT of `failures`: that list means
  // "the LLM call failed → template fallback" downstream (summary wording,
  // rollup halt_delta / round_completed_delta), which a missing edge is not —
  // the narrative was synthesized and persisted as-is. Warn-only.
  const linkWarnings: Array<{ concept: string; warning: string }> = [];
  // #3044 adoption: shared halt policy — auth/billing halt on the first
  // hit, a rate_limit streak halts after 3 consecutive failures, a
  // successful chat call resets the streak.
  const llmHalt = createGlobalLlmHaltTracker();
  let abortedGlobalError: GlobalLlmErrorClass | null = null;
  const tierCounts = { T1: 0, T2: 0, T3: 0, T4: 0 };
  const synthesisModeCounts: Record<ConceptSynthesisMode, number> = {
    llm: 0,
    deterministic_tier: 0,
    budget_fallback: 0,
    error_fallback: 0,
  };

  // v0.41.19.0 (T3): throttled yield helper. Fires `opts.yieldDuringPhase`
  // every 30s — cycle.ts threads `buildYieldDuringPhase(lock, outer)` so
  // each fire refreshes the cycle DB lock + the existing external hook.
  // Pre-v0.41.19 the bare `if (opts.yieldDuringPhase) await ...()` at
  // every iteration fired hundreds of times per phase; the 30s throttle
  // matches the actual lock-refresh budget.
  let lastYieldMs = Date.now();
  async function maybeYield(): Promise<void> {
    if (!opts.yieldDuringPhase) return;
    const now = Date.now();
    if (now - lastYieldMs < 30_000) return;
    lastYieldMs = now;
    try {
      await opts.yieldDuringPhase();
    } catch (err) {
      const msg = err instanceof Error ? err.message : String(err);
      console.error(`[synthesize_concepts] yieldDuringPhase failed (non-fatal): ${msg}`);
    }
  }

  // Honour the documented per-task routing. Without an explicit model this
  // call inherits models.chat, so models.dream.synthesize (advertised in the
  // routing table as tier.reasoning) had no effect on this path.
  const synthModel = await resolveModel(engine, {
    configKey: 'models.dream.synthesize',
    tier: 'reasoning',
    fallback: 'sonnet',
  });
  const synthMaxOutputTokens = resolveSynthMaxOutputTokens(synthModel);
  // #5484: a managed brain refuses the legacy importFromContent writer. Claim
  // maintenance authority before any model spend so a missing canonical owner
  // fails fast; null on an unmanaged brain.
  const maintenance = opts.dryRun ? null : await maintenancePreflight(engine, opts.sourceId ?? 'default', opts.brainDir);
  const conceptFiles = await derivedWriteThrough(engine, 'synthesize_concepts', opts.sourceId ?? 'default', { managed: maintenance !== null, dryRun: opts.dryRun ?? false });
  // Managed publication outcomes, kept out of `failures` (LLM fallback). A
  // deferred concept moved under a concurrent writer and is retried next run;
  // a held concept cannot be republished without losing canonical material.
  const publicationDeferred: Array<{ concept: string; reason: string }> = [];
  const publicationHeld: Array<{ concept: string; reason: string }> = [];
  const publicationPending: Array<{ concept: string; reason: string }> = [];
  const skippedHumanOwned: string[] = [];
  const skippedDeleted: string[] = [];
  const skippedUnchanged: string[] = [];
  const provenanceRestored: string[] = [];
  const rehashed: string[] = [];
  const keptExistingNarrative: string[] = [];
  const pricingOverrides = await loadPricingOverrides(engine);
  const pricingFallbackModels = new Set<string>();
  const retries = await ConceptRetryBound.load(engine); // #6260: bounded paid retries per member hash
  const skippedRetryBound: string[] = [];
  const driftSkips = await ConceptDriftSkips.load(engine, opts.sourceId ?? 'default', !!maintenance?.binding, !!opts.dryRun);
  const skippedFileDrift = driftSkips.skipped;
  const src = opts.sourceId ?? 'default';
  const countWritten = () => {
    conceptsWritten++;
    // v0.41.19.0 (T4): one tick per concept group with running count.
    opts.progress?.tick(1, `${conceptsWritten} concepts`);
  };
  const writer = conceptWriter({ engine, maintenance, sourceId: src, importSourceId: opts.sourceId, brainDir: opts.brainDir, writeThrough: conceptFiles !== null,
    retries, publicationDeferred, publicationHeld, publicationPending, linkWarnings, written: countWritten });
  // #7986: an aborted run publishes nothing more, but keeps the retry counters and the spend of the calls it finished.
  const stopIfAborted = async (): Promise<void> => {
    if (!opts.signal?.aborted) return;
    if (!opts.dryRun) {
      await retries.save(engine);
      await upsertExtractRollup(engine, { kind: 'concepts', source_id: src, cost_delta: estimatedSpendUsd });
    }
    throwIfAborted(opts.signal, 'synthesize_concepts');
  };
  for (const group of atomGroups) {
    await stopIfAborted();
    const conceptSlug = `concepts/${group.conceptSlug}`;
    // A concept page this phase did not write belongs to a human (or another
    // writer). Check before any spend; never replace its body.
    const existingSnapshot = await engine.readPageSnapshot(conceptSlug, { sourceId: opts.sourceId ?? 'default', includeDeleted: true });
    if (existingSnapshot?.page.deleted_at) { skippedDeleted.push(conceptSlug); continue; } // #6161: a deleted concept is never resurrected
    const existing = existingSnapshot?.page ?? null;
    if (existing && !String(existing.frontmatter?.synthesized_by ?? '').startsWith('synthesize_concepts')) {
      skippedHumanOwned.push(conceptSlug);
      continue;
    }
    // The narrative is a function of the member atoms' narrative (their
    // `## Facts` / `## Takes` sections excluded, D-N3), their strictest
    // visibility and the model. When none changed and the page holds a real
    // narrative, there is nothing to spend or rewrite; a fallback page is retried,
    // and so is one a chat_fallback_chain model wrote (hashed under that model).
    const hashOf = (model: string, bodies: string[]): string => createHash('sha256')
      .update(JSON.stringify([model, group.visibility, group.atomSlugs.map((s, i) => [s, group.atomTitles[i], bodies[i]])
        .sort((a, b) => a[0].localeCompare(b[0]))]))
      .digest('hex').slice(0, 16);
    const narrativeBodies = group.atomBodies.map(stripFenceSections);
    const hashFor = (model: string): string => hashOf(model, narrativeBodies);
    // #7269: a T3 narrative is the deterministic template, which never calls the model, so its hash leaves the model out.
    const hashModel = group.tier === 'T1' || group.tier === 'T2' ? synthModel : '';
    const memberHash = hashFor(hashModel);
    const priorMode = existing?.frontmatter?.synthesis_mode;
    const realNarrative = priorMode === 'llm' || priorMode === 'deterministic_tier';
    // #7269: rewriting a deterministic page to the template it already holds, at a visibility it already has, would
    // publish nothing new (a page hashed with the model before the fix keeps its body and is not rewritten).
    const priorVisibility: Visibility = existing?.frontmatter?.visibility === 'private' ? 'private' : 'world';
    if ((existing?.frontmatter?.member_hash === memberHash && realNarrative)
      || (hashModel === '' && priorMode === 'deterministic_tier' && strictestVisibility([group.visibility, priorVisibility]) === priorVisibility
        && stripFenceSections(existing!.compiled_truth).trim() === deterministicNarrative(group).trim())) {
      skippedUnchanged.push(conceptSlug);
      // #7548: the cache skips the model, not the member provenance an unchanged concept may have lost.
      if (!opts.dryRun && !await conceptProvenanceCovers(engine, src, conceptSlug, group.atomSlugs)) {
        await writer.restoreProvenance({ concept: group.conceptSlug, conceptSlug, atomSlugs: group.atomSlugs, revision: existingSnapshot!.revision });
        provenanceRestored.push(conceptSlug);
      }
      continue;
    }
    // #6643 (B1): a drifted concept would only add a conflict receipt per publication; checked live before any spend.
    if (await driftSkips.skip(conceptSlug, existingSnapshot)) continue;
    const held = existing && conceptPublicationHold(existing); // checked before any spend
    if (held) { publicationHeld.push({ concept: group.conceptSlug, reason: held }); continue; }
    // A page hashed by the pre-D-N3 formula (full atom bodies) whose inputs are
    // provably unchanged keeps its narrative: only the hash is rewritten, no spend.
    const grandfathered = realNarrative && existing?.frontmatter?.member_hash === hashOf(hashModel, group.atomBodies);
    if (!grandfathered) tierCounts[group.tier]++;
    let narrative: string;
    let synthesisMode: ConceptSynthesisMode, unverifiedClaims: UnverifiedClaim[] = [];
    let fallbackWriter: string | undefined;
    if (grandfathered) {
      narrative = stripFenceSections(existing!.compiled_truth);
      synthesisMode = priorMode as ConceptSynthesisMode;
      rehashed.push(conceptSlug);
    } else if (group.tier === 'T1' || group.tier === 'T2') {
      if (estimatedSpendUsd >= budgetCap) {
        narrative = deterministicNarrative(group);
        synthesisMode = 'budget_fallback';
        budgetCapped++;
      } else if (retries.exhausted(`${opts.sourceId ?? 'default'}:${conceptSlug}`, memberHash)) {
        skippedRetryBound.push(conceptSlug);
        continue;
      } else {
        try {
          const result = await chat({
            model: synthModel,
            system: SYNTH_PROMPT,
            messages: [
              {
                role: 'user',
                content:
                  `Concept slug: ${group.conceptSlug}\n` +
                  `${group.atomTitles.length} atoms reference this concept.\n\n` +
                  `Sample atom titles:\n${group.atomTitles.slice(0, 10).map((t) => `  - ${t}`).join('\n')}\n\n` +
                  `Sample atom bodies:\n${narrativeBodies
                    .slice(0, 5)
                    .map((b, i) => `${i + 1}. ${b.slice(0, 500)}`)
                    .join('\n\n')}`,
              },
            ],
            maxTokens: synthMaxOutputTokens,
            abortSignal: opts.signal,
          });
          // Post-await yield (T3): the LLM call is the main TTL hazard
          // codex flagged. Throttle inside maybeYield bounds the actual
          // refresh rate.
          await maybeYield();
          llmHalt.reset();
          const pricing = narrativePricing(result.model, pricingOverrides, pricingFallbackModels, budgetCap);
          estimatedSpendUsd +=
            (result.usage.input_tokens * pricing.input +
              result.usage.output_tokens * pricing.output) /
            1_000_000;
          const text = result.text.trim();
          // #6260: a narrative stopped before the end never replaces a page or advances its member hash.
          const stopped = result.stopReason === 'length' || result.stopReason === 'refusal' || result.stopReason === 'content_filter';
          if (text && !stopped) {
            ({ narrative, unverified: unverifiedClaims } = await groundConceptNarrative(engine, text, group));
            synthesisMode = 'llm';
            if (result.fallbackFrom) fallbackWriter = result.model;
          } else {
            failures.push({ concept: group.conceptSlug, error: stopped ? `output stopped before the end (stopReason=${result.stopReason})` : 'empty model response' });
            narrative = deterministicNarrative(group);
            synthesisMode = 'error_fallback';
            retries.fail(`${opts.sourceId ?? 'default'}:${conceptSlug}`, memberHash);
          }
        } catch (err) {
          await stopIfAborted();
          const msg = err instanceof Error ? err.message : String(err);
          // #3044 adoption: a whole-run LLM outage must not overwrite
          // existing concept pages with error_fallback stub narratives.
          // A halt decision stops the phase; a below-streak rate limit
          // skips this group's write (the page stays intact for the next
          // run); only non-global errors keep the per-item
          // error_fallback behavior.
          const decision = llmHalt.observe(err);
          if (decision !== 'continue') {
            abortedGlobalError = haltedClassOf(decision);
            failures.push({
              concept: group.conceptSlug,
              error: `aborting phase: ${llmHalt.note()} (${msg})`,
            });
            break;
          }
          failures.push({ concept: group.conceptSlug, error: msg });
          if (llmHalt.lastClass() === 'rate_limit') continue;
          narrative = deterministicNarrative(group);
          synthesisMode = 'error_fallback';
          retries.fail(`${opts.sourceId ?? 'default'}:${conceptSlug}`, memberHash);
        }
      }
    } else {
      narrative = deterministicNarrative(group);
      synthesisMode = 'deterministic_tier';
    }
    // Never replace an LLM narrative with a template stub; the next cycle retries.
    if (priorMode === 'llm' && (synthesisMode === 'budget_fallback' || synthesisMode === 'error_fallback')) {
      keptExistingNarrative.push(conceptSlug);
      continue;
    }
    if (!grandfathered) synthesisModeCounts[synthesisMode]++;

    await stopIfAborted();
    if (!opts.dryRun) {
      // #5525: tighten-only — a concept already private stays private.
      const prior = await engine.getPage(conceptSlug, { sourceId: src });
      const visibility = strictestVisibility([group.visibility, prior?.frontmatter?.visibility === 'private' ? 'private' : 'world']);
      // #2163: serialize to markdown and import via the canonical pipeline so
      // the page is chunked (+ embedded when a provider is configured) —
      // mirrors put_page's isAvailable('embedding') → noEmbed gate.
      const synthesizedAt = grandfathered ? String(existing!.frontmatter.synthesized_at ?? new Date().toISOString()) : new Date().toISOString();
      const synthesized = (pageVisibility: Visibility) => ({
        tier: group.tier,
        mention_count: group.atomTitles.length,
        composite_score: group.atomTitles.length,
        synthesis_mode: synthesisMode,
        member_hash: fallbackWriter ? hashFor(fallbackWriter) : memberHash,
        synthesized_at: synthesizedAt,
        synthesized_by: 'synthesize_concepts-v0.41',
        visibility: pageVisibility, ...(unverifiedClaims.length ? { unverified_claims: unverifiedClaims.map(c => ({ ...c, sources: ['member atoms'], detected_at: synthesizedAt.slice(0, 10) })) } : {}),
      });
      // #5525: a world concept stays private until every member's provenance
      // edge is durable, because a later private flip reaches it through them.
      // GBRA-60: one whose edges already cover every member publishes once, at
      // world, so a strict source never commits an intermediate private page.
      const covered = visibility === 'world' && await conceptProvenanceCovers(engine, src, conceptSlug, group.atomSlugs);
      const write: ConceptWrite = { concept: group.conceptSlug, conceptSlug, atomSlugs: group.atomSlugs, visibility, covered, synthesisMode, synthesized, narrative,
        revision: existingSnapshot?.revision ?? null, baseline: existing?.compiled_truth ?? '' };
      // GBRA-60: a managed run's concepts publish in batches (concept-writes.ts), each one publication group on Postgres.
      await stopIfAborted();
      await (maintenance ? writer.queue(write) : writer.publish(write));
    } else countWritten();

    // v0.41.19.0 (T3): replaced bare per-iteration fire with throttled
    // helper. Same hook, same cycle-lock refresh effect, just at the
    // right cadence (30s instead of every-group).
    await maybeYield();
  }
  await stopIfAborted();
  if (maintenance) await writer.flush();

  // v0.42 Wave B3: receipt + rollup for synthesize_concepts. Receipt/rollup
  // carry the cycle's resolved source (#4416, opts.sourceId); 'default'
  // survives only as the fallback for legacy unscoped callers. Receipt only
  // fires when concepts were actually written; rollup always fires so doctor
  // sees the phase ran.
  const driftActions = await driftSkips.finish();
  const warnings = budgetWarnings(budget.warning, budgetCapped, budgetCap);
  if (!opts.dryRun) await retries.save(engine);
  // Managed brains skip the receipt page (a legacy putPage), like extract_atoms;
  // the rollup row below still records the run for doctor.
  if (!opts.dryRun && !maintenance && conceptsWritten > 0) {
    const runId = `concepts-${Date.now().toString(36)}`;
    try {
      await writeReceipt(engine, {
        kind: 'concepts',
        source_id: opts.sourceId ?? 'default',
        run_id: runId,
        round: 'single',
        extracted_at: new Date().toISOString(),
        total_rows: conceptsWritten,
        cost_usd: estimatedSpendUsd,
        summary:
          `Synthesized ${conceptsWritten} concepts ` +
          `(T1=${tierCounts.T1} T2=${tierCounts.T2} T3=${tierCounts.T3}) ` +
          `(llm=${synthesisModeCounts.llm} deterministic=${synthesisModeCounts.deterministic_tier} ` +
          `budget_fallback=${synthesisModeCounts.budget_fallback} error_fallback=${synthesisModeCounts.error_fallback}) ` +
          `from ${atomGroups.length} groups across ${atoms.length} atoms.`,
      });
    } catch (err) {
      console.error(`[synthesize_concepts] receipt write failed: ${(err as Error).message}`);
    }
  }
  if (!opts.dryRun) {
    await upsertExtractRollup(engine, {
      kind: 'concepts',
      source_id: opts.sourceId ?? 'default',
      cost_delta: estimatedSpendUsd,
      round_completed_delta: failures.length === 0 && publicationDeferred.length === 0 && publicationHeld.length === 0 && publicationPending.length === 0 ? 1 : 0,
      halt_delta: failures.length > 0 ? 1 : 0, halt_reason: 'item_error',
    });
  }

  return reportDerivedWriteThrough({
    phase: 'synthesize_concepts',
    status: failures.length > 0 || linkWarnings.length > 0 || publicationDeferred.length > 0 || publicationHeld.length > 0 || publicationPending.length > 0
      || skippedFileDrift.length > 0 ? 'warn' : 'ok',
    duration_ms: 0,
    summary:
      `synthesize_concepts: ${conceptsWritten} concepts ` +
      `(T1=${tierCounts.T1} T2=${tierCounts.T2} T3=${tierCounts.T3})` +
      (failures.length > 0 ? ` (${failures.length} LLM-failed → template fallback)` : '') +
      (budgetCapped > 0 ? ` (${budgetCapped} over the $${budgetCap.toFixed(2)} budget → template fallback)` : '') +
      (linkWarnings.length > 0 ? ` (${linkWarnings.length} provenance-link warning(s))` : '') +
      (skippedHumanOwned.length > 0 ? ` (${skippedHumanOwned.length} human-owned page(s) left untouched)` : '') +
      (skippedUnchanged.length > 0 ? ` (${skippedUnchanged.length} unchanged)` : '') +
      (rehashed.length > 0 ? ` (${rehashed.length} rehashed without synthesis)` : '') +
      (keptExistingNarrative.length > 0 ? ` (${keptExistingNarrative.length} existing narrative(s) kept)` : '') +
      (skippedRetryBound.length > 0 ? ` (${skippedRetryBound.length} skipped after ${MAX_CONCEPT_ATTEMPTS} failed attempts on unchanged members)` : '') +
      (publicationDeferred.length > 0 ? ` (${publicationDeferred.length} publication(s) deferred: page changed, retried next run)` : '') +
      (publicationHeld.length > 0 ? ` (${publicationHeld.length} publication(s) held: existing page needs import/repair)` : '') +
      (publicationPending.length > 0 ? ` (${publicationPending.length} publication(s) held by the writer (pending or contended), finished later or retried next run)` : '') +
      (skippedFileDrift.length > 0 ? ` (${skippedFileDrift.length} skipped before synthesis: canonical file differs from the database; ${driftActions.join('; ')})` : ''),
    details: {
      concepts_written: conceptsWritten,
      tier_counts: tierCounts,
      synthesis_mode_counts: synthesisModeCounts,
      groups_found: atomGroups.length,
      atoms_seen: atoms.length,
      failures,
      link_warnings: linkWarnings,
      warnings,
      skipped_human_owned: skippedHumanOwned,
      skipped_deleted: skippedDeleted,
      skipped_unchanged: skippedUnchanged,
      provenance_restored: provenanceRestored,
      rehashed,
      kept_existing_narrative: keptExistingNarrative,
      skipped_retry_bound: skippedRetryBound,
      skipped_file_drift: skippedFileDrift,
      ...(driftActions.length ? { file_drift_next_actions: driftActions } : {}),
      folded_spellings: foldedSpellings,
      publication_deferred: publicationDeferred,
      publication_held: publicationHeld,
      publication_pending: publicationPending,
      ...(abortedGlobalError ? { aborted_global_error: abortedGlobalError } : {}),
      estimated_spend_usd: estimatedSpendUsd,
      budget_usd: budgetCap,
      pricing_fallback_models: [...pricingFallbackModels],
      dry_run: opts.dryRun ?? false,
    },
  }, conceptFiles, 'concept');
}

/**
 * Each atom's concept stems, one entry per atom: the ref's stem through the
 * human redirects (#6161), then through the run's spelling folds (#5965:
 * `network-effects` / `networkeffects` become one stem; the live page's
 * spelling wins, else the most frequent, tie to the hyphenated one; two live
 * spellings are left apart). The folds are reported, never written.
 */
async function resolveAtomConceptStems(engine: BrainEngine, sourceId: string, atoms: Array<{ concept_refs: string[] }>):
  Promise<{ atomStems: Set<string>[]; foldedSpellings: Array<{ from: string; to: string }> }> {
  const redirects = await loadConceptRedirects(engine, sourceId, conceptStemFor).catch(() => new Map<string, string>());
  const afterHumanHop = atoms.map((atom) => new Set(atom.concept_refs.map(conceptStemFor).filter((s): s is string => s !== null).map((stem) => canonicalConceptStem(stem, redirects))));
  const atomCounts = new Map<string, number>();
  for (const stems of afterHumanHop) for (const stem of stems) atomCounts.set(stem, (atomCounts.get(stem) ?? 0) + 1);
  const liveStems = await loadLiveConceptStems(engine, sourceId, conceptStemFor).catch(() => new Set<string>());
  const foldedSpellings = addSpellingRedirects(redirects, atomCounts, liveStems);
  return { atomStems: afterHumanHop.map((stems) => new Set([...stems].map((stem) => canonicalConceptStem(stem, redirects)))), foldedSpellings };
}

/**
 * The concept an atom's concept ref names, as the stem of its `concepts/`
 * slug. LLM refs vary in case, spacing and prefix ("Network Effects",
 * "concepts/network-effects"); they all name `network-effects`. A ref with no
 * valid slug is dropped.
 */
function conceptStemFor(ref: string): string | null {
  const stem = slugifySegment(String(ref).trim().split('/').pop() ?? '');
  if (!stem) return null;
  try {
    validatePageSlug(`concepts/${stem}`);
  } catch {
    return null;
  }
  return stem;
}

/**
 * Deterministic fallback narrative for T3/T4 concepts and budget-exhausted
 * T1/T2 groups. No LLM call. v0.41 minimal shape — v0.42 enriches with
 * dominant themes, time spread, breadth.
 */
function deterministicNarrative(group: AtomGroup): string {
  const tier = group.tier;
  const count = group.atomTitles.length;
  return (
    `${tier} concept. ${count} atom${count === 1 ? '' : 's'} reference this. ` +
    `Top mentions:\n${group.atomTitles
      .slice(0, 5)
      .map((t) => `  - ${t}`)
      .join('\n')}`
  );
}

/**
 * Ground the narrative's quoted spans against the atom titles and bodies the
 * prompt carried (quotes only: a concept narrative may count or compare).
 * A claim unit with an unverified quote is removed from the narrative and
 * kept in frontmatter `unverified_claims`. Kill switch: dream.quote_verify (default on).
 */
type UnverifiedClaim = { text: string; reason: string; detail: string };
async function groundConceptNarrative(engine: BrainEngine, narrative: string, group: { atomTitles: string[]; atomBodies: string[] }):
  Promise<{ narrative: string; unverified: UnverifiedClaim[] }> {
  const { ALL_CLAIMS_QUARANTINED_BODY, dreamQuoteVerifyEnabled, groundSource, verifyBody } = await import('./synthesize-verify.ts');
  if (!await dreamQuoteVerifyEnabled(engine)) return { narrative, unverified: [] };
  const sources = [
    groundSource('concept-atom-titles', group.atomTitles.slice(0, 10).join('\n'), { tolerant: true }),
    ...group.atomBodies.slice(0, 5).map((b, i) => groundSource(`concept-atom-${i + 1}`, b.slice(0, 500), { tolerant: true })),
  ];
  const v = verifyBody(narrative, sources, { checks: 'quotes' });
  return { narrative: v.body.trim() || ALL_CLAIMS_QUARANTINED_BODY, unverified: v.quarantined };
}