Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.0 KB
Raw Blame
/**
 * #6503: the extract_atoms input cap and what a cut item becomes.
 *
 * The model reads at most `cycle.extract_atoms.max_input_chars` characters of
 * an item (`cycle.extract_atoms.max_source_chars` is the older spelling of the
 * same knob). Before this module the cut was silent and the item was then
 * stamped complete on its full content hash, so the tail was never read and a
 * re-extraction could retire the atoms an earlier, larger-cap extraction had
 * produced from it. A cut item now gets a *partial disposition*: its head is
 * extracted once and the atoms are published, but no completion stamp is
 * written and no absent atom is retired. The disposition is recorded under
 * (content hash, cap) in `extract_atoms_page_state` /
 * `extract_atoms_transcript_state` (`partial_input_chars`) or, on a managed
 * brain, in the item's `managed-atoms` checkpoint (`partial`), so discovery
 * skips the item (it is not re-spent) until the cap rises or the content
 * changes. `releasePartialDispositions` runs before discovery with the current
 * cap and clears the records a raised cap supersedes.
 */
import type { BrainEngine } from '../engine.ts';
import type { SqlEngine } from '../persistence/model.ts';
import { truncateUtf8 } from '../text-safe.ts';
import { corpusTextForExtraction } from '../context/corpus-segments.ts';
import { AtomPageStateError, type AtomPageInput } from './extract-atoms-page-state.ts';
import { planExtraction, type ExtractionPlan, type StoredWatermark } from '../extraction-watermark.ts';
import { publicBody } from '../mentions/aliases.ts';

export const DEFAULT_EXTRACT_MAX_INPUT_CHARS = 50_000;
export const EXTRACT_MAX_INPUT_CHARS_KEY = 'cycle.extract_atoms.max_input_chars';

/**
 * The input cap: `max_source_chars` (floor 500) is read first so the newer
 * `max_input_chars` (floor 1000: below that the extractor sees a fragment too
 * small to yield atoms and every item burns budget for nothing) wins when both
 * are set. Fail-soft to the default on any config-read failure.
 */
export async function resolveExtractAtomsInputCap(engine: BrainEngine): Promise<number> {
  let cap = DEFAULT_EXTRACT_MAX_INPUT_CHARS;
  try {
    const legacy = Number(await engine.getConfig('cycle.extract_atoms.max_source_chars'));
    if (Number.isFinite(legacy) && legacy >= 500) cap = Math.floor(legacy);
    const configured = Number(await engine.getConfig(EXTRACT_MAX_INPUT_CHARS_KEY));
    if (Number.isFinite(configured) && configured >= 1_000) cap = Math.floor(configured);
  } catch { /* keep the default */ }
  return cap;
}

export type ExtractAtomsItem =
  | { kind: 'transcript'; filePath: string; content: string; contentHash: string; transcriptKey?: string }
  | ({ kind: 'page' } & AtomPageInput);

export interface ExtractAtomsInput {
  /** Exactly the text the model receives; quote provenance verifies against it. */
  promptContent: string;
  /** The model does not read the whole item (a head, an appended tail or its newest window): partial disposition. */
  truncated: boolean;
  /** Characters of the text the cap applied to (post paste-strip for a transcript). */
  inputChars: number;
  /** A page's watermark plan (extraction-watermark.ts), stored once the item's extraction succeeded. */
  plan?: ExtractionPlan;
}

/**
 * Bind the cut once. Pastes are stripped from a transcript first (#5812), the
 * cap is applied UTF-8-safely (#4529/#4540: a bare slice can split a
 * surrogate pair), and `truncated` is measured on the text the cap applied to.
 * Large pages A7: a page whose stored watermark still matches its body reads
 * the text appended since (`watermark`); a tail read is partial input too, so
 * the #6503 disposition holds (no completion stamp, nothing retired).
 * A page is read as a world reader sees it (`publicBody`: facts fence rows
 * other than world and the takes fence removed), so neither the prompt, the
 * atoms nor a verified quote can carry a row a remote read of the page hides;
 * the cap and the watermark apply to that text.
 */
export function extractAtomsInput(item: ExtractAtomsItem, maxInputChars: number, watermark: StoredWatermark | null = null): ExtractAtomsInput {
  if (item.kind === 'page') {
    const body = publicBody(item.content);
    const plan = planExtraction(body, watermark, { cap: maxInputChars, pipeline: 'extract_atoms', emptyTail: 'head' });
    return { promptContent: plan.text, truncated: plan.mode !== 'full', inputChars: body.length, plan };
  }
  const text = corpusTextForExtraction(item.filePath, item.content);
  const promptContent = truncateUtf8(text, maxInputChars);
  return { promptContent, truncated: promptContent.length < text.length, inputChars: text.length };
}

/** The one stderr line per cut item; names the knob that raises the cap. Null for a complete appended-tail read. */
export function inputTruncatedLine(originLabel: string, input: ExtractAtomsInput, maxInputChars: number): string | null {
  const cut = input.plan?.truncated;
  if (input.plan && !cut) return null;
  if (cut?.kind === 'tail_window') {
    return `[extract_atoms] ${originLabel}: ${cut.total_chars} chars were appended since the last extraction; the model read the newest ${cut.processed_chars}. `
      + `Atoms are kept, the item is not marked complete, and doctor's extraction_input_truncated lists it: gbrain config set ${EXTRACT_MAX_INPUT_CHARS_KEY} <n>`;
  }
  return `[extract_atoms] ${originLabel}: input truncated to ${maxInputChars} of ${input.inputChars} chars; the model did not read the tail. ` +
    `Atoms from the head are kept, the item is not marked complete and is skipped until the cap rises: gbrain config set ${EXTRACT_MAX_INPUT_CHARS_KEY} <n>`;
}

/**
 * Clear the partial dispositions a raised cap supersedes, so the items they
 * covered are offered again by this run's discovery. Rows recorded at or
 * above the current cap stay (the model would read the same text again).
 * Fail-soft: a failed release leaves the items skipped, never re-spent.
 */
export async function releasePartialDispositions(engine: BrainEngine, sourceId: string, maxInputChars: number): Promise<void> {
  try {
    await engine.executeRaw(
      `UPDATE extract_atoms_page_state scan SET partial_input_chars=NULL, updated_at=now()
         FROM sources s WHERE s.id=$1 AND scan.source_incarnation=s.incarnation AND scan.partial_input_chars < $2`,
      [sourceId, maxInputChars]);
    await engine.executeRaw(
      `UPDATE extract_atoms_transcript_state SET partial_input_chars=NULL, updated_at=now()
        WHERE source_id=$1 AND partial_input_chars < $2`,
      [sourceId, maxInputChars]);
    await engine.executeRaw(
      `DELETE FROM op_checkpoints WHERE op='managed-atoms' AND completed_keys->0->>'sourceId'=$1
         AND (completed_keys->0->>'partial')::integer < $2`,
      [sourceId, maxInputChars]);
  } catch (err) {
    console.error(`[extract_atoms] partial-disposition release failed (cut items stay skipped): ${err instanceof Error ? err.message : String(err)}`);
  }
}

/**
 * Stamp the transcript-side tombstone (v146). The file-backed mirror of the
 * page scan state: transcripts have no frontmatter, and their discovery is
 * gated only by `atomsExistingForHashes` and this table, so a transcript that
 * yields no atom row has nothing else to mark it done and would be
 * re-attempted every cycle. The row is (source_id, file_path, content_hash)-
 * keyed, so an edited transcript is a different row and re-eligibilizes.
 * Fail-soft: the transcript stays rediscoverable.
 */
export async function stampTranscriptTombstone(engine: BrainEngine, sourceId: string, filePath: string, contentHash: string): Promise<void> {
  try {
    await engine.executeRaw(
      `INSERT INTO extract_atoms_transcript_state (source_id, file_path, content_hash, fail_count, tombstoned, updated_at)
       VALUES ($1, $2, $3, 0, TRUE, now())
       ON CONFLICT (source_id, file_path, content_hash) DO UPDATE SET tombstoned = TRUE, updated_at = now()`,
      [sourceId, filePath, contentHash.slice(0, 16)]);
  } catch { /* fail-soft: transcript stays rediscoverable */ }
}

/**
 * Record an unmanaged item's partial disposition under its current content
 * and the cap it was read under. The page row is pinned to the page identity
 * discovery read (the same predicate `writeAtomPageState` uses), so a page
 * that changed under the run records nothing and stays retryable.
 */
export async function recordPartialDisposition(engine: SqlEngine, sourceId: string, item: ExtractAtomsItem, maxInputChars: number): Promise<void> {
  if (item.kind === 'transcript') {
    await engine.executeRaw(
      `INSERT INTO extract_atoms_transcript_state (source_id, file_path, content_hash, fail_count, tombstoned, partial_input_chars, updated_at)
       VALUES ($1, $2, $3, 0, FALSE, $4, now())
       ON CONFLICT (source_id, file_path, content_hash) DO UPDATE SET partial_input_chars=EXCLUDED.partial_input_chars, updated_at=now()`,
      [sourceId, item.filePath, item.contentHash.slice(0, 16), maxInputChars]);
    return;
  }
  if (!item.identity) throw new AtomPageStateError('unpinned');
  const rows = await engine.executeRaw(
    `INSERT INTO extract_atoms_page_state (source_incarnation, page_id, content_hash, fail_count, tombstoned, partial_input_chars)
      SELECT s.incarnation, p.id, p.content_hash, 0, false, $7::integer
      FROM pages p JOIN sources s ON s.id=p.source_id
      WHERE p.id=$1 AND s.incarnation=$2::uuid AND p.source_id=$3 AND p.slug=$4
        AND p.content_hash=$5 AND p.knowledge_revision=$6::uuid AND p.deleted_at IS NULL
      FOR SHARE OF p, s
      ON CONFLICT (source_incarnation, page_id, content_hash) DO UPDATE
        SET partial_input_chars=EXCLUDED.partial_input_chars, updated_at=now()
      RETURNING page_id`,
    [item.identity.pageId, item.identity.sourceIncarnation, sourceId, item.slug, item.contentHash, item.identity.revision, maxInputChars])
    .catch((error: unknown) => {
      const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined;
      throw new AtomPageStateError(code === '42P01' ? 'schema' : code === '42501' ? 'denied' : 'storage');
    });
  if (!rows.length) throw new AtomPageStateError('changed');
}