Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.0 KB
Raw Blame
import { bodyWriteChunkVersion } from './search/safe-chunks.ts';
import { assertInlineBody } from './page-body/page-shape.ts';
/**
 * #1856 — write-through for manual timeline entries.
 *
 * `add_timeline_entry` historically wrote ONLY the `timeline_entries` table.
 * On an FS/git-canonical brain (markdown on disk is the committable source of
 * truth) that stranded every manual entry: invisible to git, absent from
 * `gbrain get <slug>`, and silently lost on any FS→DB rebuild because no
 * bullet in the canonical markdown could reconstruct it.
 *
 * This module routes manual timeline writes through the SAME target seam
 * pages and facts use (`resolvePageWriteTarget`, the #4204 precedent): when
 * the brain resolves a disk target for the page, the entry is
 *   1. rendered as a canonical `- **date** | source — summary` bullet,
 *   2. spliced date-ordered into the ON-DISK file via a locked
 *      read-modify-write (mirrors fence-write.ts: `withPageLock` + read the
 *      current file + splice the bullet into the timeline section + atomic
 *      tmp+rename). NEVER whole-file regeneration from the DB row — the disk
 *      file is the merge point for file-only edits (facts fences, hand
 *      edits) that the pages row doesn't know about, and regenerating from
 *      the row silently reverted them (fence loss then cascaded to
 *      fact-row reconcile deletions),
 *   3. spliced date-ordered into the page row's `timeline` text, and
 *   4. inserted into `timeline_entries` with the tuple the FS extractor
 *      (`extractTimelineFromContent`) recovers from that exact bullet.
 *
 * Step 4 is the dedup-critical part the issue called out: the table's
 * uniqueness is `(page_id, date, summary, source)`, and sync re-extracts the
 * bullet on every pass — if the stored tuple differed from the re-extracted
 * one, every sync would insert a duplicate. We derive the stored tuple BY
 * RUNNING the extractor over the rendered block, so convergence holds by
 * construction. When derivation can't produce exactly one entry (pathological
 * source strings), we return `handled: false` and the caller falls back to
 * the legacy DB-only insert — an entry is never dropped. Same when the
 * on-disk file is missing or unreadable: we fall back to the DB-only path
 * rather than fabricating a file from the DB row.
 *
 * DB-only brains (no repo target, or `sync.write_through` off) always take
 * the `handled: false` path, so their behavior is byte-identical to before.
 *
 * Reconcile note: the pages row's `content_hash` is deliberately NOT updated
 * here — the on-disk file now hashes differently from the stored hash, so the
 * next `gbrain sync` re-imports the page (refreshing chunks + re-extracting
 * the timeline, which conflicts-no-ops against the row written here). Same
 * eventual-consistency contract as the facts fence lane.
 */

import { existsSync, readFileSync, writeFileSync, renameSync, unlinkSync, readdirSync } from 'node:fs';
import { basename, dirname } from 'node:path';
import type { BrainEngine } from './engine.ts';
import { sanitizeForJsonb } from './batch-rows.ts';
import {
  isWriteThroughDisabled,
  resolvePageWriteTarget,
  type WriteThroughLogger,
  type WriteThroughResult,
} from './write-through.ts';
import { withPageLock } from './page-lock.ts';
import { atomicStagingPath } from './atomic-write.ts';
import { maintenanceTransaction } from './persistence/attribution.ts';
import { assertSourceFilesystemActive, hasSourceFilesystemLock, withSourceFilesystemLock } from './minions/source-filesystem.ts';
import { findTimelineSplitIndex } from './markdown.ts';
import { isMaterializedMarkerLine } from './timeline-marker.ts';
import {
  isDurabilityHardened, commitWriteThroughFile, currentBranch, getLastPushOutcome,
  type PushLogOutcome,
} from './brain-repo-durability.ts';
import { extractTimelineFromContent } from './timeline-extract.ts';
import { readOnlyCheckout } from './persistence/mirror-read-only.ts';

/** Raw op params for one manual timeline entry (date pre-validated by caller). */
export interface TimelineEntryWriteInput {
  /** Strict YYYY-MM-DD (the op validates before calling). */
  date: string;
  summary: string;
  source?: string;
  detail?: string;
}

/** Canonical tuple actually stored (matches FS re-extraction of the bullet). */
export interface CanonicalTimelineTuple {
  date: string;
  source: string;
  summary: string;
}

export interface TimelineWriteThroughOutcome {
  /**
   * True when this helper OWNED the write: `pages.timeline` spliced, the
   * canonical `timeline_entries` row inserted, and the disk write attempted
   * (its own outcome in `file` — best-effort, like put_page's write-through).
   * False → nothing was written here; the caller MUST run the legacy DB-only
   * insert so the entry is recorded (unchanged pre-#1856 path).
   */
  handled: boolean;
  /** Disk write outcome (present when handled). */
  file?: WriteThroughResult;
  /**
   * The canonical tuple. Present when handled (it was stored in
   * timeline_entries), and ALSO present alongside `error` when the bullet
   * already reached the on-disk file before the failure — in that case the
   * caller's DB-only fallback MUST store THIS tuple, not the raw input,
   * or the next sync re-extract of the bullet inserts a duplicate.
   */
  entry?: CanonicalTimelineTuple;
  /** Why the FS-canonical path didn't apply (when handled=false). */
  skipped?:
    | 'disabled_by_config'
    | 'mirror_read_only'
    | 'url_clone'
    | 'no_repo_configured'
    | 'repo_not_found'
    | 'source_repo_belongs_to_other_source'
    | 'path_escapes_source_root'
    | 'page_not_found'
    | 'render_not_roundtrippable'
    | 'file_missing'
    | 'case_insensitive_collision'
    | 'splice_not_roundtrippable';
  /** Set when the helper threw mid-way; caller still falls back to DB-only. */
  error?: string;
}

/**
 * Default source label for manual entries with no provenance ref. A bullet
 * MUST carry a source segment: the FS extractor assigns `'markdown'` to
 * delimiter-less bullets and splits summary-first bullets on interior dashes
 * (#1856 Bug 1's fragmentation class), so rendering `| <summary>` alone can
 * never round-trip to `source: ''`. `'manual'` is honest provenance for a
 * hand-added entry and contains no delimiter the extractor could mis-split.
 */
const DEFAULT_MANUAL_SOURCE = 'manual';

/** Collapse a free-text field onto one line (the bullet is line-anchored). */
function inlineText(text: string): string {
  return text.replace(/\s+/g, ' ').trim();
}

/** Rendered markdown block + the tuple the FS extractor recovers from it. */
export interface RenderedTimelineEntry {
  /** Bullet line plus indented detail lines (when representable). */
  block: string;
  canonical: CanonicalTimelineTuple;
  detail: string;
}

/**
 * Render one entry as a canonical source-first bullet and derive the tuple
 * the FS extractor will recover from it. Returns null when the rendered
 * block does not round-trip to exactly one entry with the requested date —
 * managed callers reject an entry that cannot be represented losslessly.
 */
export function renderTimelineEntry(
  entry: TimelineEntryWriteInput,
  slug: string,
): RenderedTimelineEntry | null {
  const summary = inlineText(entry.summary ?? '');
  if (!summary) return null;
  const source = inlineText(entry.source ?? '') || DEFAULT_MANUAL_SOURCE;
  const line = `- **${entry.date}** | ${source} — ${summary}`;
  // Detail rides along as indented continuation lines so the canonical file
  // keeps the full content (the FS extractor ignores continuation lines; the
  // DB-path parser reads them back as detail).
  const detailLines = (entry.detail ?? '')
    .split('\n')
    .map((l) => l.trim())
    .filter((l) => l.length > 0)
    .map((l) => `  ${l}`);

  const derive = (block: string): CanonicalTimelineTuple | null => {
    const entries = extractTimelineFromContent(block, slug);
    if (entries.length !== 1) return null;
    if (entries[0].date !== entry.date) return null;
    return { date: entries[0].date, source: entries[0].source ?? '', summary: entries[0].summary };
  };

  const block = [line, ...detailLines].join('\n');
  const canonical = derive(block);
  if (!canonical) return null;
  return { block, canonical, detail: detailLines.map(line => line.trim()).join(' ') };
}

/** Line-anchored Format-1 bullet detector (date capture only). */
const BULLET_DATE_RE = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|/;

/**
 * Splice a rendered entry block into a page's `timeline` text, date-ordered
 * among the existing Format-1 bullets. Preserves the list's existing sort
 * direction (ascending unless the first bullet's date is greater than the
 * last's). Insertion lands immediately BEFORE the first bullet that should
 * follow the new entry, so a neighbor's indented detail lines are never
 * split; when no bullet follows, the block is appended at the end.
 */
export function spliceTimelineBlock(timelineText: string, date: string, block: string): string {
  const text = (timelineText ?? '').trimEnd();
  if (!text.trim()) {
    return `## Timeline\n\n${block}`;
  }
  // Use the same bullet-bounded splice for page snapshots and raw files, so
  // legacy facts/takes sections after the final bullet remain outside it.
  const sentinel = '<!-- timeline -->\n';
  return spliceTimelineIntoFileText(`${sentinel}${text}`, date, block).slice(sentinel.length);
}

/** True for an indented continuation line (a bullet's detail lines). */
const CONTINUATION_RE = /^\s{2,}\S/;

/**
 * Splice a rendered entry block into RAW on-disk markdown text (frontmatter +
 * body + optional timeline sentinel). Every existing byte outside the
 * insertion point is preserved — this function never regenerates the file
 * from the DB row, so file-only content (facts fences, hand edits) survives.
 *
 * Placement mirrors `spliceTimelineBlock`'s date-ordering, but is
 * BULLET-BOUNDED: when no bullet should follow the new entry, it lands right
 * after the last bullet (plus its continuation lines), NOT at the end of the
 * region — the region can carry later sections (e.g. a legacy `## Facts`
 * fence written below the sentinel by pre-#4756 upsertFactRow EOF appends)
 * that a naive tail-append would corrupt.
 */
export function spliceTimelineIntoFileText(fileText: string, date: string, block: string): string {
  const lines = fileText.split('\n');
  // Skip YAML frontmatter so its `---` delimiters can't be mistaken for the
  // bare-`---` timeline sentinel (findTimelineSplitIndex rule 3 expects body
  // lines, matching splitBody's own call shape).
  let bodyStart = 0;
  if (lines[0]?.trim() === '---') {
    for (let i = 1; i < lines.length; i++) {
      if (lines[i].trim() === '---') {
        bodyStart = i + 1;
        break;
      }
    }
  }
  const rel = findTimelineSplitIndex(lines.slice(bodyStart));
  const blockLines = block.split('\n');

  if (rel === -1) {
    // No timeline sentinel on disk yet — append a fresh section at EOF.
    const base = fileText.replace(/\s+$/, '');
    return `${base}\n\n<!-- timeline -->\n\n## Timeline\n\n${block}\n`;
  }

  const sentinel = bodyStart + rel;
  const bullets: Array<{ index: number; date: string }> = [];
  for (let i = sentinel + 1; i < lines.length; i++) {
    const m = BULLET_DATE_RE.exec(lines[i]);
    if (m) bullets.push({ index: i, date: m[1] });
  }

  // #7738: a body that ends with an empty sentinel gets the same headed section spliceTimelineBlock('') writes.
  if (lines.slice(sentinel + 1).every(line => !line.trim())) return `${lines.slice(0, sentinel + 1).join('\n')}\n\n## Timeline\n\n${block}\n`;
  let insertAt: number;
  if (bullets.length === 0) {
    // Empty timeline section (possibly followed by other sections): insert
    // right below the `## Timeline`/`## History` heading when one exists in
    // the region, else right below the sentinel line itself.
    let anchor = sentinel;
    for (let i = sentinel + 1; i < lines.length; i++) {
      const trimmed = lines[i].trim();
      if (/^##\s+(timeline|history)\b/i.test(trimmed)) {
        anchor = i;
        break;
      }
      if (/^#{1,6}\s/.test(trimmed)) break; // some OTHER section began
    }
    insertAt = anchor + 1;
  } else {
    const descending = bullets.length >= 2 && bullets[0].date > bullets[bullets.length - 1].date;
    let before = -1;
    for (const b of bullets) {
      if (descending ? b.date < date : b.date > date) {
        before = b.index;
        break;
      }
    }
    if (before !== -1) {
      insertAt = before;
      // #5567: a materialized marker belongs to the bullet below it; never split the pair.
      while (insertAt > sentinel + 1 && isMaterializedMarkerLine(lines[insertAt - 1])) insertAt--;
    } else {
      // After the LAST bullet, past its indented continuation (detail) lines.
      insertAt = bullets[bullets.length - 1].index + 1;
      while (insertAt < lines.length && CONTINUATION_RE.test(lines[insertAt])) insertAt++;
    }
  }

  return [...lines.slice(0, insertAt), ...blockLines, ...lines.slice(insertAt)].join('\n');
}

/**
 * Write one manual timeline entry through the page/facts write-through seam.
 *
 * Trust gating (subagent sandbox, dry-run) stays at the CALLER, exactly like
 * `writePageThrough` — this helper only does "target resolves → locked splice
 * into the on-disk file + row splice + insert". Never throws: failures return
 * `handled: false` so the caller's legacy DB-only insert still records the
 * entry (with `entry` carrying the canonical tuple when the bullet already
 * reached the disk file — see TimelineWriteThroughOutcome.entry).
 */
export async function writeTimelineEntryThrough(
  engine: BrainEngine,
  slug: string,
  sourceId: string,
  entry: TimelineEntryWriteInput,
  opts: { logger?: WriteThroughLogger } = {},
): Promise<TimelineWriteThroughOutcome> {
  // Set the moment the atomic rename lands: from then on the canonical bullet
  // EXISTS on disk, so any later failure must hand the canonical tuple to the
  // caller's DB-only fallback or the next sync re-extract duplicates it.
  let onDisk: CanonicalTimelineTuple | undefined;
  try {
    if (await isWriteThroughDisabled(engine)) {
      return { handled: false, skipped: 'disabled_by_config' };
    }
    const readOnly = await readOnlyCheckout(engine, sourceId);
    if (readOnly) return { handled: false, skipped: readOnly };
    const target = await resolvePageWriteTarget(engine, slug, sourceId);
    if (!target.ok) {
      return { handled: false, skipped: target.skipped };
    }
    const { filePath, writeRoot } = target;
    if (!hasSourceFilesystemLock(writeRoot)) {
      return await withSourceFilesystemLock(engine, writeRoot, () => writeTimelineEntryThrough(engine, slug, sourceId, entry, opts));
    }

    const page = await engine.getPage(slug, { sourceId });
    if (!page) {
      // Let the caller's legacy insert raise the canonical "page not found"
      // error instead of inventing a second error shape here.
      return { handled: false, skipped: 'page_not_found' };
    }
    assertInlineBody(page, 'timeline write-through');

    const rendered = renderTimelineEntry(entry, slug);
    if (!rendered) {
      return { handled: false, skipped: 'render_not_roundtrippable' };
    }

    // Same page lock as the facts fence writer (fence-write.ts) so concurrent
    // fence/timeline writers to the same page serialize their read-modify-
    // write cycles instead of dropping each other's atomic renames.
    return await withPageLock(
      slug,
      async (): Promise<TimelineWriteThroughOutcome> => {
        // Read-modify-write on the ON-DISK file: the disk copy is the merge
        // point for file-only edits (facts fences, hand edits) the pages row
        // doesn't carry — never regenerate the whole file from the DB row.
        // Missing file → DB-only fallback rather than fabricating one.
        if (!existsSync(filePath)) {
          return { handled: false, skipped: 'file_missing' };
        }
        // #2831 mirror (writePageThrough's guard): on a case-insensitive
        // filesystem the path can exist because the FS folds it onto a
        // DIFFERENTLY-cased slug's file — splicing there would edit the
        // other page's artifact.
        const dir = dirname(filePath);
        const base = basename(filePath);
        if (!readdirSync(dir).includes(base)) {
          opts.logger?.warn(
            `[timeline-write-through] case-insensitive collision for ${slug}: another entry occupies ${filePath} — falling back to DB-only`,
          );
          return { handled: false, skipped: 'case_insensitive_collision' };
        }

        const beforeText = readFileSync(filePath, 'utf8');
        const afterText = spliceTimelineIntoFileText(beforeText, entry.date, rendered.block);

        // fence-write's parse-before-rename analog: the spliced text must
        // re-extract the canonical tuple, or the file is not touched.
        const roundTrips = extractTimelineFromContent(afterText, slug).some(
          (e) =>
            e.date === rendered.canonical.date &&
            (e.source ?? '') === rendered.canonical.source &&
            e.summary === rendered.canonical.summary,
        );
        if (!roundTrips) {
          return { handled: false, skipped: 'splice_not_roundtrippable' };
        }

        // Atomic write: unique temp sibling + rename (writePageThrough's
        // convention). Clean the temp up on failure — never leak a stray.
        const tmpPath = atomicStagingPath(filePath);
        try {
          assertSourceFilesystemActive();
          writeFileSync(tmpPath, afterText, 'utf8');
          renameSync(tmpPath, filePath);
        } catch (writeErr) {
          try {
            if (existsSync(tmpPath)) unlinkSync(tmpPath);
          } catch {
            // best-effort cleanup; surface the original write error below
          }
          throw writeErr;
        }
        onDisk = rendered.canonical;

        const newTimeline = sanitizeForJsonb(
          spliceTimelineBlock(page.timeline ?? '', entry.date, rendered.block),
        );
        await maintenanceTransaction(engine, async tx => {
          await tx.executeRaw(
            `UPDATE pages SET timeline = $1, chunker_version = ${bodyWriteChunkVersion('pages.compiled_truth', '$1')}, updated_at = now()
              WHERE slug = $2 AND source_id = $3 AND deleted_at IS NULL`,
            [newTimeline, slug, sourceId],
          );

          // Store the tuple the FS extractor recovers from the bullet just
          // spliced in, so every later sync/rebuild re-extraction
          // conflicts-no-ops instead of duplicating (#1856's dedup-tuple
          // divergence).
          await tx.addTimelineEntry(slug, { // gbrain-allow-direct-insert: timeline write-through — the canonical markdown gains the same entry in this call, and the stored tuple is derived from the rendered bullet so sync/extract reconciliation dedups against it
            date: rendered.canonical.date,
            source: rendered.canonical.source,
            summary: rendered.canonical.summary,
            detail: entry.detail || '',
          }, { sourceId, skipExistenceCheck: true });
        });

        // #2426 mirror (writePageThrough): on a durability-hardened repo,
        // commit the artifact so it reaches git. Best-effort — a commit
        // failure never fails the write.
        let committed = false;
        let pushed: 'pending' | undefined;
        let lastPushStatus: PushLogOutcome | undefined;
        try {
          if (isDurabilityHardened(writeRoot)) {
            committed = commitWriteThroughFile(writeRoot, filePath, slug);
            if (committed) {
              pushed = 'pending';
              lastPushStatus = getLastPushOutcome(currentBranch(writeRoot), writeRoot);
            }
          }
        } catch { /* best-effort */ }

        const file: WriteThroughResult = {
          written: true,
          path: filePath,
          ...(committed ? { committed, pushed, lastPushStatus } : {}),
        };
        return { handled: true, file, entry: rendered.canonical };
      },
      { timeoutMs: 5_000 },
    );
  } catch (e) {
    const msg = e instanceof Error ? e.message : String(e);
    opts.logger?.warn(`[timeline-write-through] failed for ${slug}: ${msg}`);
    return { handled: false, error: msg, ...(onDisk ? { entry: onDisk } : {}) };
  }
}