Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.4 KB
Raw Blame
/**
 * Incremental sync delete lanes (refactor wave 1, W4 sync): the
 * un-syncable-modified sweep that runs before the checkpoint opens, and the
 * batched removed-file drain. Both SOFT-delete (#4587).
 */
import { softDeleteSyncPages } from '../../core/company-brain/profile.ts';
import { serr, slog } from '../../core/console-prefix.ts';
import { DELETE_BATCH_SIZE } from '../../core/engine-constants.ts';
import type { BrainEngine } from '../../core/engine.ts';
import { resumeFilter } from '../../core/op-checkpoint.ts';
import {
  resolveRemovedPathSlug,
  resolveSlugsForRemovedPaths,
  refusedRemovedPathMessage,
} from '../../core/sync-git.ts';
import { unsyncableReason, isPoisonedPath, DEFAULT_SOURCE_ID } from '../../core/sync.ts';
import type { SyncResult } from '../sync.ts';
import { partial, markCompleted, maybeYield } from './sync-run.ts';
import type { SyncPlan, SyncRun, SyncProgress } from './sync-run.ts';
import { noteCommittedProgress } from '../../core/forward-progress.ts';

export async function sweepUnsyncableModified(
  engine: BrainEngine,
  plan: Pick<SyncPlan, 'opts' | 'manifest' | 'inScope' | 'isSelectedForRun' | 'syncOpts' | 'modePath'>,
): Promise<number> {
  const { opts, manifest, inScope, isSelectedForRun, syncOpts, modePath } = plan;
  // Delete pages that became un-syncable (modified but filtered out).
  // v0.20.0 Cathedral II SP-5: resolveSlugForPath picks the right slug shape
  // (markdown vs code) based on the chunker's classifier, so a Rust file that
  // became un-syncable (e.g., moved under `.gitignore` or filtered by
  // strategy=markdown) deletes the actual code-slug page, not a ghost
  // markdown-slug that never existed.
  //
  // v0.41.13 (#1433): the original cleanup loop deleted EVERY pre-existing
  // page for unsyncable-modified paths, including `log.md`, `schema.md`,
  // `index.md`, `README.md` — files that fail `isSyncable` precisely
  // because they're metafiles by convention, not because the user
  // "removed" them from the strategy. infiniteGameExp's domain `log.md`
  // pages had been indexed by an older gbrain version (or via direct
  // put_page) and were silently dropped on every subsequent sync. The
  // fix uses `unsyncableReason` (factored from `isSyncable` so they
  // cannot drift) to skip the delete when the reason is `'metafile'`.
  //
  // Honest scope: this guard only fixes the `manifest.modified` case.
  // `manifest.deleted` is filtered upstream at sync.ts:757 via the same
  // `isSyncable` call, so `rm log.md` followed by sync also doesn't
  // delete the page. That's the same pre-fix behavior — removing the
  // page requires `gbrain pages purge-deleted` or a direct MCP delete.
  // Filed as v0.42+ follow-up for a `gbrain pages remove <slug>` surface.
  const unsyncableModified = manifest.modified.filter(p => inScope(p) && !isSelectedForRun(p, syncOpts));
  // v0.18.0+ multi-source: scope getPage + deletePage to opts.sourceId so
  // unsyncable cleanup in source A doesn't accidentally sweep same-slug
  // pages in sources B/C/D.
  const pageOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
  // #4786: pages this loop retires count as `deleted` in the result (only rows
  // that actually transitioned), so a sweep-only run never reports up_to_date.
  let swept = 0;
  for (const path of unsyncableModified) {
    // v0.41.13 #1433: never delete on metafile classification.
    // #2404 hardening: same for 'pruned-dir' — a page under a pruned
    // directory can only exist via a deliberate put_page (sync never
    // imports those paths), so "the file was modified" is not evidence
    // the page is stale. Deleting here silently destroyed put-created
    // pages every time their materialized file landed in a commit.
    const reason = unsyncableReason(path, syncOpts);
    if (reason === 'metafile' || reason === 'pruned-dir') continue;
    // Bare-bracket markdown (pre-gate imports like `notes [draft].md`) keeps
    // its row — only the poison signature (`](`/control chars) is sweepable.
    // Deleting a legit page's row while its file sits on disk is data loss.
    if (reason === 'malformed-path' && !isPoisonedPath(path)) continue;
    // #3942: guarded resolver — never delete a page whose recorded origin is
    // a DIFFERENT file just because this path re-slugifies onto its slug.
    // #4342: resolve in the mode's namespace (source-relative under
    // 'source-root'; git-root-relative otherwise).
    const slug = await resolveRemovedPathSlug(engine, modePath(path), opts.sourceId, serr);
    if (slug === undefined) continue;
    try {
      const existing = await engine.getPage(slug, pageOpts);
      if (existing) {
        // #3583 review: this loop sits ABOVE the dry-run return below, so
        // an unguarded delete made a preview under a narrower strategy
        // hard-delete previously-imported pages. A preview only reports.
        if (opts.dryRun) {
          slog(`  [dry-run] would delete un-syncable page: ${slug}`);
        } else {
          // #4587: soft-delete (72h recovery window) instead of hard delete.
          // Scope falls back to DEFAULT_SOURCE_ID to preserve deletePage's
          // old 'default' fallback; softDeletePages requires an explicit
          // sourceId. The purge phase owns the eventual hard delete.
          swept += (await softDeleteSyncPages(engine, [slug], { sourceId: opts.sourceId ?? DEFAULT_SOURCE_ID })).length;
          slog(`  Soft-deleted un-syncable page (recoverable 72h): ${slug}`);
        }
      }
    } catch { /* ignore */ }
  }
  return swept;
}

/** The removed-file drain. Returns a partial result when the run aborts mid-drain. */
export async function runDeletesPhase(
  run: SyncRun,
  plan: Pick<SyncPlan, 'opts' | 'filtered' | 'lastCommit' | 'pin'>,
  progress: SyncProgress,
): Promise<SyncResult | undefined> {
  const { engine, completed, pagesAffected, deletedSlugs, failedFiles } = run;
  const { opts, filtered } = plan;
  // v0.41.19.0 (T2/D6/D7/D16/D18 via /plan-eng-review + codex outside-voice):
  // batched delete loop. Replaces the per-file N+1 that PR #1538 originally
  // batched on Postgres only. See plan file:
  //   ~/.claude/plans/system-instruction-you-are-working-ethereal-narwhal.md
  // #4587: the lanes below SOFT-delete (deleted_at = now(), 72h recovery
  // window) via softDeletePages; the autopilot purge phase owns the eventual
  // hard delete and a re-import within the window revives via upsert.
  //
  // SHAPE (interleaved per-batch resolve + delete; caller owns chunking):
  //
  //   filtered.deleted (e.g. 73K paths)
  //       │
  //       ▼
  //   slice into batches of DELETE_BATCH_SIZE (500)
  //       │
  //       ▼  for each batch:
  //   abort-check ──► partial('timeout')
  //       │
  //       ▼
  //   resolveSlugsForRemovedPaths(batch)             ◀── exact source_path,
  //       │                                              then VERIFIED fallback;
  //       ▼                                              foreign-origin refusals
  //   slugs = deletable.map(...)                         (#3942) warned + skipped
  //       │
  //       ▼
  //   try {
  //     deleted = engine.softDeletePages(slugs, opts) ◀── 1 SQL round-trip
  //     pagesAffected.push(...deleted)                ◀── D6: only confirmed
  //   } catch {                                           transitions, not phantoms
  //     // D7 decompose: one-element softDeletePages per slug,
  //     // unrecoverable failures → failedFiles, run continues
  //   }
  //
  // ROUND-TRIP COUNTS (73K deletes):
  //   pre-fix:   73,000 SELECTs + 73,000 DELETEs = 146,000 (~5 hours)
  //   post-fix:     146 SELECTs +     146 UPDATEs =     292 (~2 minutes)
  //
  // ATOMICITY (D3): each batch is one transaction. A mid-batch abort or
  // transient connection failure rolls back up to DELETE_BATCH_SIZE - 1
  // successful deletes. Sync is idempotent — the next run picks them up
  // via git diff regenerating the deletion list.
  //
  // NO-SOURCEID FALLBACK: when opts.sourceId is undefined (legacy unscoped
  // callers, rare post-v0.34.1 source-resolution wiring), fall back to the
  // OLD per-path loop. The batch engine surface requires sourceId per D5
  // (multi-source-bug-class defense at the type level). Production callers
  // that thread sourceId via resolveSourceWithTier get the new fast path.
  // v0.42.x (#1794): resume-filter the delete set so a resumed run skips paths
  // already drained in a prior run (deletes are idempotent, but skipping avoids
  // re-resolving + re-deleting tens of thousands of already-gone pages).
  const deletesToDo = resumeFilter(filtered.deleted, [...completed]);
  if (deletesToDo.length > 0) {
    progress.start('sync.deletes', deletesToDo.length);
    if (opts.sourceId) {
      const sid = opts.sourceId;
      const deleteScopedOpts = { sourceId: sid };
      for (let i = 0; i < deletesToDo.length; i += DELETE_BATCH_SIZE) {
        if (opts.signal?.aborted) {
          progress.finish();
          return await partial(run, plan, 'timeout');
        }
        const batch = deletesToDo.slice(i, i + DELETE_BATCH_SIZE);

        // Phase A: guarded batch slug resolution (#3942 — a re-slugified
        // fallback can name a DIFFERENT page; refusals are logged + skipped).
        const resolution = await resolveSlugsForRemovedPaths(engine, batch, sid);
        for (const r of resolution.refused) {
          serr(refusedRemovedPathMessage(r));
          // Deliberately handled — checkpoint so a resume doesn't re-refuse.
          await markCompleted(run, r.path);
        }
        const deletable = batch.filter(p => resolution.slugs.has(p));
        const slugs = deletable.map(p => resolution.slugs.get(p) as string);

        // Phase B: batch soft-delete (1 round-trip per batch). #4587: the
        // removed-file drain honors the 72h recovery window — deleted_at is
        // set, the purge phase hard-deletes later, and a re-import within
        // the window revives via putPage's upsert.
        try {
          const deleted = await softDeleteSyncPages(engine, slugs, deleteScopedOpts);
          // D6: only push slugs that actually transitioned. Filters phantom
          // slugs (paths in filtered.deleted but with no DB row — or rows
          // already soft-deleted) so downstream extract/embed don't waste
          // lookups.
          pagesAffected.push(...deleted);
          for (const s of deleted) deletedSlugs.add(s);
          if (deleted.length) noteCommittedProgress();
          // v0.42.x (#1794): the whole batch is handled (soft-deleted,
          // already gone, or refused above); checkpoint every path so a
          // resume skips it.
          for (const p of deletable) await markCompleted(run, p);
        } catch (err) {
          // D7 decompose: a transient blip on this batch shouldn't lose all
          // 500 deletes. Fall back to one-element softDeletePages batches
          // for THIS batch only (per-slug isolation, same primitive);
          // unrecoverable per-slug failures land in failedFiles and the run
          // CONTINUES (--skip-failed semantics), matching the existing
          // import-loop pattern.
          for (let j = 0; j < slugs.length; j++) {
            try {
              await softDeleteSyncPages(engine, [slugs[j]], deleteScopedOpts);
              noteCommittedProgress();
              pagesAffected.push(slugs[j]);
              deletedSlugs.add(slugs[j]);
              await markCompleted(run, deletable[j]);
            } catch (perSlugErr) {
              failedFiles.push({
                path: deletable[j],
                error: `delete failed: ${perSlugErr instanceof Error ? perSlugErr.message : String(perSlugErr)} (batch error: ${err instanceof Error ? err.message : String(err)})`,
              });
            }
          }
        }
        progress.tick(batch.length, `deletes ${Math.min(i + DELETE_BATCH_SIZE, deletesToDo.length)}/${deletesToDo.length}`);
        await maybeYield(run);
      }
    } else {
      // Legacy no-sourceId path. The engine batch methods require sourceId
      // per D5 (kills the multi-source-bug-class on the new surface); when
      // sourceId is unset, fall back to the original per-path loop. Slow
      // but correct; production callers all thread sourceId so this branch
      // is functionally dead post-v0.34.1.
      for (const path of deletesToDo) {
        if (opts.signal?.aborted) {
          progress.finish();
          return await partial(run, plan, 'timeout');
        }
        // #3942: same guarded resolver as the batched lane (single-path call).
        const slug = await resolveRemovedPathSlug(engine, path, undefined, serr);
        if (slug === undefined) {
          await markCompleted(run, path);
          progress.tick(1, path);
          continue;
        }
        try {
          // #4587: soft-delete with the same 'default' fallback the old
          // optional-opts deletePage call applied on this legacy lane
          // (opts.sourceId is undefined here by construction).
          await softDeleteSyncPages(engine, [slug], { sourceId: opts.sourceId ?? DEFAULT_SOURCE_ID });
          noteCommittedProgress();
          pagesAffected.push(slug);
          deletedSlugs.add(slug);
          await markCompleted(run, path);
        } catch (err) {
          failedFiles.push({
            path,
            error: `delete failed: ${err instanceof Error ? err.message : String(err)}`,
          });
        }
        progress.tick(1, slug);
      }
    }
    progress.finish();
  }
  return undefined;
}