Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
13.7 KB
Raw Blame
/**
 * `o2b search vector-backfill` — run the vector phase on its own
 * (provenance-at-the-boundary, unit F).
 *
 * Indexing does not compute vectors unless it is asked to: no automatic
 * `indexVault` call site requests embeddings, and only the explicit
 * `--embeddings` flag on `search index` / `search reindex` reaches a
 * provider - with ONE deliberate exception: the quiet-window maintenance
 * lane asks for the embedding phase whenever the resolved config can
 * reach a provider (t_9d155d0e), announcing the predicted spend before
 * the pass and receipting the actual spend after it, leashed by the
 * embedding cost gate unless force is set. A vault therefore still
 * accumulates chunks with no `embeddings` row perfectly normally outside
 * that lane, and the only way to fill them without re-indexing the whole
 * vault is this verb. It fills them and nothing else.
 *
 * Dry-run is the DEFAULT; `--apply` is the only path that contacts a
 * provider or writes a vector. The shape follows
 * `brain/verbs/authored-at-backfill.ts` line for line, including the log
 * append whose failure is reported on stderr rather than swallowed.
 *
 * ## Where the interrupt can and cannot land
 *
 * This verb holds a real cancellation handle, and it is the only one of
 * the long verbs whose answer needed checking rather than assuming.
 * `interruptIsObservable` answers per OPERATION, and this pass runs under
 * `OPERATION.reindex`, which the table marks observable - but the table's
 * reason is about the index BUILDERS, so it has to hold here on its own
 * terms. It does, for the half of the run that is long:
 * `runEmbeddingPhase` awaits a provider round trip per super-batch and
 * calls `throwIfAborted` at every batch boundary, which is real I/O and
 * therefore a real yield to the event loop.
 *
 * It does NOT hold for the plan: counting vectorless chunks is a
 * synchronous SQLite query between two awaits, so a keystroke landing
 * inside it is not observed at a checkpoint. That is not a hole this verb
 * papers over - it is exactly what `release()` is for. An interrupt that
 * arrived and that nothing acknowledged is re-raised there, so the run
 * ends the way the un-suppressed keystroke would have, rather than
 * returning 0 for a pass the operator stopped. The plan is milliseconds;
 * the embedding phase is the part an operator would ever want to stop.
 */

import {
  COST_GATE_KEY,
  EMBEDDING_GATE_REASON,
  FORCE_COST_FLAG,
  formatEstimatedUsd,
  type EmbeddingGateReason,
} from "../../../core/search/embedding-spend.ts";
import {
  EMBEDDING_PRICE_MODEL_KEY,
  EMBEDDING_PRICE_RATE_KEY,
} from "../../../core/search/embeddings/pricing.ts";
import { appendLogEvent } from "../../../core/brain/log.ts";
import { NEXT_COMMAND_KEY, resolveNextStep } from "../../../core/brain/next-step.ts";
import {
  createSafeguard,
  OPERATION,
  resolveSafeguardTimeoutMs,
  SafeguardAbortError,
} from "../../../core/brain/safeguard.ts";
import { isoSecond } from "../../../core/brain/time.ts";
import { BRAIN_LOG_EVENT_KIND } from "../../../core/brain/types.ts";
import { resolveAgentName } from "../../../core/config.ts";
import {
  semanticCapabilityIsBlocked,
  semanticCapabilityLabel,
  SEMANTIC_VECTOR_CODE,
} from "../../../core/search/capability-tier.ts";
import {
  planVectorBackfill,
  type VectorBackfillResult,
} from "../../../core/search/vector-backfill.ts";
import { advisoryIsLegal, emitNextStep, type AdvisoryStream } from "../../advisory-rail.ts";
import { shellQuote } from "../../cron-recipe.ts";
import { onInterrupt, reportInterrupted } from "../../interrupt.ts";
import { info, ok } from "../../output.ts";
import { attachProgress, reportProgressRefusal } from "../../progress-rail.ts";
import {
  flagBoolean,
  flagString,
  flagStrings,
  parseFlags,
  resolveConfig,
  resolveConfigPath,
  searchAdvisoryStream,
  VAULT_FLAGS,
} from "../helpers.ts";

/**
 * The state this verb leaves an operator in when work remains: an index
 * holding chunks with no vector. Its registered exit is this same verb
 * under `--apply`, which is the whole point of naming it.
 */
const VECTORS_PENDING = SEMANTIC_VECTOR_CODE.pending;

/** The repeatable flag that limits the run to a vault-relative path prefix. */
const PATH_FLAG = "path";

/** A prefix that needs no shell quoting when echoed in a command. */
const PLAIN_SHELL_WORD = /^[A-Za-z0-9_./-]+$/u;

/**
 * `command` with one `--path <prefix>` per prefix, quoted only when
 * needed. Null on Windows when a prefix needs quoting: the quoting is
 * POSIX, which neither cmd.exe nor PowerShell reads back as the same
 * word, so the caller prints {@link RERUN_WITH_SCOPE_LINE}, which names
 * no command, instead of advising one that would not run as printed or
 * would run unscoped.
 */
export function scopedNextCommand(
  command: string,
  pathPrefixes: ReadonlyArray<string>,
  platform: NodeJS.Platform = process.platform,
): string | null {
  const plain = (prefix: string) => PLAIN_SHELL_WORD.test(prefix);
  if (platform === "win32" && !pathPrefixes.every(plain)) return null;
  return [
    command,
    ...pathPrefixes.map(
      (prefix) => `--${PATH_FLAG} ${plain(prefix) ? prefix : shellQuote(prefix)}`,
    ),
  ].join(" ");
}

/**
 * The line that replaces a scoped next step Windows cannot run as
 * printed. Deliberately not a command: a scope-free `--apply` copied as
 * printed would widen the spend to the whole vault.
 */
export const RERUN_WITH_SCOPE_LINE = `next: rerun this command with --apply and the same --${PATH_FLAG} flags`;

/** The next step a backfill run advises, resolved before anything prints. */
export interface BackfillNextStep {
  /** The registered exit the advice names. */
  readonly exitCode: string;
  /**
   * The runnable command, also the `next_command` JSON value. Absent
   * when the code has no command or when the scope cannot be quoted on
   * this platform.
   */
  readonly command: string | undefined;
  /** True when a scope was dropped because the platform cannot quote it. */
  readonly scopeDropped: boolean;
}

/**
 * Resolve the advice for `result`. A scoped run repeats its scope: the
 * registered `--apply` alone would widen the spend to the whole vault.
 * The command stays UNFORCED even when the gate would refuse it: the
 * gate is the operator's decision, so `--force-cost` is named only in
 * the human remedy line, never in a command a machine reader runs.
 */
export function backfillNextStep(
  result: Pick<VectorBackfillResult, "capability" | "pathPrefixes">,
  platform: NodeJS.Platform = process.platform,
): BackfillNextStep {
  // WHICH exit depends on what is in the operator's way. Naming
  // `--apply` while the credential is missing would advise a command that
  // cannot succeed, so a blocked tier names the tier's own exit instead.
  const exitCode = semanticCapabilityIsBlocked(result.capability)
    ? result.capability.code
    : VECTORS_PENDING;
  const registered = resolveNextStep(exitCode)?.nextCommand;
  if (registered === undefined || exitCode !== VECTORS_PENDING) {
    return { exitCode, command: registered, scopeDropped: false };
  }
  const scoped = scopedNextCommand(registered, result.pathPrefixes, platform);
  return scoped === null
    ? { exitCode, command: undefined, scopeDropped: true }
    : { exitCode, command: scoped, scopeDropped: false };
}

/**
 * Print `advice` on a human stream: the command, or, for a dropped
 * scope, the one rerun line that names no runnable command.
 */
export function emitBackfillNextStep(advice: BackfillNextStep, stream: AdvisoryStream): void {
  if (!advice.scopeDropped) {
    emitNextStep(advice.exitCode, stream, advice.command);
    return;
  }
  if (resolveNextStep(advice.exitCode) !== null && advisoryIsLegal(stream)) {
    info(RERUN_WITH_SCOPE_LINE);
  }
}

/** JSON payload shape. Snake case, matching every other search verb. */
function jsonForResult(result: VectorBackfillResult): Record<string, unknown> {
  return {
    dry_run: !result.applied,
    capability_tier: result.capability.tier,
    capability_code: result.capability.code,
    chunks_total: result.chunksTotal,
    pending: result.pending,
    embedded: result.embedded,
    retries: result.retries,
    // Null rather than zero when the model carries no known price: a
    // missing price is not a free run.
    estimated_cost_usd: result.estimatedCostUsd,
    price_source: result.priceSource,
    // Present only for a scoped run, so an unscoped payload is unchanged.
    ...(result.pathPrefixes.length > 0 ? { path_prefixes: result.pathPrefixes } : {}),
    ...(result.unmatchedPathPrefixes.length > 0
      ? { unmatched_path_prefixes: result.unmatchedPathPrefixes }
      : {}),
    // Present only when the unforced gate would refuse, and only for a
    // run that reached the provider, so an unblocked payload is unchanged.
    ...(result.blocked ? { gate_blocked: true, gate_reason: result.reason } : {}),
    ...(result.spend !== null
      ? {
          spend: {
            model: result.spend.model,
            tokens: result.spend.tokens,
            estimated_usd: result.spend.estimatedUsd,
            price_source: result.spend.priceSource,
            forced: result.spend.forced,
          },
        }
      : {}),
  };
}

/** What lifts a gate refusal besides `--force-cost`, per reason. */
function gateRemedy(reason: EmbeddingGateReason | null): string {
  return reason === EMBEDDING_GATE_REASON.unpriced
    ? `set ${EMBEDDING_PRICE_MODEL_KEY} and ${EMBEDDING_PRICE_RATE_KEY}`
    : `raise ${COST_GATE_KEY}`;
}

async function renderHuman(result: VectorBackfillResult): Promise<void> {
  if (result.applied) {
    ok(`vector-backfill: wrote ${result.embedded} of ${result.pending} pending vector(s)`);
  } else {
    ok(
      `vector-backfill dry-run: ${result.pending} of ${result.chunksTotal} chunk(s) have no vector`,
    );
  }
  if (result.pathPrefixes.length > 0) info(`  scope: ${result.pathPrefixes.join(", ")}`);
  if (result.pending > 0) info(`  estimated cost: ${formatEstimatedUsd(result.estimatedCostUsd)}`);
  if (!result.applied && result.blocked) {
    info(
      `  cost gate: would refuse (${result.reason}); add ${FORCE_COST_FLAG} or ${gateRemedy(result.reason)}`,
    );
  }
  if (result.retries > 0) info(`  provider retries: ${result.retries}`);
  // What the operator CONFIGURED, resolved from the registry - never a
  // sentence built here.
  if (semanticCapabilityIsBlocked(result.capability)) {
    info(`  semantic capability: ${await semanticCapabilityLabel(result.capability.code)}`);
  }
}

export async function cmdSearchVectorBackfill(argv: ReadonlyArray<string>): Promise<number> {
  const { flags } = parseFlags(argv, {
    ...VAULT_FLAGS,
    apply: { type: "boolean" },
    "force-cost": { type: "boolean" },
    // Spelled out, not `[PATH_FLAG]`: the flag census reads literal keys.
    path: { type: "string-array" },
    progress: { type: "boolean" },
    json: { type: "boolean" },
  });
  const cfg = resolveConfig(flags);
  const apply = flagBoolean(flags, "apply");
  const jsonRequested = flagBoolean(flags, "json");

  // Opt-in, for the reason the index builders give: attaching a sink by
  // default would change the stderr of every existing invocation.
  const observation = flagBoolean(flags, "progress")
    ? attachProgress({ command: "search", argv: ["vector-backfill"], jsonRequested })
    : null;
  reportProgressRefusal(observation);

  // Everything that can throw happens BEFORE the handle exists: `release`
  // removes process-global listeners and settles a signal nobody acted
  // on, and a throw between `onInterrupt()` and the `try` would skip it.
  const interrupt = onInterrupt(OPERATION.reindex);
  let result: VectorBackfillResult;
  try {
    result = await planVectorBackfill(cfg, {
      apply,
      forceCost: flagBoolean(flags, "force-cost"),
      pathPrefixes: flagStrings(flags, PATH_FLAG),
      // The three seams this module declared and nothing produced until
      // now. The deadline is the `reindex` budget off the same ladder the
      // builders read, because this pass IS that run's embedding phase on
      // its own - a second key for one phase would let the two disagree.
      safeguard: createSafeguard({
        operation: OPERATION.reindex,
        timeoutMs: resolveSafeguardTimeoutMs(OPERATION.reindex, flagString(flags, "config")),
      }),
      signal: interrupt.signal,
      ...(observation?.sink !== undefined ? { onProgress: observation.sink } : {}),
    });
  } catch (err) {
    if (err instanceof SafeguardAbortError) return reportInterrupted(interrupt, err, jsonRequested);
    throw err;
  } finally {
    interrupt.release();
  }

  if (apply && result.embedded > 0) {
    try {
      appendLogEvent(cfg.vault, {
        timestamp: isoSecond(new Date()),
        eventType: BRAIN_LOG_EVENT_KIND.vectorBackfill,
        body: {
          agent: resolveAgentName(resolveConfigPath(flags)),
          embedded: String(result.embedded),
          pending: String(result.pending),
          chunks_total: String(result.chunksTotal),
        },
      });
    } catch (err) {
      process.stderr.write(
        `warning: append vector-backfill log failed: ${(err as Error).message}\n`,
      );
    }
  }

  const advice = backfillNextStep(result);
  const pendingAfter = result.pending - result.embedded;

  // A typo'd scope has nothing pending and would otherwise read like a
  // fully embedded one; stderr, so the JSON on stdout stays parseable.
  for (const prefix of result.unmatchedPathPrefixes) {
    process.stderr.write(`warning: scope ${prefix} matches no indexed document\n`);
  }
  if (jsonRequested) {
    process.stdout.write(
      JSON.stringify({
        ...jsonForResult(result),
        ...(pendingAfter > 0 && advice.command !== undefined
          ? { [NEXT_COMMAND_KEY]: advice.command }
          : {}),
      }) + "\n",
    );
  } else {
    await renderHuman(result);
  }
  if (pendingAfter > 0) emitBackfillNextStep(advice, searchAdvisoryStream(argv, jsonRequested));
  return 0;
}