Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.6 KB
Raw Blame
/**
 * A13 vector reuse for one Markdown page import, shared by the inline import
 * (before its transaction, onto the chunks it will embed) and the prepared
 * publish (inside its publication transaction, under the page guard).
 */
import type { BrainEngine } from './engine.ts';
import { embeddingWriteTarget, embeddingInputContext } from './page-state/projections.ts';
import { canReuseMarkdownVector, planEmbeddingReuse } from './embed-reuse.ts';
import { modeRequiresSynopsis } from './embedding-context.ts';
import type { Chunk, ChunkInput, CRMode } from './types.ts';
import type { SegmentChunkInput } from './page-body/types.ts';
import { quoteIdentifier } from './search/embedding-column.ts';
import { chunkGenerationFilter } from './search/safe-chunks.ts';
import { parseEmbedding } from './utils.ts';
import { segmentSynopsisInputHashes, segmentSynopsisLead } from './page-body/segment-synopsis.ts';

/**
 * A stored row an import keeps: its id and the index it holds now (a kept row whose new chunk sits elsewhere moves
 * there). `vector`: the row holds one, so moving it is a vector-index write (page-state/chunk-moves.ts).
 */
export interface KeptChunkRow { id: number; chunk_index: number; vector: boolean }

/**
 * A13 reuse for one page import: copies each stored vector the reuse gate
 * admits onto its matching new chunk in `into`, and maps the chunk's index to
 * the stored row when that row already holds exactly what the chunk would
 * insert, at any index (#7172: an entry inserted above it shifts its index, not
 * its content), so an import can keep it, else to null. A stored row at the
 * chunk's own index is matched first. With `keepUnembedded` (the
 * in-transaction pass), an identical stored row with no vector at the same
 * index also maps to its row when the new chunk carries no vector. A prepared
 * import reuses nothing from a page stored under per-chunk synopsis.
 */
export async function reuseStoredChunkVectors(
  exec: BrainEngine,
  into: ChunkInput[],
  { slug, sourceId, title, corpusGeneration, tier, prepared, keepUnembedded = false, requireRecordedInput = false }: { slug: string; sourceId: string; title: string; corpusGeneration: string | null; tier: 'title' | 'none'; prepared: boolean; keepUnembedded?: boolean;
    /** #6961: a fenced page admits only vectors whose embedding input was recorded, never the legacy raw-text match. */
    requireRecordedInput?: boolean },
): Promise<Map<number, KeptChunkRow | null>> {
  const reused = new Map<number, KeptChunkRow | null>();
  const target = await embeddingWriteTarget(exec);
  const provenance = embeddingInputContext(target, title, corpusGeneration, into);
  const rows = await exec.executeRaw<{ chunk_index: number; embedding_input_hash: string | null; contextual_retrieval_mode: string | null; text_only: boolean; unembedded: boolean }>(
    `SELECT c.chunk_index, c.embedding_input_hash, p.contextual_retrieval_mode,
        c.embedding_image IS NULL AND c.embedding_multimodal IS NULL AND c.embedded_text_hash = md5(c.chunk_text) AS text_only,
        c.embedding IS NULL AND c.embedding_image IS NULL AND c.embedding_multimodal IS NULL
          AND c.embedding_input_hash IS NULL AND c.embedded_text_hash IS NULL AS unembedded
      FROM content_chunks c JOIN pages p ON p.id = c.page_id
      WHERE p.source_id = $1 AND p.slug = $2`, [sourceId, slug]);
  if (rows.length === 0) return reused;
  // A page stored under per-chunk synopsis re-embeds every chunk on its next
  // prepared edit: its vectors carry synopses this import does not recompute.
  if (prepared && rows[0]!.contextual_retrieval_mode && modeRequiresSynopsis(rows[0]!.contextual_retrieval_mode as CRMode)) return reused;
  const recorded = new Map(rows.map(row => [Number(row.chunk_index), row.embedding_input_hash]));
  const textOnly = new Set(rows.filter(row => row.text_only).map(row => Number(row.chunk_index)));
  // Reuse is keyed on chunk source + text, so a stored chunk's current-input
  // hash is the hash its matching new chunk would record.
  const all = await exec.getChunks(slug, { sourceId, includeEmbedding: true, requireSafeChunks: true });
  const stored = all.filter(chunk => chunk.embedding && (!requireRecordedInput || recorded.get(chunk.chunk_index) != null) && canReuseMarkdownVector(recorded.get(chunk.chunk_index), rows[0]!.contextual_retrieval_mode, tier, provenance, chunk));
  const identity = (chunk: ChunkInput | Chunk) => JSON.stringify([chunk.chunk_text, chunk.chunk_source,
    chunk.language ?? null, chunk.symbol_name ?? null, chunk.symbol_type ?? null, chunk.start_line ?? null, chunk.end_line ?? null,
    chunk.parent_symbol_path ?? null, chunk.doc_comment ?? null, chunk.symbol_name_qualified ?? null, chunk.modality ?? 'text']);
  const key = (chunk: { chunk_source?: string | null; chunk_text: string }) => `${chunk.chunk_source}\0${chunk.chunk_text}`;
  // A stored row at the chunk's own index first, then FIFO by source + text: one page can hold two identical chunks.
  const atIndex = new Map(stored.map(chunk => [chunk.chunk_index, chunk]));
  const matches = new Map<number, Chunk>();
  into.forEach((chunk, i) => { const row = atIndex.get(chunk.chunk_index); if (row && key(row) === key(chunk)) matches.set(i, row); });
  const taken = new Set([...matches.values()].map(row => row.id));
  const byKey = new Map<string, Chunk[]>();
  for (const row of stored) if (!taken.has(row.id)) byKey.set(key(row), [...(byKey.get(key(row)) ?? []), row]);
  into.forEach((chunk, i) => { const row = matches.has(i) ? undefined : byKey.get(key(chunk))?.shift(); if (row) matches.set(i, row); });
  for (const [i, row] of matches) {
    into[i].embedding = row.embedding as Float32Array;
    into[i].token_count = row.token_count ?? undefined;
    if (row.model) into[i].model = row.model;
    // The stored row already holds exactly what this chunk would insert (identity, vector, recorded input and text
    // hash, no image or multimodal vector the insert would drop), so it can stay, moved to this chunk's index.
    reused.set(i, recorded.get(row.chunk_index) != null && textOnly.has(row.chunk_index) && identity(row) === identity(into[i]) ? { id: row.id, chunk_index: row.chunk_index, vector: true } : null);
  }
  // A stored row that never got a vector stays in place under a new chunk that
  // will not get one either (a --no-embed or deferred write): the same upsert
  // over it would keep it as it is.
  if (keepUnembedded) {
    const unembedded = new Set(rows.filter(row => row.unembedded).map(row => Number(row.chunk_index)));
    const byIndex = new Map(into.map((chunk, i) => [chunk.chunk_index, i]));
    for (const row of all) {
      const i = byIndex.get(row.chunk_index);
      if (i === undefined || reused.has(i) || into[i]!.embedding || !unembedded.has(row.chunk_index)) continue;
      if (identity(row) === identity(into[i]!)) reused.set(i, { id: row.id, chunk_index: row.chunk_index, vector: false });
    }
  }
  return reused;
}

/**
 * Wave B: vector reuse for a segmented write, scoped to the live chunk rows of
 * the occurrences the write retires (never the whole page): one statement
 * reads those rows with their vectors, the same gate as
 * `reuseStoredChunkVectors` admits each (recorded input hash for the write's
 * tier), and each admitted vector is copied, with its recorded input, onto the
 * new chunk with the same source and text (`chunk_source\0chunk_text`, FIFO for
 * duplicates); on a page stored under per-chunk synopsis, onto the new chunk
 * whose synopsis input hashes to the recorded one (its window is unchanged). So an
 * unchanged chunk inside a rewritten tail keeps its vector and only new text
 * reaches the provider. Returns how many chunks got one.
 */
export async function reuseSegmentChunkVectors(exec: BrainEngine, { pageId, retiredOccurrences, chunks, title, corpusGeneration, tier, truthLead }: {
  pageId: number; retiredOccurrences: readonly number[]; chunks: SegmentChunkInput[]; title: string; corpusGeneration: string | null; tier: 'title' | 'none';
  /** The truth_lead the write stores (a synopsis window opens with it); omitted when the body is unchanged (the stored one). */
  truthLead?: string }): Promise<number> {
  if (retiredOccurrences.length === 0 || chunks.length === 0) return 0;
  const target = await embeddingWriteTarget(exec);
  const provenance = embeddingInputContext(target, title, corpusGeneration, []);
  const col = quoteIdentifier(target.column.name);
  const rows = await exec.executeRaw<{ chunk_text: string; chunk_source: string; embedding: unknown; token_count: number | null; model: string | null;
    embedding_input_hash: string | null; contextual_retrieval_mode: string | null; slug: string; source_id: string; corpus_generation: string | null }>(
    `SELECT c.chunk_text, c.chunk_source, c.${col} AS embedding, c.token_count, c.model, c.embedding_input_hash, p.contextual_retrieval_mode,
            p.slug, p.source_id, p.corpus_generation
       FROM content_chunks c JOIN pages p ON p.id = c.page_id
      WHERE c.page_id = $1 AND c.occurrence_id = ANY($2::bigint[]) AND c.${col} IS NOT NULL AND ${chunkGenerationFilter('p', 'c')}
      ORDER BY c.chunk_index`, [pageId, [...retiredOccurrences]]);
  const mode = rows[0]?.contextual_retrieval_mode ?? null;
  if (rows.length === 0) return 0;
  // Wave B spec 8.6: on a page stored under per-chunk synopsis, a vector carries over only onto a new chunk whose
  // synopsis input (its segment window over the page lead) hashes to the stored row's recorded input.
  const synopsis = mode && modeRequiresSynopsis(mode as CRMode)
    ? segmentSynopsisInputHashes(embeddingInputContext(target, title, rows[0]!.corpus_generation, []),
      await segmentSynopsisLead(exec, { id: pageId, slug: rows[0]!.slug, source_id: rows[0]!.source_id }, truthLead), chunks)
    : null;
  const stored = rows.map(row => ({ chunk_text: row.chunk_text, chunk_source: row.chunk_source, embedding: parseEmbedding(row.embedding),
    token_count: row.token_count, model: row.model, embedding_input_hash: row.embedding_input_hash }))
    .filter(chunk => synopsis ? chunk.embedding_input_hash !== null : canReuseMarkdownVector(chunk.embedding_input_hash, mode, tier, provenance, chunk));
  const { reuse } = planEmbeddingReuse(stored, chunks, synopsis
    ? c => synopsis.get(c as SegmentChunkInput) ?? (c as { embedding_input_hash?: string | null }).embedding_input_hash ?? ''
    : c => `${c.chunk_source}\0${c.chunk_text}`);
  for (const [i, matched] of reuse) {
    const row = matched as typeof stored[number];
    chunks[i]!.embedding = row.embedding as Float32Array;
    chunks[i]!.token_count = row.token_count ?? undefined;
    if (row.model) chunks[i]!.model = row.model;
    if (row.embedding_input_hash) chunks[i]!.embedding_input_hash = row.embedding_input_hash;
  }
  return reuse.size;
}