Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.5 KB
Raw Blame
/**
 * Settles queued embedding effects that have nothing left to do, in bulk.
 *
 * A classic import followed by `gbrain embed --stale` (or a restored or moved
 * brain) can leave tens of thousands of queued page embedding effects whose
 * chunks already carry current vectors. Each one would otherwise take a claim,
 * a guard transaction, a projection read and a completion update before
 * reaching the outcome the effect runner records when nothing is pending.
 *
 * An effect is settled here only when the runner would find nothing to embed:
 * a plain page effect (no targets, scan, retry slugs or parked targets) never
 * attempted, ready and claimable by this host as `claimPersistenceEffect`
 * would claim it, whose source passes `guardEffectSource`, whose page is live
 * at the effect's revision with its text projection sealed there, and whose
 * every chunk passes `readEmbeddingEffectProjection`'s completion test for the
 * current signature, write column and model. The row ends exactly as the
 * runner's own claim and completion leave it (state, outcome, attempts,
 * cleared claim). Every other effect is left for the runner.
 *
 * No effect kind is ordered after a page embedding effect: the claim order is
 * `next_attempt_at, id` across kinds, only Git groups and withdrawal mirrors
 * gate other effects, and an embedding effect gates nothing.
 */
import { trueUpRequests } from './receipt-true-up.ts';
import type { BrainEngine } from '../engine.ts';
import { OperationError } from '../ops/contract.ts';
import { embeddingWriteTarget } from '../page-state/projections.ts';
import { quoteIdentifier } from '../search/embedding-column.ts';
import { guardEffectSource } from './effect-recovery.ts';
import { embeddingCompletionModel } from './effects.ts';
import type { PersistenceEffect } from './effect-model.ts';
import { faultPoint } from './fault-points.ts';
import { declarePersistenceProtocol, PERSISTENCE_PROTOCOL_PREDICATE } from './protocol.ts';
import { refreshFenceClear } from './worktree-refresh-schema.ts';

/**
 * At most this many effects settle in one statement. A pass and its reservation true-up hold the event loop on PGLite,
 * so the batch bounds how long a request waits behind the drain.
 */
export const NOOP_SETTLE_BATCH = 50;

/** Guard failures that leave an effect for the runner, which records them as it does today. */
const LEFT_FOR_RUNNER = new Set(['owner_unavailable', 'source_changed', 'recovery_required', 'owner_marker_missing']);

type Candidate = Pick<PersistenceEffect, 'id' | 'kind' | 'source_id' | 'source_incarnation' | 'worktree_id'>;

/**
 * How far a process's settle passes have looked, for one write column, model and signature. Effects at or below
 * `through` were each considered once; one that becomes a no-op later (vectors installed while it waited) takes the
 * runner path. Without it, a backlog that cannot settle (stale vectors, an unconfigured provider) would be walked in
 * full on every drain.
 */
export interface NoopSettleCursor { key: string; through: string }
export interface NoopSettlePass { settled: number; cursor: NoopSettleCursor; done: boolean }

/** One settle pass over effects past `after`, in id order; `done` once it reached the newest effect. */
export async function settleNoopEmbeddingEffects(engine: BrainEngine, hostId: string, signature: string,
  after?: NoopSettleCursor, limit = NOOP_SETTLE_BATCH): Promise<NoopSettlePass> {
  const { column } = await embeddingWriteTarget(engine);
  const expectedModel = embeddingCompletionModel(column, signature) ?? null;
  const key = `${column.name}\0${expectedModel ?? ''}\0${signature}`;
  const from = after?.key === key ? after.through : '0';
  // A drain with nothing new to consider costs one indexed read and opens no transaction.
  // Ordered by id so a generic plan (a prepared statement past its first runs) still walks the primary key from `from`.
  const [due] = await engine.executeRaw(`SELECT 1 FROM persistence_effects e WHERE e.id>$1::bigint AND e.kind='embedding' AND e.state='queued'
    AND e.attempts=0 AND e.recovery IS NULL ORDER BY e.id LIMIT 1`, [from]);
  if (!due) return { settled: 0, cursor: { key, through: from }, done: true };
  const requestIds: string[] = [];
  const pass = await engine.transaction(async tx => {
    await declarePersistenceProtocol(tx);
    const [{ newest }] = await tx.executeRaw<{ newest: string }>('SELECT COALESCE(max(id),0)::text AS newest FROM persistence_effects');
    // The window is the next `limit` unattempted embedding effects by id, read from the primary key and locked;
    // the per-effect tests then run on that window only, so a pass costs at most `limit` effects whatever the
    // backlog holds and whatever the planner estimates for it.
    const rows = await tx.executeRaw<{ id: string; page_id: number | null }>(`SELECT e.id::text AS id, (e.data->>'page_id')::int AS page_id
      FROM persistence_effects e WHERE e.id>$1::bigint AND e.kind='embedding' AND e.state='queued' AND e.attempts=0 AND e.recovery IS NULL
      ORDER BY e.id LIMIT $2 FOR UPDATE SKIP LOCKED`, [from, limit]);
    const window = rows.map(row => row.id);
    // A short window reached the newest effect; a full one continues after its last effect.
    const done = window.length < limit;
    const cursor = { key, through: done ? newest : window[window.length - 1]! };
    if (!window.length) return { settled: 0, cursor, done };
    const vector = quoteIdentifier(column.name);
    const candidates = await tx.executeRaw<Candidate>(`SELECT e.id::text AS id, e.kind, e.source_id, e.source_incarnation, e.worktree_id
      FROM persistence_effects e
      JOIN pages p ON p.id=(e.data->>'page_id')::int AND p.source_id=e.source_id
      WHERE e.id=ANY($4::text[]::bigint[]) AND p.id=ANY($5::int[]) AND e.next_attempt_at<=now()
        AND NOT (e.data ? 'targets') AND NOT (e.data ? 'source_scan') AND NOT (e.data ? 'retry_slugs') AND NOT (e.data ? 'parked')
        AND (e.worktree_id IS NULL OR (EXISTS (SELECT 1 FROM persistence_worktrees w WHERE w.id=e.worktree_id AND w.owner_host_id=$1::uuid)
          AND ${refreshFenceClear('e')}))
        AND NOT EXISTS (SELECT 1 FROM persistence_effects blocked WHERE blocked.worktree_id=e.worktree_id AND blocked.recovery IS NOT NULL)
        AND NOT EXISTS (SELECT 1 FROM persistence_requests blocked WHERE blocked.worktree_id=e.worktree_id AND blocked.recovery IS NOT NULL)
        AND NOT EXISTS (SELECT 1 FROM persistence_effects mirror
          WHERE mirror.request_id=e.request_id AND mirror.kind='withdrawal-mirror' AND mirror.state<>'committed')
        AND p.deleted_at IS NULL AND p.knowledge_revision=e.revision AND p.text_projection_revision=p.knowledge_revision
        AND p.embedding_signature=$2
        AND NOT EXISTS (SELECT 1 FROM content_chunks cc WHERE cc.page_id=p.id AND NOT (cc.${vector} IS NOT NULL
          AND cc.embedded_at IS NOT NULL AND cc.embedded_text_hash IS NOT DISTINCT FROM md5(cc.chunk_text) AND cc.model IS NOT DISTINCT FROM $3))
      ORDER BY e.id
      FOR SHARE OF p`, [hostId, signature, expectedModel, window, rows.flatMap(row => row.page_id === null ? [] : [row.page_id])]);
    if (!candidates.length) return { settled: 0, cursor, done };
    const guarded = new Map<string, boolean>();
    const ids: string[] = [];
    for (const effect of candidates) {
      const scope = `${effect.source_id}\0${effect.source_incarnation}\0${effect.worktree_id ?? ''}`;
      if (!guarded.has(scope)) {
        guarded.set(scope, await guardEffectSource(tx, effect as PersistenceEffect, hostId).then(() => true, error => {
          if (error instanceof OperationError && LEFT_FOR_RUNNER.has(error.code)) return false;
          throw error;
        }));
      }
      if (guarded.get(scope)) ids.push(String(effect.id));
    }
    if (!ids.length) return { settled: 0, cursor, done };
    const settled = await tx.executeRaw(`UPDATE persistence_effects SET state='committed', error_code=NULL, attempts=attempts+1,
        data=data-'retry_slugs'-'target_failures'-'failing_target', execution_token=NULL, claim_expires_at=NULL, claim_owner=NULL, outcome='{}'::jsonb, updated_at=now()
      WHERE id=ANY($1::text[]::bigint[]) AND state='queued' AND recovery IS NULL AND ${PERSISTENCE_PROTOCOL_PREDICATE} RETURNING id, request_id`, [ids]);
    await faultPoint('effect:embedding:settle', { effectId: ids[0] });
    for (const row of settled as Array<{ request_id: string }>) requestIds.push(row.request_id);
    return { settled: settled.length, cursor, done };
  });
  // #6696 (A1): the settled requests give back their unused reservation, outside the settle transaction.
  await trueUpRequests(engine, requestIds);
  return pass;
}