Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.4 KB
Raw Blame
/**
 * `gbrain repair embedding-effects` settlement (#5629, #5734). A committed
 * write can keep an embedding effect that never finishes: queued with no owner
 * run for over an hour, or failed after its retry allowance. Either one blocks
 * receipt compaction and activation (`writer_not_quiesced`). Each candidate
 * ends in exactly one outcome and no obligation is dropped without one:
 *
 * - `reconciled`: the page's current chunks pass the effect projection
 *   verifier (sealed revision, selected column, model and hashes), so the
 *   effect commits with no provider work.
 * - `superseded`: the page was deleted, or a newer revision of the same page
 *   (same page id) owns its own embedding effect or has independently
 *   verified current vectors; the effect commits with
 *   `{ embedding: 'superseded' }`, as the owner's own run would.
 * - `retry_queued`: the owner embeds it. A stale queued effect is re-queued;
 *   a failed one gets the `retry-effects` allowance, or, when that allowance
 *   is consumed (`embedding_retry_exhausted`), one new bounded retry cycle
 *   under a durable authorization id bound to this repair run, so a resumed
 *   run replays it instead of granting another. Not success: doctor keeps the
 *   effect pending until it settles.
 * - `blocked`: nothing changed; `reason` says why (`owner_unavailable`,
 *   `embedding_disabled`, `embedding_unconfigured`, `projection_pending`,
 *   `no_replacement_obligation`).
 *
 * Every state change is a compare-and-set on state, attempts, execution token
 * and recovery. A candidate that changed since planning is `changed_since_preview`.
 * Two failed effects are not candidates (GBRA-130 A-15): one with parked targets
 * (`targets_parked`; `gbrain sources writer retry-effects` retries those) and a
 * zero-norm refusal, which a retry repeats by design.
 */
import { trueUpRequests } from './receipt-true-up.ts';
import { EMBEDDING_ZERO_NORM } from '../ai/embedding-guard.ts';
import type { BrainEngine } from '../engine.ts';
import type { GBrainConfig } from '../config.ts';
import { opError, OperationError } from '../ops/contract.ts';
import { EmbeddingDisabledError } from '../embedding-dim-check.ts';
import { MAX_RATE_LIMIT_RETRIES } from '../embed-retry.ts';
import { digest } from './digest.ts';
import { guardEffectSource } from './effect-recovery.ts';
import { assertEmbeddingEffectEnabled, readEmbeddingEffectProjection, selectedEffectPage } from './effects.ts';
import { targetedWithdrawalEffect } from './effect-targets.ts';
import type { PersistenceEffect } from './effect-model.ts';
import { declarePersistenceProtocol } from './protocol.ts';

export type EmbeddingSettlementOutcome = 'reconciled' | 'superseded' | 'retry_queued' | 'blocked' | 'changed_since_preview';

export interface EmbeddingSettlement {
  outcome: EmbeddingSettlementOutcome;
  reason?: string;
  /** A retry grant that lets the owner spend on the provider. */
  paid?: boolean;
  /** Pending chunks the verifier rejected (missing or signature-mismatched vectors). */
  pending_chunks?: number;
}

export interface EmbeddingCandidate {
  effect_id: string; source_id: string; slug: string; request_id: string; state: 'queued' | 'failed'; attempts: number; chars: number;
}

/** Stale queued (over an hour, or re-queued by this repair and not yet settled) and failed embedding effects of committed writes. */
export const EMBEDDING_CANDIDATE_WHERE = `e.kind='embedding' AND r.state='committed' AND e.recovery IS NULL AND e.execution_token IS NULL
  AND (e.state='failed' OR e.state='queued' AND (e.updated_at < now() - interval '1 hour' OR e.data ? 'repair_requeued_at'))
  AND NOT (e.state='failed' AND e.error_code IN ('targets_parked','${EMBEDDING_ZERO_NORM}'))`;

export async function listEmbeddingCandidates(engine: Pick<BrainEngine, 'executeRaw'>, sourceIds: string[]): Promise<EmbeddingCandidate[]> {
  const rows = await engine.executeRaw<EmbeddingCandidate>(`SELECT e.id::text AS effect_id, r.source_id, r.slug, r.request_id::text AS request_id,
      e.state, e.attempts, COALESCE(CASE
        WHEN e.data ? 'page_id' THEN (SELECT length(p.compiled_truth) + length(COALESCE(p.timeline,'')) FROM pages p WHERE p.id = (e.data->>'page_id')::int)
        WHEN e.data ? 'targets' THEN (SELECT SUM(length(p.compiled_truth) + length(COALESCE(p.timeline,''))) FROM pages p
          JOIN jsonb_array_elements(e.data->'targets') t ON p.id = (t->>'page_id')::int)
        ELSE (SELECT SUM(length(p.compiled_truth) + length(COALESCE(p.timeline,''))) FROM pages p WHERE p.source_id = e.source_id AND p.deleted_at IS NULL)
      END, 0)::bigint AS chars
    FROM persistence_effects e JOIN persistence_requests r ON r.id = e.request_id
    WHERE ${EMBEDDING_CANDIDATE_WHERE} AND (r.source_id = ANY($1::text[])
      OR NOT EXISTS (SELECT 1 FROM sources s WHERE s.id = r.source_id AND s.incarnation = r.source_incarnation)) ORDER BY e.id`, [sourceIds]);
  return rows.map(row => ({ ...row, attempts: Number(row.attempts), chars: Number(row.chars) }));
}

function configuredSignature(config: GBrainConfig | null): string | null {
  return config?.embedding_model && config.embedding_dimensions ? `${config.embedding_model}:${config.embedding_dimensions}` : null;
}

/** The authorization id of one repair run's grant for one effect. */
export function embeddingGrantId(runId: string, effectId: string | number): string {
  return digest(['embedding-effects-grant-v1', runId, String(effectId)]);
}

/**
 * Settle one candidate. With `dryRun` it only predicts the outcome. `runId`
 * binds an exhausted-retry grant to the repair run that authorized it.
 */
export async function settleEmbeddingEffect(engine: BrainEngine, candidate: Pick<EmbeddingCandidate, 'effect_id' | 'state' | 'attempts'>,
  opts: { dryRun: boolean; config: GBrainConfig | null; hostId: string; runId?: string }): Promise<EmbeddingSettlement> {
  let requestId: string | undefined;
  const settlement = await settleInTransaction(engine, candidate, opts, id => { requestId = id; });
  // #6696 (A1): the request's reservation shrinks once this commit settled its last effect, outside the settling transaction.
  if (requestId && !opts.dryRun) await trueUpRequests(engine, [requestId]);
  return settlement;
}

function settleInTransaction(engine: BrainEngine, candidate: Pick<EmbeddingCandidate, 'effect_id' | 'state' | 'attempts'>,
  opts: { dryRun: boolean; config: GBrainConfig | null; hostId: string; runId?: string }, settled: (requestId: string) => void): Promise<EmbeddingSettlement> {
  return engine.transaction(async tx => {
    await declarePersistenceProtocol(tx);
    const [effect] = await tx.executeRaw<PersistenceEffect & { request_state: string }>(`SELECT e.*, r.state AS request_state
      FROM persistence_effects e JOIN persistence_requests r ON r.id = e.request_id WHERE e.id = $1 FOR UPDATE OF e`, [candidate.effect_id]);
    if (!effect || effect.kind !== 'embedding' || effect.request_state !== 'committed' || effect.state !== candidate.state
      || Number(effect.attempts) !== candidate.attempts || effect.execution_token !== null || effect.recovery) {
      return { outcome: 'changed_since_preview' };
    }
    const commit = async (outcome: Record<string, unknown>): Promise<boolean> => {
      if (opts.dryRun) return true;
      const rows = await tx.executeRaw(`UPDATE persistence_effects SET state='committed', error_code=NULL, claim_expires_at=NULL,claim_owner=NULL,
          outcome=$3::text::jsonb, data=data-'repair_requeued_at', updated_at=now()
        WHERE id=$1 AND state=$2 AND attempts=$4 AND execution_token IS NULL AND recovery IS NULL RETURNING id`,
      [effect.id, effect.state, JSON.stringify(outcome), effect.attempts]);
      if (rows.length) settled(effect.request_id);
      return rows.length > 0;
    };
    // The obligation of a removed (or replaced) source incarnation is moot: nothing can read those vectors.
    const [source] = await tx.executeRaw<{ incarnation: string }>('SELECT incarnation FROM sources WHERE id=$1 FOR SHARE', [effect.source_id]);
    if (!source || source.incarnation !== effect.source_incarnation) {
      return await commit({ embedding: 'superseded', reason: 'source_removed' }) ? { outcome: 'superseded', reason: 'source_removed' } : { outcome: 'changed_since_preview' };
    }
    try { await guardEffectSource(tx, effect, opts.hostId); }
    catch (error) {
      if (error instanceof OperationError && ['owner_unavailable', 'source_changed'].includes(error.code)) return { outcome: 'blocked', reason: 'owner_unavailable' };
      throw error;
    }
    const scanning = targetedWithdrawalEffect(effect) || effect.data.source_scan === true;
    const snapshot = await selectedEffectPage(tx, effect);
    let pendingChunks: number | undefined;
    if (scanning) {
      // A finished scan has nothing left to embed; one with pages left needs a configured model like a page effect.
      if (!snapshot) return await commit({ embedding: 'reconciled' }) ? { outcome: 'reconciled' } : { outcome: 'changed_since_preview' };
      if (!configuredSignature(opts.config)) return { outcome: 'blocked', reason: 'embedding_unconfigured' };
    } else {
      if (!snapshot || snapshot.page.deleted_at || snapshot.page.id !== effect.data.page_id || snapshot.revision !== effect.revision) {
        const deleted = !snapshot || snapshot.page.deleted_at || snapshot.page.id !== effect.data.page_id;
        const [replacement] = deleted ? [] : await tx.executeRaw<{ id: string }>(`SELECT id::text FROM persistence_effects
          WHERE kind='embedding' AND source_id=$1 AND source_incarnation=$2::uuid AND revision=$3::uuid AND (data->>'page_id')::int=$4 AND id<>$5 LIMIT 1`,
        [effect.source_id, effect.source_incarnation, snapshot!.revision, snapshot!.page.id, effect.id]);
        if (!deleted && !replacement) {
          // Classic import/embed can advance the canonical revision without
          // scheduling a replacement outbox obligation. Verify its sealed,
          // same-page projection under the existing source/page guards; never
          // grant paid retries for an effect targeting the obsolete revision.
          const signature = configuredSignature(opts.config);
          if (!signature) return { outcome: 'blocked', reason: 'no_replacement_obligation' };
          try {
            const { pending } = await readEmbeddingEffectProjection(tx, effect, snapshot!, opts.hostId, signature);
            if (pending.length) return { outcome: 'blocked', reason: 'no_replacement_obligation', pending_chunks: pending.length };
          } catch (error) {
            if (error instanceof OperationError && error.code === 'projection_pending') return { outcome: 'blocked', reason: 'no_replacement_obligation' };
            throw error;
          }
          return await commit({ embedding: 'superseded', reason: 'current_vectors_verified', verified_revision: snapshot!.revision })
            ? { outcome: 'superseded', reason: 'current_vectors_verified' } : { outcome: 'changed_since_preview' };
        }
        const settled = await commit(deleted ? { embedding: 'superseded', reason: 'page_deleted' }
          : { embedding: 'superseded', reason: 'revision_changed', replaced_by: replacement.id });
        return settled ? { outcome: 'superseded', reason: deleted ? 'page_deleted' : 'revision_changed' } : { outcome: 'changed_since_preview' };
      }
      const signature = configuredSignature(opts.config);
      if (!signature) return { outcome: 'blocked', reason: 'embedding_unconfigured' };
      let pending: unknown[];
      try { pending = (await readEmbeddingEffectProjection(tx, effect, snapshot, opts.hostId, signature)).pending; }
      catch (error) {
        if (error instanceof OperationError && error.code === 'projection_pending') return { outcome: 'blocked', reason: 'projection_pending' };
        throw error;
      }
      if (pending.length === 0) {
        return await commit({ embedding: 'reconciled' }) ? { outcome: 'reconciled' } : { outcome: 'changed_since_preview' };
      }
      pendingChunks = pending.length;
    }
    await tx.executeRaw("SELECT key FROM config WHERE key='embedding_disabled' FOR SHARE");
    try { await assertEmbeddingEffectEnabled(tx, opts.config); }
    catch (error) {
      if (error instanceof EmbeddingDisabledError) return { outcome: 'blocked', reason: 'embedding_disabled' };
      throw error;
    }
    return queueRetry(tx, effect, opts, pendingChunks);
  });
}

async function queueRetry(tx: BrainEngine, effect: PersistenceEffect, opts: { dryRun: boolean; runId?: string }, pendingChunks: number | undefined): Promise<EmbeddingSettlement> {
  const base = { outcome: 'retry_queued' as const, ...(pendingChunks !== undefined ? { pending_chunks: pendingChunks } : {}) };
  const exhausted = effect.state === 'failed' && effect.data.embedding_retry_base !== undefined;
  const grants: string[] = (effect.data as { repair_authorizations?: string[] }).repair_authorizations ?? [];
  // Every retry this run authorizes for a failed effect is recorded, the first one included.
  const paid = effect.state === 'failed';
  const grantId = paid && opts.runId ? embeddingGrantId(opts.runId, effect.id) : null;
  if (grantId && grants.includes(grantId)) return { ...base, reason: 'grant_replayed' };
  if (opts.dryRun) return { ...base, paid, ...(exhausted ? { reason: 'grant_new_retry_cycle' } : {}) };
  if (paid && !grantId) {
    throw opError('invalid_params', 'An embedding retry grant needs a repair run id.',
      'Paid embedding retries run only through gbrain repair embedding-effects, which supplies the run id. Preview it with gbrain repair embedding-effects --json and apply it after the user approves the cost.');
  }
  const now = new Date().toISOString();
  const data = { ...effect.data, repair_requeued_at: now,
    ...(effect.state === 'failed' ? { embedding_attempt_base: effect.attempts, embedding_retry_base: effect.attempts } : {}),
    ...(grantId ? { repair_authorizations: [...grants, grantId] } : {}) };
  const rows = await tx.executeRaw(`UPDATE persistence_effects SET state='queued', data=$3::text::jsonb, error_code=NULL, claim_expires_at=NULL,claim_owner=NULL,
      next_attempt_at=now(), outcome=NULL, updated_at=now()
    WHERE id=$1 AND state=$2 AND attempts=$4 AND execution_token IS NULL AND recovery IS NULL RETURNING id`,
  [effect.id, effect.state, JSON.stringify(data), effect.attempts]);
  if (!rows.length) return { outcome: 'changed_since_preview' };
  return { ...base, paid, ...(exhausted ? { reason: 'granted_new_retry_cycle' } : {}) };
}

/** The retry budget one grant authorizes, for the `--max-usd` estimate. */
export const EMBEDDING_RETRY_BUDGET_ATTEMPTS = MAX_RATE_LIMIT_RETRIES;