Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.4 KB
Raw Blame
/**
 * #6803 part 2: bounded resident projection selection. On master the
 * resident's selector walked the whole `page_projection_jobs` table in
 * `updated_at` order and probed each row's page, so every tick of every
 * resident probed all 93K undrainable image rows v153 queued. Each tick now
 * looks at most at `RESIDENT_PROBE_BUDGET` rows:
 *
 * 1. a recency lane: rows updated after `high water - 10 minutes` (newest
 *    first, at most `RESIDENT_RECENT_ROWS`), so a fresh edit behind a large
 *    queue, or a row whose writer transaction started before the previous
 *    tick, is reached within two ticks. The mark starts at `max(updated_at)`;
 *    the lane scans the queue table only, never `pages`.
 * 2. a primary-key window over the rest of the budget, from a per-process
 *    cursor that starts at a random key and wraps around, so every row
 *    (a `rebuild_failed` retry after its cooldown, a restored source's rows)
 *    is reached within one lap. The consumer's idle `hasWork` probe walks
 *    the same window and advances the same cursor.
 *
 * A bounded negative cache skips (source incarnation, slug, revision) of a page
 * whose kind the resident never builds (images: their seal is the importer's
 * or the explicit `gbrain projections drain`'s). Archive, missing-source and
 * cooldown states are never cached. The window and the recency lane bound the
 * inline selector only; the segmented route keeps its own probe and lease, and
 * the explicit drain keeps oldest-first.
 */
import { randomUUID } from 'node:crypto';

export const RESIDENT_PROBE_BUDGET = 500;
export const RESIDENT_RECENT_ROWS = 100;
const WINDOW_ROWS = RESIDENT_PROBE_BUDGET - RESIDENT_RECENT_ROWS;
const NEGATIVE_CACHE_MAX = 100_000;

export interface ProjectionScanCursor {
  /** The last primary key the window covered; the next window starts after it. */
  after: { source_incarnation: string; slug: string };
  /** `max(updated_at)` the previous selection saw (text), or null before the first. */
  highWater: string | null;
  /** Keys (incarnation, slug, revision) of pages whose kind the resident never builds. */
  ineligible: Set<string>;
}

export function newProjectionScanCursor(): ProjectionScanCursor {
  return { after: { source_incarnation: randomUUID(), slug: '' }, highWater: null, ineligible: new Set() };
}

const cursors = new WeakMap<object, ProjectionScanCursor>();

/** The per-process cursor of one engine, shared by its resident invocations and its consumer's idle probe. */
export function residentProjectionCursor(engine: object): ProjectionScanCursor {
  let cursor = cursors.get(engine);
  if (!cursor) cursors.set(engine, cursor = newProjectionScanCursor());
  return cursor;
}

export function ineligibleKey(row: { source_incarnation: string; slug: string; revision: string }): string {
  return `${row.source_incarnation}\u0000${row.slug}\u0000${row.revision}`;
}

export function rememberIneligible(cursor: ProjectionScanCursor, row: { source_incarnation: string; slug: string; revision: string }): void {
  if (cursor.ineligible.size >= NEGATIVE_CACHE_MAX) cursor.ineligible.clear();
  cursor.ineligible.add(ineligibleKey(row));
}

/**
 * The CTEs `recent` and `win` over `page_projection_jobs` for one cursor: each
 * row carries `lane` ('recent' or 'window') and `ord` (window order). `filter`
 * is an extra predicate over `j` (internal SQL); parameters are appended to
 * `params`. `high_water` (the queue's current `max(updated_at)`) rides along so
 * the caller can move the mark.
 */
export function residentWindowCtes(cursor: ProjectionScanCursor, params: unknown[], filter = 'TRUE'): string {
  const inc = params.push(cursor.after.source_incarnation);
  const slug = params.push(cursor.after.slug);
  const mark = params.push(cursor.highWater);
  const cols = 'j.source_incarnation,j.slug,j.revision,j.updated_at,j.reason';
  return `mark AS (SELECT COALESCE($${mark}::text::timestamptz,(SELECT max(updated_at) FROM page_projection_jobs)) AS at),
    recent AS (SELECT ${cols},'recent'::text AS lane,0 AS ord FROM page_projection_jobs j, mark
      WHERE j.updated_at > mark.at - interval '10 minutes' AND ${filter} ORDER BY j.updated_at DESC LIMIT ${RESIDENT_RECENT_ROWS}),
    fwd AS (SELECT ${cols} FROM page_projection_jobs j WHERE (j.source_incarnation,j.slug) > ($${inc}::uuid,$${slug}) AND ${filter}
      ORDER BY j.source_incarnation,j.slug LIMIT ${WINDOW_ROWS}),
    wrap AS (SELECT ${cols} FROM page_projection_jobs j WHERE (j.source_incarnation,j.slug) <= ($${inc}::uuid,$${slug}) AND ${filter}
      ORDER BY j.source_incarnation,j.slug LIMIT ${WINDOW_ROWS}),
    win AS (SELECT *,'window'::text AS lane,row_number() OVER (ORDER BY part,source_incarnation,slug) AS ord FROM (
      (SELECT *,0 AS part FROM fwd ORDER BY source_incarnation,slug) UNION ALL (SELECT *,1 AS part FROM wrap ORDER BY source_incarnation,slug)
      ORDER BY part,source_incarnation,slug LIMIT ${WINDOW_ROWS}) w),
    high_water AS (SELECT max(updated_at)::text AS at FROM page_projection_jobs)`;
}

/** Move the cursor past the window's last row a caller has finished with (none: the window was empty). */
export function advanceProjectionCursor(cursor: ProjectionScanCursor, last: { source_incarnation: string; slug: string } | null | undefined, highWater: string | null | undefined): void {
  if (last) cursor.after = { source_incarnation: String(last.source_incarnation), slug: String(last.slug) };
  if (highWater) cursor.highWater = highWater;
}