Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.0 KB
Raw Blame
/**
 * #6803 (1): an embed-backfill whose only remaining work sits on pages it
 * cannot project dead-lettered every ~15 minutes forever (23 dead jobs a day
 * on a production brain, nothing embedded), because every automatic submitter
 * queued the next run as soon as the 10-minute cooldown passed.
 *
 * Real Postgres job queue and worker: each submitted job runs the real
 * handler (only the stale drain is stubbed to report the blocked shape) and
 * fails through the worker's retry/dead-letter path; the submit gate reads
 * the resulting rows. Identical no-progress failures back off, stop at the
 * limit, and a changed blocker or real progress re-arms the backfill.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { getEngine, hasDatabase, setupDB, teardownDB } from './helpers.ts';
import { MinionQueue } from '../../src/core/minions/queue.ts';
import { MinionWorker } from '../../src/core/minions/worker.ts';
import { makeEmbedBackfillHandler } from '../../src/core/minions/handlers/embed-backfill.ts';
import { submitEmbedBackfill, type SubmitEmbedBackfillResult } from '../../src/core/embed-backfill-submit.ts';
import { EMBED_BACKFILL_STALL_LIMIT } from '../../src/core/embed-backfill-stall.ts';
import { embedBackfillStalledCheck } from '../../src/commands/doctor/checks/embed-backfill-stalled.ts';
import type { EmbedStaleResult } from '../../src/core/embed-stale.ts';

const describeE2E = hasDatabase() ? describe : describe.skip;
const SOURCE = 'stall-example';
const MIN = 60_000;

let embedded = 0;
const runStale = async (): Promise<EmbedStaleResult> => ({ embedded, chunksProcessed: embedded, pagesProcessed: embedded ? 1 : 0, invalidated: 0,
  lastCursor: null, done: true, aborted: false, remaining: 2, blocked: 1, complete: false });

/** A page with an unembedded chunk whose text projection is behind its revision: what the backfill's readiness pass reports blocked. */
async function blockPage(slug: string) {
  const engine = getEngine();
  await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `Body of ${slug}.` }, { sourceId: SOURCE });
  const [page] = await engine.executeRaw<{ id: number }>('SELECT id FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, slug]);
  const [chunks] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM content_chunks WHERE page_id=$1', [page.id]);
  if (chunks.n === 0) {
    await engine.executeRaw(`INSERT INTO content_chunks (page_id, chunk_index, chunk_text, chunk_source) VALUES ($1, 0, $2, 'compiled_truth')`, [page.id, `Body of ${slug}.`]);
  }
  await engine.executeRaw('UPDATE pages SET text_projection_revision=NULL WHERE id=$1', [page.id]);
}

/** Submit through the gate, then run the job to its terminal state on a real worker. */
async function submitAndRun(nowMs: number): Promise<SubmitEmbedBackfillResult> {
  const engine = getEngine();
  const result = await submitEmbedBackfill(engine, SOURCE, { reason: 'test', cooldownMinOverride: 10, spendCapUsdOverride: 1_000_000, nowMs });
  if (result.status !== 'submitted') return result;
  await engine.executeRaw('UPDATE minion_jobs SET backoff_delay=10 WHERE id=$1', [result.jobId]);
  const queue = new MinionQueue(engine);
  const worker = new MinionWorker(engine, { pollInterval: 25 });
  worker.register('embed-backfill', makeEmbedBackfillHandler(engine, { runStale }));
  const running = worker.start();
  try {
    for (let i = 0; i < 600; i++) {
      const status = (await queue.getJob(result.jobId))?.status;
      if (status === 'dead' || status === 'failed' || status === 'completed') break;
      await new Promise(r => setTimeout(r, 25));
    }
  } finally {
    worker.stop();
    await running;
  }
  return result;
}

async function lastJob() {
  const [row] = await getEngine().executeRaw<{ status: string; attempts_made: number; error_text: string | null; finished_at: Date }>(
    "SELECT status, attempts_made, error_text, finished_at FROM minion_jobs WHERE name='embed-backfill' ORDER BY id DESC LIMIT 1");
  return row;
}

describeE2E('E2E: embed-backfill stall cap (#6803)', () => {
  beforeAll(async () => {
    await setupDB();
    await getEngine().executeRaw("INSERT INTO sources (id, name, config) VALUES ($1, $1, '{}'::jsonb) ON CONFLICT (id) DO NOTHING", [SOURCE]);
    await blockPage('notes/blocked-a');
  });
  afterAll(async () => { await teardownDB(); });

  test('identical no-progress failures back off, stop at the limit, and re-arm when the blocker changes', async () => {
    embedded = 0;
    // Run 1: the job fails, its retry sees the same blockers and dead-letters instead of failing a third time.
    expect((await submitAndRun(Date.now())).status).toBe('submitted');
    const first = await lastJob();
    expect(first.status).toBe('dead');
    expect(first.attempts_made).toBe(2);
    expect(first.error_text).toStartWith('embed-backfill: incomplete; 2 chunks remain and 1 projections are blocked.');
    expect(first.error_text).toContain('No progress since the previous attempt');

    // The plain 10-minute cooldown would admit a run at +11 min; one identical failure doubles it.
    let finished = new Date(first.finished_at).getTime();
    const held = await submitAndRun(finished + 11 * MIN);
    expect(held).toMatchObject({ status: 'cooldown', reason: 'recently_finished' });
    expect((await submitAndRun(finished + 21 * MIN)).status).toBe('submitted');
    finished = new Date((await lastJob()).finished_at).getTime();
    // Two identical failures: 40 minutes.
    expect((await submitAndRun(finished + 39 * MIN)).status).toBe('cooldown');
    expect((await submitAndRun(finished + 41 * MIN)).status).toBe('submitted');
    finished = new Date((await lastJob()).finished_at).getTime();

    // At the limit, automatic submission stops however long it waits.
    const stalled = await submitAndRun(finished + 7 * 24 * 60 * MIN);
    expect(stalled).toMatchObject({ status: 'stalled', failures: EMBED_BACKFILL_STALL_LIMIT, remaining: 1, blocked: 1 });
    const jobs = await getEngine().executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM minion_jobs WHERE name='embed-backfill'");
    expect(jobs[0].n).toBe(EMBED_BACKFILL_STALL_LIMIT);

    // doctor names the blocked page and the preview-first repair.
    const check = await embedBackfillStalledCheck(getEngine());
    expect(check.status).toBe('warn');
    expect(check.message).toContain('notes/blocked-a');
    expect(check.fix).toMatchObject({ argv: ['gbrain', 'projections', 'drain'], preview_argv: ['gbrain', 'projections', 'drain', '--dry-run', '--json'] });

    // A changed blocker re-arms the backfill.
    await blockPage('notes/blocked-b');
    expect((await submitAndRun(finished + 7 * 24 * 60 * MIN)).status).toBe('submitted');

    // Real progress ends the streak: the next identical failure starts over at the plain backoff.
    embedded = 1;
    expect((await submitAndRun(Date.now() + 7 * 24 * 60 * MIN)).status).toBe('submitted');
    embedded = 0;
    finished = new Date((await lastJob()).finished_at).getTime();
    expect((await submitAndRun(finished + 11 * MIN)).status).toBe('submitted');
  }, 120_000);
});