Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
82.3 KB
Raw Blame
/**
 * google-source reconcile + resilience sweeps — sibling of
 * test/google-source-materialize.test.ts (same real-PGLite + mutable
 * fake-Google + in-memory-vault harness), covering the paths that file does
 * not: `--full` reconcile deletes (incl. the >200 mass-delete guard and its
 * GBRAIN_ALLOW_MASS_RECONCILE escape hatch), the backfill floor freeze on a
 * mid-backfill failure, 404-vanished threads in both lanes, calendar +
 * contacts syncToken 410 recovery, and the loops_extract enqueue cap.
 *
 * NOTE on funnel/heartbeat rows (coverage item 1e): runGoogleSync writes NO
 * heartbeat.jsonl rows — the funnel (`appendGoogleHeartbeat`) belongs to the
 * COMMANDS layer (gbrain google connect), asserted in
 * test/google-connect-cmd.serial.test.ts. The sweep-side observable here is
 * the enqueue-cap drop log on stderr, asserted below.
 *
 * Synthetic data only: example.com addresses, hex message/thread ids.
 */
import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import type { SyncOpts } from '../src/commands/sync.ts';
import type {
  CredentialEntry,
  CredentialMeta,
  CredentialVault,
  ProviderClientRecord,
} from '../src/core/creds/vault.ts';
import type { FetchImpl } from '../src/core/google/google-clients.ts';
import { __clearSuppressionCacheForTests } from '../src/core/google/loop-detect.ts';
import { __setChatTransportForTests, configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
import {
  LOOPS_EXTRACT_ENQUEUE_CEILING,
  LOOPS_EXTRACT_MAX_PER_SWEEP,
} from '../src/core/google/loops-extract.ts';
import {
  googleStateFile,
  parseGoogleSourceConfig,
  readGoogleState,
  runGoogleSync,
} from '../src/core/google/google-source.ts';

let engine: PGLiteEngine;
let schemaVersion: string;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  schemaVersion = (await engine.getConfig('version')) ?? '7';
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
  // MinionQueue.ensureSchema gates on the 'version' config row (same re-seed
  // as the sibling materialize suite).
  await engine.setConfig('version', schemaVersion);
  __clearSuppressionCacheForTests();
});

// ── Time helpers (exact-second timestamps so backfill floors are stable) ────

const NOW_MS = Math.floor(Date.now() / 1000) * 1000;
const daysAgoMs = (n: number): number => NOW_MS - n * 86_400_000;
const hoursAgoMs = (n: number): number => NOW_MS - n * 3_600_000;

// ── Thread / message id constants (hex only) ────────────────────────────────

const T_A = '17aa00000000a001';
const T_B = '17aa00000000b002';
const T_V = '17aa00000000e005'; // the vanished (404) thread

// ── Fake Google API behind fetchImpl ─────────────────────────────────────────

interface FakeMessage {
  id: string;
  threadId: string;
  internalDateMs: number;
  headers: Record<string, string>;
  labelIds: string[];
  body: string;
}

interface FakeGoogle {
  profileHistoryId: string;
  messages: FakeMessage[];
  messagesPageSize: number;
  history: string[][];
  historyResponseId: string;
  failThreads: Set<string>;
  /** getThread → 404 for these ids (deleted between listing and fetch). */
  vanishedThreads: Set<string>;
  contacts: unknown[];
  contactsDelta: unknown[];
  contactsExpireSyncToken: boolean;
  contactsExpireSyncStatus?: number;
  calendarEvents: unknown[];
  calendarDelta: unknown[];
  calendarExpireSyncToken: boolean;
  /**
   * Sees every timeMin/timeMax (non-delta) calendar list before it is served;
   * returning 'fail' answers that list with HTTP 400.
   */
  calendarRangeHook?: (query: URLSearchParams) => 'fail' | void;
  /** Counters so each windowed/full re-list mints a DISTINCT sync token. */
  calendarWindowedLists: number;
  contactsFullLists: number;
  calls: string[];
  threadFetches: number;
  /** #5445: the account's label catalog (`users/me/labels`); `'fail'` answers the list with HTTP 500. */
  labels: Array<{ id: string; name: string; type: string }> | 'fail';
}

function emptyFx(): FakeGoogle {
  return {
    profileHistoryId: '1000',
    messages: [],
    messagesPageSize: 100,
    history: [],
    historyResponseId: '1000',
    failThreads: new Set(),
    vanishedThreads: new Set(),
    contacts: [],
    contactsDelta: [],
    contactsExpireSyncToken: false,
    calendarEvents: [],
    calendarDelta: [],
    calendarExpireSyncToken: false,
    calendarWindowedLists: 0,
    contactsFullLists: 0,
    calls: [],
    threadFetches: 0,
    labels: [],
  };
}

function b64url(s: string): string {
  return Buffer.from(s, 'utf-8').toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
}

function buildFetch(fx: FakeGoogle): FetchImpl {
  const json = (body: unknown, status = 200): Response =>
    new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });

  return async (url: string): Promise<Response> => {
    const u = new URL(url);
    fx.calls.push(u.pathname + (u.search ? u.search : ''));

    if (u.hostname === 'oauth2.googleapis.com') {
      return json({ access_token: 't2', expires_in: 3600 });
    }

    if (u.pathname.endsWith('/users/me/profile')) {
      return json({ emailAddress: '[email protected]', historyId: fx.profileHistoryId });
    }

    if (u.pathname.endsWith('/users/me/messages')) {
      const q = u.searchParams.get('q') ?? '';
      const after = /after:(\d+)/.exec(q);
      const before = /before:(\d+)/.exec(q);
      let msgs = [...fx.messages];
      if (after) msgs = msgs.filter((m) => Math.floor(m.internalDateMs / 1000) >= Number(after[1]));
      if (before) msgs = msgs.filter((m) => Math.floor(m.internalDateMs / 1000) < Number(before[1]));
      msgs.sort((a, b) => b.internalDateMs - a.internalDateMs); // newest first
      const start = Number(u.searchParams.get('pageToken') ?? '0');
      const page = msgs.slice(start, start + fx.messagesPageSize);
      const next = start + fx.messagesPageSize < msgs.length ? String(start + fx.messagesPageSize) : undefined;
      return json({
        messages: page.map((m) => ({ id: m.id, threadId: m.threadId })),
        ...(next ? { nextPageToken: next } : {}),
      });
    }

    if (u.pathname.endsWith('/users/me/labels')) {
      if (fx.labels === 'fail') return json({ error: { code: 500, message: 'backend error' } }, 500);
      return json({ labels: fx.labels });
    }

    if (u.pathname.endsWith('/users/me/history')) {
      return json({
        historyId: fx.historyResponseId,
        history: fx.history.map((tids) => ({ messages: tids.map((tid) => ({ threadId: tid })) })),
      });
    }

    const threadMatch = u.pathname.match(/\/users\/me\/threads\/([^/]+)$/);
    if (threadMatch) {
      const tid = threadMatch[1];
      fx.threadFetches++;
      if (fx.vanishedThreads.has(tid)) return json({ error: { code: 404, message: 'Not Found' } }, 404);
      if (fx.failThreads.has(tid)) return json({ error: { code: 500, message: 'backend error' } }, 500);
      const msgs = fx.messages
        .filter((m) => m.threadId === tid)
        .sort((a, b) => b.internalDateMs - a.internalDateMs);
      return json({
        id: tid,
        messages: msgs.map((m) => ({
          id: m.id,
          threadId: tid,
          labelIds: m.labelIds,
          internalDate: String(m.internalDateMs),
          payload: {
            mimeType: 'multipart/alternative',
            headers: Object.entries(m.headers).map(([name, value]) => ({ name, value })),
            parts: [{ mimeType: 'text/plain', body: { data: b64url(m.body) } }],
          },
        })),
      });
    }

    if (/\/calendars\/[^/]+\/events$/.test(u.pathname)) {
      if (u.searchParams.get('syncToken')) {
        if (fx.calendarExpireSyncToken) return json({ error: { code: 410, message: 'Sync token expired' } }, 410);
        return json({ items: fx.calendarDelta, nextSyncToken: 'cal-sync-delta' });
      }
      if (fx.calendarRangeHook?.(u.searchParams) === 'fail') return json({ error: { code: 400, message: 'Bad Request' } }, 400);
      fx.calendarWindowedLists++;
      return json({ items: fx.calendarEvents, nextSyncToken: `cal-sync-w${fx.calendarWindowedLists}` });
    }

    if (u.pathname.includes('/people/me/connections')) {
      if (u.searchParams.get('syncToken')) {
        if (fx.contactsExpireSyncToken) {
          const status = fx.contactsExpireSyncStatus ?? 410;
          return json({ error: { code: status, message: 'Sync token expired' } }, status);
        }
        return json({ connections: fx.contactsDelta, nextSyncToken: 'ppl-sync-delta' });
      }
      fx.contactsFullLists++;
      return json({ connections: fx.contacts, nextSyncToken: `ppl-sync-w${fx.contactsFullLists}` });
    }

    return json({ error: { message: `unhandled ${u.pathname}` } }, 400);
  };
}

// ── In-memory vault ──────────────────────────────────────────────────────────

class FakeVault implements CredentialVault {
  entries = new Map<string, CredentialEntry>();
  clients = new Map<string, ProviderClientRecord>();
  async get(id: string): Promise<CredentialEntry | null> {
    return this.entries.get(id) ?? null;
  }
  async put(entry: CredentialEntry): Promise<void> {
    this.entries.set(entry.id, entry);
  }
  async list(): Promise<CredentialMeta[]> {
    return [];
  }
  async delete(id: string): Promise<boolean> {
    return this.entries.delete(id);
  }
  async getClient(provider: string): Promise<ProviderClientRecord | null> {
    return this.clients.get(provider) ?? null;
  }
  async putClient(rec: ProviderClientRecord): Promise<void> {
    this.clients.set(rec.provider, rec);
  }
  async deleteClient(provider: string): Promise<boolean> {
    return this.clients.delete(provider);
  }
}

const CLIENT_ID = '123-abc.apps.googleusercontent.com';

function makeVault(): FakeVault {
  const v = new FakeVault();
  v.entries.set('google:[email protected]', {
    id: 'google:[email protected]',
    provider: 'google',
    kind: 'oauth2',
    client_ref: 'byo',
    secret: {
      access_token: 't',
      refresh_token: 'r',
      expiry: new Date(Date.now() + 3_600_000).toISOString(),
    },
    meta: {
      account: '[email protected]',
      connected_at: new Date().toISOString(),
      client_id: CLIENT_ID,
    },
  });
  v.clients.set('google', {
    provider: 'google',
    client_id: CLIENT_ID,
    client_secret: 'GOCSPX-test-secret-0000',
    created_at: new Date().toISOString(),
  });
  return v;
}

// ── Shared setup helpers ─────────────────────────────────────────────────────

async function insertGoogleSource(dir: string): Promise<void> {
  await engine.executeRaw(
    `INSERT INTO sources (id, name, local_path, config) VALUES ($1, $2, $3, $4::text::jsonb)`,
    [
      'gsrc',
      'google',
      dir,
      JSON.stringify({
        kind: 'google',
        g_account: '[email protected]',
        g_services: 'gmail,calendar,contacts',
        g_history_days: 90,
        g_dir: dir,
      }),
    ],
  );
}

async function sweep(
  dir: string,
  fx: FakeGoogle,
  vault: FakeVault,
  opts: Partial<SyncOpts> = {},
  services = 'gmail',
  extra: { cfg?: Record<string, unknown>; engine?: PGLiteEngine } = {},
) {
  const cfg = parseGoogleSourceConfig(
    { kind: 'google', g_account: '[email protected]', g_services: services, g_history_days: 90, g_dir: dir, ...extra.cfg },
    dir,
  );
  return runGoogleSync(
    extra.engine ?? engine,
    'gsrc',
    cfg,
    { sourceId: 'gsrc', noEmbed: true, noExtract: true, ...opts },
    buildFetch(fx),
    vault,
  );
}

async function emailSlugs(): Promise<string[]> {
  const rows = await engine.executeRaw<{ slug: string }>(
    `SELECT slug FROM pages WHERE source_id = 'gsrc' AND deleted_at IS NULL AND slug LIKE 'emails/%' ORDER BY slug`,
  );
  return rows.map((r) => r.slug);
}

function gmsg(id: string, threadId: string, ms: number, over: Partial<FakeMessage> = {}): FakeMessage {
  return {
    id,
    threadId,
    internalDateMs: ms,
    headers: { From: 'Charlie Example <[email protected]>', To: '[email protected]', Subject: 'Quarterly zephyr roadmap' },
    labelIds: [],
    body: 'Sharing the zephyr roadmap draft for review.',
    ...over,
  };
}

async function withHome<T>(fn: () => Promise<T>): Promise<T> {
  return withEnv({ GBRAIN_HOME: mkdtempSync(join(tmpdir(), 'gbrain-home-')) }, fn);
}

/** Capture stderr for the duration of `fn` (progress + sweep logs land there). */
async function capturedStderr<T>(fn: () => Promise<T>): Promise<{ result: T; err: string }> {
  const errOrig = process.stderr.write.bind(process.stderr);
  const chunks: string[] = [];
  process.stderr.write = ((chunk: string | Uint8Array): boolean => {
    chunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf-8'));
    return true;
  }) as typeof process.stderr.write;
  try {
    const result = await fn();
    return { result, err: chunks.join('') };
  } finally {
    process.stderr.write = errOrig;
  }
}

/** Pre-mark the gmail backfill as complete so a sweep goes straight to delta. */
function writeBackfilledState(dir: string): void {
  writeFileSync(
    googleStateFile(dir),
    JSON.stringify({
      gmail_history_id: '1000',
      gmail_backfill_floor_ms: null,
      gmail_backfill_done: true,
      gmail_newest_ms: daysAgoMs(1),
      calendar_sync_token: null,
      contacts_sync_token: null,
      last_full_at: null,
    }),
    'utf-8',
  );
}

/** Seed N pending loops_extract jobs carrying the real payload shape for one source. */
async function seedWaitingLoopsJobs(
  sourceId: string,
  n: number,
  keyPrefix: string,
  status: 'waiting' | 'delayed' | 'active' = 'waiting',
): Promise<void> {
  await engine.executeRaw(
    `INSERT INTO minion_jobs (submission_authority, name, queue, status, data, idempotency_key, delay_until)
     SELECT '{"version":1,"kind":"application"}'::jsonb, 'loops_extract', 'default', $4::text, jsonb_build_object('sourceId', $2::text), $3::text || '-' || i,
            CASE WHEN $4::text = 'delayed' THEN now() + interval '10 minutes' END
       FROM generate_series(1, $1) AS i`,
    [n, sourceId, keyPrefix, status],
  );
}

// ── --full reconcile ─────────────────────────────────────────────────────────

describe('gmail --full reconcile', () => {
  test('a thread missing from the live listing loses its page + file; out-of-window pages survive', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-reconcile-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.messages.push(
      gmsg('18c2f4a9b3d21e01', T_A, daysAgoMs(2)),
      gmsg('18c2f4a9b3d21e03', T_B, daysAgoMs(3), {
        headers: { From: 'Dana Example <[email protected]>', To: '[email protected]', Subject: 'Contract question' },
        body: 'Could you confirm the contract terms?',
      }),
    );
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res1 = await sweep(dir, fx, vault);
        expect(res1.added).toBe(2);
        const before = await emailSlugs();
        expect(before).toHaveLength(2);
        const bSlug = before.find((s) => s.includes('contract-question'))!;
        const aSlug = before.find((s) => s.includes('quarterly-zephyr-roadmap'))!;
        expect(existsSync(join(dir, `${bSlug}.md`))).toBe(true);

        // A page whose first_message_date PREDATES the enumeration window is
        // out of reconcile scope and must survive even though its thread id
        // is not in the live listing.
        await engine.putPage(
          'emails/2020/01/2020-01-05-ancient-thread-deadbeef',
          {
            type: 'email',
            title: 'Ancient thread',
            compiled_truth: 'A thread from before the window.',
            frontmatter: {
              thread_id: '17aa00000000ffff',
              first_message_date: new Date(daysAgoMs(100)).toISOString(),
            },
          },
          { sourceId: 'gsrc' },
        );

        // Thread B vanishes upstream (deleted/trash); the live window now
        // only lists thread A.
        fx.messages = fx.messages.filter((m) => m.threadId !== T_B);
        const res2 = await sweep(dir, fx, vault, { full: true });
        expect(res2.status).toBe('synced');
        expect(res2.deleted).toBe(1);

        const after = await emailSlugs();
        expect(after).toContain(aSlug);
        expect(after).toContain('emails/2020/01/2020-01-05-ancient-thread-deadbeef'); // survives
        expect(after).not.toContain(bSlug);
        expect(existsSync(join(dir, `${bSlug}.md`))).toBe(false); // file removed too
        expect(existsSync(join(dir, `${aSlug}.md`))).toBe(true);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('>200 stale threads: the mass-delete guard refuses; GBRAIN_ALLOW_MASS_RECONCILE=1 proceeds', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-massguard-'));
    const fx = emptyFx(); // live listing is EMPTY → every seeded page is stale
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      writeBackfilledState(dir); // skip the backfill; go straight to delta + reconcile
      // 201 email pages inside the window, thread ids the live listing lacks.
      for (let i = 0; i < 201; i++) {
        const tid = `17dd0000${i.toString(16).padStart(8, '0')}`;
        await engine.putPage(
          `emails/2026/bulk/thread-${String(i).padStart(3, '0')}`,
          {
            type: 'email',
            title: `Bulk thread ${i}`,
            compiled_truth: 'Bulk body.',
            frontmatter: {
              thread_id: tid,
              first_message_date: new Date(daysAgoMs(5)).toISOString(),
            },
          },
          { sourceId: 'gsrc' },
        );
      }
      await withHome(async () => {
        // Without the escape hatch: zero deletions + the guard log.
        const { result: res1, err } = await capturedStderr(() => sweep(dir, fx, vault, { full: true }));
        expect(res1.deleted).toBe(0);
        expect(err).toContain('mass-delete guard refused 201 deletes');
        expect(await emailSlugs()).toHaveLength(201);

        // With it: the deletions proceed.
        await withEnv({ GBRAIN_ALLOW_MASS_RECONCILE: '1' }, async () => {
          const res2 = await sweep(dir, fx, vault, { full: true });
          expect(res2.deleted).toBe(201);
        });
        expect(await emailSlugs()).toHaveLength(0);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ── Backfill floor freeze ────────────────────────────────────────────────────

describe('backfill floor freeze on failure', () => {
  test('a failed thread freezes the floor at the last good batch; the retry completes', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-freeze-'));
    const fx = emptyFx();
    const vault = makeVault();
    // 26 single-message threads, days 40..65 ago: batch 1 = the 25 newest
    // (days 40..64), batch 2 = the day-65 thread alone (BACKFILL_BATCH=25).
    const tids: string[] = [];
    for (let i = 0; i < 26; i++) {
      const tid = `17cc00000000${(0xa00 + i).toString(16)}`;
      tids.push(tid);
      fx.messages.push(
        gmsg(`18dd00000000${(0xb00 + i).toString(16)}`, tid, daysAgoMs(40 + i), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Archive topic ${i}` },
          body: `Archive body ${i}.`,
        }),
      );
    }
    const failedTid = tids[25]; // the day-65 thread — batch 2
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        fx.failThreads.add(failedTid);
        const res1 = await sweep(dir, fx, vault);
        expect(res1.status).toBe('partial');
        expect(res1.failedFiles).toBe(1);
        expect(res1.added).toBe(25);

        let state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(false);
        // The floor froze at batch 1's oldest thread (day 64) — it did NOT
        // advance past the failed thread's batch. A floor below day 65 would
        // orphan the failed thread outside every future resume window.
        expect(state.gmail_backfill_floor_ms).toBe(daysAgoMs(64));
        expect(state.gmail_history_id).toBe('1000'); // delta anchor intact

        // Failure clears → the resume window (before:floor) retries the
        // failed thread and the backfill completes.
        fx.failThreads.clear();
        const res2 = await sweep(dir, fx, vault);
        expect(res2.status).toBe('synced');
        expect(res2.added).toBe(1); // just the previously-failed thread
        state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(true);
        expect(state.gmail_backfill_floor_ms).toBeNull();
        const slugs = await emailSlugs();
        expect(slugs).toHaveLength(26);
        expect(slugs.some((s) => s.includes('archive-topic-25'))).toBe(true); // the failed thread's page
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ── 404-vanished threads ─────────────────────────────────────────────────────

describe('404-vanished threads', () => {
  test('delta lane: a 404 thread is skipped, the sync is NOT partial, and the cursor advances', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-vanish-delta-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.messages.push(gmsg('18c2f4a9b3d21e01', T_A, daysAgoMs(2)));
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault);
        expect(readGoogleState(dir).gmail_history_id).toBe('1000');

        // History flags a thread that was deleted before we could fetch it.
        fx.history = [[T_V]];
        fx.historyResponseId = '1010';
        fx.vanishedThreads.add(T_V);
        const res = await sweep(dir, fx, vault);
        expect(res.status).toBe('up_to_date'); // skipped ≠ partial
        expect(res.failedFiles).toBeUndefined();
        expect(readGoogleState(dir).gmail_history_id).toBe('1010'); // cursor advanced
        // No page ever materialized for the vanished thread.
        expect((await emailSlugs())).toHaveLength(1);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('backfill lane: a 404 thread is skipped and the backfill still completes', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-vanish-backfill-'));
    const fx = emptyFx();
    const vault = makeVault();
    // One good thread (day 40) + one whose listing entry survives but whose
    // getThread 404s (day 41).
    fx.messages.push(
      gmsg('18c2f4a9b3d21f01', '17cc00000000f001', daysAgoMs(40), {
        headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Good archive thread' },
        body: 'Archive body.',
      }),
      gmsg('18c2f4a9b3d21f02', T_V, daysAgoMs(41), {
        headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Vanished archive thread' },
        body: 'Gone before fetch.',
      }),
    );
    fx.vanishedThreads.add(T_V);
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res = await sweep(dir, fx, vault);
        expect(res.status).toBe('first_sync'); // NOT partial
        expect(res.added).toBe(1);
        expect(res.failedFiles).toBeUndefined();
        const state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(true);
        expect(state.gmail_backfill_floor_ms).toBeNull();
        const slugs = await emailSlugs();
        expect(slugs).toHaveLength(1);
        expect(slugs[0]).toContain('good-archive-thread');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ── syncToken 410 recovery (calendar + contacts) ─────────────────────────────

describe('syncToken 410 recovery', () => {
  test('calendar: an expired syncToken re-lists windowed and stores the fresh token', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-cal410-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.calendarEvents = [
      {
        id: 'evt00000000000001',
        status: 'confirmed',
        summary: 'Zephyr planning sync',
        start: { dateTime: new Date(daysAgoMs(1)).toISOString() },
        end: { dateTime: new Date(daysAgoMs(1) + 3_600_000).toISOString() },
        organizer: { email: '[email protected]' },
        attendees: [{ email: '[email protected]', self: true, responseStatus: 'accepted' }],
      },
    ];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res1 = await sweep(dir, fx, vault, {}, 'calendar');
        expect(res1.added).toBe(1);
        expect(readGoogleState(dir).calendar_sync_token).toBe('cal-sync-w1');

        // The stored token expires upstream: 410 → windowed re-list.
        fx.calendarExpireSyncToken = true;
        const res2 = await sweep(dir, fx, vault, {}, 'calendar');
        expect(res2.status).not.toBe('partial'); // recovery, not failure
        const state = readGoogleState(dir);
        expect(state.calendar_sync_token).toBe('cal-sync-w2'); // fresh token banked
        // The windowed re-list actually ran (second windowed call).
        expect(fx.calendarWindowedLists).toBe(2);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('calendar: a 410 re-list stays on the SECONDARY calendar the source is bound to', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-cal410sec-'));
    const fx = emptyFx();
    const vault = makeVault();
    const secondary = '[email protected]';
    fx.calendarEvents = [
      {
        id: 'evt00000000000002',
        status: 'confirmed',
        summary: 'Family dinner',
        start: { dateTime: new Date(daysAgoMs(1)).toISOString() },
        end: { dateTime: new Date(daysAgoMs(1) + 3_600_000).toISOString() },
        organizer: { email: '[email protected]' },
        attendees: [{ email: '[email protected]', self: true, responseStatus: 'accepted' }],
      },
    ];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const cfg = { g_calendar_id: secondary };
        const res1 = await sweep(dir, fx, vault, {}, 'calendar', { cfg });
        expect(res1.added).toBe(1);
        expect(readGoogleState(dir).calendar_sync_token).toBe('cal-sync-w1');

        fx.calendarExpireSyncToken = true;
        const res2 = await sweep(dir, fx, vault, {}, 'calendar', { cfg });
        expect(res2.status).not.toBe('partial');
        expect(readGoogleState(dir).calendar_sync_token).toBe('cal-sync-w2');
        expect(fx.calendarWindowedLists).toBe(2);

        // Every calendar call — initial window, the 410'd syncToken attempt,
        // and the windowed re-list — addressed the secondary; none fell back
        // to primary.
        const calendarCalls = fx.calls.filter((c) => c.includes('/calendars/'));
        expect(calendarCalls.length).toBe(3);
        for (const c of calendarCalls) {
          expect(c).toContain(`/calendars/${encodeURIComponent(secondary)}/events`);
          expect(c).not.toContain('/calendars/primary/');
        }
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('calendar: the stored sync token is bound to its calendar id — re-pointing g_calendar_id discards it and re-lists windowed', async () => {
    // Ship-review fix: the persisted token was not bound to the calendar it
    // was minted for, so changing g_calendar_id paired the NEW calendar with
    // the OLD token (a foreign delta cursor) instead of a fresh window.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-calrebind-'));
    const fx = emptyFx();
    const vault = makeVault();
    const secondary = '[email protected]';
    fx.calendarEvents = [
      {
        id: 'evt00000000000003',
        status: 'confirmed',
        summary: 'Family dinner',
        start: { dateTime: new Date(daysAgoMs(1)).toISOString() },
        end: { dateTime: new Date(daysAgoMs(1) + 3_600_000).toISOString() },
        organizer: { email: '[email protected]' },
        attendees: [{ email: '[email protected]', self: true, responseStatus: 'accepted' }],
      },
    ];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        // State minted against PRIMARY, then the source is re-pointed.
        writeFileSync(
          googleStateFile(dir),
          JSON.stringify({
            gmail_history_id: null,
            gmail_backfill_floor_ms: null,
            gmail_backfill_done: false,
            gmail_newest_ms: null,
            calendar_sync_token: 'cal-sync-stale-primary',
            calendar_id: 'primary',
            contacts_sync_token: null,
            last_full_at: null,
          }),
          'utf-8',
        );
        const { result: res, err } = await capturedStderr(() =>
          sweep(dir, fx, vault, {}, 'calendar', { cfg: { g_calendar_id: secondary } }),
        );
        expect(res.status).not.toBe('partial');
        expect(res.added).toBe(1);
        // Fresh window: no syncToken on the wire, exactly one windowed list.
        const calendarCalls = fx.calls.filter((c) => c.includes('/calendars/'));
        expect(calendarCalls.length).toBe(1);
        expect(calendarCalls[0]).not.toContain('syncToken=');
        expect(fx.calendarWindowedLists).toBe(1);
        // Token rebound to the calendar it now belongs to; the switch is logged.
        const state = readGoogleState(dir);
        expect(state.calendar_sync_token).toBe('cal-sync-w1');
        expect(state.calendar_id).toBe(secondary);
        expect(err).toContain('[google] calendar changed');
        expect(err).toContain(secondary);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test("calendar: a legacy token with no calendar_id is primary's — primary keeps its delta, a secondary re-lists windowed", async () => {
    const legacyState = {
      gmail_history_id: null,
      gmail_backfill_floor_ms: null,
      gmail_backfill_done: false,
      gmail_newest_ms: null,
      calendar_sync_token: 'cal-sync-legacy',
      contacts_sync_token: null,
      last_full_at: null,
    };
    const dirA = mkdtempSync(join(tmpdir(), 'gsrc-callegacy-a-'));
    const dirB = mkdtempSync(join(tmpdir(), 'gsrc-callegacy-b-'));
    try {
      await withHome(async () => {
        // A: still primary → the legacy token is used as a delta cursor and
        // gets bound to primary on the way out.
        const fxA = emptyFx();
        await insertGoogleSource(dirA);
        writeFileSync(googleStateFile(dirA), JSON.stringify(legacyState), 'utf-8');
        await sweep(dirA, fxA, makeVault(), {}, 'calendar');
        expect(fxA.calls.filter((c) => c.includes('/calendars/'))[0]).toContain('syncToken=cal-sync-legacy');
        // Legacy state has no calendar horizon either: one bounded [now, now + 60d] coverage list, never the window floor.
        expect(fxA.calendarWindowedLists).toBe(1);
        const coverageMin = Date.parse(calendarQueries(fxA)[1].get('timeMin')!);
        expect(Math.abs(coverageMin - Date.now())).toBeLessThan(60_000);
        const stateA = readGoogleState(dirA);
        expect(stateA.calendar_sync_token).toBe('cal-sync-delta');
        expect(stateA.calendar_id).toBe('primary');

        // B: re-pointed at a secondary → the primary token is discarded.
        const fxB = emptyFx();
        await engine.executeRaw(`DELETE FROM sources WHERE id = 'gsrc'`);
        await insertGoogleSource(dirB);
        writeFileSync(googleStateFile(dirB), JSON.stringify(legacyState), 'utf-8');
        const secondary = '[email protected]';
        await sweep(dirB, fxB, makeVault(), {}, 'calendar', { cfg: { g_calendar_id: secondary } });
        expect(fxB.calendarWindowedLists).toBe(1);
        expect(fxB.calls.filter((c) => c.includes('/calendars/'))[0]).not.toContain('syncToken=');
        expect(readGoogleState(dirB).calendar_id).toBe(secondary);
      });
    } finally {
      rmSync(dirA, { recursive: true, force: true });
      rmSync(dirB, { recursive: true, force: true });
    }
  });

  test('contacts: an expired syncToken re-lists in full and stores the fresh token', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-ppl410-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.contacts = [
      {
        resourceName: 'people/c000000001',
        names: [{ displayName: 'Alice Example', metadata: { primary: true } }],
        emailAddresses: [{ value: '[email protected]' }],
      },
    ];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res1 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res1.added).toBe(1);
        expect(readGoogleState(dir).contacts_sync_token).toBe('ppl-sync-w1');

        fx.contactsExpireSyncToken = true;
        const res2 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res2.status).not.toBe('partial');
        expect(readGoogleState(dir).contacts_sync_token).toBe('ppl-sync-w2');
        expect(fx.contactsFullLists).toBe(2);
        // The re-listed contact page is still there (idempotent re-import).
        const people = await engine.executeRaw<{ slug: string }>(
          `SELECT slug FROM pages WHERE source_id = 'gsrc' AND deleted_at IS NULL AND slug = 'people/alice-example'`,
        );
        expect(people).toHaveLength(1);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('contacts (#4845): g_contacts_dir moves contact pages; a page the contact owned under the old dir is retired by contact id', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-ppldir-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.contacts = [{
      resourceName: 'people/c000000001',
      names: [{ displayName: 'Alice Example', metadata: { primary: true } }],
      emailAddresses: [{ value: '[email protected]' }],
    }];
    const slugs = async () => (await engine.executeRaw<{ slug: string }>(
      `SELECT slug FROM pages WHERE source_id = 'gsrc' AND deleted_at IS NULL AND type = 'person' ORDER BY slug`)).map((r) => r.slug);
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'contacts');
        expect(await slugs()).toEqual(['people/alice-example']);
        // The same source now renders contacts under contacts/: the old page goes, the new one lands, one page per contact.
        const moved = await sweep(dir, fx, vault, { full: true }, 'contacts', { cfg: { g_contacts_dir: 'contacts' } });
        expect(moved.added).toBe(1);
        expect(moved.deleted).toBe(1);
        expect(await slugs()).toEqual(['contacts/alice-example']);
        expect(existsSync(join(dir, 'people/alice-example.md'))).toBe(false);
        expect(existsSync(join(dir, 'contacts/alice-example.md'))).toBe(true);
        // A deletion tombstone (resourceName only) still finds the page under the new dir by contact id.
        fx.contactsDelta = [{ resourceName: 'people/c000000001', metadata: { deleted: true } }];
        const gone = await sweep(dir, fx, vault, {}, 'contacts', { cfg: { g_contacts_dir: 'contacts' } });
        expect(gone.deleted).toBe(1);
        expect(await slugs()).toEqual([]);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('contacts: HTTP 400 expiry recovers without restarting Gmail or losing the contact', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-ppl400-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.contacts = [{
      resourceName: 'people/c000000001',
      names: [{ displayName: 'Alice Example', metadata: { primary: true } }],
      emailAddresses: [{ value: '[email protected]' }],
    }];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'contacts');
        expect(readGoogleState(dir).contacts_sync_token).toBe('ppl-sync-w1');
        fx.contactsExpireSyncToken = true;
        fx.contactsExpireSyncStatus = 400;
        const result = await sweep(dir, fx, vault, {}, 'contacts');
        expect(result.status).not.toBe('partial');
        expect(readGoogleState(dir).contacts_sync_token).toBe('ppl-sync-w2');
        expect(fx.contactsFullLists).toBe(2);
        expect(fx.calls.some(call => call.includes('/people/me/connections?') && call.includes('syncToken=ppl-sync-w1'))).toBe(true);
        expect(fx.calls.some(call => call.includes('/gmail/'))).toBe(false);
        const rows = await engine.executeRaw<{ slug: string }>(
          `SELECT slug FROM pages WHERE source_id = 'gsrc' AND deleted_at IS NULL AND slug = 'people/alice-example'`,
        );
        expect(rows).toHaveLength(1);
        fx.contactsExpireSyncToken = false;
        await sweep(dir, fx, vault, {}, 'contacts');
        expect(fx.contactsFullLists).toBe(2);
        expect(readGoogleState(dir).contacts_sync_token).toBe('ppl-sync-delta');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('contacts: HTTP 400 recovery preserves incremental Gmail in an all-services sync', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-ppl400-all-'));
    const fx = emptyFx();
    fx.contactsExpireSyncToken = true;
    fx.contactsExpireSyncStatus = 400;
    fx.historyResponseId = '2000';
    try {
      await insertGoogleSource(dir);
      writeBackfilledState(dir);
      const before = readGoogleState(dir);
      writeFileSync(googleStateFile(dir), JSON.stringify({ ...before, contacts_sync_token: 'expired' }));
      await withHome(async () => {
        const result = await sweep(dir, fx, makeVault(), {}, 'gmail,calendar,contacts');
        expect(result.status).not.toBe('partial');
        expect(fx.contactsFullLists).toBe(1);
        const after = readGoogleState(dir);
        expect(after.contacts_sync_token).toBe('ppl-sync-w1');
        expect(after.gmail_backfill_done).toBe(true);
        expect(after.gmail_backfill_floor_ms).toBe(before.gmail_backfill_floor_ms);
        expect(after.gmail_history_id).toBe('2000');
        expect(fx.calls.filter(call => call.includes('/users/me/history'))).toEqual([
          expect.stringContaining('startHistoryId=1000'),
        ]);
        expect(fx.calls.some(call => call.includes('/users/me/messages') || call.includes('/users/me/profile'))).toBe(false);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ── Calendar window (#5442) ──────────────────────────────────────────────────
// The source keeps pages for events from historyDays back to 60 days ahead.
// Google applies that range only to windowed lists: a syncToken delta has no
// timeMin/timeMax, and with singleEvents=true a touched recurring series comes
// back as every expanded instance. These cases pin the sweep's own filter on
// every path, the catch-up list for instances crossing the leading edge, and
// the --full reconcile scope.
//
// The fake serves fx.calendarEvents for every timeMin/timeMax list and
// fx.calendarDelta for every delta, without filtering, exactly like the
// unfiltered delta Google returns.

const DAY = 86_400_000;

/** A confirmed one-hour meeting `startOffsetDays` from NOW_MS. */
function meeting(id: string, startOffsetDays: number, opts: { title?: string; lengthMs?: number; status?: string } = {}) {
  const startMs = NOW_MS + startOffsetDays * DAY;
  return {
    id,
    status: opts.status ?? 'confirmed',
    summary: opts.title ?? `Zephyr meeting ${id}`,
    start: { dateTime: new Date(startMs).toISOString() },
    end: { dateTime: new Date(startMs + (opts.lengthMs ?? 3_600_000)).toISOString() },
    organizer: { email: '[email protected]' },
    attendees: [{ email: '[email protected]', self: true, responseStatus: 'accepted' }],
  };
}

/** Live calendar pages keyed by event id (pages without one keyed by slug). */
async function calendarPages(): Promise<Map<string, { slug: string; title: string }>> {
  const rows = await engine.executeRaw<{ slug: string; title: string; event_id: string | null }>(
    `SELECT slug, title, frontmatter->>'event_id' AS event_id FROM pages
      WHERE source_id = 'gsrc' AND deleted_at IS NULL AND slug LIKE 'calendar/%'`,
  );
  return new Map(rows.map((r) => [r.event_id ?? r.slug, { slug: r.slug, title: r.title }]));
}

/** Query strings of the calendar list calls the sweep made, in order. */
function calendarQueries(fx: FakeGoogle): URLSearchParams[] {
  return fx.calls.filter((c) => c.includes('/calendars/')).map((c) => new URL(c, 'https://www.googleapis.com').searchParams);
}

function storeHorizon(dir: string, horizonMs: number | null): void {
  writeFileSync(googleStateFile(dir), JSON.stringify({ ...readGoogleState(dir), calendar_horizon_ms: horizonMs }), 'utf-8');
}

const near = (actualMs: number, expectedMs: number, toleranceMs = 120_000): boolean => Math.abs(actualMs - expectedMs) < toleranceMs;

/** A page the sweep would have written for `eventId` (or a hand-made one when null), at slug.md. */
async function seedMeetingPage(slug: string, eventId: string | null, startOffsetDays: number): Promise<void> {
  const startMs = NOW_MS + startOffsetDays * DAY;
  await engine.putPage(
    slug,
    {
      type: 'meeting',
      title: `Seeded ${slug.split('/').pop()}`,
      compiled_truth: 'Seeded for the calendar window tests.',
      frontmatter: {
        ...(eventId === null ? {} : { event_id: eventId }),
        start: new Date(startMs).toISOString(),
        end: new Date(startMs + 3_600_000).toISOString(),
      },
    },
    { sourceId: 'gsrc' },
  );
  await engine.executeRaw(`UPDATE pages SET source_path = slug || '.md' WHERE source_id = 'gsrc' AND slug = $1`, [slug]);
}

/** Runs `body` against a fresh google source directory with an isolated GBRAIN_HOME. */
async function withCalendarSource(body: (h: { dir: string; fx: FakeGoogle; vault: FakeVault }) => Promise<void>): Promise<void> {
  const dir = mkdtempSync(join(tmpdir(), 'gsrc-calwindow-'));
  try {
    await insertGoogleSource(dir);
    await withHome(() => body({ dir, fx: emptyFx(), vault: makeVault() }));
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

describe('calendar window (#5442)', () => {
  test('a delta full of series instances materializes only those inside the window', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('seed_tomorrow', 1)];
      await sweep(dir, fx, vault, {}, 'calendar');

      fx.calendarDelta = [
        meeting('inst_4y_ago', -4 * 365),
        meeting('inst_ended_before_floor', -91),
        // Starts before the 90-day floor but ends after it: inside, as Google's timeMin compares the end.
        meeting('inst_spans_floor', -95, { lengthMs: 10 * DAY }),
        meeting('inst_next_week', 7),
        meeting('inst_day_59', 59),
        meeting('inst_day_61', 61),
        meeting('inst_8y_ahead', 8 * 365),
      ];
      const { result, err } = await capturedStderr(() => sweep(dir, fx, vault, {}, 'calendar'));

      expect(result.status).not.toBe('partial');
      expect(result.added).toBe(3);
      expect([...(await calendarPages()).keys()].sort()).toEqual(['inst_day_59', 'inst_next_week', 'inst_spans_floor', 'seed_tomorrow']);
      expect(err).toContain('[google] calendar: 4 listed event(s) outside the sync window (90 days back, 60 ahead) were not imported');
      expect(readGoogleState(dir).calendar_sync_token).toBe('cal-sync-delta');
    });
  });

  test('negative control: a delta with only in-window changes logs no skips and imports them all', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      fx.calendarDelta = [meeting('only_soon', 2), meeting('only_recent', -2)];
      const { result, err } = await capturedStderr(() => sweep(dir, fx, vault, {}, 'calendar'));
      expect(result.added).toBe(2);
      expect(err).not.toContain('outside the sync window');
    });
  });

  test('a move past the horizon removes the page; a change to aged-out history leaves its page as it was; a cancellation always removes', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('old_offsite', -150, { title: 'Old offsite' }), meeting('old_cancelled', -160), meeting('next_review', 12)];
      expect((await sweep(dir, fx, vault, {}, 'calendar', { cfg: { g_history_days: 365 } })).added).toBe(3);
      const before = await calendarPages();

      fx.calendarDelta = [
        meeting('old_offsite', -150, { title: 'Old offsite (renamed)' }),
        meeting('old_cancelled', -160, { status: 'cancelled' }),
        meeting('next_review', 120),
      ];
      const result = await sweep(dir, fx, vault, {}, 'calendar');

      expect(result.added).toBe(0);
      expect(result.deleted).toBe(2);
      const after = await calendarPages();
      expect([...after.keys()]).toEqual(['old_offsite']);
      expect(after.get('old_offsite')).toEqual(before.get('old_offsite')!);
      expect(existsSync(join(dir, `${before.get('next_review')!.slug}.md`))).toBe(false);
    });
  });

  test('a cancelled skeleton (id and status only) removes its own page and nothing else', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('keep_me', 3), meeting('drop_me', 4)];
      await sweep(dir, fx, vault, {}, 'calendar');
      fx.calendarDelta = [{ id: 'drop_me', status: 'cancelled' }];
      const result = await sweep(dir, fx, vault, {}, 'calendar');
      expect(result.deleted).toBe(1);
      expect([...(await calendarPages()).keys()]).toEqual(['keep_me']);
    });
  });

  test('a lagging horizon gets one catch-up list from the horizon to the ceiling; the delta token stays the cursor', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      expect(calendarQueries(fx)).toHaveLength(1);
      expect(near(readGoogleState(dir).calendar_horizon_ms!, Date.now() + 60 * DAY)).toBe(true);

      const lagging = NOW_MS + 48 * DAY;
      storeHorizon(dir, lagging);
      fx.calls = [];
      fx.calendarEvents = [meeting('crossed_edge', 52, { title: 'Crossed the edge' })];
      const result = await sweep(dir, fx, vault, {}, 'calendar');

      expect(result.added).toBe(1);
      const [delta, catchUp, ...rest] = calendarQueries(fx);
      expect(rest).toHaveLength(0);
      expect(delta.get('syncToken')).toBe('cal-sync-w1');
      expect(catchUp.get('syncToken')).toBeNull();
      expect(catchUp.get('timeMin')).toBe(new Date(lagging).toISOString());
      expect(near(Date.parse(catchUp.get('timeMax')!), Date.now() + 60 * DAY)).toBe(true);
      const state = readGoogleState(dir);
      expect(state.calendar_sync_token).toBe('cal-sync-delta');
      expect(near(state.calendar_horizon_ms!, Date.now() + 60 * DAY)).toBe(true);
    });
  });

  test('a horizon less than a day behind is neither listed nor advanced, so short gaps add up to a catch-up list', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      const ceiling = NOW_MS + 60 * DAY;

      const hoursBehind = ceiling - 20 * 3_600_000;
      storeHorizon(dir, hoursBehind);
      fx.calls = [];
      await sweep(dir, fx, vault, {}, 'calendar');
      expect(calendarQueries(fx)).toHaveLength(1);
      expect(readGoogleState(dir).calendar_horizon_ms).toBe(hoursBehind);

      const dayBehind = ceiling - 26 * 3_600_000;
      storeHorizon(dir, dayBehind);
      fx.calls = [];
      await sweep(dir, fx, vault, {}, 'calendar');
      const queries = calendarQueries(fx);
      expect(queries).toHaveLength(2);
      expect(queries[1].get('timeMin')).toBe(new Date(dayBehind).toISOString());
      expect(readGoogleState(dir).calendar_horizon_ms!).toBeGreaterThan(ceiling - 60_000);
    });
  });

  test('a failed catch-up list marks the sweep partial and keeps the horizon; the next sweep catches up', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      const lagging = NOW_MS + 45 * DAY;
      storeHorizon(dir, lagging);
      fx.calendarEvents = [meeting('late_arrival', 50)];
      fx.calendarRangeHook = () => 'fail';

      const failed = await sweep(dir, fx, vault, {}, 'calendar');
      expect(failed.status).toBe('partial');
      expect(readGoogleState(dir).calendar_horizon_ms).toBe(lagging);
      expect((await calendarPages()).size).toBe(0);

      fx.calendarRangeHook = undefined;
      const retried = await sweep(dir, fx, vault, {}, 'calendar');
      expect(retried.status).not.toBe('partial');
      expect([...(await calendarPages()).keys()]).toEqual(['late_arrival']);
      expect(readGoogleState(dir).calendar_horizon_ms!).toBeGreaterThan(lagging);
    });
  });

  test('an abort during the catch-up list keeps the horizon, and an aborted --full deletes nothing', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      const lagging = NOW_MS + 45 * DAY;
      storeHorizon(dir, lagging);
      fx.calendarEvents = [meeting('never_lands', 50)];
      const stop = new AbortController();
      fx.calendarRangeHook = () => { stop.abort(); };

      const aborted = await sweep(dir, fx, vault, { signal: stop.signal }, 'calendar');
      expect(aborted.status).toBe('partial');
      expect(readGoogleState(dir).calendar_horizon_ms).toBe(lagging);
      expect((await calendarPages()).size).toBe(0);

      await seedMeetingPage('calendar/seeded/unlisted-but-aborted', 'unlisted_evt', 6);
      const stopFull = new AbortController();
      fx.calendarRangeHook = () => { stopFull.abort(); };
      const abortedFull = await sweep(dir, fx, vault, { full: true, signal: stopFull.signal }, 'calendar');
      expect(abortedFull.status).toBe('partial');
      expect([...(await calendarPages()).keys()]).toEqual(['unlisted_evt']);
    });
  });

  test('an expired token re-lists the whole window, which counts as caught up', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      await sweep(dir, fx, vault, {}, 'calendar');
      const lagging = NOW_MS + 30 * DAY;
      storeHorizon(dir, lagging);
      fx.calls = [];
      fx.calendarExpireSyncToken = true;
      await sweep(dir, fx, vault, {}, 'calendar');
      const [expired, relist, ...rest] = calendarQueries(fx);
      expect(rest).toHaveLength(0);
      expect(expired.get('syncToken')).toBe('cal-sync-w1');
      expect(near(Date.parse(relist.get('timeMin')!), Date.now() - 90 * DAY)).toBe(true);
      expect(readGoogleState(dir).calendar_horizon_ms!).toBeGreaterThan(lagging);
    });
  });

  test('the catch-up list reads the secondary calendar the source is bound to', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      const team = '[email protected]';
      await sweep(dir, fx, vault, {}, 'calendar', { cfg: { g_calendar_id: team } });
      storeHorizon(dir, NOW_MS + 40 * DAY);
      fx.calls = [];
      await sweep(dir, fx, vault, {}, 'calendar', { cfg: { g_calendar_id: team } });
      const paths = fx.calls.filter((c) => c.includes('/calendars/')).map((c) => new URL(c, 'https://www.googleapis.com').pathname);
      expect(paths).toEqual([0, 1].map(() => `/calendar/v3/calendars/${encodeURIComponent(team)}/events`));
    });
  });

  test('state saved before horizons existed lists [now, ceiling] once; a failed list leaves it unset for a retry', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      writeFileSync(
        googleStateFile(dir),
        JSON.stringify({ ...readGoogleState(dir), calendar_sync_token: 'cal-sync-legacy', calendar_id: 'primary' }),
        'utf-8',
      );
      fx.calendarEvents = [meeting('unchanged_since_legacy', 35)];
      fx.calendarRangeHook = () => 'fail';
      expect((await sweep(dir, fx, vault, {}, 'calendar')).status).toBe('partial');
      expect(readGoogleState(dir).calendar_horizon_ms ?? null).toBeNull();

      fx.calendarRangeHook = undefined;
      fx.calls = [];
      const result = await sweep(dir, fx, vault, {}, 'calendar');
      expect(result.added).toBe(1);
      const [, coverage] = calendarQueries(fx);
      expect(near(Date.parse(coverage.get('timeMin')!), Date.now())).toBe(true);
      expect(near(Date.parse(coverage.get('timeMax')!), Date.now() + 60 * DAY)).toBe(true);
      expect(near(readGoogleState(dir).calendar_horizon_ms!, Date.now() + 60 * DAY)).toBe(true);
    });
  });

  test('--full deletes in-window pages the list no longer names and keeps every page outside that scope', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('still_listed', 2)];
      await seedMeetingPage('calendar/seeded/upcoming-cancelled', 'gone_upcoming', 8);
      await seedMeetingPage('calendar/seeded/recent-deleted', 'gone_recent', -20);
      await seedMeetingPage('calendar/seeded/before-floor', 'history_evt', -180);
      await seedMeetingPage('calendar/seeded/beyond-horizon', 'far_evt', 9 * 365);
      await seedMeetingPage('calendar/seeded/hand-written', null, 5);

      const result = await sweep(dir, fx, vault, { full: true }, 'calendar');

      expect(result.status).not.toBe('partial');
      // P4.2 (#5442, contract change): a sweep page past the horizon is now reconciled too,
      // so `far_evt` goes with the two unlisted in-window pages; history and hand-written stay.
      expect(result.deleted).toBe(3);
      expect([...(await calendarPages()).keys()].sort()).toEqual(
        ['calendar/seeded/hand-written', 'history_evt', 'still_listed'].sort(),
      );
    });
  });

  test('--full removes sweep pages that start at or past the horizon and keeps pages inside it (#5442)', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('soon', 30), meeting('inside_59', 59)];
      expect((await sweep(dir, fx, vault, {}, 'calendar')).added).toBe(2);
      await seedMeetingPage('calendar/seeded/overflow-400', 'overflow_400', 400);
      await seedMeetingPage('calendar/seeded/overflow-at-ceiling', 'overflow_60', 60);
      await seedMeetingPage('calendar/seeded/hand-far', null, 400);

      const result = await sweep(dir, fx, vault, { full: true }, 'calendar');

      expect(result.status).not.toBe('partial');
      expect(result.deleted).toBe(2);
      expect([...(await calendarPages()).keys()].sort()).toEqual(['calendar/seeded/hand-far', 'inside_59', 'soon'].sort());
    });
  });

  test('--full with g_future_days widens the horizon: pages inside the wider window stay (#5442)', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('soon', 30), meeting('day_120', 120)];
      expect((await sweep(dir, fx, vault, {}, 'calendar', { cfg: { g_future_days: 180 } })).added).toBe(2);
      await seedMeetingPage('calendar/seeded/day-400', 'day_400', 400);

      const result = await sweep(dir, fx, vault, { full: true }, 'calendar', { cfg: { g_future_days: 180 } });

      expect(result.deleted).toBe(1);
      expect([...(await calendarPages()).keys()].sort()).toEqual(['day_120', 'soon']);
      const listed = calendarQueries(fx).find((q) => q.has('timeMax'))!;
      expect(near(Date.parse(listed.get('timeMax')!), NOW_MS + 180 * DAY)).toBe(true);
    });
  });

  test('--full refuses more than 200 past-horizon deletions through the mass-delete guard and reports partial (#5442)', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      for (let i = 0; i < 201; i++) await seedMeetingPage(`calendar/seeded/future-${i}`, `future_${i}`, 365 + i);

      const { result: refused, err } = await capturedStderr(() => sweep(dir, fx, vault, { full: true }, 'calendar'));
      expect(refused.status).toBe('partial');
      expect(refused.deleted).toBe(0);
      expect(err).toContain('mass-delete guard refused 201 deletes for source gsrc');
      expect((await calendarPages()).size).toBe(201);
    });
  });

  test('--full after historyDays shrinks keeps pages from the old, wider window', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [meeting('wide_window_only', -250), meeting('both_windows', -40)];
      expect((await sweep(dir, fx, vault, { full: true }, 'calendar', { cfg: { g_history_days: 365 } })).added).toBe(2);
      fx.calendarEvents = [meeting('both_windows', -40)];
      const result = await sweep(dir, fx, vault, { full: true }, 'calendar');
      expect(result.deleted).toBe(0);
      expect([...(await calendarPages()).keys()].sort()).toEqual(['both_windows', 'wide_window_only']);
    });
  });

  test('--full refuses more than 200 calendar deletions without GBRAIN_ALLOW_MASS_RECONCILE and reports partial', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      for (let i = 0; i < 205; i++) await seedMeetingPage(`calendar/seeded/bulk-${i}`, `bulk_${i}`, 1 + (i % 40));

      const { result: refused, err } = await capturedStderr(() => sweep(dir, fx, vault, { full: true }, 'calendar'));
      expect(refused.status).toBe('partial');
      expect(refused.deleted).toBe(0);
      expect(err).toContain('mass-delete guard refused 205 deletes for source gsrc');
      expect((await calendarPages()).size).toBe(205);

      const allowed = await withEnv({ GBRAIN_ALLOW_MASS_RECONCILE: '1' }, () => sweep(dir, fx, vault, { full: true }, 'calendar'));
      expect(allowed.deleted).toBe(205);
      expect((await calendarPages()).size).toBe(0);
    });
  });
});

// ── loops_extract enqueue completeness ───────────────────────────────────────

describe('loops_extract enqueue completeness', () => {
  // The sweep only enqueues when a chat provider is available (a job the
  // handler can never run would complete as a no-work row and consume the
  // thread's revision slot). The gateway test seam makes chat "available" for
  // every test here; the dedicated test below clears it again.
  const chatStub = async () => {
    throw new Error('chat transport must not be called by the sweep');
  };
  beforeEach(() => __setChatTransportForTests(chatStub));
  afterAll(() => {
    // Shard hygiene (same as facts-extract-junk-filter.test.ts): a reset
    // gateway must not leak a dimensionless config into later files.
    __setChatTransportForTests(null);
    configureGateway({
      embedding_model: 'openai:text-embedding-3-large',
      embedding_dimensions: 1536,
      env: { ...process.env },
    });
  });

  test('chat provider unavailable (keyless install / outage) → 0 jobs enqueued, one stderr line names the reason', async () => {
    // Ship-review fix: a job enqueued with no chat provider ran to
    // `llm_unavailable`, completed, and its revision-keyed idempotency row
    // blocked re-enqueue until the thread changed — every eligible thread
    // swept during the outage was silently never extracted. Gate at enqueue.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-nochat-'));
    const fx = emptyFx();
    const vault = makeVault();
    for (let i = 0; i < 2; i++) {
      const tid = `17ee88000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff88000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Nochat topic ${i}` },
          body: `Nochat body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      resetGateway();
      __setChatTransportForTests(null); // no transport stub, no config → isAvailable('chat') === false
      await withHome(async () => {
        const { result: res, err } = await capturedStderr(() => sweep(dir, fx, vault));
        expect(res.added).toBe(2); // pages still import — only the LLM lane is skipped
        const jobs = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        expect(Number(jobs[0].n)).toBe(0);
        expect(err).toContain('loops_extract: chat provider unavailable');
        expect(err).toContain('2 eligible thread(s)');
        expect(err).not.toContain('loops_extract: enqueued');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  describe('label exclusion (#5445)', () => {
    const seedThreads = (fx: FakeGoogle) => {
      fx.labels = [{ id: 'INBOX', name: 'INBOX', type: 'system' }, { id: 'Label_7', name: 'Newsletters', type: 'user' }];
      fx.messages.push(
        gmsg('18ff55000000a201', '17ee55000000a101', hoursAgoMs(1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Excluded topic' },
          body: 'Could you look at the attached draft?', labelIds: ['INBOX', 'Label_7'],
        }),
        gmsg('18ff55000000a202', '17ee55000000a102', hoursAgoMs(2), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Plain topic' },
          body: 'Could you confirm the plan?', labelIds: ['INBOX'],
        }),
      );
    };
    const jobCount = async () => Number((await engine.executeRaw<{ n: string }>(`SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract'`))[0].n);
    /** The per-reason counts the sweep logs (`loops_extract eligibility: a=1 b=2`). */
    const eligibilityCounts = (err: string): Record<string, number> => {
      const line = /loops_extract eligibility: ([^\n]+)/.exec(err)?.[1] ?? '';
      return Object.fromEntries(line.trim().split(/\s+/).filter(Boolean).map((kv) => { const [k, v] = kv.split('='); return [k, Number(v)]; }));
    };

    test('a name resolves once per sweep: one labels call, the labelled thread is excluded_label and not queued, the plain one is queued', async () => {
      const dir = mkdtempSync(join(tmpdir(), 'gsrc-excl-'));
      const fx = emptyFx();
      const vault = makeVault();
      seedThreads(fx);
      try {
        await insertGoogleSource(dir);
        await withHome(async () => {
          const { result: res, err } = await capturedStderr(() => sweep(dir, fx, vault, {}, 'gmail', { cfg: { g_loops_exclude_labels: 'newsletters' } }));
          expect(res.added).toBe(2);
          expect(fx.calls.filter((c) => c.includes('/users/me/labels'))).toHaveLength(1);
          expect(eligibilityCounts(err)).toEqual({ excluded_label: 1, human_correspondence: 1 });
          expect(res.loops_enqueue).toMatchObject({ enqueued: 1, skipped_reason: null });
          expect(await jobCount()).toBe(1);
          expect(err).not.toContain('excluded_label_unresolved');
          const stored = await engine.executeRaw<{ s: { ids: string[]; unresolved: string[] } }>(
            `SELECT completed_keys->0 AS s FROM op_checkpoints WHERE op = 'loops-exclusion'`);
          expect(stored[0]?.s).toMatchObject({ ids: ['Label_7'], unresolved: [] });
        });
      } finally {
        rmSync(dir, { recursive: true, force: true });
      }
    }, 120_000);

    test('an unknown name fails closed: every thread counts excluded_label_unresolved, nothing is queued, stderr names the label', async () => {
      const dir = mkdtempSync(join(tmpdir(), 'gsrc-excl-unres-'));
      const fx = emptyFx();
      const vault = makeVault();
      seedThreads(fx);
      try {
        await insertGoogleSource(dir);
        await withHome(async () => {
          const { result: res, err } = await capturedStderr(() => sweep(dir, fx, vault, {}, 'gmail', { cfg: { g_loops_exclude_labels: 'Newsletters, Vendor-Noise' } }));
          expect(res.added).toBe(2);
          expect(eligibilityCounts(err)).toEqual({ excluded_label: 1, excluded_label_unresolved: 1 });
          expect(res.loops_enqueue).toMatchObject({ enqueued: 0, skipped_reason: 'excluded_label_unresolved' });
          expect(await jobCount()).toBe(0);
          expect(err).toContain('"Vendor-Noise" is not a label of [email protected]');
          expect(err).toContain('excluded_label_unresolved');
        });
      } finally {
        rmSync(dir, { recursive: true, force: true });
      }
    }, 120_000);

    test('a failed label list resolves ids only: names stay unresolved this sweep and the brain-wide key is the fallback', async () => {
      const dir = mkdtempSync(join(tmpdir(), 'gsrc-excl-fail-'));
      const fx = emptyFx();
      const vault = makeVault();
      seedThreads(fx);
      fx.labels = 'fail';
      try {
        await insertGoogleSource(dir);
        await engine.setConfig('loops.extraction_exclude_labels', 'Label_7');
        await withHome(async () => {
          const { result: byId, err: errById } = await capturedStderr(() => sweep(dir, fx, vault));
          expect(eligibilityCounts(errById)).toEqual({ excluded_label: 1, human_correspondence: 1 });
          expect(byId.loops_enqueue).toMatchObject({ enqueued: 1 });
          await engine.setConfig('loops.extraction_exclude_labels', 'Newsletters');
          const { result: byName, err } = await capturedStderr(() => sweep(dir, fx, vault, { full: true }));
          expect(byName.loops_enqueue).toMatchObject({ enqueued: 0, skipped_reason: 'excluded_label_unresolved' });
          expect(err).toContain("could not read the account's labels");
          // #7073: a failed list never claims the name is not a label of the account.
          expect(err).toContain('"Newsletters" could not be checked against the labels of [email protected] (the label list failed');
          expect(err).not.toContain('is not a label');
        });
      } finally {
        await engine.setConfig('loops.extraction_exclude_labels', '');
        rmSync(dir, { recursive: true, force: true });
      }
    }, 120_000);
  });

  test(`>${LOOPS_EXTRACT_MAX_PER_SWEEP} eligible threads → EVERY one is queued exactly once`, async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-cap-'));
    const fx = emptyFx();
    const vault = makeVault();
    const total = LOOPS_EXTRACT_MAX_PER_SWEEP + 3; // 53 recent threads
    for (let i = 0; i < total; i++) {
      const tid = `17ee00000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff00000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Recent topic ${i}` },
          body: `Recent body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const { result: res, err } = await capturedStderr(() => sweep(dir, fx, vault));
        expect(res.added).toBe(total);
        // The old cap kept only the newest 50 and logged the rest as
        // "deferring … (they re-candidate on next touch)". That was silent
        // loss: a thread only re-candidates when it CHANGES, so an untouched
        // overflow thread was never extracted at all.
        const jobs = await engine.executeRaw<{ data: unknown }>(
          `SELECT data FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        expect(jobs).toHaveLength(total);
        expect(err).toContain(`loops_extract: enqueued ${total} eligible thread(s)`);
        const slugs = jobs
          .map((j) => (typeof j.data === 'string' ? JSON.parse(j.data) : j.data) as { slug: string })
          .map((p) => p.slug);
        // The previously-dropped tail is present, and so is the newest.
        for (const kept of ['recent-topic-50', 'recent-topic-51', 'recent-topic-52', 'recent-topic-0']) {
          expect(slugs.some((s) => s.includes(kept))).toBe(true);
        }
        // Every slug distinct — one job per thread, no duplicates.
        expect(new Set(slugs).size).toBe(total);
        // NOTE (item 1e): runGoogleSync writes NO heartbeat.jsonl rows — the
        // connect funnel is a commands-layer concern; see
        // test/google-connect-cmd.serial.test.ts for those assertions.
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('a repeated sweep re-touching unchanged threads adds no duplicate jobs', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-dup-'));
    const fx = emptyFx();
    const vault = makeVault();
    const total = 5;
    const threadIds: string[] = [];
    for (let i = 0; i < total; i++) {
      const tid = `17ee11000000${(0x100 + i).toString(16)}`;
      threadIds.push(tid);
      fx.messages.push(
        gmsg(`18ff11000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Dup topic ${i}` },
          body: `Dup body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await capturedStderr(() => sweep(dir, fx, vault));
        const after1 = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        // Second sweep RE-TOUCHES every thread via the history delta with
        // unchanged content — the revision-keyed idempotency key is now the
        // ONLY dedupe in play (no maxWaiting), so it must hold on its own.
        fx.history.push([...threadIds]);
        fx.historyResponseId = '2000';
        await capturedStderr(() => sweep(dir, fx, vault));
        const after2 = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        expect(Number(after1[0].n)).toBe(total);
        expect(Number(after2[0].n)).toBe(total);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('a deep waiting backlog shrinks the sweep enqueue budget (defer, not stack)', async () => {
    // The maxWaiting removal left the per-sweep ceiling as the ONLY bound —
    // with a stalled worker, repeated pathological sweeps could stack another
    // 500 waiting jobs each. The sweep now counts already-waiting
    // loops_extract jobs and shrinks this sweep's budget by that depth;
    // overflow is a deferral (a deferred thread re-candidates on next touch).
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-depth-'));
    const fx = emptyFx();
    const vault = makeVault();
    for (let i = 0; i < 2; i++) {
      const tid = `17ee33000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff33000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Depth topic ${i}` },
          body: `Depth body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      // Seed a waiting backlog at the ceiling — the stalled-worker shape.
      // Real loops_extract payloads carry the enqueuing source's id.
      await seedWaitingLoopsJobs('gsrc', LOOPS_EXTRACT_ENQUEUE_CEILING, 'depthseed');
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault));
        const after = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract' AND status = 'waiting'`,
        );
        // Budget exhausted by the backlog: NOTHING new stacks on top.
        expect(Number(after[0].n)).toBe(LOOPS_EXTRACT_ENQUEUE_CEILING);
        expect(err).toContain('deferring 2');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('a DELAYED (retry-backoff) backlog counts against the budget too — a flapping provider cannot stack past the ceiling', async () => {
    // Ship-review fix: the depth probe counted status='waiting' only. During a
    // provider outage every claimed job fails and parks as 'delayed' (backoff),
    // so the probe read ~0 and each sweep stacked another ceiling's worth of
    // jobs on top of the backlog. Pending = waiting + delayed + active.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-delayed-'));
    const fx = emptyFx();
    const vault = makeVault();
    for (let i = 0; i < 2; i++) {
      const tid = `17ee77000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff77000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Delayed topic ${i}` },
          body: `Delayed body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await seedWaitingLoopsJobs('gsrc', LOOPS_EXTRACT_ENQUEUE_CEILING, 'delayedseed', 'delayed');
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault));
        const fresh = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs
            WHERE name = 'loops_extract' AND idempotency_key NOT LIKE 'delayedseed-%'`,
        );
        expect(Number(fresh[0].n)).toBe(0);
        expect(err).toContain('deferring 2');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test("another source's waiting backlog does NOT starve this source's budget (per-source depth)", async () => {
    // Pre-fix the probe counted EVERY waiting loops_extract job brain-wide,
    // so one Google account's stalled backlog drove every other source's
    // budget to 0 forever — nothing new was ever enqueued for them.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-otherdepth-'));
    const fx = emptyFx();
    const vault = makeVault();
    for (let i = 0; i < 2; i++) {
      const tid = `17ee44000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff44000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Other depth topic ${i}` },
          body: `Other depth body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await seedWaitingLoopsJobs('google-other-account', LOOPS_EXTRACT_ENQUEUE_CEILING, 'otherseed');
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault));
        const mine = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs
            WHERE name = 'loops_extract' AND status = 'waiting' AND data->>'sourceId' = 'gsrc'`,
        );
        expect(Number(mine[0].n)).toBe(2);
        expect(err).toContain('loops_extract: enqueued 2 eligible thread(s)');
        expect(err).not.toContain('deferring');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('partial budget keeps the NEWEST thread and logs the deferral count', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-partial-'));
    const fx = emptyFx();
    const vault = makeVault();
    // Three threads, ages 1h/2h/3h — only ONE slot is left in the budget.
    for (let i = 0; i < 3; i++) {
      const tid = `17ee55000000${(0x100 + i).toString(16)}`;
      fx.messages.push(
        gmsg(`18ff55000000${(0x200 + i).toString(16)}`, tid, hoursAgoMs(i + 1), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Partial topic ${i}` },
          body: `Partial body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await seedWaitingLoopsJobs('gsrc', LOOPS_EXTRACT_ENQUEUE_CEILING - 1, 'partialseed');
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault));
        expect(err).toContain('deferring 2');
        expect(err).toContain(`${LOOPS_EXTRACT_ENQUEUE_CEILING - 1} already pending`);
        const fresh = await engine.executeRaw<{ data: unknown }>(
          `SELECT data FROM minion_jobs
            WHERE name = 'loops_extract' AND idempotency_key NOT LIKE 'partialseed-%'`,
        );
        expect(fresh).toHaveLength(1);
        const payload = (typeof fresh[0].data === 'string' ? JSON.parse(fresh[0].data) : fresh[0].data) as { slug: string; sourceId: string };
        expect(payload.slug).toContain('partial-topic-0'); // the 1h-old thread, not 2h/3h
        expect(payload.sourceId).toBe('gsrc');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('a failing waiting-depth probe fails OPEN — the sweep still enqueues', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-probefail-'));
    const fx = emptyFx();
    const vault = makeVault();
    const tid = '17ee66000000a100';
    fx.messages.push(
      gmsg('18ff66000000b200', tid, hoursAgoMs(1), {
        headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Probe topic' },
        body: 'Probe body.',
      }),
    );
    // Only the depth probe (`… AS n FROM minion_jobs …`) throws; queue.add and
    // every other query reach the real engine.
    const probeFailing = new Proxy(engine, {
      get(target, prop) {
        if (prop === 'executeRaw') {
          return (sql: string, params?: unknown[]) => {
            if (/AS n FROM minion_jobs/.test(sql)) throw new Error('probe unavailable');
            return target.executeRaw(sql, params);
          };
        }
        const v = Reflect.get(target, prop, target);
        return typeof v === 'function' ? v.bind(target) : v;
      },
    }) as PGLiteEngine;
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault, {}, 'gmail', { engine: probeFailing }));
        expect(err).toContain('loops_extract: enqueued 1 eligible thread(s)');
        expect(err).not.toContain('enqueue failed');
        const jobs = await engine.executeRaw<{ n: string }>(
          `SELECT count(*)::text AS n FROM minion_jobs WHERE name = 'loops_extract' AND data->>'sourceId' = 'gsrc'`,
        );
        expect(Number(jobs[0].n)).toBe(1);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);

  test('bulk mail the owner never answered is filtered before the queue', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-elig-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.messages.push(
      gmsg('18ff220000000001', '17ee220000000001', hoursAgoMs(2), {
        headers: {
          From: 'Peer Example <[email protected]>',
          To: '[email protected]',
          Subject: 'Real question',
          'List-Unsubscribe': '<https://example.com/u>',
        },
        body: 'Newsletter body.',
      }),
    );
    fx.messages.push(
      gmsg('18ff220000000002', '17ee220000000002', hoursAgoMs(3), {
        headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: 'Human topic' },
        body: 'A real message.',
      }),
    );
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const { err } = await capturedStderr(() => sweep(dir, fx, vault));
        const jobs = await engine.executeRaw<{ data: unknown }>(
          `SELECT data FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        const slugs = jobs
          .map((j) => (typeof j.data === 'string' ? JSON.parse(j.data) : j.data) as { slug: string })
          .map((p) => p.slug);
        expect(slugs.some((s) => s.includes('human-topic'))).toBe(true);
        expect(slugs.some((s) => s.includes('real-question'))).toBe(false);
        // Auditable counts, no mail content: one filtered, one queued.
        expect(err).toContain('loops_extract eligibility:');
        expect(err).toContain('list_mail=1');
        expect(err).toContain('human_correspondence=1');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }, 120_000);
});

// ── Sweep progress (#5349) ──────────────────────────────────────────────────

/** Forward-progress notes recorded while `fn` runs (what the progress-aware sync deadline extends on). */
async function countForwardProgress<T>(fn: () => Promise<T>): Promise<{ result: T; notes: number }> {
  const { onForwardProgress } = await import('../src/core/forward-progress.ts');
  let notes = 0;
  const off = onForwardProgress(() => { notes++; });
  try {
    return { result: await fn(), notes };
  } finally {
    off();
  }
}

function fxContact(n: number): unknown {
  return {
    resourceName: `people/c00000000${n}`,
    names: [{ displayName: `Contact Example ${n}`, metadata: { primary: true } }],
    emailAddresses: [{ value: `contact${n}@example.com` }],
  };
}

describe('sweep progress (#5349)', () => {
  test('a contacts sweep notes forward progress once per imported contact, so the progress-aware deadline extends', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-pplprog-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.contacts = [fxContact(1), fxContact(2), fxContact(3)];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const { result, notes } = await countForwardProgress(() => sweep(dir, fx, vault, {}, 'contacts'));
        expect(result.added).toBe(3);
        expect(notes).toBeGreaterThanOrEqual(3);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('a calendar sweep notes forward progress once per listed event, and a list with nothing to import notes none', async () => {
    await withCalendarSource(async ({ dir, fx, vault }) => {
      fx.calendarEvents = [1, 2, 3, 4].map((n) => meeting(`evt0000000000000${n}`, n));
      const first = await countForwardProgress(() => sweep(dir, fx, vault, {}, 'calendar'));
      expect(first.result.added).toBe(4);
      expect(first.notes).toBeGreaterThanOrEqual(4);
      // Heartbeats are not progress: an empty delta must not extend the deadline.
      fx.calendarDelta = [];
      const second = await countForwardProgress(() => sweep(dir, fx, vault, {}, 'calendar'));
      expect(second.result.status).toBe('up_to_date');
      expect(second.notes).toBe(0);
    });
  });
});