Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
62.9 KB
Raw Blame
/**
 * google-source materialize — end-to-end sweeps on a real PGLite engine.
 *
 * Mirrors test/github-source-materialize.test.ts: one shared engine
 * (beforeAll + resetPgliteState), a mutable fake Google API behind
 * fetchImpl, temp managed dirs, and an in-memory credential vault passed
 * via runGoogleSync's vaultOverride (no token-refresh HTTP ever fires —
 * the vault entry carries a fresh access token).
 *
 * Synthetic data only: example.com addresses, hex message/thread ids.
 */
import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';

import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import type { SyncOpts } from '../src/commands/sync.ts';
import type {
  CredentialEntry,
  CredentialMeta,
  CredentialVault,
  ProviderClientRecord,
} from '../src/core/creds/vault.ts';
import type { FetchImpl } from '../src/core/google/google-clients.ts';
import { __clearSuppressionCacheForTests } from '../src/core/google/loop-detect.ts';
import { __setChatTransportForTests } from '../src/core/ai/gateway.ts';
import {
  googleStateFile,
  myAddressSet,
  parseGoogleSourceConfig,
  readGoogleState,
  runGoogleSync,
} from '../src/core/google/google-source.ts';
import { importFile } from '../src/core/import-file.ts';

let engine: PGLiteEngine;
let schemaVersion: string;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  schemaVersion = (await engine.getConfig('version')) ?? '7';
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
  // resetPgliteState truncates config; MinionQueue.ensureSchema gates on the
  // 'version' row (same re-seed pattern as the other minion-using tests).
  await engine.setConfig('version', schemaVersion);
  __clearSuppressionCacheForTests();
});

// ── Time helpers (exact-second timestamps so backfill floors are stable) ────

const NOW_MS = Math.floor(Date.now() / 1000) * 1000;
const daysAgoMs = (n: number): number => NOW_MS - n * 86_400_000;
/** Fix wave 4: a thread's item-hold record in the cursor state. */
const heldRecord = (state: { item_holds?: unknown }, tid: string) =>
  (state.item_holds as { items?: Record<string, { attempts: number; state: string; code: string }> } | undefined)?.items?.[tid];
const hoursAgoMs = (n: number): number => NOW_MS - n * 3_600_000;

// ── Thread / message id constants (hex only — emailCitation validates) ──────

const T_A = '17aa00000000a001'; // conversational thread (search + SENT + delta)
const T_B = '17aa00000000b002'; // old unanswered inbound (open-loop source)
const T_N = '17aa00000000c003'; // noise thread (noreply@)
const T_C = '17aa00000000d004'; // appears later (history-expired fallback)

// ── Fake Google API behind fetchImpl ─────────────────────────────────────────

interface FakeMessage {
  id: string;
  threadId: string;
  internalDateMs: number;
  headers: Record<string, string>;
  labelIds: string[];
  body: string;
}

interface FakeGoogle {
  profileHistoryId: string;
  messages: FakeMessage[];
  messagesPageSize: number;
  /** Each entry is one history record's touched threadIds. */
  history: string[][];
  historyResponseId: string;
  historyExpired: boolean;
  failThreads: Set<string>;
  /** Threads that 429 (rate-limited) on every fetch — retry-after: 0 so tests
   *  don't actually sleep through the client's real backoff schedule. */
  rateLimitThreads: Set<string>;
  contacts: unknown[];
  contactsDelta: unknown[];
  calendarEvents: unknown[];
  calendarDelta: unknown[];
  calendarExpireSyncToken: boolean;
  calls: string[];
  threadFetches: number;
  tokenPosts: number;
  onThreadFetch?: () => void;
  /** Real Gmail semantics: history.list from the latest historyId is empty. */
  historyHonorsStart?: boolean;
}

function emptyFx(): FakeGoogle {
  return {
    profileHistoryId: '1000',
    messages: [],
    messagesPageSize: 100,
    history: [],
    historyResponseId: '1000',
    historyExpired: false,
    failThreads: new Set(),
    rateLimitThreads: new Set(),
    contacts: [],
    contactsDelta: [],
    calendarEvents: [],
    calendarDelta: [],
    calendarExpireSyncToken: false,
    calls: [],
    threadFetches: 0,
    tokenPosts: 0,
  };
}

function b64url(s: string): string {
  return Buffer.from(s, 'utf-8').toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
}

function buildFetch(fx: FakeGoogle): FetchImpl {
  const json = (body: unknown, status = 200, headers: Record<string, string> = {}): Response =>
    new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json', ...headers } });

  return async (url: string): Promise<Response> => {
    const u = new URL(url);
    fx.calls.push(u.pathname + (u.search ? u.search : ''));

    if (u.hostname === 'oauth2.googleapis.com') {
      fx.tokenPosts++;
      return json({ access_token: 't2', expires_in: 3600 });
    }

    if (u.pathname.endsWith('/users/me/profile')) {
      return json({ emailAddress: '[email protected]', historyId: fx.profileHistoryId });
    }

    if (u.pathname.endsWith('/users/me/messages')) {
      const q = u.searchParams.get('q') ?? '';
      const after = /after:(\d+)/.exec(q);
      const before = /before:(\d+)/.exec(q);
      let msgs = [...fx.messages];
      if (after) msgs = msgs.filter((m) => Math.floor(m.internalDateMs / 1000) >= Number(after[1]));
      if (before) msgs = msgs.filter((m) => Math.floor(m.internalDateMs / 1000) < Number(before[1]));
      msgs.sort((a, b) => b.internalDateMs - a.internalDateMs); // newest first, like Gmail
      const start = Number(u.searchParams.get('pageToken') ?? '0');
      const page = msgs.slice(start, start + fx.messagesPageSize);
      const next = start + fx.messagesPageSize < msgs.length ? String(start + fx.messagesPageSize) : undefined;
      return json({
        messages: page.map((m) => ({ id: m.id, threadId: m.threadId })),
        ...(next ? { nextPageToken: next } : {}),
      });
    }

    if (u.pathname.endsWith('/users/me/history')) {
      if (fx.historyExpired) return json({ error: { code: 404, message: 'Start history id is too old' } }, 404);
      return json({
        historyId: fx.historyResponseId,
        history: (fx.historyHonorsStart && u.searchParams.get('startHistoryId') === fx.historyResponseId ? [] : fx.history)
          .map((tids) => ({ messages: tids.map((tid) => ({ threadId: tid })) })),
      });
    }

    const threadMatch = u.pathname.match(/\/users\/me\/threads\/([^/]+)$/);
    if (threadMatch) {
      const tid = threadMatch[1];
      fx.threadFetches++;
      fx.onThreadFetch?.();
      if (fx.failThreads.has(tid)) return json({ error: { code: 500, message: 'backend error' } }, 500);
      if (fx.rateLimitThreads.has(tid)) {
        // retry-after: 0 → the client's real (patient) retry loop still
        // runs its full attempt budget, but with zero actual delay.
        return json({ error: { message: 'rate limit' } }, 429, { 'retry-after': '0' });
      }
      const msgs = fx.messages
        .filter((m) => m.threadId === tid)
        .sort((a, b) => b.internalDateMs - a.internalDateMs); // served newest-first
      return json({
        id: tid,
        messages: msgs.map((m) => ({
          id: m.id,
          threadId: tid,
          labelIds: m.labelIds,
          internalDate: String(m.internalDateMs),
          payload: {
            mimeType: 'multipart/alternative',
            headers: Object.entries(m.headers).map(([name, value]) => ({ name, value })),
            parts: [{ mimeType: 'text/plain', body: { data: b64url(m.body) } }],
          },
        })),
      });
    }

    // Any calendar id — the real API serves /calendars/<id>/events for every
    // calendar the account can read, not just 'primary'.
    if (/\/calendars\/[^/]+\/events/.test(u.pathname)) {
      if (u.searchParams.get('syncToken')) {
        if (fx.calendarExpireSyncToken) return json({ error: { code: 410, message: 'Sync token expired' } }, 410);
        return json({ items: fx.calendarDelta, nextSyncToken: 'cal-sync-2' });
      }
      return json({ items: fx.calendarEvents, nextSyncToken: 'cal-sync-1' });
    }

    if (u.pathname.includes('/people/me/connections')) {
      if (u.searchParams.get('syncToken')) {
        return json({ connections: fx.contactsDelta, nextSyncToken: 'ppl-sync-2' });
      }
      return json({ connections: fx.contacts, nextSyncToken: 'ppl-sync-1' });
    }

    return json({ error: { message: `unhandled ${u.pathname}` } }, 400);
  };
}

// ── In-memory vault (runGoogleSync's vaultOverride — no ~/.gbrain reads) ────

class FakeVault implements CredentialVault {
  entries = new Map<string, CredentialEntry>();
  clients = new Map<string, ProviderClientRecord>();
  async get(id: string): Promise<CredentialEntry | null> {
    return this.entries.get(id) ?? null;
  }
  async put(entry: CredentialEntry): Promise<void> {
    this.entries.set(entry.id, entry);
  }
  async list(): Promise<CredentialMeta[]> {
    return [];
  }
  async delete(id: string): Promise<boolean> {
    return this.entries.delete(id);
  }
  async getClient(provider: string): Promise<ProviderClientRecord | null> {
    return this.clients.get(provider) ?? null;
  }
  async putClient(rec: ProviderClientRecord): Promise<void> {
    this.clients.set(rec.provider, rec);
  }
  async deleteClient(provider: string): Promise<boolean> {
    return this.clients.delete(provider);
  }
}

const CLIENT_ID = '123-abc.apps.googleusercontent.com';

function makeVault(): FakeVault {
  const v = new FakeVault();
  v.entries.set('google:[email protected]', {
    id: 'google:[email protected]',
    provider: 'google',
    kind: 'oauth2',
    client_ref: 'byo',
    secret: {
      access_token: 't',
      refresh_token: 'r',
      expiry: new Date(Date.now() + 3_600_000).toISOString(),
    },
    meta: {
      account: '[email protected]',
      sendas_aliases: ['[email protected]'],
      connected_at: new Date().toISOString(),
      client_id: CLIENT_ID,
    },
  });
  v.clients.set('google', {
    provider: 'google',
    client_id: CLIENT_ID,
    client_secret: 'GOCSPX-test-secret-0000',
    created_at: new Date().toISOString(),
  });
  return v;
}

// ── Shared setup helpers ─────────────────────────────────────────────────────

async function insertGoogleSource(dir: string): Promise<void> {
  await engine.executeRaw(
    `INSERT INTO sources (id, name, local_path, config) VALUES ($1, $2, $3, $4::text::jsonb)`,
    [
      'gsrc',
      'google',
      dir,
      JSON.stringify({
        kind: 'google',
        g_account: '[email protected]',
        g_services: 'gmail,calendar,contacts',
        g_history_days: 90,
        g_dir: dir,
      }),
    ],
  );
}

function cfgFor(dir: string, services = 'gmail,calendar,contacts') {
  return parseGoogleSourceConfig(
    { kind: 'google', g_account: '[email protected]', g_services: services, g_history_days: 90, g_dir: dir },
    dir,
  );
}

async function sweep(
  dir: string,
  fx: FakeGoogle,
  vault: FakeVault,
  opts: Partial<SyncOpts> = {},
  services = 'gmail,calendar,contacts',
) {
  return runGoogleSync(
    engine,
    'gsrc',
    cfgFor(dir, services),
    { sourceId: 'gsrc', noEmbed: true, noExtract: true, ...opts },
    buildFetch(fx),
    vault,
  );
}

async function slugsWhere(where: string): Promise<string[]> {
  const rows = await engine.executeRaw<{ slug: string }>(
    `SELECT slug FROM pages WHERE source_id = 'gsrc' AND deleted_at IS NULL AND ${where} ORDER BY slug`,
  );
  return rows.map((r) => r.slug);
}

function gmsg(id: string, threadId: string, ms: number, over: Partial<FakeMessage> = {}): FakeMessage {
  return {
    id,
    threadId,
    internalDateMs: ms,
    headers: { From: 'Charlie Example <[email protected]>', To: '[email protected]', Subject: 'Quarterly zephyr roadmap' },
    labelIds: [],
    body: 'Sharing the zephyr roadmap draft for review.',
    ...over,
  };
}

/** Threads A (2 msgs, one SENT), B (old unanswered inbound), N (noise). */
function gmailFixture(fx: FakeGoogle): void {
  fx.messages.push(
    gmsg('18c2f4a9b3d21e01', T_A, daysAgoMs(2)),
    gmsg('18c2f4a9b3d21e02', T_A, daysAgoMs(2) + 3_600_000, {
      headers: { From: 'A Example <[email protected]>', To: '[email protected]', Subject: 'Re: Quarterly zephyr roadmap' },
      labelIds: ['SENT'],
      body: 'Looks good, shipping it this week.',
    }),
    gmsg('18c2f4a9b3d21e03', T_B, daysAgoMs(3), {
      headers: { From: 'Dana Example <[email protected]>', To: '[email protected]', Subject: 'Contract question' },
      body: 'Could you confirm the contract terms?',
    }),
    gmsg('18c2f4a9b3d21e04', T_N, daysAgoMs(1), {
      headers: { From: 'Notifier <[email protected]>', To: '[email protected]', Subject: 'Automated notification' },
      body: 'This is an automated message.',
    }),
  );
}

function contactsFixture(fx: FakeGoogle): void {
  fx.contacts = [
    {
      resourceName: 'people/c000000001',
      names: [{ displayName: 'Alice Example', metadata: { primary: true } }],
      emailAddresses: [{ value: '[email protected]' }],
      organizations: [{ name: 'Acme Example', title: 'Engineer', metadata: { primary: true } }],
    },
    {
      resourceName: 'people/c000000002',
      names: [{ displayName: 'Dana Example' }],
      emailAddresses: [{ value: '[email protected]' }],
    },
  ];
}

function calendarFixture(fx: FakeGoogle): void {
  fx.calendarEvents = [
    {
      id: 'evt00000000000001',
      status: 'confirmed',
      summary: 'Zephyr planning sync',
      description: 'Review the zephyr roadmap.',
      start: { dateTime: new Date(daysAgoMs(1)).toISOString() },
      end: { dateTime: new Date(daysAgoMs(1) + 3_600_000).toISOString() },
      organizer: { email: '[email protected]' },
      attendees: [
        { email: '[email protected]', self: true, responseStatus: 'accepted' },
        { email: '[email protected]', displayName: 'Charlie Example', responseStatus: 'accepted' },
      ],
      htmlLink: 'https://calendar.google.com/calendar/event?eid=evt1',
    },
    {
      id: 'evt00000000000002',
      status: 'cancelled',
      summary: 'Dropped sync',
      start: { dateTime: new Date(daysAgoMs(2)).toISOString() },
      end: { dateTime: new Date(daysAgoMs(2) + 1_800_000).toISOString() },
    },
  ];
}

async function withHome<T>(fn: () => Promise<T>): Promise<T> {
  return withEnv({ GBRAIN_HOME: mkdtempSync(join(tmpdir(), 'gbrain-home-')) }, fn);
}

// ── Pure config/state units (google-source exports) ─────────────────────────

describe('google-source config + state units', () => {
  test('parseGoogleSourceConfig normalizes account, services, historyDays, dir', () => {
    const cfg = parseGoogleSourceConfig(
      { kind: 'google', g_account: ' [email protected] ', g_services: 'Gmail, bogus , CALENDAR', g_history_days: 30.7, g_dir: '/tmp/gdir' },
      '/tmp/fallback',
    );
    expect(cfg.account).toBe('[email protected]');
    expect(cfg.services).toEqual(['gmail', 'calendar']);
    expect(cfg.historyDays).toBe(30);
    expect(cfg.dir).toBe('/tmp/gdir');
  });

  test('parseGoogleSourceConfig defaults: all services, 90 days, fallback dir', () => {
    const cfg = parseGoogleSourceConfig({ kind: 'google' }, '/tmp/fallback');
    expect(cfg.account).toBe('');
    expect(cfg.services).toEqual(['gmail', 'calendar', 'contacts']);
    expect(cfg.historyDays).toBe(90);
    expect(cfg.dir).toBe('/tmp/fallback');
  });

  test('parseGoogleSourceConfig: all-invalid services fall back to all; historyDays clamps', () => {
    const cfg = parseGoogleSourceConfig({ kind: 'google', g_services: 'bogus', g_history_days: 99999 }, '/tmp/x');
    expect(cfg.services).toEqual(['gmail', 'calendar', 'contacts']);
    expect(cfg.historyDays).toBe(3650);
    expect(parseGoogleSourceConfig({ kind: 'google', g_history_days: -5 }, '/tmp/x').historyDays).toBe(90);
  });

  test('googleStateFile + readGoogleState: missing file → empty state; partial file merges', () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-state-'));
    try {
      expect(googleStateFile(dir)).toBe(join(dir, '.google-source.json'));
      const empty = readGoogleState(dir);
      expect(empty.gmail_history_id).toBeNull();
      expect(empty.gmail_backfill_done).toBe(false);
      writeFileSync(googleStateFile(dir), JSON.stringify({ gmail_history_id: '42' }), 'utf-8');
      const partial = readGoogleState(dir);
      expect(partial.gmail_history_id).toBe('42');
      expect(partial.gmail_backfill_done).toBe(false);
      expect(partial.calendar_sync_token).toBeNull();
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('myAddressSet includes the account and sendAs aliases, lowercased', () => {
    const vault = makeVault();
    const entry = vault.entries.get('google:[email protected]')!;
    const set = myAddressSet({ ...entry, meta: { ...entry.meta, account: '[email protected]', sendas_aliases: ['[email protected]'] } });
    expect(set.has('[email protected]')).toBe(true);
    expect(set.has('[email protected]')).toBe(true);
    expect(set.size).toBe(2);
  });
});

// ── End-to-end sweeps ────────────────────────────────────────────────────────

describe('google-source materialize', () => {
  test('first sweep materializes contacts, calendar, and gmail; state + search projection land', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-first-'));
    const fx = emptyFx();
    gmailFixture(fx);
    contactsFixture(fx);
    calendarFixture(fx);
    fx.messagesPageSize = 3; // force messages.list pagination during backfill
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res = await sweep(dir, fx, vault);
        expect(res.status).toBe('first_sync');
        // 2 person pages + 1 meeting + 2 threads (noise thread excluded).
        expect(res.added).toBe(5);
        expect(res.deleted).toBe(0);
        expect(res.chunksCreated).toBeGreaterThan(0);
        expect(fx.tokenPosts).toBe(0); // fresh vault token — no refresh HTTP
        // #5621: noEmbed connector imports stamp the wrapping convention.
        const unstamped = await engine.executeRaw<{ n: number }>(
          `SELECT count(*)::int AS n FROM pages
            WHERE source_id = 'gsrc' AND page_kind = 'markdown' AND contextual_retrieval_mode IS NULL`,
        );
        expect(unstamped[0].n).toBe(0);

        // Contacts → person pages with aliases projected into page_aliases.
        expect(await slugsWhere(`slug LIKE 'people/%'`)).toEqual(['people/alice-example', 'people/dana-example']);
        const aliasRows = await engine.executeRaw<{ alias_norm: string }>(
          `SELECT alias_norm FROM page_aliases WHERE source_id = 'gsrc' AND slug = 'people/alice-example' ORDER BY alias_norm`,
        );
        const norms = aliasRows.map((r) => r.alias_norm);
        expect(norms).toContain('[email protected]');
        expect(norms).toContain('alice example');

        // Calendar → meeting page; cancelled event never materializes.
        const meetings = await slugsWhere(`slug LIKE 'calendar/%'`);
        expect(meetings.length).toBe(1);
        expect(meetings[0]).toContain('zephyr-planning-sync');
        const meetingMd = readFileSync(join(dir, `${meetings[0]}.md`), 'utf-8');
        expect(meetingMd).toContain('type: meeting');
        expect(meetingMd).toContain('- "[email protected]"');

        // Gmail → thread pages with correct frontmatter; noise thread → no page.
        const emails = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(emails.length).toBe(2);
        expect(emails.some((s) => s.includes('automated-notification'))).toBe(false);
        const aSlug = emails.find((s) => s.includes('quarterly-zephyr-roadmap'))!;
        expect(aSlug).toBeDefined();
        const aMd = readFileSync(join(dir, `${aSlug}.md`), 'utf-8');
        expect(aMd).toContain('type: email');
        expect(aMd).toContain(`thread_id: "${T_A}"`);
        expect(aMd).toContain('message_id: "18c2f4a9b3d21e02"'); // latest message
        expect(aMd).toContain('account: "[email protected]"');
        expect(aMd).toContain('## → A Example <[email protected]>'); // SENT marker
        expect(aMd).toContain('https://mail.google.com/mail/u/?authuser=a%40example.com#inbox/18c2f4a9b3d21e01');

        // State file: history anchored, backfill done, service cursors banked.
        const state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('1000');
        expect(state.gmail_backfill_done).toBe(true);
        expect(state.gmail_backfill_floor_ms).toBeNull();
        expect(state.gmail_newest_ms).toBe(daysAgoMs(1)); // noise thread is newest seen
        expect(state.calendar_sync_token).toBe('cal-sync-1');
        expect(state.contacts_sync_token).toBe('ppl-sync-1');

        // SearchResult projection: subject word → message_id + thread_id.
        const results = await engine.searchKeyword('zephyr', { sourceId: 'gsrc' });
        const emailHit = results.find((r) => r.slug.startsWith('emails/'));
        expect(emailHit).toBeDefined();
        expect(emailHit!.message_id).toBe('18c2f4a9b3d21e02');
        expect(emailHit!.thread_id).toBe(T_A);

        // Sources row touched.
        const row = await engine.executeRaw<{ last_sync_at: string | null }>(
          `SELECT last_sync_at FROM sources WHERE id = 'gsrc'`,
        );
        expect(row[0].last_sync_at).not.toBeNull();
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('delta sweep re-imports only the replied thread and advances the cursor', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-delta-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');
        expect(readGoogleState(dir).gmail_history_id).toBe('1000');

        // A reply lands on thread A; history flags only that thread.
        fx.messages.push(
          gmsg('18c2f4a9b3d21e05', T_A, hoursAgoMs(1), {
            headers: { From: 'Charlie Example <[email protected]>', To: '[email protected]', Subject: 'Re: Quarterly zephyr roadmap' },
            body: 'One more question about rollout timing?',
          }),
        );
        fx.history = [[T_A]];
        fx.historyResponseId = '1010';
        const fetchesBefore = fx.threadFetches;

        const res = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res.status).toBe('synced');
        expect(res.modified).toBe(1);
        expect(res.added).toBe(0);
        expect(fx.threadFetches - fetchesBefore).toBe(1); // only thread A re-fetched

        const state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('1010'); // cursor advanced
        expect(state.gmail_newest_ms).toBe(hoursAgoMs(1));

        const aSlug = (await slugsWhere(`slug LIKE 'emails/%'`)).find((s) => s.includes('quarterly-zephyr-roadmap'))!;
        const aMd = readFileSync(join(dir, `${aSlug}.md`), 'utf-8');
        expect(aMd).toContain('One more question about rollout timing?');
        expect(aMd).toContain('message_id: "18c2f4a9b3d21e05"'); // latest id moved
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('delta lane: a rate-limited thread is not poisoned, the sweep stops at it, and the cursor is held (wave review)', async () => {
    // The backfill loop broke out of a batch on the first rate-limited thread;
    // the delta loop kept walking every remaining flagged thread through the
    // client's full retry budget against the same exhausted per-user quota.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-delta-ratelimit-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');
        expect(readGoogleState(dir).gmail_history_id).toBe('1000');

        // History flags A then B; A is throttled on every fetch.
        fx.history = [[T_A], [T_B]];
        fx.historyResponseId = '1010';
        fx.rateLimitThreads.add(T_A);
        const callsBefore = fx.calls.length;

        const res = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res.status).toBe('partial');
        expect(res.failedFiles).toBe(1);
        const state = readGoogleState(dir);
        expect(state.gmail_fail_counts?.[T_A]).toBeUndefined(); // never poisoned
        expect(heldRecord(state, T_A)).toBeUndefined(); // a rate limit never counts toward a hold
        expect(state.gmail_history_id).toBe('1000'); // cursor held
        // The sweep stopped at the throttled thread: B (behind A) was not fetched.
        expect(fx.calls.slice(callsBefore).some((c) => c.includes(`/threads/${T_B}`))).toBe(false);

        // Quota back: the same window drains (unchanged content → up_to_date)
        // and the cursor advances.
        fx.rateLimitThreads.clear();
        const res2 = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res2.status).not.toBe('partial');
        expect(readGoogleState(dir).gmail_history_id).toBe('1010');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('history 404 falls back to a bookmark window and re-anchors a fresh historyId', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-expired-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');

        // Cursor expires upstream; a brand-new thread arrived meanwhile.
        fx.historyExpired = true;
        fx.profileHistoryId = '2000';
        fx.messages.push(
          gmsg('18c2f4a9b3d21e06', T_C, hoursAgoMs(1), {
            headers: { From: 'Erin Example <[email protected]>', To: '[email protected]', Subject: 'Fresh zephyr kickoff' },
            body: 'Kicking off the fresh thread here.',
          }),
        );
        const callsBefore = fx.calls.length;
        const res = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res.status).toBe('synced'); // sync still succeeds
        expect(res.added).toBe(1); // the new thread landed via messages.list fallback
        // Fallback used messages.list + getProfile (re-anchor), not history.
        const newCalls = fx.calls.slice(callsBefore);
        expect(newCalls.some((c) => c.includes('/users/me/messages?'))).toBe(true);
        expect(newCalls.some((c) => c.endsWith('/users/me/profile'))).toBe(true);
        // H6 anchor ordering: getProfile fires BEFORE listMessageIds. A
        // message arriving between the two calls is then either in the
        // listing (post-anchor arrival) or replayed by history.list from the
        // anchor; anchoring AFTER the listing would drop it forever.
        const profileIdx = newCalls.findIndex((c) => c.endsWith('/users/me/profile'));
        const messagesIdx = newCalls.findIndex((c) => c.includes('/users/me/messages?'));
        expect(profileIdx).toBeGreaterThanOrEqual(0);
        expect(messagesIdx).toBeGreaterThanOrEqual(0);
        expect(profileIdx).toBeLessThan(messagesIdx);

        const state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('2000'); // fresh anchor
        expect(state.gmail_backfill_done).toBe(true);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('a failed thread fetch yields partial and does NOT advance the history cursor', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-fail-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');

        fx.history = [[T_A]];
        fx.historyResponseId = '1010';
        fx.failThreads.add(T_A);
        const res = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res.status).toBe('partial');
        expect(res.failedFiles).toBe(1);
        expect(readGoogleState(dir).gmail_history_id).toBe('1000'); // NOT advanced

        // Failure clears; the same window re-lists and the cursor advances.
        fx.failThreads.clear();
        const retry = await sweep(dir, fx, vault, {}, 'gmail');
        expect(retry.status).toBe('up_to_date'); // unchanged content re-import skips
        expect(readGoogleState(dir).gmail_history_id).toBe('1010');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('aborted backfill persists a resume floor; the next run completes it', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-floor-'));
    const fx = emptyFx();
    const vault = makeVault();
    // Six single-message threads, days 40..45 ago (outside the extract window).
    const tids = ['17aa00000000f001', '17aa00000000f002', '17aa00000000f003', '17aa00000000f004', '17aa00000000f005', '17aa00000000f006'];
    for (let i = 0; i < 6; i++) {
      fx.messages.push(
        gmsg(`18c2f4a9b3d21f0${i + 1}`, tids[i], daysAgoMs(40 + i), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Archive topic ${i}` },
          body: `Archive body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        // Abort after the 2nd thread fetch (mid first batch).
        const controller = new AbortController();
        fx.onThreadFetch = () => {
          if (fx.threadFetches >= 2) controller.abort();
        };
        const res1 = await sweep(dir, fx, vault, { signal: controller.signal }, 'gmail');
        expect(res1.status).toBe('partial');
        expect(res1.added).toBe(2);

        let state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(false);
        expect(state.gmail_backfill_floor_ms).toBe(daysAgoMs(41)); // oldest fully-attempted thread
        expect(state.gmail_history_id).toBe('1000'); // delta anchor captured BEFORE backfill

        // Resume without the abort: only the remaining window is listed.
        fx.onThreadFetch = undefined;
        const res2 = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res2.added).toBe(4); // the four threads below the floor
        state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(true);
        expect(state.gmail_backfill_floor_ms).toBeNull();
        expect((await slugsWhere(`slug LIKE 'emails/%'`)).length).toBe(6);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('aborted delta drain banks landed threads; the next run resumes only the remainder', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-delta-abort-'));
    const fx = emptyFx();
    fx.historyHonorsStart = true;
    const vault = makeVault();
    const tids = ['17aa00000000e001', '17aa00000000e002', '17aa00000000e003', '17aa00000000e004', '17aa00000000e005', '17aa00000000e006'];
    for (let i = 0; i < 6; i++) {
      fx.messages.push(
        gmsg(`18c2f4a9b3d21e1${i + 1}`, tids[i], daysAgoMs(40 + i), {
          headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Delta topic ${i}` },
          body: `Delta body ${i}.`,
        }),
      );
    }
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');
        expect(readGoogleState(dir).gmail_history_id).toBe('1000');

        // History flags all six threads; the wall-clock budget expires after two.
        fx.history = [tids];
        fx.historyResponseId = '1010';
        const controller = new AbortController();
        fx.onThreadFetch = () => {
          if (fx.threadFetches >= 8) controller.abort();
        };
        const res1 = await sweep(dir, fx, vault, { signal: controller.signal }, 'gmail');
        expect(res1.status).toBe('partial');
        let state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('1010');
        expect(state.gmail_pending_thread_ids).toEqual(tids.slice(2));

        // Resume: only the four unlanded threads are fetched, then the backlog clears.
        fx.onThreadFetch = undefined;
        const fetchesBefore = fx.threadFetches;
        await sweep(dir, fx, vault, {}, 'gmail');
        expect(fx.threadFetches - fetchesBefore).toBe(4);
        state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('1010');
        expect(state.gmail_pending_thread_ids).toEqual([]);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('contacts: deleted contact removes its page; hand-authored person pages are never rewritten', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-contacts-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.contacts = [
      {
        resourceName: 'people/c000000001',
        names: [{ displayName: 'Alice Example', metadata: { primary: true } }],
        emailAddresses: [{ value: '[email protected]' }],
      },
      {
        resourceName: 'people/c000000003',
        names: [{ displayName: 'Bob Example', metadata: { primary: true } }],
        emailAddresses: [{ value: '[email protected]' }],
      },
    ];
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        // Hand-authored page at the exact path the Bob contact would take —
        // no google_contact_id marker, so the connector must not touch it.
        const bobPath = join(dir, 'people/bob-example.md');
        mkdirSync(dirname(bobPath), { recursive: true });
        const handAuthored = [
          '---',
          'type: person',
          'title: "Bob Example"',
          'aliases:',
          '  - "[email protected]"',
          '---',
          '',
          '# Bob Example',
          '',
          'Hand-written notes that must survive the connector.',
          '',
        ].join('\n');
        writeFileSync(bobPath, handAuthored, 'utf-8');
        const imported = await importFile(engine, bobPath, 'people/bob-example.md', { sourceId: 'gsrc', noEmbed: true });
        expect(imported.status).toBe('imported');

        const res1 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res1.added).toBe(1); // Alice only — Bob skipped as hand-authored
        expect(readFileSync(bobPath, 'utf-8')).toBe(handAuthored); // body preserved
        expect(existsSync(join(dir, 'people/alice-example.md'))).toBe(true);
        const alicePage = readFileSync(join(dir, 'people/alice-example.md'), 'utf-8');
        expect(alicePage).toContain('google_contact_id: "people/c000000001"');

        // Delta: Alice is deleted upstream.
        fx.contactsDelta = [
          {
            resourceName: 'people/c000000001',
            names: [{ displayName: 'Alice Example' }],
            emailAddresses: [{ value: '[email protected]' }],
            metadata: { deleted: true },
          },
        ];
        const res2 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res2.deleted).toBe(1);
        expect(existsSync(join(dir, 'people/alice-example.md'))).toBe(false);
        expect(await slugsWhere(`slug = 'people/alice-example'`)).toEqual([]);
        // The hand-authored page survives both sweeps untouched.
        expect(readFileSync(bobPath, 'utf-8')).toBe(handAuthored);
        expect(await slugsWhere(`slug = 'people/bob-example'`)).toEqual(['people/bob-example']);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('item holds (formerly the poison ledger): failed runs never stamp last_sync_at and count toward a hold; the 4th run skips the held thread; --full retries it', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-poison-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const vault = makeVault();
    const seededSyncAt = '2026-01-01T00:00:00.000Z';
    const lastSyncAt = async (): Promise<string | null> => {
      const rows = await engine.executeRaw<{ last_sync_at: Date | string | null }>(
        `SELECT last_sync_at FROM sources WHERE id = 'gsrc'`,
      );
      const v = rows[0].last_sync_at;
      return v === null ? null : new Date(v).toISOString();
    };
    try {
      await insertGoogleSource(dir);
      await engine.executeRaw(
        `UPDATE sources SET last_sync_at = $1::timestamptz WHERE id = 'gsrc'`,
        [seededSyncAt],
      );
      await withHome(async () => {
        // (a) Thread B persistently 500s: every run is partial, the ledger
        // increments ACROSS runs, and last_sync_at is never stamped — a
        // wedged sweep must read stale to the staleness gate, not fresh.
        fx.failThreads.add(T_B);
        for (let run = 1; run <= 3; run++) {
          const res = await sweep(dir, fx, vault, {}, 'gmail');
          expect(res.status).toBe('partial');
          expect(res.failedFiles).toBe(1);
          const state = readGoogleState(dir);
          expect(heldRecord(state, T_B)).toMatchObject({ attempts: run, state: run < 3 ? 'failing' : 'held', code: 'http_5xx' });
          expect(state.gmail_fail_counts).toBeUndefined();
          expect(state.gmail_backfill_done).toBe(false);
          expect(await lastSyncAt()).toBe(seededSyncAt); // NOT updated
        }

        // (b) Run 4: three recorded failures = poisoned. The thread is
        // SKIPPED (never even fetched), the backfill completes around it,
        // and last_sync_at finally stamps.
        const callsBefore = fx.calls.length;
        const res4 = await sweep(dir, fx, vault, {}, 'gmail');
        expect(res4.failedFiles).toBeUndefined();
        const run4Calls = fx.calls.slice(callsBefore);
        expect(run4Calls.some((c) => c.includes(`/threads/${T_B}`))).toBe(false);
        const state4 = readGoogleState(dir);
        expect(state4.gmail_backfill_done).toBe(true);
        expect(state4.gmail_backfill_floor_ms).toBeNull();
        expect(heldRecord(state4, T_B)).toMatchObject({ attempts: 3, state: 'held' }); // skip is not forgiveness
        expect(await lastSyncAt()).not.toBe(seededSyncAt); // stamped at last
        // The poisoned thread never materialized.
        const slugs4 = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(slugs4.some((s) => s.includes('contract-question'))).toBe(false);
        expect(slugs4.some((s) => s.includes('quarterly-zephyr-roadmap'))).toBe(true);

        // (c) --full resets the ledger, so the (now healthy) thread is
        // retried instead of being skipped forever.
        fx.failThreads.clear();
        fx.history = [[T_B]];
        fx.historyResponseId = '1010';
        const res5 = await sweep(dir, fx, vault, { full: true }, 'gmail');
        expect(res5.status).toBe('synced');
        expect(res5.added).toBe(1);
        const state5 = readGoogleState(dir);
        expect(heldRecord(state5, T_B)).toBeUndefined(); // reset + success
        const slugs5 = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(slugs5.some((s) => s.includes('contract-question'))).toBe(true);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('rate-limited thread: partial every run like a real failure, but NEVER poisoned — the ledger stays empty across many runs', async () => {
    // The production bug this pins: a transient 429/403 rate-limit failure
    // used to count toward MAX_THREAD_FAILURES exactly like a hard failure,
    // so a burst of rate limiting during a big backfill would poison
    // (silently skip, without --full) mail that was never actually bad.
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-ratelimit-'));
    const fx = emptyFx();
    gmailFixture(fx);
    // A thread listed BEHIND the rate-limited one in the same batch (older, so
    // the newest-first listing puts it after T_B).
    fx.messages.push(
      gmsg('18c2f4a9b3d21e08', T_C, daysAgoMs(4), {
        headers: { From: 'Erin Example <[email protected]>', To: '[email protected]', Subject: 'Older thread' },
        body: 'An older thread behind the rate-limited one.',
      }),
    );
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        fx.rateLimitThreads.add(T_B);
        // Run this MORE times than MAX_THREAD_FAILURES (3) would take to
        // poison a hard failure — if rate-limit failures were still counted,
        // run 4 would already skip T_B instead of retrying it.
        for (let run = 1; run <= 5; run++) {
          const res = await sweep(dir, fx, vault, {}, 'gmail');
          expect(res.status).toBe('partial');
          expect(res.failedFiles).toBe(1);
          const state = readGoogleState(dir);
          // The ledger entry for T_B never appears — a rate-limit failure is
          // never counted toward the poison threshold, no matter how many
          // times it happens in a row.
          expect(state.gmail_fail_counts?.[T_B]).toBeUndefined();
          expect(heldRecord(state, T_B)).toBeUndefined();
          expect(state.gmail_backfill_done).toBe(false);
        }

        // The thread was actually FETCHED every run (never skipped as
        // poisoned) — proof the loop kept retrying it, not silently giving up.
        const fetchCalls = fx.calls.filter((c) => c.includes(`/threads/${T_B}`));
        expect(fetchCalls.length).toBeGreaterThan(0);
        // ...and once one thread in the batch is rate limited, the rest of the
        // batch is deferred to the next run instead of hammering the same
        // exhausted quota: T_C (listed behind T_B) is never fetched while T_B
        // is throttled. Nothing is lost — the floor didn't advance, so the
        // next run re-lists it.
        expect(fx.calls.some((c) => c.includes(`/threads/${T_C}`))).toBe(false);

        // Once the rate limit clears, the thread imports normally and the
        // backfill completes — nothing was permanently lost.
        fx.rateLimitThreads.clear();
        const resFinal = await sweep(dir, fx, vault, {}, 'gmail');
        expect(resFinal.status).toBe('synced');
        const stateFinal = readGoogleState(dir);
        expect(stateFinal.gmail_backfill_done).toBe(true);
        const slugsFinal = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(slugsFinal.some((s) => s.includes('contract-question'))).toBe(true);
        expect(slugsFinal.some((s) => s.includes('older-thread'))).toBe(true);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('contacts: bare tombstone deletes by google_contact_id; a rename moves the page; an unknown tombstone is a no-op', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-contacts-tomb-'));
    const fx = emptyFx();
    const vault = makeVault();
    contactsFixture(fx); // alice (c000000001) + dana (c000000002)
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const res1 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res1.added).toBe(2);
        expect(existsSync(join(dir, 'people/alice-example.md'))).toBe(true);
        expect(existsSync(join(dir, 'people/dana-example.md'))).toBe(true);

        // (a) Deletion tombstone carrying ONLY resourceName + deleted — no
        // names, no emails, so slug derivation yields nothing. The page must
        // still die via the google_contact_id-keyed DB lookup.
        fx.contactsDelta = [{ resourceName: 'people/c000000002', metadata: { deleted: true } }];
        const res2 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res2.deleted).toBe(1);
        expect(existsSync(join(dir, 'people/dana-example.md'))).toBe(false); // file removed
        expect(await slugsWhere(`slug = 'people/dana-example'`)).toEqual([]); // live row gone
        // Honest current behavior: deletePageByRelPath routes through
        // engine.deletePages, which HARD-deletes the row (DELETE FROM pages),
        // so nothing remains — deleted or otherwise.
        const gone = await engine.executeRaw<{ n: number | string }>(
          `SELECT count(*) AS n FROM pages
           WHERE source_id = 'gsrc' AND slug = 'people/dana-example'`,
        );
        expect(Number(gone[0].n)).toBe(0);

        // (b) Rename: same resourceName, new displayName → the new slug
        // imports AND the old page is deleted instead of stranding.
        fx.contactsDelta = [
          {
            resourceName: 'people/c000000001',
            names: [{ displayName: 'Alicia Example', metadata: { primary: true } }],
            emailAddresses: [{ value: '[email protected]' }],
          },
        ];
        const res3 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res3.added).toBe(1);
        expect(res3.deleted).toBe(1);
        expect(existsSync(join(dir, 'people/alicia-example.md'))).toBe(true);
        expect(existsSync(join(dir, 'people/alice-example.md'))).toBe(false);
        expect(await slugsWhere(`slug LIKE 'people/%'`)).toEqual(['people/alicia-example']);
        const md = readFileSync(join(dir, 'people/alicia-example.md'), 'utf-8');
        expect(md).toContain('google_contact_id: "people/c000000001"');
        expect(md).toContain('title: "Alicia Example"');

        // (c) Tombstone for a never-imported contact: a clean no-op.
        fx.contactsDelta = [{ resourceName: 'people/c999999999', metadata: { deleted: true } }];
        const res4 = await sweep(dir, fx, vault, {}, 'contacts');
        expect(res4.status).toBe('up_to_date');
        expect(res4.deleted).toBe(0);
        expect(await slugsWhere(`slug LIKE 'people/%'`)).toEqual(['people/alicia-example']);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('open_loops: an old unanswered inbound opens a loop; the reply closes it as reply_detected', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-loops-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.messages.push(
      gmsg('18c2f4a9b3d21e03', T_B, daysAgoMs(3), {
        headers: { From: 'Dana Example <[email protected]>', To: '[email protected]', Subject: 'Contract question' },
        body: 'Could you confirm the contract terms?',
      }),
    );
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');
        const open = await engine.executeRaw<{ loop_type: string; status: string; counterparty_email: string; thread_id: string }>(
          `SELECT loop_type, status, counterparty_email, thread_id FROM open_loops WHERE source_id = 'gsrc'`,
        );
        expect(open.length).toBe(1);
        expect(open[0].loop_type).toBe('unanswered_inbound');
        expect(open[0].status).toBe('open');
        expect(open[0].counterparty_email).toBe('[email protected]');
        expect(open[0].thread_id).toBe(T_B);

        // My reply lands in the delta sweep → the loop auto-closes.
        fx.messages.push(
          gmsg('18c2f4a9b3d21e07', T_B, hoursAgoMs(2), {
            headers: { From: 'A Example <[email protected]>', To: '[email protected]', Subject: 'Re: Contract question' },
            labelIds: ['SENT'],
            body: 'Confirmed, terms attached.',
          }),
        );
        fx.history = [[T_B]];
        fx.historyResponseId = '1010';
        await sweep(dir, fx, vault, {}, 'gmail');

        const after = await engine.executeRaw<{ status: string; closed_by: string | null }>(
          `SELECT status, closed_by FROM open_loops WHERE source_id = 'gsrc' AND loop_type = 'unanswered_inbound'`,
        );
        expect(after.length).toBe(1);
        expect(after[0].status).toBe('done');
        expect(after[0].closed_by).toBe('reply_detected');
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('command-mode access: sweep succeeds with NO vault entry; threads import and loop detection runs on the account-only identity', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-cmdaccess-'));
    const fx = emptyFx();
    gmailFixture(fx);
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        // g_access/g_token_command map through parseGoogleSourceConfig.
        const cfg = parseGoogleSourceConfig(
          {
            kind: 'google',
            g_account: '[email protected]',
            g_services: 'gmail',
            g_history_days: 90,
            g_dir: dir,
            g_access: 'command',
            g_token_command: 'echo fixture-token',
          },
          dir,
        );
        expect(cfg.access).toBe('command');
        expect(cfg.tokenCommand).toBe('echo fixture-token');

        // Every API call carries the command-minted bearer; NO vault is
        // passed (vaultOverride undefined) and no refresh POST ever fires —
        // the command IS the refresher.
        const inner = buildFetch(fx);
        const authHeaders: string[] = [];
        const spyFetch: FetchImpl = async (url, init) => {
          const h = (init?.headers ?? {}) as Record<string, string>;
          const auth = h.authorization ?? h.Authorization;
          if (auth) authHeaders.push(auth);
          return inner(url, init);
        };
        const res = await runGoogleSync(
          engine,
          'gsrc',
          cfg,
          { sourceId: 'gsrc', noEmbed: true, noExtract: true },
          spyFetch,
        );
        expect(res.status).toBe('first_sync');
        expect(fx.tokenPosts).toBe(0);
        expect(authHeaders.length).toBeGreaterThan(0);
        for (const a of authHeaders) expect(a).toBe('Bearer fixture-token');

        // Threads imported (noise excluded), same as vault mode.
        const emails = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(emails.length).toBe(2);

        // Loop detection ran with myAddresses = account-only (the fake API
        // serves no sendAs endpoint; fetchSendAsAliases degrades to []):
        // T_A's SENT reply from [email protected] still reads as mine, so only
        // T_B's 3-day-old inbound opens a loop.
        const loops = await engine.executeRaw<{ loop_type: string; status: string; counterparty_email: string }>(
          `SELECT loop_type, status, counterparty_email FROM open_loops WHERE source_id = 'gsrc'`,
        );
        expect(loops.length).toBe(1);
        expect(loops[0].loop_type).toBe('unanswered_inbound');
        expect(loops[0].status).toBe('open');
        expect(loops[0].counterparty_email).toBe('[email protected]');

        // gmail sweep succeeded → the trust-critical freshness stamp landed.
        const row = await engine.executeRaw<{ last_sync_at: unknown }>(
          `SELECT last_sync_at FROM sources WHERE id = 'gsrc'`,
        );
        expect(row[0].last_sync_at).not.toBeNull();
        expect(readGoogleState(dir).gmail_backfill_done).toBe(true);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('env-mode access: sweep succeeds when the var is set; unset var → partial with access_env_missing on stderr and NO freshness stamp', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-envaccess-'));
    const fx = emptyFx();
    gmailFixture(fx);
    const ENV_KEY = 'GBRAIN_TEST_GSYNC_ACCESS_TOKEN';
    const cfg = () =>
      parseGoogleSourceConfig(
        {
          kind: 'google',
          g_account: '[email protected]',
          g_services: 'gmail',
          g_history_days: 90,
          g_dir: dir,
          g_access: 'env',
          g_token_env: ENV_KEY,
        },
        dir,
      );
    const stderrCaptured = async (fn: () => Promise<void>): Promise<string> => {
      const orig = process.stderr.write.bind(process.stderr);
      const chunks: string[] = [];
      process.stderr.write = ((chunk: string | Uint8Array): boolean => {
        chunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf-8'));
        return true;
      }) as typeof process.stderr.write;
      try {
        await fn();
      } finally {
        process.stderr.write = orig;
      }
      return chunks.join('');
    };
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        expect(cfg().access).toBe('env');
        expect(cfg().tokenEnv).toBe(ENV_KEY);

        // (a) Unset var: the sweep FAILS honestly. Current behavior (pinned):
        // the CredentialError surfaces per-service inside runGoogleSync (the
        // same catch that handles scope_missing/upstream credential errors) —
        // status 'partial', toHuman() on stderr, nothing imported, and the
        // staleness-gate stamp is withheld.
        await withEnv({ [ENV_KEY]: undefined }, async () => {
          let res: Awaited<ReturnType<typeof runGoogleSync>> | undefined;
          const err = await stderrCaptured(async () => {
            res = await runGoogleSync(
              engine,
              'gsrc',
              cfg(),
              { sourceId: 'gsrc', noEmbed: true, noExtract: true },
              buildFetch(fx),
            );
          });
          expect(res!.status).toBe('partial');
          expect(res!.added).toBe(0);
          expect(err).toContain('token environment variable is empty');
          expect(err).toContain(`$${ENV_KEY}`);
          const row = await engine.executeRaw<{ last_sync_at: unknown }>(
            `SELECT last_sync_at FROM sources WHERE id = 'gsrc'`,
          );
          expect(row[0].last_sync_at).toBeNull(); // stale gate stays honest
          expect((await slugsWhere(`slug LIKE 'emails/%'`)).length).toBe(0);
        });

        // (b) Var set: the same source syncs end-to-end, no vault involved.
        await withEnv({ [ENV_KEY]: 'env-token-abc123' }, async () => {
          const res = await runGoogleSync(
            engine,
            'gsrc',
            cfg(),
            { sourceId: 'gsrc', noEmbed: true, noExtract: true },
            buildFetch(fx),
          );
          expect(res.status).toBe('first_sync');
          expect(fx.tokenPosts).toBe(0);
          expect((await slugsWhere(`slug LIKE 'emails/%'`)).length).toBe(2);
          const row = await engine.executeRaw<{ last_sync_at: unknown }>(
            `SELECT last_sync_at FROM sources WHERE id = 'gsrc'`,
          );
          expect(row[0].last_sync_at).not.toBeNull();
        });
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('loops_extract enqueue: recent threads get idempotency-keyed jobs; unchanged re-sweeps add no duplicates', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-extract-'));
    const fx = emptyFx();
    const vault = makeVault();
    fx.messages.push(
      gmsg('18c2f4a9b3d21e01', T_A, daysAgoMs(2)),
      gmsg('18c2f4a9b3d21e02', T_A, daysAgoMs(2) + 3_600_000, {
        headers: { From: 'A Example <[email protected]>', To: '[email protected]', Subject: 'Re: Quarterly zephyr roadmap' },
        labelIds: ['SENT'],
        body: 'Looks good, shipping it this week.',
      }),
    );
    // The sweep enqueues only while a chat provider is available (a job the
    // handler cannot run would burn the thread's revision slot); the gateway
    // test seam makes chat "available" without a key or network.
    __setChatTransportForTests(async () => {
      throw new Error('chat transport must not be called by the sweep');
    });
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweep(dir, fx, vault, {}, 'gmail');
        const jobs1 = await engine.executeRaw<{ id: number; idempotency_key: string | null }>(
          `SELECT id, idempotency_key FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        expect(jobs1.length).toBe(1);
        // Key folds the SOURCE first (red-team: the same account registered
        // twice must not coalesce source B's job onto source A's).
        expect(jobs1[0].idempotency_key).toMatch(/^loops:[^:]+:emails\//);

        // History touches the thread again with NO content change: the
        // page-revision-keyed idempotency key dedupes the enqueue.
        fx.history = [[T_A]];
        fx.historyResponseId = '1011';
        await sweep(dir, fx, vault, {}, 'gmail');
        const jobs2 = await engine.executeRaw<{ id: number }>(
          `SELECT id FROM minion_jobs WHERE name = 'loops_extract'`,
        );
        expect(jobs2.length).toBe(1);
        expect(jobs2[0].id).toBe(jobs1[0].id);
      });
    } finally {
      __setChatTransportForTests(null);
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ── Secondary calendars (one calendar per source) ────────────────────────────

describe('google-source gmail recent-first cursors', () => {
  const sweepDays = (dir: string, fx: FakeGoogle, vault: FakeVault, days: number, opts: Partial<SyncOpts> = {}) =>
    runGoogleSync(
      engine,
      'gsrc',
      parseGoogleSourceConfig({ kind: 'google', g_account: '[email protected]', g_services: 'gmail', g_history_days: days, g_dir: dir }, dir),
      { sourceId: 'gsrc', noEmbed: true, noExtract: true, ...opts },
      buildFetch(fx),
      vault,
    );
  const tid = (p: string, i: number) => `17aa0000000${p}${String(i).padStart(4, '0')}`;
  const msg = (p: string, i: number, ms: number) =>
    gmsg(`18c2f4a9b3d${p}${String(i).padStart(4, '0')}`, tid(p, i), ms, {
      headers: { From: 'Peer Example <[email protected]>', To: '[email protected]', Subject: `Topic ${p} ${i}` },
      body: `Body ${p} ${i}.`,
    });
  const lastSyncAt = async () =>
    (await engine.executeRaw<{ last_sync_at: unknown }>(`SELECT last_sync_at FROM sources WHERE id = 'gsrc'`))[0].last_sync_at;

  test('expired history: a gap larger than one bounded listing drains fully, resumes after an abort, and is never called fresh early', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-gap-'));
    const fx = emptyFx();
    fx.messages.push(msg('e', 0, daysAgoMs(20)));
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        await sweepDays(dir, fx, vault, 90);
        const stamped = await lastSyncAt();

        // History expires while 30 messages arrive; one listing holds only 20
        // ids (maxPages 20 x page 1), which the old bookmark fallback dropped.
        fx.historyExpired = true;
        fx.profileHistoryId = '2000';
        fx.messagesPageSize = 1;
        for (let i = 1; i <= 30; i++) fx.messages.push(msg('e', i, hoursAgoMs(31 - i)));
        const controller = new AbortController();
        const before = fx.threadFetches;
        fx.onThreadFetch = () => { if (fx.threadFetches - before >= 10) controller.abort(); };
        const r1 = await sweepDays(dir, fx, vault, 90, { signal: controller.signal });
        expect(r1.status).toBe('partial');
        let state = readGoogleState(dir);
        expect(state.gmail_history_id).toBe('2000'); // re-anchored with the gap, atomically
        expect(state.gmail_gap_floor_ms).not.toBeNull();
        expect(await lastSyncAt()).toEqual(stamped); // gap open → not fresh

        fx.onThreadFetch = undefined;
        fx.historyExpired = false;
        fx.historyHonorsStart = true;
        fx.historyResponseId = '2000';
        await sweepDays(dir, fx, vault, 90);
        state = readGoogleState(dir);
        expect(state.gmail_gap_floor_ms).toBeNull();
        expect(state.gmail_gap_after_ms).toBeNull();
        expect(await lastSyncAt()).not.toEqual(stamped);
        const slugs = await slugsWhere(`slug LIKE 'emails/%'`);
        expect(slugs.length).toBe(31); // every gap message landed; none skipped
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('history window: narrowing mid-backfill keeps the covered bound; widening again resumes below it', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-window-'));
    const fx = emptyFx();
    for (let i = 0; i < 6; i++) fx.messages.push(msg('d', i, daysAgoMs(40 + 10 * i))); // 40..90 days
    const vault = makeVault();
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        // Abort after two archive threads: floor sits at 50 days.
        const controller = new AbortController();
        fx.onThreadFetch = () => { if (fx.threadFetches >= 2) controller.abort(); };
        await sweepDays(dir, fx, vault, 120, { signal: controller.signal });
        expect(readGoogleState(dir).gmail_backfill_floor_ms).toBe(daysAgoMs(50));

        // A 4-day scoped window: nothing to fetch, and the done bound records
        // what was really covered (50 days), not the narrow 4-day cutoff.
        fx.onThreadFetch = undefined;
        const fetches = fx.threadFetches;
        await sweepDays(dir, fx, vault, 4);
        expect(fx.threadFetches).toBe(fetches);
        let state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(true);
        expect(state.gmail_backfill_cutoff_ms).toBe(daysAgoMs(50));

        // Back to 120 days: only the four unimported archive threads are fetched.
        await sweepDays(dir, fx, vault, 120);
        expect(fx.threadFetches - fetches).toBe(4);
        state = readGoogleState(dir);
        expect(state.gmail_backfill_done).toBe(true);
        expect((await slugsWhere(`slug LIKE 'emails/%'`)).length).toBe(6);

        // Wider still: 200 days reopens below the covered 120-day bound.
        fx.messages.push(msg('d', 9, daysAgoMs(150)));
        await sweepDays(dir, fx, vault, 200);
        expect((await slugsWhere(`slug LIKE 'emails/%'`)).length).toBe(7);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

describe('google-source secondary calendar', () => {
  test('a source with g_calendar_id sweeps THAT calendar, URL-encoded, and materializes its events', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gsrc-cal2-'));
    const fx = emptyFx();
    const vault = makeVault();
    calendarFixture(fx);
    const calId = '[email protected]';
    try {
      await insertGoogleSource(dir);
      await withHome(async () => {
        const cfg = parseGoogleSourceConfig(
          {
            kind: 'google',
            g_account: '[email protected]',
            g_services: 'calendar',
            g_history_days: 90,
            g_dir: dir,
            g_calendar_id: calId,
          },
          dir,
        );
        const res = await runGoogleSync(
          engine,
          'gsrc',
          cfg,
          { sourceId: 'gsrc', noEmbed: true, noExtract: true },
          buildFetch(fx),
          vault,
        );
        expect(res.status).toBe('first_sync');
        // The sweep hit the SECONDARY calendar's path (percent-encoded '@'),
        // never the hardcoded primary.
        const calCalls = fx.calls.filter((c) => c.includes('/calendars/'));
        expect(calCalls.length).toBeGreaterThan(0);
        for (const c of calCalls) {
          expect(c).toContain('/calendars/family0123456789%40group.calendar.google.com/events');
        }
        // Its events materialized through the normal pipeline.
        expect((await slugsWhere(`slug LIKE 'calendar/%zephyr%'`)).length).toBe(1);
      });
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});