Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(google): Gmail sync no longer imports threads older than the history window
garrytan28ee99872cb2
diff --git a/src/core/google/google-source.ts b/src/core/google/google-source.tsindex f8ac5059be..43c208aa6d 100644--- a/src/core/google/google-source.ts+++ b/src/core/google/google-source.ts@@ -55,6 +55,7 @@ import {type FetchImpl,} from './google-clients.ts';import { renderThreadPage, threadRelPath } from './google-render.ts';+import { slugifyPath } from '../sync.ts';import {ALL_GOOGLE_SERVICES,isGmailHoldKey,@@ -173,12 +174,20 @@ async function processThread(summary: GoogleSyncSummary,countedSlugs: Set<string>,seen: { thread?: GmailThreadData } = {},+ cutoffMs?: number,): Promise<GmailThreadData | null> {const thread = await gmail.getThread(threadId, deps.cfg.account, {...(deps.opts.signal ? { signal: deps.opts.signal } : {}),});seen.thread = thread;summary.threadsSeen++;+ const newestMs = thread.messages[thread.messages.length - 1]?.internalDateMs ?? 0;+ // #8482: the change feed and held retries name threads of any age; the history window keeps a thread with no page out, as the backfill does.+ if (cutoffMs !== undefined && newestMs < cutoffMs) {+ const [page] = await deps.engine.executeRaw('SELECT 1 FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL',+ [deps.sourceId, slugifyPath(threadRelPath(thread))]);+ if (!page) return null;+ }const rendered = renderThreadPage(thread);// Pure noise renders no page AND skips detection — an all-noise thread// produces an empty verdict anyway, so nothing opens and nothing closes.@@ -196,7 +205,6 @@ async function processThread(if (verdict?.withheld) summary.loopsWithheld[verdict.withheld.reason] = (summary.loopsWithheld[verdict.withheld.reason] ?? 0) + 1;// LLM extraction candidates: trickle + the bounded recent window only —// the deep historical backfill is never extracted (spend honesty, F9).- const newestMs = thread.messages[thread.messages.length - 1]?.internalDateMs ?? 0;const windowMs = LOOPS_EXTRACT_WINDOW_DAYS * 86_400_000;if (newestMs > 0 && Date.now() - newestMs <= windowMs) {// Structural eligibility, not "everything recent": bulk mail the owner@@ -267,14 +275,14 @@ type ThreadOutcome = { kind: 'landed'; newestMs: number } | { kind: 'gone' | 'sk* block the cursor). On a managed brain a write error aborts the run after* the abort path publishes the holds.*/-async function attemptThread(g: GmailSweep, tid: string, version: string | null): Promise<ThreadOutcome> {+async function attemptThread(g: GmailSweep, tid: string, version: string | null, cutoffMs?: number): Promise<ThreadOutcome> {if (!g.holds.shouldAttempt(tid, version)) {g.deps.tick(`thread ${tid} held`);return { kind: 'skipped' };}const seen: { thread?: GmailThreadData } = {};try {- const thread = await processThread(g.deps, g.gmail, tid, g.activePack, g.summary, g.countedSlugs, seen);+ const thread = await processThread(g.deps, g.gmail, tid, g.activePack, g.summary, g.countedSlugs, seen, cutoffMs);g.holds.succeed(tid);const newestMs = thread?.messages[thread.messages.length - 1]?.internalDateMs ?? 0;if (newestMs > (g.state.gmail_newest_ms ?? 0)) g.state.gmail_newest_ms = newestMs;@@ -417,7 +425,7 @@ async function sweepGmail(g: GmailSweep, onCurrent?: () => Promise<void>): Promi// hold and never blocks the cursor.for (const tid of g.holds.dueHeldKeys().filter(isGmailHoldKey)) {if (deps.opts.signal?.aborted) return false;- await attemptThread(g, tid, null);+ await attemptThread(g, tid, null, cutoffMs);}if (await settleGraceHolds(g, cutoffMs) === 'aborted') return false;const isCurrent = (): boolean => {@@ -543,7 +551,7 @@ async function drainGmailDelta(g: GmailSweep, cutoffMs: number): Promise<WalkOut}// Thread-specific version (the latest history record touching it), so unrelated mail never resets a count.const version = fromHistory.has(tid) && versions.get(tid) ? `history:${versions.get(tid)}` : null;- const outcome = await attemptThread(g, tid, version);+ const outcome = await attemptThread(g, tid, version, cutoffMs);if (outcome.kind === 'failed' || outcome.kind === 'rate_limited') {failed++;// Per-user quota: the remaining threads would burn the client's retrydiff --git a/test/google-gmail-delta-window-8482.test.ts b/test/google-gmail-delta-window-8482.test.tsnew file mode 100644index 0000000000..e9d67ed5be--- /dev/null+++ b/test/google-gmail-delta-window-8482.test.ts@@ -0,0 +1,120 @@+/**+ * #8482: the Gmail change feed reports a thread whenever anything on it+ * changes (a label, a read flag), however old its mail is. A thread whose+ * newest message is older than g_history_days stays out of the brain, the+ * same bound the backfill walk applies with `after:<cutoff>`. A thread the brain already holds keeps getting its+ * changes after it ages out of the window.+ *+ * Synthetic mailbox only: example.invalid addresses, no network.+ */+import { afterAll, beforeAll, expect, test } from 'bun:test';+import { parseGoogleSourceConfig, runGoogleSync } from '../src/core/google/google-source.ts';+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';+import { withEnv } from './helpers/with-env.ts';+import { createConnectorFixture, options, json, googleConfig, withGoogleAccount } from './helpers/connector-fixture.ts';++const { engines, env, source, setup, teardown } = createConnectorFixture();+beforeAll(setup, 120_000);+afterAll(teardown);++const b64url = (s: string) => Buffer.from(s, 'utf-8').toString('base64url');+const NOW_MS = Math.floor(Date.now() / 1000) * 1000;+const DAY = 86_400_000;+const ARCHIVED = { tid: '17ee000000000a01', ms: NOW_MS - 400 * DAY };+const CURRENT = { tid: '17ee000000000c01', ms: NOW_MS - 2 * DAY };++test('a change-feed thread older than g_history_days is not imported; a current one is', async () => withEnv(env, async () => {+ for (const engine of engines) {+ const config = { ...googleConfig, g_services: 'gmail', g_history_days: 30 };+ const f = await source(engine, config);+ const cfg = parseGoogleSourceConfig(config, f.dir);+ let historyId = '3000';+ let changed: Array<typeof ARCHIVED> = [];+ const fetched: string[] = [];+ const fetcher = async (url: string) => {+ const u = new URL(url);+ if (u.pathname.endsWith('/settings/sendAs')) return json({ sendAs: [] });+ if (u.pathname.endsWith('/users/me/profile')) return json({ emailAddress: cfg.account, historyId: '3000' });+ if (u.pathname.endsWith('/users/me/messages')) return json({ messages: [] });+ if (u.pathname.endsWith('/users/me/history')) {+ const replay = u.searchParams.get('startHistoryId') !== historyId;+ return json({ historyId, history: replay ? [{ id: historyId, messages: changed.map(({ tid }) => ({ id: `18ee${tid.slice(-8)}`, threadId: tid })) }] : [] });+ }+ const m = u.pathname.match(/\/users\/me\/threads\/([^/]+)$/);+ if (m) {+ fetched.push(m[1]);+ const t = [ARCHIVED, CURRENT].find(({ tid }) => tid === m[1])!;+ return json({ id: t.tid, messages: [{ id: `18ee${t.tid.slice(-8)}`, threadId: t.tid, labelIds: ['INBOX'], internalDate: String(t.ms),+ payload: { mimeType: 'text/plain', headers: [{ name: 'From', value: 'Alice Example <[email protected]>' },+ { name: 'To', value: cfg.account }, { name: 'Subject', value: `Topic ${t.tid}` }], body: { data: b64url(`Body ${t.tid}.`) } } }] });+ }+ return json({ error: { message: 'unexpected fixture route' } }, 400);+ };+ const run = () => runGoogleSync(engine, f.id, cfg, options, withGoogleAccount(fetcher, cfg.account));+ const imported = async () => (await engine.executeRaw<{ slug: string }>(+ "SELECT slug FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' ORDER BY slug", [f.id])).map((r) => r.slug);++ expect((await run()).status).not.toBe('partial');+ expect(await imported()).toEqual([]);++ historyId = '3010';+ changed = [ARCHIVED, CURRENT];+ const delta = await run();+ expect(delta.status).not.toBe('partial');+ expect(fetched.sort()).toEqual([ARCHIVED.tid, CURRENT.tid].sort());+ const slugs = await imported();+ expect(slugs).toHaveLength(1);+ expect(slugs[0]).toContain(CURRENT.tid);++ // The skipped thread is settled, not parked or held: the next run does not fetch it again.+ fetched.length = 0;+ changed = [];+ historyId = '3020';+ expect((await run()).status).not.toBe('partial');+ expect(fetched).toEqual([]);+ expect((await imported()).some((slug) => slug.includes(ARCHIVED.tid))).toBe(false);+ await disposePersistenceConsumer(engine);+ }+}), 120_000);++test('a thread already in the brain keeps updating after it ages out of the window', async () => withEnv(env, async () => {+ for (const engine of engines) {+ const config = { ...googleConfig, g_services: 'gmail', g_history_days: 30 };+ const f = await source(engine, config);+ const cfg = parseGoogleSourceConfig(config, f.dir);+ const tid = '17ee000000000e01';+ const started = NOW_MS - 400 * DAY;+ let reply = { ms: NOW_MS - DAY, body: 'Second message.' };+ let historyId = '4000';+ const fetcher = async (url: string) => {+ const u = new URL(url);+ if (u.pathname.endsWith('/settings/sendAs')) return json({ sendAs: [] });+ if (u.pathname.endsWith('/users/me/profile')) return json({ emailAddress: cfg.account, historyId: '4000' });+ if (u.pathname.endsWith('/users/me/messages')) return json({ messages: [] });+ if (u.pathname.endsWith('/users/me/history')) {+ return json({ historyId, history: u.searchParams.get('startHistoryId') === historyId ? [] : [{ id: historyId, messages: [{ id: '18ee00000000e002', threadId: tid }] }] });+ }+ if (u.pathname.endsWith(`/users/me/threads/${tid}`)) {+ const msg = (id: string, ms: number, body: string) => ({ id, threadId: tid, labelIds: ['INBOX'], internalDate: String(ms),+ payload: { mimeType: 'text/plain', headers: [{ name: 'From', value: 'Alice Example <[email protected]>' },+ { name: 'To', value: cfg.account }, { name: 'Subject', value: 'Long thread' }], body: { data: b64url(body) } } });+ return json({ id: tid, messages: [msg('18ee00000000e001', started, 'First message.'), msg('18ee00000000e002', reply.ms, reply.body)] });+ }+ return json({ error: { message: 'unexpected fixture route' } }, 400);+ };+ const run = () => runGoogleSync(engine, f.id, cfg, options, withGoogleAccount(fetcher, cfg.account));+ const imports = async () => (await engine.executeRaw<{ n: number }>(+ "SELECT count(*)::int AS n FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' AND state='committed'", [f.id]))[0]!.n;++ expect((await run()).status).not.toBe('partial');+ historyId = '4010';+ expect((await run()).status).not.toBe('partial');+ expect(await imports()).toBe(1);++ reply = { ms: NOW_MS - 60 * DAY, body: 'Second message, edited upstream.' };+ historyId = '4020';+ expect((await run()).status).not.toBe('partial');+ expect(await imports()).toBe(2);+ await disposePersistenceConsumer(engine);+ }+}), 120_000);