Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(google): Gmail sync no longer imports threads older than the history window

garrytan28ee99872cb2
diff --git a/src/core/google/google-source.ts b/src/core/google/google-source.ts
index f8ac5059be..43c208aa6d 100644
--- a/src/core/google/google-source.ts
+++ b/src/core/google/google-source.ts
@@ -55,6 +55,7 @@ import {
type FetchImpl,
} from './google-clients.ts';
import { renderThreadPage, threadRelPath } from './google-render.ts';
+import { slugifyPath } from '../sync.ts';
import {
ALL_GOOGLE_SERVICES,
isGmailHoldKey,
@@ -173,12 +174,20 @@ async function processThread(
summary: GoogleSyncSummary,
countedSlugs: Set<string>,
seen: { thread?: GmailThreadData } = {},
+ cutoffMs?: number,
): Promise<GmailThreadData | null> {
const thread = await gmail.getThread(threadId, deps.cfg.account, {
...(deps.opts.signal ? { signal: deps.opts.signal } : {}),
});
seen.thread = thread;
summary.threadsSeen++;
+ const newestMs = thread.messages[thread.messages.length - 1]?.internalDateMs ?? 0;
+ // #8482: the change feed and held retries name threads of any age; the history window keeps a thread with no page out, as the backfill does.
+ if (cutoffMs !== undefined && newestMs < cutoffMs) {
+ const [page] = await deps.engine.executeRaw('SELECT 1 FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL',
+ [deps.sourceId, slugifyPath(threadRelPath(thread))]);
+ if (!page) return null;
+ }
const rendered = renderThreadPage(thread);
// Pure noise renders no page AND skips detection — an all-noise thread
// produces an empty verdict anyway, so nothing opens and nothing closes.
@@ -196,7 +205,6 @@ async function processThread(
if (verdict?.withheld) summary.loopsWithheld[verdict.withheld.reason] = (summary.loopsWithheld[verdict.withheld.reason] ?? 0) + 1;
// LLM extraction candidates: trickle + the bounded recent window only —
// the deep historical backfill is never extracted (spend honesty, F9).
- const newestMs = thread.messages[thread.messages.length - 1]?.internalDateMs ?? 0;
const windowMs = LOOPS_EXTRACT_WINDOW_DAYS * 86_400_000;
if (newestMs > 0 && Date.now() - newestMs <= windowMs) {
// Structural eligibility, not "everything recent": bulk mail the owner
@@ -267,14 +275,14 @@ type ThreadOutcome = { kind: 'landed'; newestMs: number } | { kind: 'gone' | 'sk
* block the cursor). On a managed brain a write error aborts the run after
* the abort path publishes the holds.
*/
-async function attemptThread(g: GmailSweep, tid: string, version: string | null): Promise<ThreadOutcome> {
+async function attemptThread(g: GmailSweep, tid: string, version: string | null, cutoffMs?: number): Promise<ThreadOutcome> {
if (!g.holds.shouldAttempt(tid, version)) {
g.deps.tick(`thread ${tid} held`);
return { kind: 'skipped' };
}
const seen: { thread?: GmailThreadData } = {};
try {
- const thread = await processThread(g.deps, g.gmail, tid, g.activePack, g.summary, g.countedSlugs, seen);
+ const thread = await processThread(g.deps, g.gmail, tid, g.activePack, g.summary, g.countedSlugs, seen, cutoffMs);
g.holds.succeed(tid);
const newestMs = thread?.messages[thread.messages.length - 1]?.internalDateMs ?? 0;
if (newestMs > (g.state.gmail_newest_ms ?? 0)) g.state.gmail_newest_ms = newestMs;
@@ -417,7 +425,7 @@ async function sweepGmail(g: GmailSweep, onCurrent?: () => Promise<void>): Promi
// hold and never blocks the cursor.
for (const tid of g.holds.dueHeldKeys().filter(isGmailHoldKey)) {
if (deps.opts.signal?.aborted) return false;
- await attemptThread(g, tid, null);
+ await attemptThread(g, tid, null, cutoffMs);
}
if (await settleGraceHolds(g, cutoffMs) === 'aborted') return false;
const isCurrent = (): boolean => {
@@ -543,7 +551,7 @@ async function drainGmailDelta(g: GmailSweep, cutoffMs: number): Promise<WalkOut
}
// Thread-specific version (the latest history record touching it), so unrelated mail never resets a count.
const version = fromHistory.has(tid) && versions.get(tid) ? `history:${versions.get(tid)}` : null;
- const outcome = await attemptThread(g, tid, version);
+ const outcome = await attemptThread(g, tid, version, cutoffMs);
if (outcome.kind === 'failed' || outcome.kind === 'rate_limited') {
failed++;
// Per-user quota: the remaining threads would burn the client's retry
diff --git a/test/google-gmail-delta-window-8482.test.ts b/test/google-gmail-delta-window-8482.test.ts
new file mode 100644
index 0000000000..e9d67ed5be
--- /dev/null
+++ b/test/google-gmail-delta-window-8482.test.ts
@@ -0,0 +1,120 @@
+/**
+ * #8482: the Gmail change feed reports a thread whenever anything on it
+ * changes (a label, a read flag), however old its mail is. A thread whose
+ * newest message is older than g_history_days stays out of the brain, the
+ * same bound the backfill walk applies with `after:<cutoff>`. A thread the brain already holds keeps getting its
+ * changes after it ages out of the window.
+ *
+ * Synthetic mailbox only: example.invalid addresses, no network.
+ */
+import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { parseGoogleSourceConfig, runGoogleSync } from '../src/core/google/google-source.ts';
+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
+import { withEnv } from './helpers/with-env.ts';
+import { createConnectorFixture, options, json, googleConfig, withGoogleAccount } from './helpers/connector-fixture.ts';
+
+const { engines, env, source, setup, teardown } = createConnectorFixture();
+beforeAll(setup, 120_000);
+afterAll(teardown);
+
+const b64url = (s: string) => Buffer.from(s, 'utf-8').toString('base64url');
+const NOW_MS = Math.floor(Date.now() / 1000) * 1000;
+const DAY = 86_400_000;
+const ARCHIVED = { tid: '17ee000000000a01', ms: NOW_MS - 400 * DAY };
+const CURRENT = { tid: '17ee000000000c01', ms: NOW_MS - 2 * DAY };
+
+test('a change-feed thread older than g_history_days is not imported; a current one is', async () => withEnv(env, async () => {
+ for (const engine of engines) {
+ const config = { ...googleConfig, g_services: 'gmail', g_history_days: 30 };
+ const f = await source(engine, config);
+ const cfg = parseGoogleSourceConfig(config, f.dir);
+ let historyId = '3000';
+ let changed: Array<typeof ARCHIVED> = [];
+ const fetched: string[] = [];
+ const fetcher = async (url: string) => {
+ const u = new URL(url);
+ if (u.pathname.endsWith('/settings/sendAs')) return json({ sendAs: [] });
+ if (u.pathname.endsWith('/users/me/profile')) return json({ emailAddress: cfg.account, historyId: '3000' });
+ if (u.pathname.endsWith('/users/me/messages')) return json({ messages: [] });
+ if (u.pathname.endsWith('/users/me/history')) {
+ const replay = u.searchParams.get('startHistoryId') !== historyId;
+ return json({ historyId, history: replay ? [{ id: historyId, messages: changed.map(({ tid }) => ({ id: `18ee${tid.slice(-8)}`, threadId: tid })) }] : [] });
+ }
+ const m = u.pathname.match(/\/users\/me\/threads\/([^/]+)$/);
+ if (m) {
+ fetched.push(m[1]);
+ const t = [ARCHIVED, CURRENT].find(({ tid }) => tid === m[1])!;
+ return json({ id: t.tid, messages: [{ id: `18ee${t.tid.slice(-8)}`, threadId: t.tid, labelIds: ['INBOX'], internalDate: String(t.ms),
+ payload: { mimeType: 'text/plain', headers: [{ name: 'From', value: 'Alice Example <[email protected]>' },
+ { name: 'To', value: cfg.account }, { name: 'Subject', value: `Topic ${t.tid}` }], body: { data: b64url(`Body ${t.tid}.`) } } }] });
+ }
+ return json({ error: { message: 'unexpected fixture route' } }, 400);
+ };
+ const run = () => runGoogleSync(engine, f.id, cfg, options, withGoogleAccount(fetcher, cfg.account));
+ const imported = async () => (await engine.executeRaw<{ slug: string }>(
+ "SELECT slug FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' ORDER BY slug", [f.id])).map((r) => r.slug);
+
+ expect((await run()).status).not.toBe('partial');
+ expect(await imported()).toEqual([]);
+
+ historyId = '3010';
+ changed = [ARCHIVED, CURRENT];
+ const delta = await run();
+ expect(delta.status).not.toBe('partial');
+ expect(fetched.sort()).toEqual([ARCHIVED.tid, CURRENT.tid].sort());
+ const slugs = await imported();
+ expect(slugs).toHaveLength(1);
+ expect(slugs[0]).toContain(CURRENT.tid);
+
+ // The skipped thread is settled, not parked or held: the next run does not fetch it again.
+ fetched.length = 0;
+ changed = [];
+ historyId = '3020';
+ expect((await run()).status).not.toBe('partial');
+ expect(fetched).toEqual([]);
+ expect((await imported()).some((slug) => slug.includes(ARCHIVED.tid))).toBe(false);
+ await disposePersistenceConsumer(engine);
+ }
+}), 120_000);
+
+test('a thread already in the brain keeps updating after it ages out of the window', async () => withEnv(env, async () => {
+ for (const engine of engines) {
+ const config = { ...googleConfig, g_services: 'gmail', g_history_days: 30 };
+ const f = await source(engine, config);
+ const cfg = parseGoogleSourceConfig(config, f.dir);
+ const tid = '17ee000000000e01';
+ const started = NOW_MS - 400 * DAY;
+ let reply = { ms: NOW_MS - DAY, body: 'Second message.' };
+ let historyId = '4000';
+ const fetcher = async (url: string) => {
+ const u = new URL(url);
+ if (u.pathname.endsWith('/settings/sendAs')) return json({ sendAs: [] });
+ if (u.pathname.endsWith('/users/me/profile')) return json({ emailAddress: cfg.account, historyId: '4000' });
+ if (u.pathname.endsWith('/users/me/messages')) return json({ messages: [] });
+ if (u.pathname.endsWith('/users/me/history')) {
+ return json({ historyId, history: u.searchParams.get('startHistoryId') === historyId ? [] : [{ id: historyId, messages: [{ id: '18ee00000000e002', threadId: tid }] }] });
+ }
+ if (u.pathname.endsWith(`/users/me/threads/${tid}`)) {
+ const msg = (id: string, ms: number, body: string) => ({ id, threadId: tid, labelIds: ['INBOX'], internalDate: String(ms),
+ payload: { mimeType: 'text/plain', headers: [{ name: 'From', value: 'Alice Example <[email protected]>' },
+ { name: 'To', value: cfg.account }, { name: 'Subject', value: 'Long thread' }], body: { data: b64url(body) } } });
+ return json({ id: tid, messages: [msg('18ee00000000e001', started, 'First message.'), msg('18ee00000000e002', reply.ms, reply.body)] });
+ }
+ return json({ error: { message: 'unexpected fixture route' } }, 400);
+ };
+ const run = () => runGoogleSync(engine, f.id, cfg, options, withGoogleAccount(fetcher, cfg.account));
+ const imports = async () => (await engine.executeRaw<{ n: number }>(
+ "SELECT count(*)::int AS n FROM persistence_requests WHERE source_id=$1 AND intent->>'kind'='connector_v2_import' AND state='committed'", [f.id]))[0]!.n;
+
+ expect((await run()).status).not.toBe('partial');
+ historyId = '4010';
+ expect((await run()).status).not.toBe('partial');
+ expect(await imports()).toBe(1);
+
+ reply = { ms: NOW_MS - 60 * DAY, body: 'Second message, edited upstream.' };
+ historyId = '4020';
+ expect((await run()).status).not.toBe('partial');
+ expect(await imports()).toBe(2);
+ await disposePersistenceConsumer(engine);
+ }
+}), 120_000);