Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(google): a purged Google item no longer fails and holds every classic sync

garrytanb27c4fe0f247
diff --git a/src/core/google/google-source.ts b/src/core/google/google-source.ts
index 43c208aa6d..cad4348a4c 100644
--- a/src/core/google/google-source.ts
+++ b/src/core/google/google-source.ts
@@ -194,6 +194,12 @@ async function processThread(
if (!rendered) return thread;
const slug = await importRendered(deps, rendered.relPath, rendered.markdown, activePack, summary, countedSlugs,
{ thread_id: thread.threadId, message_ids: thread.messages.map((m) => m.id) });
+ // #8644: an unmanaged purged thread is neither detected nor extracted, and its grace hold is dropped, so its text does
+ // not come back through open loops.
+ if (slug === null) {
+ delete deps.loopState?.loop_grace_holds?.[thread.threadId];
+ return thread;
+ }
for (const message of thread.messages) {
const state = message.attachmentInspection?.state ?? 'not_inspected';
summary.attachmentInspection[state] = (summary.attachmentInspection[state] ?? 0) + 1;
diff --git a/src/core/google/sweep-shared.ts b/src/core/google/sweep-shared.ts
index 5e91da5a7d..d58ce70253 100644
--- a/src/core/google/sweep-shared.ts
+++ b/src/core/google/sweep-shared.ts
@@ -23,6 +23,7 @@ import { isPrivatePage } from '../search/private-visibility.ts';
import { withPrivateVisibility } from './google-render.ts';
import { isWriteThroughDisabled } from '../write-through.ts';
import { isSourceDbOnlySlug } from '../persistence/source-storage.ts';
+import { intentCarriesPurgedContent } from '../persistence/purged-intent.ts';
import type { GoogleSourceConfig, GoogleSourceState } from './types.ts';
import type { LoopsExclusionPolicy } from './loops-exclusion.ts';
@@ -139,7 +140,7 @@ export async function importRendered(
summary: GoogleSyncSummary,
countedSlugs: Set<string>,
identity: Record<string, string | readonly string[]> = {},
-): Promise<string> {
+): Promise<string | null> {
markdown = connectorRender(markdown, { path: relPath, account: deps.cfg.account, ...identity });
const slug = slugifyPath(relPath);
const [labels] = await deps.engine.executeRaw<{ type: string; frontmatter: unknown }>(
@@ -162,6 +163,11 @@ export async function importRendered(
const existing = await deps.engine.getPage(slug, { sourceId: deps.sourceId });
if (existing) assertInlineBody(existing, 'Google connector materialization');
markdown = await carryStoredFences(deps.sourceId, slug, relPath, existing, markdown, 'Google connector fence carry');
+ // #8644: content the owner purged is skipped before any file is touched, as the managed connector skips it (#6831).
+ if (await intentCarriesPurgedContent(deps.engine, { source_id: deps.sourceId, slug, intent: { content: markdown } }, { facts: false, pages: true })) {
+ deps.log(`[google] skipped ${slug}: its content matches a page purge tombstone; the rest of the sync continues. To let it back in: gbrain pages unpurge ${slug}`);
+ return null;
+ }
const databaseOnly = await isWriteThroughDisabled(deps.engine) || isSourceDbOnlySlug(deps.cfg.dir, slug, 'refuse');
const guard = databaseOnly ? null : await replaceableFile(deps.engine, deps.sourceId, slug, filePath, existing ? withStoredFactsFence : undefined);
const drift = () => new OperationError('file_database_drift',
diff --git a/test/google-classic-purged-item-8644.test.ts b/test/google-classic-purged-item-8644.test.ts
new file mode 100644
index 0000000000..4709d05ff6
--- /dev/null
+++ b/test/google-classic-purged-item-8644.test.ts
@@ -0,0 +1,159 @@
+/**
+ * #8644: on an unmanaged (classic) brain, a Google item whose page the owner purged keeps arriving unchanged from
+ * the provider. The purge leaves a connector page's checkout file (write_through skipped: connector_database), so every
+ * later sync failed that item file_database_drift, or purged_content from the import guard after a temporary write
+ * once the file was removed, and held it after three runs. It is now skipped before any file is written, the run stays
+ * clean, and the page stays purged, as on a managed brain (#6831). A skipped Gmail thread is not run through loop
+ * detection or extraction either, so the open loops the purge removed do not come back with the thread's text.
+ * Real PGLite and Postgres (when DATABASE_URL is set), the real Google importer and purge, a fake People API.
+ */
+import { describe, expect, test } from 'bun:test';
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { parseGoogleSourceConfig, readGoogleState, runGoogleSync } from '../src/core/google/google-source.ts';
+import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';
+import { unpurgePage } from '../src/core/persistence/page-purge.ts';
+import { _resetWriteThroughCacheForTest } from '../src/core/write-through.ts';
+import { contact, json, options, withGoogleAccount } from './helpers/connector-fixture.ts';
+import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const SOURCE = 'source-a';
+const SLUG = 'people/alice-example';
+const CONFIG = { kind: 'google', g_account: '[email protected]', g_services: 'contacts', g_access: 'env', g_token_env: 'CONNECTOR_TEST_TOKEN' };
+const ctx = (engine: BrainEngine) => ({ engine, sourceId: SOURCE, config: { engine: engine.kind }, remote: false, dryRun: false,
+ logger: { info() {}, warn() {}, error() {} } });
+
+interface Brain { engine: BrainEngine; root: string; path: string; render: string; sync(): ReturnType<typeof runGoogleSync> }
+
+async function withPurgedPage(databaseUrl: string | undefined, run: (b: Brain) => Promise<void>): Promise<void> {
+ const root = mkdtempSync(join(tmpdir(), 'gbrain-8644-'));
+ try {
+ await withEnv({ GBRAIN_HOME: root, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined, CONNECTOR_TEST_TOKEN: 'synthetic-local-fixture' }, async () => {
+ _resetWriteThroughCacheForTest();
+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);
+ try {
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ const dir = join(root, 'google');
+ await engine.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,$3::text::jsonb)', [SOURCE, dir, JSON.stringify(CONFIG)]);
+ const fetch = withGoogleAccount(async (url: string) => {
+ if (new URL(url).pathname === '/v1/people/me/connections') return json({ connections: [
+ { ...contact('c1', 'Alice Example'), organizations: [{ name: 'Acme Example' }] }], nextSyncToken: 'token-1' });
+ throw new Error(`Unexpected Google fixture route: ${url}`);
+ }, CONFIG.g_account);
+ const cfg = parseGoogleSourceConfig(CONFIG, dir);
+ const sync = () => runGoogleSync(engine, SOURCE, cfg, { ...options, sourceId: SOURCE }, fetch);
+ expect((await sync()).failedFiles ?? 0).toBe(0);
+ const path = join(dir, `${SLUG}.md`);
+ const render = readFileSync(path, 'utf8');
+ const snapshot = await engine.readPageSnapshot(SLUG, { sourceId: SOURCE });
+ await submitPageMutation(ctx(engine), { operation: 'delete_page', params: { slug: SLUG, purge: true, expected_revision: snapshot!.revision } });
+ expect(await engine.getPage(SLUG, { sourceId: SOURCE })).toBeNull();
+ await run({ engine, root, path, render, sync });
+ } finally { await close(); }
+ });
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}
+
+const holds = (root: string) => ((readGoogleState(join(root, 'google')).item_holds ?? { items: {} }) as { items: Record<string, { code: string; state: string }> }).items;
+const pageRows = async (engine: BrainEngine) => (await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND slug=$2', [SOURCE, SLUG]))[0]!.n;
+
+for (const backend of testBackends()) describe(`[${backend}] #8644: a purged item on an unmanaged Google source`, () => {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+
+ test('the unchanged item is skipped on every run while the purge left its file: no failure, no hold, no page', async () => withPurgedPage(databaseUrl, async b => {
+ expect(readFileSync(b.path, 'utf8')).toBe(b.render);
+ for (let i = 0; i < 3; i++) {
+ const result = await b.sync();
+ expect(result.failedFiles ?? 0).toBe(0);
+ expect(result.status).not.toBe('partial');
+ }
+ expect(holds(b.root)['contact:people/c1']).toBeUndefined();
+ expect(readFileSync(b.path, 'utf8')).toBe(b.render);
+ expect(await pageRows(b.engine)).toBe(0);
+ }), 180_000);
+
+ test('with the checkout file removed, nothing is written, not even a temporary file', async () => withPurgedPage(databaseUrl, async b => {
+ rmSync(b.path);
+ for (let i = 0; i < 3; i++) expect((await b.sync()).failedFiles ?? 0).toBe(0);
+ expect(holds(b.root)['contact:people/c1']).toBeUndefined();
+ expect(existsSync(b.path)).toBe(false);
+ expect(existsSync(`${b.path}.tmp`)).toBe(false);
+ expect(await pageRows(b.engine)).toBe(0);
+ }), 180_000);
+
+ test('after unpurge the item imports again', async () => withPurgedPage(databaseUrl, async b => {
+ rmSync(b.path);
+ await unpurgePage(ctx(b.engine) as never, { slug: SLUG, source_id: SOURCE });
+ expect((await b.sync()).failedFiles ?? 0).toBe(0);
+ expect(await pageRows(b.engine)).toBe(1);
+ expect(existsSync(b.path)).toBe(true);
+ }), 180_000);
+});
+
+const THREAD = '17ff00000000b001';
+const GMAIL_CONFIG = { ...CONFIG, g_services: 'gmail' };
+const b64url = (text: string) => Buffer.from(text, 'utf-8').toString('base64url');
+
+for (const backend of testBackends()) describe(`[${backend}] #8644: a purged Gmail thread on an unmanaged Google source`, () => {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+
+ test('a change to the purged thread brings back neither its page, its open loop nor its grace hold', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'gbrain-8644-gmail-'));
+ try {
+ await withEnv({ GBRAIN_HOME: root, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined, CONNECTOR_TEST_TOKEN: 'synthetic-local-fixture' }, async () => {
+ _resetWriteThroughCacheForTest();
+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);
+ try {
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ const dir = join(root, 'google');
+ await engine.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,$3::text::jsonb)', [SOURCE, dir, JSON.stringify(GMAIL_CONFIG)]);
+ const sentAt = Date.now() - 3 * 86_400_000;
+ let historyId = '5000';
+ let changed: string[] = [];
+ const message = { id: '18ff00000000b001', threadId: THREAD, labelIds: ['INBOX'], internalDate: String(sentAt),
+ payload: { mimeType: 'text/plain', headers: [{ name: 'From', value: 'Alice Example <[email protected]>' },
+ { name: 'To', value: GMAIL_CONFIG.g_account }, { name: 'Subject', value: 'Contract question' }],
+ body: { data: b64url('Could you confirm the acme-example contract terms?') } } };
+ const fetch = withGoogleAccount(async (url: string) => {
+ const u = new URL(url);
+ if (u.pathname.endsWith('/settings/sendAs')) return json({ sendAs: [] });
+ if (u.pathname.endsWith('/users/me/profile')) return json({ emailAddress: GMAIL_CONFIG.g_account, historyId: '5000' });
+ if (u.pathname.endsWith('/users/me/messages')) return json({ messages: Number(/after:(\d+)/.exec(u.searchParams.get('q') ?? '')?.[1] ?? 0) * 1000 <= sentAt
+ && sentAt < Number(/before:(\d+)/.exec(u.searchParams.get('q') ?? '')?.[1] ?? Infinity) * 1000 ? [{ id: message.id, threadId: THREAD }] : [] });
+ if (u.pathname.endsWith('/users/me/history')) return json({ historyId, history: u.searchParams.get('startHistoryId') === historyId ? []
+ : [{ id: historyId, messages: changed.map(threadId => ({ id: message.id, threadId })) }] });
+ if (u.pathname.endsWith(`/users/me/threads/${THREAD}`)) return json({ id: THREAD, messages: [message] });
+ throw new Error(`Unexpected Google fixture route: ${url}`);
+ }, GMAIL_CONFIG.g_account);
+ const cfg = parseGoogleSourceConfig(GMAIL_CONFIG, dir);
+ const sync = () => runGoogleSync(engine, SOURCE, cfg, { ...options, sourceId: SOURCE }, fetch);
+ const loops = async () => (await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM open_loops WHERE source_id=$1 AND thread_id=$2', [SOURCE, THREAD]))[0]!.n;
+ expect((await sync()).failedFiles ?? 0).toBe(0);
+ const [page] = await engine.executeRaw<{ slug: string }>('SELECT slug FROM pages WHERE source_id=$1', [SOURCE]);
+ expect(await loops()).toBe(1);
+ const snapshot = await engine.readPageSnapshot(page!.slug, { sourceId: SOURCE });
+ await submitPageMutation(ctx(engine), { operation: 'delete_page', params: { slug: page!.slug, purge: true, expected_revision: snapshot!.revision } });
+ expect(await loops()).toBe(0);
+ const statePath = join(dir, '.google-source.json');
+ const state = JSON.parse(readFileSync(statePath, 'utf8'));
+ state.loop_grace_holds = { ...state.loop_grace_holds, [THREAD]: { due_ms: Date.now() + 30 * 86_400_000, slug: page!.slug, rev: null,
+ spec: { loopType: 'unanswered_inbound', counterpartyEmail: '[email protected]', summary: 'Contract question',
+ evidence: [{ quote: 'Could you confirm the acme-example contract terms?' }], lastActivityMs: sentAt, openedMs: sentAt } } };
+ writeFileSync(statePath, JSON.stringify(state), { mode: 0o600 });
+
+ historyId = '5010';
+ changed = [THREAD];
+ const replay = await sync();
+ expect(replay.failedFiles ?? 0).toBe(0);
+ expect(await loops()).toBe(0);
+ expect((await engine.executeRaw('SELECT 1 FROM pages WHERE source_id=$1', [SOURCE])).length).toBe(0);
+ expect(readGoogleState(dir).loop_grace_holds?.[THREAD]).toBeUndefined();
+ } finally { await close(); }
+ });
+ } finally { rmSync(root, { recursive: true, force: true }); }
+ }, 180_000);
+});
diff --git a/test/google-connector-private-hold-7880.test.ts b/test/google-connector-private-hold-7880.test.ts
index 279497122d..d73a014cf7 100644
--- a/test/google-connector-private-hold-7880.test.ts
+++ b/test/google-connector-private-hold-7880.test.ts
@@ -49,8 +49,8 @@ describe('google connector database-only import holds a non-canonical private re
test('visibility: private stores a private page, and a page that does not ask for private stores', async () => {
const privateSlug = await run('calendar/kept-private.md', page('visibility: private'));
- expect((await engine.getPage(privateSlug, { sourceId: 'default' }))?.frontmatter.visibility).toBe('private');
+ expect((await engine.getPage(privateSlug!, { sourceId: 'default' }))?.frontmatter.visibility).toBe('private');
const plainSlug = await run('calendar/plain.md', page('owner: alice-example'));
- expect((await engine.getPage(plainSlug, { sourceId: 'default' }))?.title).toBe('Example Event');
+ expect((await engine.getPage(plainSlug!, { sourceId: 'default' }))?.title).toBe('Example Event');
}, 120_000);
});
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 17fa3b9c91..5c2c07b8fa 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -119,6 +119,7 @@ test/git-door-strict-gbra133.test.ts
test/git-door-write-cost.test.ts
test/git-gap-census.test.ts
test/git-gap-direction-evidence-d4.test.ts
+test/google-classic-purged-item-8644.test.ts
test/google-connector-facts-fence-7292.test.ts
test/google-withdrawn-fence-carry-8645.test.ts
test/grade-takes-window.test.ts
diff --git a/test/visibility-fail-closed-derived-writes.test.ts b/test/visibility-fail-closed-derived-writes.test.ts
index 4ada400077..d9f8a194c5 100644
--- a/test/visibility-fail-closed-derived-writes.test.ts
+++ b/test/visibility-fail-closed-derived-writes.test.ts
@@ -333,7 +333,7 @@ describe('connector pages keep a private label', () => {
const deps = { engine, sourceId: s.sourceId, cfg: { account: '[email protected]', dir }, opts: {}, log() {}, tick() {} } as never;
const summary = { pagesAffected: [], chunksCreated: 0, added: 0, modified: 0 } as never;
const rendered = renderCalendarEventPage(event)!;
- const slug = await importRendered(deps, rendered.relPath, rendered.markdown, undefined as never, summary, new Set());
+ const slug = (await importRendered(deps, rendered.relPath, rendered.markdown, undefined as never, summary, new Set()))!;
const page = (await engine.readPageSnapshot(slug, { sourceId: s.sourceId }))!.page;
expect(page.frontmatter.visibility).toBeUndefined();
await engine.transaction(tx => withCoordinatedWrite(tx, [s.sourceId], () => tx.putPage(slug, { ...page, frontmatter: { ...page.frontmatter, visibility: 'private' } },