OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M36-004): gsc.getQueries / getPages: stop the memory cliff without changing a single row
ralph00495b3ae03e
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.tsindex 6897f8e1c..47aecfad7 100644--- a/packages/core/src/modules/gsc/gsc.service.test.ts+++ b/packages/core/src/modules/gsc/gsc.service.test.ts@@ -388,6 +388,33 @@ test('getGscOverview returns the ClickHouse rows as-is', async () => {expect(result).toEqual(rows);});+const TOP_N_READS = [+ ['getGscPages', 'GROUP BY page'],+ ['getGscQueries', 'GROUP BY query'],+] as const;++for (const [name, groupBy] of TOP_N_READS) {+ test(`${name} spills its GROUP BY at 1 GiB and caps the read at 3 GiB`, async () => {+ const rows = [+ { clicks: 3, impressions: 9, ctr: 0.3, position: 2, page: '/a' },+ ];+ ch.query.mockClear();+ ch.query.mockImplementationOnce(async () => ({ json: async () => rows }));++ const result = await subject[name](deps, 'p1', '2026-09-01', '2026-09-03');++ expect(result).toEqual(rows);+ const [params] = ch.query.mock.calls[0] as unknown as [+ { query: string; clickhouse_settings: Record<string, string> },+ ];+ expect(params.query).toContain(groupBy);+ expect(params.clickhouse_settings).toEqual({+ max_bytes_before_external_group_by: String(1024 ** 3),+ max_memory_usage: String(3 * 1024 ** 3),+ });+ });+}+test('gscGetOverviewCore aggregates the daily rows into a summary', async () => {ch.query.mockImplementationOnce(async () => ({json: async () => [diff --git a/packages/core/src/modules/gsc/gsc.service.ts b/packages/core/src/modules/gsc/gsc.service.tsindex 725018f06..3e62925fd 100644--- a/packages/core/src/modules/gsc/gsc.service.ts+++ b/packages/core/src/modules/gsc/gsc.service.ts@@ -24,6 +24,7 @@// tag: ADR-013 converts the analytics read path one query per P7 task, and// this module's queries haven't been converted yet.+import type { ClickHouseSettings } from '@clickhouse/client';import { decrypt, encrypt } from '@openpanel/shared/server';import { cacheablePerDeps } from '../../cacheable-per-deps';import type { CoreConfig } from '../../config';@@ -37,6 +38,20 @@ const BACKFILL_MONTHS = 6;const CHUNK_DAYS = 14;const CANNIBALIZATION_CACHE_TTL_SEC = 60 * 60 * 4;const GSC_ROW_LIMIT = 25_000;+const GIB = 1024 ** 3;+const TOP_N_SPILL_BYTES = GIB;+const TOP_N_MEMORY_LIMIT_BYTES = 3 * GIB;++/**+ * The top-N page/query reads hash every distinct key in the window before+ * `LIMIT` applies (a year of a large property: 24.5M keys, 6.8 GiB peak), and+ * the SEO page issues both at once. Spilling the GROUP BY to disk bounds that+ * without changing a row; the cap stops one read from exhausting the server.+ */+const GSC_TOP_N_QUERY_SETTINGS: ClickHouseSettings = {+ max_bytes_before_external_group_by: String(TOP_N_SPILL_BYTES),+ max_memory_usage: String(TOP_N_MEMORY_LIMIT_BYTES),+};export interface GscSite {siteUrl: string;@@ -412,6 +427,7 @@ export async function getGscPages(`,query_params: { projectId, startDate, endDate, limit },format: 'JSONEachRow',+ clickhouse_settings: GSC_TOP_N_QUERY_SETTINGS,});return result.json();}@@ -697,6 +713,7 @@ export async function getGscQueries(`,query_params: { projectId, startDate, endDate, limit },format: 'JSONEachRow',+ clickhouse_settings: GSC_TOP_N_QUERY_SETTINGS,});return result.json();}