Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M36-004): gsc.getQueries / getPages: stop the memory cliff without changing a single row

ralph00495b3ae03e
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.ts
index 6897f8e1c..47aecfad7 100644
--- a/packages/core/src/modules/gsc/gsc.service.test.ts
+++ b/packages/core/src/modules/gsc/gsc.service.test.ts
@@ -388,6 +388,33 @@ test('getGscOverview returns the ClickHouse rows as-is', async () => {
expect(result).toEqual(rows);
});
+const TOP_N_READS = [
+ ['getGscPages', 'GROUP BY page'],
+ ['getGscQueries', 'GROUP BY query'],
+] as const;
+
+for (const [name, groupBy] of TOP_N_READS) {
+ test(`${name} spills its GROUP BY at 1 GiB and caps the read at 3 GiB`, async () => {
+ const rows = [
+ { clicks: 3, impressions: 9, ctr: 0.3, position: 2, page: '/a' },
+ ];
+ ch.query.mockClear();
+ ch.query.mockImplementationOnce(async () => ({ json: async () => rows }));
+
+ const result = await subject[name](deps, 'p1', '2026-09-01', '2026-09-03');
+
+ expect(result).toEqual(rows);
+ const [params] = ch.query.mock.calls[0] as unknown as [
+ { query: string; clickhouse_settings: Record<string, string> },
+ ];
+ expect(params.query).toContain(groupBy);
+ expect(params.clickhouse_settings).toEqual({
+ max_bytes_before_external_group_by: String(1024 ** 3),
+ max_memory_usage: String(3 * 1024 ** 3),
+ });
+ });
+}
+
test('gscGetOverviewCore aggregates the daily rows into a summary', async () => {
ch.query.mockImplementationOnce(async () => ({
json: async () => [
diff --git a/packages/core/src/modules/gsc/gsc.service.ts b/packages/core/src/modules/gsc/gsc.service.ts
index 725018f06..3e62925fd 100644
--- a/packages/core/src/modules/gsc/gsc.service.ts
+++ b/packages/core/src/modules/gsc/gsc.service.ts
@@ -24,6 +24,7 @@
// tag: ADR-013 converts the analytics read path one query per P7 task, and
// this module's queries haven't been converted yet.
+import type { ClickHouseSettings } from '@clickhouse/client';
import { decrypt, encrypt } from '@openpanel/shared/server';
import { cacheablePerDeps } from '../../cacheable-per-deps';
import type { CoreConfig } from '../../config';
@@ -37,6 +38,20 @@ const BACKFILL_MONTHS = 6;
const CHUNK_DAYS = 14;
const CANNIBALIZATION_CACHE_TTL_SEC = 60 * 60 * 4;
const GSC_ROW_LIMIT = 25_000;
+const GIB = 1024 ** 3;
+const TOP_N_SPILL_BYTES = GIB;
+const TOP_N_MEMORY_LIMIT_BYTES = 3 * GIB;
+
+/**
+ * The top-N page/query reads hash every distinct key in the window before
+ * `LIMIT` applies (a year of a large property: 24.5M keys, 6.8 GiB peak), and
+ * the SEO page issues both at once. Spilling the GROUP BY to disk bounds that
+ * without changing a row; the cap stops one read from exhausting the server.
+ */
+const GSC_TOP_N_QUERY_SETTINGS: ClickHouseSettings = {
+ max_bytes_before_external_group_by: String(TOP_N_SPILL_BYTES),
+ max_memory_usage: String(TOP_N_MEMORY_LIMIT_BYTES),
+};
export interface GscSite {
siteUrl: string;
@@ -412,6 +427,7 @@ export async function getGscPages(
`,
query_params: { projectId, startDate, endDate, limit },
format: 'JSONEachRow',
+ clickhouse_settings: GSC_TOP_N_QUERY_SETTINGS,
});
return result.json();
}
@@ -697,6 +713,7 @@ export async function getGscQueries(
`,
query_params: { projectId, startDate, endDate, limit },
format: 'JSONEachRow',
+ clickhouse_settings: GSC_TOP_N_QUERY_SETTINGS,
});
return result.json();
}