OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M31-002): Group C fix 7: bound event.pagesTimeseries with LIMIT n BY origin, path
ralpha1e9529bba89
diff --git a/packages/core/src/modules/event/event.rpc.ts b/packages/core/src/modules/event/event.rpc.tsindex ec3fb7cd5..5f825483b 100644--- a/packages/core/src/modules/event/event.rpc.ts+++ b/packages/core/src/modules/event/event.rpc.ts@@ -38,6 +38,16 @@ import {const DEFAULT_BOTS_LIMIT = 8;+/**+ * `pagesTimeseries` returns every page in the project, so its size is the+ * project's page cardinality, not its traffic: unbounded it is 5,050,763 rows+ * / 486 MiB on the busiest anchor (M31-002). 50 is enough to carry any chart+ * legend or table page the result could feed, and caps the response at+ * `50 x buckets`. The origin+path-filtered `pageTimeseries` below is bounded+ * by its own filters and stays unbounded here.+ */+const PAGES_TIMESERIES_TOP_PAGES_PER_BUCKET = 50;+const zEventRef = z.object({id: z.string(),projectId: z.string(),@@ -224,6 +234,7 @@ export const eventRouter = createTRPCRouter({endDate,timezone,interval: input.interval,+ topPagesPerBucket: PAGES_TIMESERIES_TOP_PAGES_PER_BUCKET,});}),diff --git a/packages/core/src/modules/overview/pages.service.ts b/packages/core/src/modules/overview/pages.service.tsindex 2b8e3b099..41cbd997a 100644--- a/packages/core/src/modules/overview/pages.service.ts+++ b/packages/core/src/modules/overview/pages.service.ts@@ -64,6 +64,14 @@ export function getTopPages();}+export interface IGetPageTimeseriesInput extends IGetPagesInput {+ interval: IInterval;+ filterOrigin?: string;+ filterPath?: string;+ /** Unbounded when omitted; see `PageTimeseriesQueryInput`. */+ topPagesPerBucket?: number;+}+export function getPageTimeseries(deps: ServiceDeps,{@@ -74,11 +82,8 @@ export function getPageTimeseries(interval,filterOrigin,filterPath,- }: IGetPagesInput & {- interval: IInterval;- filterOrigin?: string;- filterPath?: string;- }+ topPagesPerBucket,+ }: IGetPageTimeseriesInput): Promise<IPageTimeseriesRow[]> {return runQuery<IPageTimeseriesRow>(deps,@@ -89,6 +94,7 @@ export function getPageTimeseries(interval,filterOrigin,filterPath,+ topPagesPerBucket,}),timezone);@@ -227,11 +233,7 @@ export function createPagesService(getTopPages: (input: IGetPagesInput): Promise<ITopPage[]> =>getTopPages(deps, input),getPageTimeseries: (- input: IGetPagesInput & {- interval: IInterval;- filterOrigin?: string;- filterPath?: string;- }+ input: IGetPageTimeseriesInput): Promise<IPageTimeseriesRow[]> => getPageTimeseries(deps, input),getTopPagesCore: (input: Parameters<typeof getTopPagesCore>[1]diff --git a/packages/core/src/modules/overview/src/pages.sql.proof.md b/packages/core/src/modules/overview/src/pages.sql.proof.mdindex 3f3c9974a..8284b5876 100644--- a/packages/core/src/modules/overview/src/pages.sql.proof.md+++ b/packages/core/src/modules/overview/src/pages.sql.proof.md@@ -60,3 +60,58 @@ GROUP BY vbc.path, vbc.origin ORDER BY unique_converters DESC LIMIT {p12:UInt64}```Full renders exercised by `pages.sql.test.ts`'s `EXPLAIN`-based parse gate.++## M31-002 — `topPagesPerBucket`, the optional bound on `pageTimeseriesQuery`++- **Date**: 2026-09-16. **Data**: local prod-copy `openpanel` (read-only `SELECT`/`EXPLAIN`).+ Window `2026-07-26 00:00:00`..`2026-08-25 00:00:00` UTC, `interval: 'day'`,+ `use_query_condition_cache=0`, `max_memory_usage=6 GB`, statements rendered by the builder+ itself (`toStatement()`) and executed with their bound `param_pN`.++Unbounded, the query is one row per `(origin, path, bucket)`, so its size is the project's page+cardinality rather than its traffic. `topPagesPerBucket` keeps the `n` busiest pages per bucket.++**Result sets — bounded vs unbounded, `sum(cityHash64(...))` over the whole set:**++| project | unbounded rows | `n = 50` rows | verdict |+|---|---:|---:|---|+| `chatpaper` | 5,050,763 | 1,500 | truncated (2.68 M distinct pages) |+| `website-8103` | 200,687 | 1,500 | truncated |+| `bayse` | 14,604 | 1,500 | truncated |+| `earlysalary-production` | 33 | 33 | **IDENTICAL hash** — 2 pages, always under the bound |+| `verdict` | 30 | 30 | **IDENTICAL hash** — no `screen_view`; all 30 are `WITH FILL` rows |++`verdict` is the proof that the bound does not eat the fill: `EXPLAIN PLAN` shows+`Filling > LimitBy`, i.e. ClickHouse fills **after** `LIMIT BY`, so an empty bucket still gets+its filled row and never spends the bucket's quota.++The truncated set is exactly the top `n`: an independent `row_number() OVER (PARTITION BY date+ORDER BY pageviews DESC, origin ASC, path ASC) <= 50` produced the identical hash on all three+truncated projects (`bayse` 13709211731794099286, `website-8103` 6888520080080211176,+`chatpaper` 566149299824503912).++**Why `origin ASC, path ASC` is part of the ranking, not decoration.** On `chatpaper` the 50th+place is 4–5 pageviews with **17–30 pages tied at that value** in every bucket. Without the+tiebreak the endpoint returned a different top-50 on every single call (3 runs, 3 different+hashes); with it, 3 runs, 1 hash. It costs nothing on four of five anchors and ~0.9–1.1 s on+`chatpaper`, whose 2.68 M-row sort is the only one large enough to notice.++**Timing and response size** (median of 3, `FORMAT Null`):++| project | unbounded | `n = 50` | response unbounded -> bounded |+|---|---|---|---|+| `chatpaper` | 1,603 ms / 1,556 MiB peak | 2,539 ms / 1,556 MiB peak | **485.7 MiB -> 0.67 MiB** |+| `website-8103` | 512 ms | 545 ms | 23.5 MiB -> 0.29 MiB |+| `bayse` | 336 ms | 344 ms | 2.06 MiB -> 0.20 MiB |+| `earlysalary-production` | 378 ms | 385 ms | unchanged |+| `verdict` | 39 ms | 34 ms | unchanged |++ClickHouse server time is **not** the win here and was not expected to be: the scan is+unchanged (13,401,914 rows read either way on `chatpaper`) and the added sort columns cost+~1 s there. What disappears is the response — and with it the ~5 s of serialisation M25-001+measured between this query's ClickHouse time and `event.pagesTimeseries`' procedure time.++Omitting `topPagesPerBucket` renders the query exactly as before: token-identical and+parameter-identical to the version at `4b7afb8e`, differing only by trailing whitespace where+the two conditional fragments render empty. `event.pageTimeseries` (the `origin`+`path`-filtered+variant) never passes it and is unchanged.diff --git a/packages/core/src/modules/overview/src/pages.sql.test.ts b/packages/core/src/modules/overview/src/pages.sql.test.tsindex 64712ef59..50c38c264 100644--- a/packages/core/src/modules/overview/src/pages.sql.test.ts+++ b/packages/core/src/modules/overview/src/pages.sql.test.ts@@ -61,6 +61,21 @@ describe('pages.sql — parses against ClickHouse', () => {);});+ it('pageTimeseriesQuery — topPagesPerBucket (WITH FILL then LIMIT n BY)', async () => {+ // `WITH FILL` binds to the ORDER BY expression it follows, so a `DESC`+ // ranking column appended after it is rejected outright — this case pins+ // the working spelling.+ await explain(+ PG.pageTimeseriesQuery({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ interval: 'day',+ topPagesPerBucket: 50,+ })+ );+ });+it('pageConversionsQuery', async () => {await explain(PG.pageConversionsQuery({diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.tsindex 3fb89df6b..1874f73e2 100644--- a/packages/core/src/modules/overview/src/pages.sql.ts+++ b/packages/core/src/modules/overview/src/pages.sql.ts@@ -108,6 +108,15 @@ export interface PageTimeseriesQueryInput {interval: IInterval;filterOrigin?: string;filterPath?: string;+ /**+ * Keep only the `n` busiest pages in each date bucket. Omitted means+ * unbounded, which is one row per (origin, path, bucket) and on a project+ * with per-user URL segments is millions of rows — see pages.sql.proof.md.+ * `origin`/`path` break the ranking's ties so the truncated set is stable+ * between calls; without them ClickHouse returns a different top-n on every+ * run (measured, M31-002).+ */+ topPagesPerBucket?: number;}export function pageTimeseriesQuery(@@ -120,6 +129,19 @@ export function pageTimeseriesQuery(? sql`AND e.path = ${sql.string(input.filterPath)}`: sql.empty;+ // `WITH FILL` attaches to the ORDER BY expression it follows, so the+ // ranking columns go after it, not before. ClickHouse fills AFTER+ // `LIMIT BY` (EXPLAIN PLAN: Filling > LimitBy), so empty buckets are still+ // filled and never spend the bucket's quota.+ const bucketRanking =+ input.topPagesPerBucket === undefined+ ? sql.empty+ : sql`, pageviews DESC, origin ASC, path ASC`;+ const bucketLimit =+ input.topPagesPerBucket === undefined+ ? sql.empty+ : sql`LIMIT ${sql.uint64(input.topPagesPerBucket)} BY date`;+return sql`SELECTe.origin as origin,@@ -136,7 +158,8 @@ export function pageTimeseriesQuery(${pathFilter}GROUP BY e.origin, e.path, dateORDER BY date ASC- ${fillClause(input.interval, input.startDate, input.endDate)}+ ${fillClause(input.interval, input.startDate, input.endDate)}${bucketRanking}+ ${bucketLimit}`;}