OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat(lists): per-list ceilings for the empty-result lookback (#443)
Ritik Jainc63ef738435a
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex b9fee9836..43091e359 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -1001,6 +1001,32 @@ FUNNEL_NON_STRICT_ORDERING=1## Performance & Tuning+### EVENT_LIST_MAX_LOOKBACK_DAYS++**Type**: `number` (whole days, positive integer)+**Required**: No+**Default**: `1825` (5 years)++Ceiling for the empty-result lookback on the event list. When a page's cursor window matches nothing, the window doubles and retries until it reaches this many days. With filters that no index can prune (e.g. a properties value) each retry is a full scan of its window, so large deployments can bound the worst case with a small, human-sized value. Invalid values (non-integer, zero, negative) keep the default.++**Example**:+```bash+EVENT_LIST_MAX_LOOKBACK_DAYS=7+```++### SESSION_LIST_MAX_LOOKBACK_DAYS++**Type**: `number` (whole days, positive integer)+**Required**: No+**Default**: `365`++Same ceiling for the session list, which has its own default. Invalid values keep the default.++**Example**:+```bash+SESSION_LIST_MAX_LOOKBACK_DAYS=7+```+### IMPORT_BATCH_SIZE**Type**: `number`diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 981c17f1d..68527f501 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -16,6 +16,7 @@ import { clix, type Query } from '../clickhouse/query-builder';import type { EventMeta, Prisma } from '../prisma-client';import { db } from '../prisma-client';import { createSqlBuilder, type SqlBuilderObject } from '../sql-builder';+import { resolveMaxLookbackDays } from './lookback';import { getEventFiltersWhereClause } from './chart.service';import { buildFilterWhere } from './filter-where.service';import type { IServiceProfile, IServiceUpsertProfile } from './profile.service';@@ -499,7 +500,11 @@ export async function getEventList(options: GetEventListOptions) {} = options;const { sb, getSql, join } = createSqlBuilder();- const MAX_DATE_INTERVAL_IN_DAYS = 365 * 5;+ // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).+ const MAX_DATE_INTERVAL_IN_DAYS = resolveMaxLookbackDays(+ 'EVENT_LIST_MAX_LOOKBACK_DAYS',+ 365 * 5,+ );// Cap the date interval to prevent infinityconst safeDateIntervalInDays = Math.min(dateIntervalInDays,diff --git a/packages/db/src/services/lookback.test.ts b/packages/db/src/services/lookback.test.tsnew file mode 100644index 000000000..d0c96890a--- /dev/null+++ b/packages/db/src/services/lookback.test.ts@@ -0,0 +1,46 @@+import { afterEach, describe, expect, it, vi } from 'vitest';+import { resolveMaxLookbackDays } from './lookback';++describe('resolveMaxLookbackDays', () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it('returns the caller default when unset', () => {+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '');+ expect(resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365 * 5)).toBe(+ 365 * 5,+ );+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(+ 365,+ );+ });++ it('replaces the ceiling when set to a positive integer', () => {+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '7');+ expect(resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365 * 5)).toBe(+ 7,+ );+ // Raising past the default is allowed too — it's a ceiling, not a min.+ vi.stubEnv('SESSION_LIST_MAX_LOOKBACK_DAYS', '3650');+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(+ 3650,+ );+ });++ it('reads each list its own variable — no cross-talk', () => {+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '7');+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(+ 365,+ );+ });++ it('falls back to the default for malformed values', () => {+ for (const bad of ['0', '-1', '7.5', '7days', 'junk']) {+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', bad);+ expect(+ resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365),+ ).toBe(365);+ }+ });+});diff --git a/packages/db/src/services/lookback.ts b/packages/db/src/services/lookback.tsnew file mode 100644index 000000000..970ba9fc3--- /dev/null+++ b/packages/db/src/services/lookback.ts@@ -0,0 +1,30 @@+/**+ * Ceiling (in days) for the empty-result lookback expansion on the event and+ * session lists.+ *+ * When a cursor window matches nothing, those lists double the window and+ * retry until the ceiling is reached. The doubling geometry keeps the total+ * cost at roughly 2x the final window, so the ceiling is what actually+ * bounds the worst case: with a filter no index can prune (e.g. a+ * properties value) that matches nothing at all, every step is a full scan+ * of its window, and a multi-year ceiling turns one page view into minutes+ * of scanning.+ *+ * Each list has its own variable (EVENT_LIST_MAX_LOOKBACK_DAYS,+ * SESSION_LIST_MAX_LOOKBACK_DAYS) because their built-in ceilings differ —+ * the surfaces have different cost profiles and a shared knob would couple+ * them. Values are human-sized day counts (1 / 7 / 365 / ...); unset or+ * invalid (non-integer, zero, negative) keeps the caller's default, so+ * behavior is unchanged unless a variable is set.+ */+export function resolveMaxLookbackDays(+ envName: string,+ defaultDays: number+): number {+ const raw = process.env[envName];+ if (!raw || !/^\d+$/.test(raw)) {+ return defaultDays;+ }+ const parsed = Number(raw);+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : defaultDays;+}diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex 7a3321e47..a8a2167af 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -11,6 +11,7 @@ import {} from '../clickhouse/client';import { clix } from '../clickhouse/query-builder';import { createSqlBuilder } from '../sql-builder';+import { resolveMaxLookbackDays } from './lookback';import { buildFilterWhere } from './filter-where.service';import { getProfilesCached, type IServiceProfile } from './profile.service';@@ -172,7 +173,11 @@ export async function getSessionList(options: GetSessionListOptions) {sb.limit = take;sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;- const MAX_DATE_INTERVAL_IN_DAYS = 365;+ // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).+ const MAX_DATE_INTERVAL_IN_DAYS = resolveMaxLookbackDays(+ 'SESSION_LIST_MAX_LOOKBACK_DAYS',+ 365,+ );// Cap the date interval to prevent infinityconst safeDateIntervalInDays = Math.min(dateIntervalInDays,@@ -243,7 +248,7 @@ export async function getSessionList(options: GetSessionListOptions) {});sb.select.has_replay = `toBool(src.session_id != '') as hasReplay`;- sb.joins.has_replay = `LEFT JOIN (SELECT DISTINCT session_id FROM ${TABLE_NAMES.session_replay_chunks} WHERE project_id = ${sqlstring.escape(projectId)} AND started_at > now() - INTERVAL ${dateIntervalInDays} DAY) AS src ON src.session_id = id`;+ sb.joins.has_replay = `LEFT JOIN (SELECT DISTINCT session_id FROM ${TABLE_NAMES.session_replay_chunks} WHERE project_id = ${sqlstring.escape(projectId)} AND started_at > now() - INTERVAL ${safeDateIntervalInDays} DAY) AS src ON src.session_id = id`;const sql = getSql();const data = await chQuery<