Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat(lists): per-list ceilings for the empty-result lookback (#443)

Ritik Jainc63ef738435a
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index b9fee9836..43091e359 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -1001,6 +1001,32 @@ FUNNEL_NON_STRICT_ORDERING=1
## Performance & Tuning
+### EVENT_LIST_MAX_LOOKBACK_DAYS
+
+**Type**: `number` (whole days, positive integer)
+**Required**: No
+**Default**: `1825` (5 years)
+
+Ceiling for the empty-result lookback on the event list. When a page's cursor window matches nothing, the window doubles and retries until it reaches this many days. With filters that no index can prune (e.g. a properties value) each retry is a full scan of its window, so large deployments can bound the worst case with a small, human-sized value. Invalid values (non-integer, zero, negative) keep the default.
+
+**Example**:
+```bash
+EVENT_LIST_MAX_LOOKBACK_DAYS=7
+```
+
+### SESSION_LIST_MAX_LOOKBACK_DAYS
+
+**Type**: `number` (whole days, positive integer)
+**Required**: No
+**Default**: `365`
+
+Same ceiling for the session list, which has its own default. Invalid values keep the default.
+
+**Example**:
+```bash
+SESSION_LIST_MAX_LOOKBACK_DAYS=7
+```
+
### IMPORT_BATCH_SIZE
**Type**: `number`
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index 981c17f1d..68527f501 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -16,6 +16,7 @@ import { clix, type Query } from '../clickhouse/query-builder';
import type { EventMeta, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
import { createSqlBuilder, type SqlBuilderObject } from '../sql-builder';
+import { resolveMaxLookbackDays } from './lookback';
import { getEventFiltersWhereClause } from './chart.service';
import { buildFilterWhere } from './filter-where.service';
import type { IServiceProfile, IServiceUpsertProfile } from './profile.service';
@@ -499,7 +500,11 @@ export async function getEventList(options: GetEventListOptions) {
} = options;
const { sb, getSql, join } = createSqlBuilder();
- const MAX_DATE_INTERVAL_IN_DAYS = 365 * 5;
+ // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).
+ const MAX_DATE_INTERVAL_IN_DAYS = resolveMaxLookbackDays(
+ 'EVENT_LIST_MAX_LOOKBACK_DAYS',
+ 365 * 5,
+ );
// Cap the date interval to prevent infinity
const safeDateIntervalInDays = Math.min(
dateIntervalInDays,
diff --git a/packages/db/src/services/lookback.test.ts b/packages/db/src/services/lookback.test.ts
new file mode 100644
index 000000000..d0c96890a
--- /dev/null
+++ b/packages/db/src/services/lookback.test.ts
@@ -0,0 +1,46 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { resolveMaxLookbackDays } from './lookback';
+
+describe('resolveMaxLookbackDays', () => {
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ });
+
+ it('returns the caller default when unset', () => {
+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '');
+ expect(resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365 * 5)).toBe(
+ 365 * 5,
+ );
+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(
+ 365,
+ );
+ });
+
+ it('replaces the ceiling when set to a positive integer', () => {
+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '7');
+ expect(resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365 * 5)).toBe(
+ 7,
+ );
+ // Raising past the default is allowed too — it's a ceiling, not a min.
+ vi.stubEnv('SESSION_LIST_MAX_LOOKBACK_DAYS', '3650');
+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(
+ 3650,
+ );
+ });
+
+ it('reads each list its own variable — no cross-talk', () => {
+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', '7');
+ expect(resolveMaxLookbackDays('SESSION_LIST_MAX_LOOKBACK_DAYS', 365)).toBe(
+ 365,
+ );
+ });
+
+ it('falls back to the default for malformed values', () => {
+ for (const bad of ['0', '-1', '7.5', '7days', 'junk']) {
+ vi.stubEnv('EVENT_LIST_MAX_LOOKBACK_DAYS', bad);
+ expect(
+ resolveMaxLookbackDays('EVENT_LIST_MAX_LOOKBACK_DAYS', 365),
+ ).toBe(365);
+ }
+ });
+});
diff --git a/packages/db/src/services/lookback.ts b/packages/db/src/services/lookback.ts
new file mode 100644
index 000000000..970ba9fc3
--- /dev/null
+++ b/packages/db/src/services/lookback.ts
@@ -0,0 +1,30 @@
+/**
+ * Ceiling (in days) for the empty-result lookback expansion on the event and
+ * session lists.
+ *
+ * When a cursor window matches nothing, those lists double the window and
+ * retry until the ceiling is reached. The doubling geometry keeps the total
+ * cost at roughly 2x the final window, so the ceiling is what actually
+ * bounds the worst case: with a filter no index can prune (e.g. a
+ * properties value) that matches nothing at all, every step is a full scan
+ * of its window, and a multi-year ceiling turns one page view into minutes
+ * of scanning.
+ *
+ * Each list has its own variable (EVENT_LIST_MAX_LOOKBACK_DAYS,
+ * SESSION_LIST_MAX_LOOKBACK_DAYS) because their built-in ceilings differ —
+ * the surfaces have different cost profiles and a shared knob would couple
+ * them. Values are human-sized day counts (1 / 7 / 365 / ...); unset or
+ * invalid (non-integer, zero, negative) keeps the caller's default, so
+ * behavior is unchanged unless a variable is set.
+ */
+export function resolveMaxLookbackDays(
+ envName: string,
+ defaultDays: number
+): number {
+ const raw = process.env[envName];
+ if (!raw || !/^\d+$/.test(raw)) {
+ return defaultDays;
+ }
+ const parsed = Number(raw);
+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : defaultDays;
+}
diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts
index 7a3321e47..a8a2167af 100644
--- a/packages/db/src/services/session.service.ts
+++ b/packages/db/src/services/session.service.ts
@@ -11,6 +11,7 @@ import {
} from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import { createSqlBuilder } from '../sql-builder';
+import { resolveMaxLookbackDays } from './lookback';
import { buildFilterWhere } from './filter-where.service';
import { getProfilesCached, type IServiceProfile } from './profile.service';
@@ -172,7 +173,11 @@ export async function getSessionList(options: GetSessionListOptions) {
sb.limit = take;
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
- const MAX_DATE_INTERVAL_IN_DAYS = 365;
+ // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).
+ const MAX_DATE_INTERVAL_IN_DAYS = resolveMaxLookbackDays(
+ 'SESSION_LIST_MAX_LOOKBACK_DAYS',
+ 365,
+ );
// Cap the date interval to prevent infinity
const safeDateIntervalInDays = Math.min(
dateIntervalInDays,
@@ -243,7 +248,7 @@ export async function getSessionList(options: GetSessionListOptions) {
});
sb.select.has_replay = `toBool(src.session_id != '') as hasReplay`;
- sb.joins.has_replay = `LEFT JOIN (SELECT DISTINCT session_id FROM ${TABLE_NAMES.session_replay_chunks} WHERE project_id = ${sqlstring.escape(projectId)} AND started_at > now() - INTERVAL ${dateIntervalInDays} DAY) AS src ON src.session_id = id`;
+ sb.joins.has_replay = `LEFT JOIN (SELECT DISTINCT session_id FROM ${TABLE_NAMES.session_replay_chunks} WHERE project_id = ${sqlstring.escape(projectId)} AND started_at > now() - INTERVAL ${safeDateIntervalInDays} DAY) AS src ON src.session_id = id`;
const sql = getSql();
const data = await chQuery<