OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(cohorts): aggregate property cohorts without FINAL so they can spill to disk (#444)
Ritik Jain73411b1859f8
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex 43091e359..9507dc49b 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -15,8 +15,8 @@ For deployment-specific configuration, see the [deployment guides](/docs/self-ho### DATABASE_URL-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: NonePostgreSQL connection string for the main database.@@ -28,8 +28,8 @@ DATABASE_URL=postgres://user:password@localhost:5432/openpanel?schema=public### DATABASE_URL_DIRECT-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: Same as `DATABASE_URL`Direct PostgreSQL connection string (bypasses connection pooling). Used for migrations and administrative operations.@@ -41,8 +41,8 @@ DATABASE_URL_DIRECT=postgres://user:password@localhost:5432/openpanel?schema=pub### DATABASE_URL_REPLICA-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: Same as `DATABASE_URL`Read replica connection string for read-heavy operations. If not set, uses the main database.@@ -54,8 +54,8 @@ DATABASE_URL_REPLICA=postgres://user:password@replica-host:5432/openpanel?schema### REDIS_URL-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: `redis://localhost:6379`Redis connection string for caching and queue management.@@ -69,8 +69,8 @@ REDIS_URL=redis://:password@localhost:6379### CLICKHOUSE_URL-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: `http://localhost:8123/openpanel`ClickHouse HTTP connection URL for analytics data storage.@@ -82,8 +82,8 @@ CLICKHOUSE_URL=http://localhost:8123/openpanel### CLICKHOUSE_CLUSTER-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Enable ClickHouse cluster mode. Set to `true` or `1` if using a ClickHouse cluster.@@ -95,8 +95,8 @@ CLICKHOUSE_CLUSTER=true### CLICKHOUSE_SETTINGS-**Type**: `string` (JSON)-**Required**: No+**Type**: `string` (JSON)+**Required**: No**Default**: `{}`Additional ClickHouse settings as a JSON object.@@ -108,8 +108,8 @@ CLICKHOUSE_SETTINGS='{"max_execution_time": 300}'### CLICKHOUSE_SETTINGS_REMOVE_CONVERT_ANY_JOIN-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Remove `convert_any_join` from ClickHouse settings. Used for compatibility with certain ClickHouse versions. This needs to be set if you use any clickhouse version below 25!@@ -118,8 +118,8 @@ Remove `convert_any_join` from ClickHouse settings. Used for compatibility with### API_URL-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: NonePublic API URL exposed to the browser. Used by the dashboard frontend and API service.@@ -131,8 +131,8 @@ API_URL=https://analytics.example.com/api### API_URL_SSR-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: Same as `API_URL`Internal API URL used **only** for the dashboard's server-side rendering (SSR) requests. Set this when the dashboard server can't resolve or reach the public `API_URL` and needs an internal address instead — for example a Docker Compose service name or a Kubernetes cluster-internal address.@@ -154,8 +154,8 @@ This only affects the dashboard's server-side requests. It does not change the A### DASHBOARD_URL-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: NonePublic dashboard URL exposed to the browser. Used by the dashboard frontend and API service.@@ -167,8 +167,8 @@ DASHBOARD_URL=https://analytics.example.com### API_CORS_ORIGINS-**Type**: `string` (comma-separated)-**Required**: No+**Type**: `string` (comma-separated)+**Required**: No**Default**: NoneAdditional CORS origins allowed for API requests. Comma-separated list of origins.@@ -182,8 +182,8 @@ API_CORS_ORIGINS=https://app.example.com,https://another-app.com### COOKIE_SECRET-**Type**: `string`-**Required**: Yes+**Type**: `string`+**Required**: Yes**Default**: NoneSecret key for encrypting session cookies. Generate a secure random string (32+ characters).@@ -200,8 +200,8 @@ Never use the default value in production! Always generate a unique secret.### COOKIE_TLDS-**Type**: `string` (comma-separated)-**Required**: No+**Type**: `string` (comma-separated)+**Required**: No**Default**: NoneCustom multi-part TLDs for cookie domain handling. Use this when deploying on domains with public suffixes that aren't recognized by default (e.g., `.my.id`, `.web.id`, `.co.id`).@@ -221,8 +221,8 @@ This is required when using domain suffixes that are public suffixes (like `.co.### CUSTOM_COOKIE_DOMAIN-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneOverride the automatic cookie domain detection and set a specific domain for authentication cookies. Useful when proxying the API through your main domain or when you need precise control over cookie scope.@@ -242,8 +242,8 @@ When set, this completely bypasses the automatic domain parsing logic. The cooki### DEMO_USER_ID-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneUser ID for demo mode. When set, creates a demo session for testing.@@ -255,8 +255,8 @@ DEMO_USER_ID=user_1234567890### ALLOW_REGISTRATION-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false` (after first user is created)Allow new user registrations. Set to `true` to enable public registration.@@ -272,8 +272,8 @@ Registration is automatically disabled after the first user is created. Set this### ALLOW_INVITATION-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `true`Allow user invitations. Set to `false` to disable invitation functionality.@@ -298,8 +298,8 @@ The AI assistant is optional. Without `OPENAI_API_KEY` or `ANTHROPIC_API_KEY` se### OPENAI_API_KEY-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneOpenAI API key. When set, the OpenAI models appear in the chat model picker.@@ -311,8 +311,8 @@ OPENAI_API_KEY=sk-your-openai-api-key-here### OPENAI_BASE_URL-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: OpenAI default API base URLOverride the OpenAI API base URL. Useful for proxies, gateways, Azure-compatible endpoints, or self-hosted OpenAI-compatible providers.@@ -324,8 +324,8 @@ OPENAI_BASE_URL=https://your-openai-compatible-endpoint.example.com/v1### OPENAI_PROJECT-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneOptional OpenAI project identifier to send with requests.@@ -337,8 +337,8 @@ OPENAI_PROJECT=proj_1234567890### OPENAI_ORGANIZATION-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneOptional OpenAI organization identifier to send with requests.@@ -350,8 +350,8 @@ OPENAI_ORGANIZATION=org_1234567890### ANTHROPIC_API_KEY-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneAnthropic API key. When set, the Claude models appear in the chat model picker.@@ -363,8 +363,8 @@ ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here### ANTHROPIC_BASE_URL-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: Anthropic default API base URLOverride the Anthropic API base URL. Useful for proxies, gateways, or Anthropic-compatible endpoints.@@ -376,8 +376,8 @@ ANTHROPIC_BASE_URL=https://your-anthropic-endpoint.example.com### ANTHROPIC_TOKEN-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneOptional auth token sent to the Anthropic provider in addition to the API key. Use this only if your Anthropic-compatible gateway requires it.@@ -389,8 +389,8 @@ ANTHROPIC_TOKEN=your-auth-token### ANTHROPIC_VERSION-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: Provider defaultOverride the Anthropic API version header sent with requests.@@ -404,8 +404,8 @@ ANTHROPIC_VERSION=2023-06-01### RESEND_API_KEY-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneResend API key for sending transactional emails (password resets, invitations, etc.).@@ -421,8 +421,8 @@ Get your API key from [resend.com](https://resend.com). Make sure to verify your### EMAIL_SENDER-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: `[email protected]`Email address used as the sender for transactional emails. Applies to both Resend and SMTP transports.@@ -434,8 +434,8 @@ [email protected]### SMTP_HOST-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSMTP server hostname. When set, OpenPanel uses SMTP to send emails instead of Resend. Takes priority over `RESEND_API_KEY` if both are configured.@@ -447,8 +447,8 @@ SMTP_HOST=smtp.example.com### SMTP_PORT-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `587`SMTP server port.@@ -460,8 +460,8 @@ SMTP_PORT=587### SMTP_SECURE-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Use TLS for the SMTP connection. Set to `true` when using port `465`.@@ -473,8 +473,8 @@ SMTP_SECURE=true### SMTP_USER-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSMTP authentication username. Leave unset if your server does not require authentication.@@ -486,8 +486,8 @@ [email protected]### SMTP_PASS-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSMTP authentication password.@@ -510,8 +510,8 @@ If `SMTP_HOST` is set and an SMTP send attempt fails, the system will **not** au### SLACK_CLIENT_ID-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSlack OAuth client ID for Slack integration.@@ -523,8 +523,8 @@ SLACK_CLIENT_ID=1234567890.1234567890### SLACK_CLIENT_SECRET-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSlack OAuth client secret for Slack integration.@@ -536,8 +536,8 @@ SLACK_CLIENT_SECRET=your-slack-client-secret### SLACK_OAUTH_REDIRECT_URL-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSlack OAuth redirect URL. Must match the redirect URI configured in your Slack app.@@ -549,8 +549,8 @@ SLACK_OAUTH_REDIRECT_URL=https://analytics.example.com/api/integrations/slack/ca### SLACK_STATE_SECRET-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneSecret for signing Slack OAuth state parameter.@@ -564,8 +564,8 @@ SLACK_STATE_SECRET=your-state-secret### SELF_HOSTED-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Enable self-hosted mode. Set to `true` or `1` to enable self-hosting features. Used by both the dashboard frontend and API service.@@ -579,8 +579,8 @@ SELF_HOSTED=true### WORKER_PORT-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `3000`Port for the worker service to listen on.@@ -592,8 +592,8 @@ WORKER_PORT=3000### DISABLE_BULLBOARD-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Disable BullMQ board UI. Set to `true` or `1` to disable the queue monitoring dashboard.@@ -605,8 +605,8 @@ DISABLE_BULLBOARD=true### DISABLE_WORKERS-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Disable all worker processes. Set to `true` or `1` to disable background job processing.@@ -618,8 +618,8 @@ DISABLE_WORKERS=true### ENABLED_QUEUES-**Type**: `string` (comma-separated)-**Required**: No+**Type**: `string` (comma-separated)+**Required**: No**Default**: All queues enabledComma-separated list of queue names to enable. Available queues: `events`, `events_kafka`, `sessions`, `cron`, `notification`, `misc`, `import`.@@ -635,8 +635,8 @@ Enable `events_kafka` to run the Kafka-based events consumer alongside (or inste### EVENT_JOB_CONCURRENCY-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `10`Number of concurrent event processing jobs per worker.@@ -648,8 +648,8 @@ EVENT_JOB_CONCURRENCY=20### EVENT_BLOCKING_TIMEOUT_SEC-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `1`Blocking timeout in seconds for event queue workers.@@ -661,8 +661,8 @@ EVENT_BLOCKING_TIMEOUT_SEC=2### EVENTS_GROUP_QUEUES_SHARDS-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `1`Number of shards for the events group queue. Increase for better performance with high event volume.@@ -674,8 +674,8 @@ EVENTS_GROUP_QUEUES_SHARDS=4### QUEUE_CLUSTER-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Enable Redis cluster mode for queues. When enabled, queue names are wrapped with `{}` for Redis cluster sharding.@@ -687,8 +687,8 @@ QUEUE_CLUSTER=true### ORDERING_DELAY_MS-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `100`Delay in milliseconds to hold events for correct ordering when events arrive out of order.@@ -704,8 +704,8 @@ Should not exceed 500ms. Higher values may cause delays in event processing.### AUTO_BATCH_MAX_WAIT_MS-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `0` (disabled)Maximum wait time in milliseconds for auto-batching events. Experimental feature.@@ -721,8 +721,8 @@ AUTO_BATCH_MAX_WAIT_MS=100### AUTO_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `0` (disabled)Batch size for auto-batching events. Experimental feature.@@ -882,8 +882,8 @@ KAFKA_HEARTBEAT_INTERVAL_MS=3000### SESSION_BUFFER_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultBatch size for session buffer operations.@@ -895,8 +895,8 @@ SESSION_BUFFER_BATCH_SIZE=5000### SESSION_BUFFER_CHUNK_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultChunk size for session buffer operations.@@ -908,8 +908,8 @@ SESSION_BUFFER_CHUNK_SIZE=1000### EVENT_BUFFER_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `4000`Batch size for event buffer operations.@@ -921,8 +921,8 @@ EVENT_BUFFER_BATCH_SIZE=5000### EVENT_BUFFER_CHUNK_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `1000`Chunk size for event buffer operations.@@ -934,8 +934,8 @@ EVENT_BUFFER_CHUNK_SIZE=2000### PROFILE_BUFFER_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultBatch size for profile buffer operations.@@ -947,8 +947,8 @@ PROFILE_BUFFER_BATCH_SIZE=5000### PROFILE_BUFFER_CHUNK_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultChunk size for profile buffer operations.@@ -960,8 +960,8 @@ PROFILE_BUFFER_CHUNK_SIZE=1000### PROFILE_BUFFER_TTL_IN_SECONDS-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultTime-to-live in seconds for profile buffer entries.@@ -973,8 +973,8 @@ PROFILE_BUFFER_TTL_IN_SECONDS=3600### BOT_BUFFER_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: Buffer-specific defaultBatch size for bot detection buffer operations.@@ -988,8 +988,8 @@ BOT_BUFFER_BATCH_SIZE=1000### FUNNEL_NON_STRICT_ORDERING-**Type**: `boolean` (`1` or `true`)-**Required**: No+**Type**: `boolean` (`1` or `true`)+**Required**: No**Default**: unset (strict ordering)Funnels use ClickHouse's `windowFunnel` in `strict_increase` mode by default: every step's timestamp must be strictly greater than the previous step's. Setting this variable switches funnels to non-strict ordering (`>=`), where steps sharing the same timestamp still count as an ordered sequence — matching how Mixpanel and Amplitude evaluate funnels. Useful for server-side senders, batched SDKs, or imported data with coarse timestamps.@@ -1003,8 +1003,8 @@ FUNNEL_NON_STRICT_ORDERING=1### EVENT_LIST_MAX_LOOKBACK_DAYS-**Type**: `number` (whole days, positive integer)-**Required**: No+**Type**: `number` (whole days, positive integer)+**Required**: No**Default**: `1825` (5 years)Ceiling for the empty-result lookback on the event list. When a page's cursor window matches nothing, the window doubles and retries until it reaches this many days. With filters that no index can prune (e.g. a properties value) each retry is a full scan of its window, so large deployments can bound the worst case with a small, human-sized value. Invalid values (non-integer, zero, negative) keep the default.@@ -1016,8 +1016,8 @@ EVENT_LIST_MAX_LOOKBACK_DAYS=7### SESSION_LIST_MAX_LOOKBACK_DAYS-**Type**: `number` (whole days, positive integer)-**Required**: No+**Type**: `number` (whole days, positive integer)+**Required**: No**Default**: `365`Same ceiling for the session list, which has its own default. Invalid values keep the default.@@ -1027,10 +1027,36 @@ Same ceiling for the session list, which has its own default. Invalid values keeSESSION_LIST_MAX_LOOKBACK_DAYS=7```+### COHORT_QUERY_MEMORY_LIMIT_BYTES++**Type**: `number` (bytes, positive integer)+**Required**: No+**Default**: unset (server defaults govern)++Hard memory cap applied to property-cohort queries, which aggregate every profile row for a project and are the one cohort query that can outgrow the server's headroom. When set without `COHORT_QUERY_SPILL_BYTES`, the spill threshold is derived as a third of this limit so the query spills to disk before it can be killed. Unset (together with the spill variable) applies no per-query settings at all. Invalid values are ignored.++**Example**:+```bash+COHORT_QUERY_MEMORY_LIMIT_BYTES=1400000000+```++### COHORT_QUERY_SPILL_BYTES++**Type**: `number` (bytes, positive integer)+**Required**: No+**Default**: unset (derived as `COHORT_QUERY_MEMORY_LIMIT_BYTES / 3` when that is set)++Point past which a property-cohort GROUP BY spills to disk instead of growing in memory. Must sit below the memory limit — a threshold at or above the kill limit means the query dies before it ever spills, so inverted pairs are re-derived. Invalid values are ignored.++**Example**:+```bash+COHORT_QUERY_SPILL_BYTES=314572800+```+### IMPORT_BATCH_SIZE-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `5000`Batch size for data import operations.@@ -1042,8 +1068,8 @@ IMPORT_BATCH_SIZE=10000### IP_HEADER_ORDER-**Type**: `string` (comma-separated)-**Required**: No+**Type**: `string` (comma-separated)+**Required**: No**Default**: See [default order](https://github.com/Openpanel-dev/openpanel/blob/main/packages/common/server/get-client-ip.ts)Custom order of HTTP headers to check for client IP address. Useful when behind specific proxies or CDNs.@@ -1059,8 +1085,8 @@ The default order includes: `openpanel-client-ip`, `cf-connecting-ip`, `true-cli### SHUTDOWN_GRACE_PERIOD_MS-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `5000`Grace period in milliseconds for graceful shutdown of services.@@ -1072,8 +1098,8 @@ SHUTDOWN_GRACE_PERIOD_MS=10000### API_PORT-**Type**: `number`-**Required**: No+**Type**: `number`+**Required**: No**Default**: `3000`Port for the API service to listen on.@@ -1085,8 +1111,8 @@ API_PORT=3000### API_HOST-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: `0.0.0.0` (production) / `localhost` (development)Host address for the API service to bind to. Set to `::` to enable IPv6 support (useful for platforms like Railway that use IPv6 for internal networking).@@ -1108,8 +1134,8 @@ Use `API_HOST=::` when deploying on platforms like Railway where private network### LOG_LEVEL-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: `info`Logging level. Options: `error`, `warn`, `info`, `debug`.@@ -1121,8 +1147,8 @@ LOG_LEVEL=debug### LOG_SILENT-**Type**: `boolean`-**Required**: No+**Type**: `boolean`+**Required**: No**Default**: `false`Disable all logging output. Set to `true` to silence logs.@@ -1134,8 +1160,8 @@ LOG_SILENT=true### LOG_PREFIX-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NonePrefix for log messages. Useful for identifying logs from different services.@@ -1147,8 +1173,8 @@ LOG_PREFIX=api### HYPERDX_API_KEY-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneHyperDX API key for sending logs to HyperDX for monitoring and analysis.@@ -1162,8 +1188,8 @@ HYPERDX_API_KEY=your-hyperdx-api-key### MAXMIND_LICENSE_KEY-**Type**: `string`-**Required**: No+**Type**: `string`+**Required**: No**Default**: NoneMaxMind GeoLite2 license key for downloading GeoIP databases.@@ -1201,5 +1227,4 @@ For a basic self-hosted installation, these variables are required:- [Deploy with Docker Compose](/docs/self-hosting/deploy-docker-compose)- [Deploy with Coolify](/docs/self-hosting/deploy-coolify)- [Deploy with Dokploy](/docs/self-hosting/deploy-dokploy)-- [Deploy on Kubernetes](/docs/self-hosting/deploy-kubernetes)-+- [Deploy on Kubernetes](/docs/self-hosting/deploy-kubernetes)\ No newline at end of filediff --git a/packages/db/src/services/cohort-property-sql.test.ts b/packages/db/src/services/cohort-property-sql.test.tsnew file mode 100644index 000000000..76c1f0d92--- /dev/null+++ b/packages/db/src/services/cohort-property-sql.test.ts@@ -0,0 +1,278 @@+/**+ * SQL-shape tests for property-based cohort queries.+ *+ * Same strategy as chart-sql.test.ts / funnel-sql.test.ts: string assertions+ * always run; `EXPLAIN` validation runs against a locally reachable+ * ClickHouse (`pnpm dock:up`) and skips otherwise.+ */+import type { PropertyBasedCohortDefinition } from '@openpanel/validation';+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';++vi.mock('../prisma-client', () => ({+ db: {},+}));++import { ch } from '../clickhouse/client';+import {+ buildPropertyBasedCohortQuery,+ PROFILE_COHORT_QUERY_SETTINGS,+} from './cohort.service';++const PROJECT_ID = 'test-sql-validation';++let chReachable = false;++beforeAll(async () => {+ vi.spyOn(console, 'log').mockImplementation(() => {});+ try {+ await ch.command({ query: 'SELECT 1' });+ chReachable = true;+ } catch {+ chReachable = false;+ }+});++afterAll(() => {+ vi.restoreAllMocks();+});++const itCH = (name: string, fn: () => Promise<void>) =>+ it(name, async (ctx) => {+ if (!chReachable) {+ ctx.skip('ClickHouse not reachable at CLICKHOUSE_URL');+ }+ await fn();+ });++function buildSql(+ criteria: PropertyBasedCohortDefinition['criteria'],+ limit?: number,+) {+ return buildPropertyBasedCohortQuery(+ PROJECT_ID,+ { type: 'property', criteria } as PropertyBasedCohortDefinition,+ limit,+ );+}++const mapFilter = {+ id: 'a',+ name: 'profile.properties.experiment',+ operator: 'is' as const,+ value: ['control'],+};++describe('buildPropertyBasedCohortQuery', () => {+ it('resolves the newest row per profile without FINAL', () => {+ const sql = buildSql({ operator: 'and', properties: [mapFilter] });++ // FINAL cannot spill to disk, so wide projects OOM on the dedup itself.+ expect(sql).not.toContain('FINAL');+ expect(sql).toContain('GROUP BY id');+ expect(sql).toContain(+ "argMax(profiles.properties['experiment'], tuple(last_seen_at, cityHash64(profiles.properties['experiment'])))",+ );+ });++ it('orders every aggregate by ONE shared row key', () => {+ // Per-column tie-breaking lets equal-version rows with conflicting+ // fields each win a different column — an AND cohort could then match a+ // synthetic combination no stored row contains. The shared key makes+ // all aggregates read the same winning row.+ const sql = buildSql({+ operator: 'and',+ properties: [+ mapFilter,+ {+ id: 'b',+ name: 'profile.email',+ operator: 'is' as const,+ value: ['[email protected]'],+ },+ ],+ });++ const sharedKey =+ "tuple(last_seen_at, cityHash64(profiles.properties['experiment'], profiles.email))";+ expect(sql).toContain(+ `argMax(profiles.properties['experiment'], ${sharedKey})`,+ );+ expect(sql).toContain(`argMax(profiles.email, ${sharedKey})`);+ });++ it('filters aggregates in HAVING, not WHERE', () => {+ const sql = buildSql({ operator: 'and', properties: [mapFilter] });++ const having = sql.indexOf('HAVING');+ expect(having).toBeGreaterThan(-1);+ expect(sql.indexOf('argMax')).toBeGreaterThan(having);+ });++ it('wraps plain columns too, so mixed cohorts stay on one scan', () => {+ const sql = buildSql({+ operator: 'or',+ properties: [+ mapFilter,+ {+ id: 'b',+ name: 'profile.email',+ operator: 'contains' as const,+ value: ['@example.com'],+ },+ ],+ });++ expect(sql).toContain('argMax(profiles.email, tuple(last_seen_at,');+ expect(sql).toContain(' OR ');+ expect(sql).not.toContain('FINAL');+ });++ it('wraps numeric comparisons inside the cast', () => {+ const sql = buildSql({+ operator: 'and',+ properties: [+ {+ id: 'n',+ name: 'profile.properties.age',+ operator: 'gt' as const,+ value: ['30'],+ },+ ],+ });++ expect(sql).toContain(+ "toFloat64OrNull(argMax(profiles.properties['age'], tuple(last_seen_at,",+ );+ });++ it('escapes quotes in user-controlled property keys', () => {+ const sql = buildSql({+ operator: 'and',+ properties: [+ {+ id: 'q',+ name: "profile.properties.pl'an",+ operator: 'is' as const,+ value: ['x'],+ },+ ],+ });++ // The raw quote must never appear inside the literal unescaped.+ expect(sql).toContain("profiles.properties['pl\\'an']");+ expect(sql).not.toContain("properties['pl'an']");+ });++ it('applies the limit', () => {+ const sql = buildSql({ operator: 'and', properties: [mapFilter] }, 10);+ expect(sql).toContain('LIMIT 10');+ });++ it('matches nothing when every filter was dropped as empty', () => {+ const sql = buildSql({+ operator: 'and',+ properties: [+ {+ id: 'a',+ name: 'profile.properties.x',+ operator: 'is' as const,+ value: [],+ },+ ],+ });++ expect(sql).toContain('WHERE 1=0');+ expect(sql).not.toContain('argMax');+ });++ describe('PROFILE_COHORT_QUERY_SETTINGS', () => {+ const loadSettings = async (env: Record<string, string>) => {+ vi.resetModules();+ vi.stubEnv('COHORT_QUERY_MEMORY_LIMIT_BYTES', env.limit ?? '');+ vi.stubEnv('COHORT_QUERY_SPILL_BYTES', env.spill ?? '');+ const mod = await import('./cohort.service');+ return mod.PROFILE_COHORT_QUERY_SETTINGS;+ };++ afterAll(() => {+ vi.unstubAllEnvs();+ });++ it('applies NO settings when neither variable is set (upstream defaults govern)', async () => {+ expect(await loadSettings({})).toEqual({});+ });++ it('derives the spill threshold as limit/3 when only the limit is set', async () => {+ expect(await loadSettings({ limit: '3000000000' })).toEqual({+ max_bytes_before_external_group_by: '1000000000',+ max_memory_usage: '3000000000',+ });+ });++ it('respects both values when both are set', async () => {+ expect(+ await loadSettings({ limit: '2000000000', spill: '500000000' }),+ ).toEqual({+ max_bytes_before_external_group_by: '500000000',+ max_memory_usage: '2000000000',+ });+ });++ it('applies only the spill threshold when only it is set', async () => {+ expect(await loadSettings({ spill: '500000000' })).toEqual({+ max_bytes_before_external_group_by: '500000000',+ });+ });++ it('re-derives an inverted pair (spill >= limit would never spill)', async () => {+ // A GROUP BY only starts spilling once it crosses the threshold, so a+ // threshold at/above the kill limit means the query dies before it+ // ever writes to disk — the exact inversion ClickHouse Cloud ships.+ expect(+ await loadSettings({ limit: '900000000', spill: '900000000' }),+ ).toEqual({+ max_bytes_before_external_group_by: '300000000',+ max_memory_usage: '900000000',+ });+ });++ it('never derives a zero spill threshold (0 would DISABLE spilling)', async () => {+ for (const limit of ['1', '2', '3']) {+ const settings = await loadSettings({ limit });+ expect(+ Number(settings.max_bytes_before_external_group_by),+ ).toBeGreaterThanOrEqual(1);+ }+ });++ it('ignores malformed values', async () => {+ expect(await loadSettings({ limit: '2gb', spill: '-1' })).toEqual({});+ });+ });++ itCH('parses and resolves against ClickHouse', async () => {+ for (const sql of [+ buildSql({ operator: 'and', properties: [mapFilter] }, 10),+ buildSql({+ operator: 'or',+ properties: [+ mapFilter,+ {+ id: 'n',+ name: 'profile.properties.age',+ operator: 'gte' as const,+ value: ['30'],+ },+ {+ id: 'e',+ name: 'profile.email',+ operator: 'isNotNull' as const,+ value: [],+ },+ ],+ }),+ ]) {+ await ch.command({ query: `EXPLAIN ${sql}` });+ }+ });+});diff --git a/packages/db/src/services/cohort.service.ts b/packages/db/src/services/cohort.service.tsindex ff2c005cb..8a20fe06d 100644--- a/packages/db/src/services/cohort.service.ts+++ b/packages/db/src/services/cohort.service.ts@@ -10,6 +10,7 @@ import type {} from '@openpanel/validation';import { cohortComputeQueue } from '@openpanel/queue';+import type { ClickHouseSettings } from '@clickhouse/client';import {TABLE_NAMES,ch,@@ -44,6 +45,60 @@ export const COHORT_MATERIALIZE_LIMIT =? COHORT_MATERIALIZE_LIMIT_PARSED: 10000;+// Strictly a positive safe integer, or undefined — same validation rationale+// as COHORT_MATERIALIZE_LIMIT above.+function parsePositiveIntEnv(name: string): number | undefined {+ const raw = process.env[name];+ if (!raw || !/^\d+$/.test(raw)) {+ return undefined;+ }+ const parsed = Number(raw);+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined;+}++// Property cohorts aggregate every profile row for the project, so they are+// the one cohort query that can outgrow the server's memory headroom. Two+// opt-in knobs bound them; with NEITHER set, no per-query settings are+// applied and the server's own defaults govern — upstream behavior is+// unchanged.+//+// COHORT_QUERY_MEMORY_LIMIT_BYTES hard cap for these queries+// COHORT_QUERY_SPILL_BYTES GROUP BY spills to disk past this+//+// A GROUP BY only starts spilling once it crosses the threshold, so the+// spill threshold must sit BELOW the memory limit — inverted, the query is+// killed before it ever writes to disk (ClickHouse Cloud ships exactly that+// inversion by default, which is how these queries OOM'd instead of+// spilling). When only the limit is set — or the pair is inverted — the+// threshold derives as limit/3. Spilling early costs little: the volume+// spilled is set by the data, not the threshold (measured on 8.3M profiles,+// ~281MB spilled whether the threshold was 300, 512 or 768MB, at+// 6.9s/6.7s/6.0s, while peak memory climbed 410/695/893MiB).+const COHORT_QUERY_MEMORY_LIMIT_BYTES = parsePositiveIntEnv(+ 'COHORT_QUERY_MEMORY_LIMIT_BYTES',+);+const COHORT_QUERY_SPILL_BYTES_RAW = parsePositiveIntEnv(+ 'COHORT_QUERY_SPILL_BYTES',+);+const COHORT_QUERY_SPILL_BYTES =+ COHORT_QUERY_MEMORY_LIMIT_BYTES !== undefined &&+ (COHORT_QUERY_SPILL_BYTES_RAW === undefined ||+ COHORT_QUERY_SPILL_BYTES_RAW >= COHORT_QUERY_MEMORY_LIMIT_BYTES)+ ? // Clamped to 1: a (nonsensical) limit below 3 would derive 0, and+ // max_bytes_before_external_group_by = 0 means spilling DISABLED —+ // the exact inversion this derivation exists to prevent.+ Math.max(1, Math.floor(COHORT_QUERY_MEMORY_LIMIT_BYTES / 3))+ : COHORT_QUERY_SPILL_BYTES_RAW;++export const PROFILE_COHORT_QUERY_SETTINGS: ClickHouseSettings = {+ ...(COHORT_QUERY_SPILL_BYTES !== undefined+ ? { max_bytes_before_external_group_by: String(COHORT_QUERY_SPILL_BYTES) }+ : {}),+ ...(COHORT_QUERY_MEMORY_LIMIT_BYTES !== undefined+ ? { max_memory_usage: String(COHORT_QUERY_MEMORY_LIMIT_BYTES) }+ : {}),+};+function buildTimeConstraint(timeframe: Timeframe): string {if (timeframe.type === 'relative') {const match = timeframe.value.match(/^(\d+)d$/);@@ -179,27 +234,74 @@ export function buildEventCriteriaQuery(`;}-export function buildPropertyBasedCohortQuery(- projectId: string,+// SQL for a profile filter's column: either a properties Map lookup or a+// plain column, qualified with the table name.+function profileColumnAccess(name: string): string {+ const normalizedName = name.replace(/^profile\./, 'profiles.');+ if (normalizedName.startsWith('profiles.properties.')) {+ const propKey = normalizedName.replace('profiles.properties.', '');+ // Escaped: cohort definitions come from the API, so the key is+ // user-controlled — a quote in it must not terminate the literal.+ return `profiles.properties[${sqlstring.escape(propKey)}]`;+ }+ return normalizedName;+}++function buildProfileCohortHavingClause(definition: PropertyBasedCohortDefinition,-): string {+): string | null {const { properties, operator } = definition.criteria;- const filterWhere = getProfileFiltersWhereClause(properties);++ // Every argMax below must order the candidate rows IDENTICALLY, or+ // equal-version rows with conflicting fields could each win a different+ // column — matching an AND cohort against a synthetic combination no+ // stored row contains. One shared key — the version column, tie-broken by+ // a hash of every referenced column — makes all aggregates pick their+ // value from the same winning row, deterministically. The hash (rather+ // than the raw value tuple) keeps the per-group comparison state at a+ // fixed 8 bytes: measured on 8.8M profiles, the raw-tuple key cost ~40%+ // extra query time while the hashed key is free. A wrong tie-break would+ // need a version tie AND a 64-bit collision between different rows — and+ // even then every aggregate in the query still elects the same row.+ const referencedColumns = Array.from(+ new Set(properties.map((f) => profileColumnAccess(f.name))),+ );+ const latestRowKey = `tuple(last_seen_at, cityHash64(${referencedColumns.join(', ')}))`;++ const filterWhere = getProfileFiltersWhereClause(properties, {+ latestPerProfileKey: latestRowKey,+ });const filterClauses = Object.values(filterWhere);if (filterClauses.length === 0) {- return `SELECT id as profile_id FROM ${TABLE_NAMES.profiles} FINAL WHERE 1=0`;+ return null;}- const filterClause = filterClauses.join(- operator === 'and' ? ' AND ' : ' OR ',- );+ return filterClauses.join(operator === 'and' ? ' AND ' : ' OR ');+}++export function buildPropertyBasedCohortQuery(+ projectId: string,+ definition: PropertyBasedCohortDefinition,+ limit?: number,+): string {+ const havingClause = buildProfileCohortHavingClause(definition);++ if (!havingClause) {+ return `SELECT id as profile_id FROM ${TABLE_NAMES.profiles} WHERE 1=0`;+ }+ // Resolve each profile's newest row with GROUP BY + argMax instead of+ // FINAL: FINAL cannot spill to disk, so on wide projects the dedup itself+ // is what runs out of memory. The aggregate shape spills normally under+ // PROFILE_COHORT_QUERY_SETTINGS, and filters on aggregates move to HAVING.return `SELECT id as profile_id- FROM ${TABLE_NAMES.profiles} FINAL+ FROM ${TABLE_NAMES.profiles}WHERE project_id = ${sqlstring.escape(projectId)}- AND (${filterClause})+ GROUP BY id+ HAVING (${havingClause})+ ${limit ? `LIMIT ${limit}` : ''}`;}@@ -247,6 +349,7 @@ export async function countEventBasedCohort(function getProfileFiltersWhereClause(filters: IChartEventFilter[],+ { latestPerProfileKey }: { latestPerProfileKey?: string } = {},): Record<string, string> {const where: Record<string, string> = {};@@ -262,14 +365,20 @@ function getProfileFiltersWhereClause(return;}- const normalizedName = name.replace(/^profile\./, 'profiles.');- let columnAccess: string;-- if (normalizedName.startsWith('profiles.properties.')) {- const propKey = normalizedName.replace('profiles.properties.', '');- columnAccess = `profiles.properties['${propKey}']`;- } else {- columnAccess = normalizedName;+ let columnAccess = profileColumnAccess(name);++ if (latestPerProfileKey) {+ // Resolve the profile's newest row inside a GROUP BY instead of+ // reading through FINAL. The key is shared by every wrapped column+ // (see buildProfileCohortHavingClause), so all aggregates read the+ // SAME winning row: last_seen_at is the table's version column but is+ // not unique, and per-column tie-breaking would let equal-version+ // rows with conflicting fields produce a synthetic combination no+ // stored row contains. FINAL breaks the same ties by part order,+ // which is not derivable from the data and can shift under a+ // background merge — the shared value-tuple tie-break is+ // deterministic instead.+ columnAccess = `argMax(${columnAccess}, ${latestPerProfileKey})`;}switch (operator) {@@ -372,27 +481,14 @@ export async function computePropertyBasedCohort(definition: PropertyBasedCohortDefinition,limit?: number,): Promise<string[]> {- const { properties, operator } = definition.criteria;- const filterWhere = getProfileFiltersWhereClause(properties);- const filterClauses = Object.values(filterWhere);-- if (filterClauses.length === 0) {+ if (!buildProfileCohortHavingClause(definition)) {return [];}- const filterClause = filterClauses.join(- operator === 'and' ? ' AND ' : ' OR ',+ const results = await chQuery<{ profile_id: string }>(+ buildPropertyBasedCohortQuery(projectId, definition, limit),+ PROFILE_COHORT_QUERY_SETTINGS,);-- const query = `- SELECT id as profile_id- FROM ${TABLE_NAMES.profiles} FINAL- WHERE project_id = ${sqlstring.escape(projectId)}- AND (${filterClause})- ${limit ? `LIMIT ${limit}` : ''}- `;-- const results = await chQuery<{ profile_id: string }>(query);return results.map((r) => r.profile_id);}@@ -400,26 +496,14 @@ export async function countPropertyBasedCohort(projectId: string,definition: PropertyBasedCohortDefinition,): Promise<number> {- const { properties, operator } = definition.criteria;- const filterWhere = getProfileFiltersWhereClause(properties);- const filterClauses = Object.values(filterWhere);-- if (filterClauses.length === 0) {+ if (!buildProfileCohortHavingClause(definition)) {return 0;}- const filterClause = filterClauses.join(- operator === 'and' ? ' AND ' : ' OR ',+ const results = await chQuery<{ count: number }>(+ `SELECT count() as count FROM (${buildPropertyBasedCohortQuery(projectId, definition)})`,+ PROFILE_COHORT_QUERY_SETTINGS,);-- const query = `- SELECT count() as count- FROM ${TABLE_NAMES.profiles} FINAL- WHERE project_id = ${sqlstring.escape(projectId)}- AND (${filterClause})- `;-- const results = await chQuery<{ count: number }>(query);return results[0]?.count ?? 0;}