Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M5-003): cohort module: service (enqueue wrappers move in), trpc, cohortCompute queue, cohortRefresh cron

ralphdf05fbd79ea9
diff --git a/apps/worker/src/jobs/cohort.compute.ts b/apps/worker/src/jobs/cohort.compute.ts
index 9665ef1e1..8bea88292 100644
--- a/apps/worker/src/jobs/cohort.compute.ts
+++ b/apps/worker/src/jobs/cohort.compute.ts
@@ -1,7 +1,10 @@
-import type { Job } from 'bullmq';
-
-import { updateCohortMembership } from '@openpanel/db';
+// Dissolved into @openpanel/core's cohort module (M5-003): the compute body
+// moved to packages/core/src/modules/cohort/cohort.service.ts#updateCohortMembership.
+// This file stays (DELEGATE PATTERN) — it is the BullMQ job body V1's worker
+// registers (boot-workers.ts), a thin wrapper around the core function.
+import { updateCohortMembership } from '@openpanel/core';
import type { CohortComputePayload } from '@openpanel/queue';
+import type { Job } from 'bullmq';
export async function cohortComputeJob(job: Job<CohortComputePayload>) {
await updateCohortMembership(job.data.cohortId);
diff --git a/apps/worker/src/jobs/cron.cohort-refresh.ts b/apps/worker/src/jobs/cron.cohort-refresh.ts
index 192522646..f9c037e64 100644
--- a/apps/worker/src/jobs/cron.cohort-refresh.ts
+++ b/apps/worker/src/jobs/cron.cohort-refresh.ts
@@ -1,10 +1,25 @@
-import { db, enqueueCohortCompute } from '@openpanel/db';
+// Dissolved into @openpanel/core's cohort module (M5-003): the non-static
+// cohort lookup moved to
+// packages/core/src/modules/cohort/cohort.service.ts#listRefreshableCohortIds.
+// This file stays (DELEGATE PATTERN) — it is the `cron.ts` dispatcher's
+// `cohortRefresh` case, a thin wrapper around the core function. The enqueue
+// itself stays inline against @openpanel/queue's cohortComputeQueue directly
+// — same as gsc's worker file does for `gscProjectSync` (M5-002); see
+// packages/core/src/modules/cohort/cohort.service.ts's header for why core
+// cannot import @openpanel/queue back.
+import { listRefreshableCohortIds } from '@openpanel/core';
+import { cohortComputeQueue } from '@openpanel/queue';
export async function cohortRefreshCronJob() {
- const cohorts = await db.cohort.findMany({
- where: { isStatic: false },
- select: { id: true },
- });
+ const cohortIds = await listRefreshableCohortIds();
- await Promise.all(cohorts.map((cohort) => enqueueCohortCompute(cohort.id)));
+ await Promise.all(
+ cohortIds.map((cohortId) =>
+ cohortComputeQueue.add(
+ 'cohortCompute',
+ { cohortId },
+ { deduplication: { id: `cohort-${cohortId}` } }
+ )
+ )
+ );
}
diff --git a/packages/core/package.json b/packages/core/package.json
index 2c24357f1..5eba5c77e 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -49,6 +49,7 @@
"pino-pretty": "catalog:",
"prom-client": "^15.1.3",
"qrcode": "^1.5.4",
+ "sqlstring": "^2.3.3",
"superjson": "^1.13.3",
"ua-parser-js": "^2.0.6",
"undici": "^8.10.0",
@@ -58,6 +59,7 @@
"@clickhouse/client": "^1.18.5",
"@openpanel/tsconfig": "workspace:*",
"@types/qrcode": "^1.5.6",
+ "@types/sqlstring": "^2.3.2",
"bun-types": "1.4.0",
"tar": "^7.4.3",
"typescript": "catalog:"
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index e8819ced7..dbf5153c4 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -145,6 +145,27 @@ export {
verifyPasswordHash,
verifyTotpCode,
} from './modules/auth/auth.service';
+// Dissolved from @openpanel/db's services/cohort.service.ts (M5-003) —
+// packages/trpc's cohort router and apps/worker's cohort job files call
+// these directly, the same way V1 reaches every other dissolved service
+// here. Nothing else in the tree reached cohort.service.ts through
+// @openpanel/db's barrel, so packages/db loses the file entirely rather than
+// keeping a re-export shim (unlike gsc.ts/gsc.service.ts).
+export type { CohortService } from './modules/cohort/cohort.service';
+export {
+ computeCohort,
+ countCohort,
+ deleteCohortMembership,
+ getCohortCount,
+ getCohortEventsPerDay,
+ getCohortMemberEvents,
+ getCohortMemberRoutes,
+ getCohortMembers,
+ getProfilesInCohort,
+ listCohortMemberProfiles,
+ listRefreshableCohortIds,
+ updateCohortMembership,
+} from './modules/cohort/cohort.service';
// Dissolved from @openpanel/db's src/gsc.ts + services/gsc.service.ts
// (M5-002) — apps/worker's gsc job file, apps/api's gsc OAuth callback
// controller and packages/trpc's gsc router call these directly, the same
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.ts
index f0c8be344..d853841a0 100644
--- a/packages/core/src/jobs.registry.test.ts
+++ b/packages/core/src/jobs.registry.test.ts
@@ -82,7 +82,13 @@ test('the insights queue carries the insight module job', () => {
// job's `type` straight onto its name, so these three ids double as job names.
test('the cron queue carries the insight module cron fragment', () => {
expect(Object.keys(queues.cron.jobs).sort()).toEqual(
- ['gscSync', 'insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()
+ [
+ 'cohortRefresh',
+ 'gscSync',
+ 'insightCleanup',
+ 'insightsDaily',
+ 'weeklyDigest',
+ ].sort()
);
for (const name of ['insightsDaily', 'insightCleanup', 'weeklyDigest']) {
expect(
@@ -111,3 +117,21 @@ test('the cron queue carries the gsc module cron fragment', () => {
name: 'gscSync',
});
});
+
+// M5-003: the cohort module's own queue job, plus its cron fan-out fragment.
+// legacyCompat.cohortCompute already discriminates on a bare `{cohortId}`
+// (jobs/compat.ts), so this pins the two in agreement.
+test('the cohortCompute queue carries the cohort module job', () => {
+ expect(Object.keys(queues.cohortCompute.jobs)).toEqual(['cohortCompute']);
+ expect(queues.cohortCompute.jobs.cohortCompute).toMatchObject({
+ queue: 'cohortCompute',
+ name: 'cohortCompute',
+ });
+});
+
+test('the cron queue carries the cohort module cron fragment', () => {
+ expect(queues.cron.jobs.cohortRefresh).toMatchObject({
+ queue: 'cron',
+ name: 'cohortRefresh',
+ });
+});
diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.ts
index 79f898545..5444a7d2d 100644
--- a/packages/core/src/jobs.registry.ts
+++ b/packages/core/src/jobs.registry.ts
@@ -14,6 +14,7 @@ import { legacyCompat } from './jobs/compat';
import type { Producers } from './jobs/define';
import { defineQueue } from './jobs/define';
import type { ProducerHandle } from './jobs/producers';
+import { cohortCronJobs, cohortQueueJobs } from './modules/cohort/cohort.jobs';
import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';
import {
insightCronJobs,
@@ -55,6 +56,7 @@ const cron = defineQueue('cron', {
jobs: {
...insightCronJobs,
...gscCronJobs,
+ ...cohortCronJobs,
},
});
@@ -101,7 +103,9 @@ const cohortCompute = defineQueue('cohortCompute', {
},
worker: { concurrency: CONCURRENCY.cohortCompute },
compat: legacyCompat.cohortCompute,
- jobs: {},
+ jobs: {
+ ...cohortQueueJobs,
+ },
});
export const queues = {
diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.ts
index b94cfc668..21353ca8d 100644
--- a/packages/core/src/jobs/schedulers.ts
+++ b/packages/core/src/jobs/schedulers.ts
@@ -5,6 +5,7 @@
// locking of our own needed.
import type { Logger } from '../logger';
+import { cohortCronSchedules } from '../modules/cohort/cohort.jobs';
import { gscCronSchedules } from '../modules/gsc/gsc.jobs';
import { insightCronSchedules } from '../modules/insight/insight.jobs';
import { wrap } from './envelope';
@@ -67,7 +68,9 @@ export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [
// gscSync — owned by the gsc module, declared next to its jobs
// (modules/gsc/gsc.jobs.ts).
...gscCronSchedules,
- { id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },
+ // cohortRefresh — owned by the cohort module, declared next to its jobs
+ // (modules/cohort/cohort.jobs.ts).
+ ...cohortCronSchedules,
{ id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },
// Daily 04:00 UTC — backstop for cleanup leaks.
{ id: 'sessionVacuum', schedule: { pattern: '0 4 * * *' } },
diff --git a/packages/core/src/modules/cohort/cohort.constants.ts b/packages/core/src/modules/cohort/cohort.constants.ts
new file mode 100644
index 000000000..0a6d0e071
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.constants.ts
@@ -0,0 +1,162 @@
+// Moved from packages/validation/src/cohort.validation.ts (M5-003, ADR-008's
+// module map: cohort owns "C"). packages/validation/src/cohort.validation.ts
+// becomes a re-export shim of this file's `./modules/cohort/cohort.constants`
+// subpath (same shape as packages/db/src/gsc.ts since M5-002), so
+// apps/start and packages/db/src/types.ts keep resolving these symbols
+// through @openpanel/validation's existing barrel unchanged.
+//
+// Isomorphic by the AGENTS.md rule: zod and nothing else.
+
+import { operators } from '@openpanel/constants';
+import { z } from 'zod';
+
+// TODO(P7): the canonical zChartEventFilter lives in
+// packages/validation/src/index.ts:31 and is due to move to
+// core/src/modules/report/report.constants.ts. That module does not exist
+// yet (creating it early is out of this task's scope), so this stays a
+// diverged local copy — defined here, rather than imported, to avoid a
+// runtime TDZ from this file being evaluated before validation's index.ts
+// finishes initializing zChartEventFilter. Once report.constants.ts exists,
+// delete this duplicate and import zChartEventFilter from there instead.
+function objectToZodEnums<K extends string>(
+ obj: Record<K, unknown>
+): [K, ...K[]] {
+ const [firstKey, ...otherKeys] = Object.keys(obj) as K[];
+ return [firstKey!, ...otherKeys];
+}
+
+const zChartEventFilter = z.object({
+ id: z.string().optional().describe('Unique identifier for the filter'),
+ name: z.string().describe('The property name to filter on'),
+ operator: z
+ .enum(objectToZodEnums(operators))
+ .describe('The operator to use for the filter'),
+ value: z
+ .array(z.string().or(z.number()).or(z.boolean()).or(z.null()))
+ .describe('The values to filter on'),
+ cohortId: z
+ .string()
+ .optional()
+ .describe('Cohort ID when using inCohort/notInCohort operators'),
+});
+
+export const zRelativeTimeframe = z.object({
+ type: z.literal('relative'),
+ value: z.enum(['7d', '30d', '90d', '180d', '365d']),
+});
+
+export const zAbsoluteTimeframe = z.object({
+ type: z.literal('absolute'),
+ start: z.string(),
+ end: z.string().optional(),
+});
+
+export const zTimeframe = z.discriminatedUnion('type', [
+ zRelativeTimeframe,
+ zAbsoluteTimeframe,
+]);
+
+export type Timeframe = z.infer<typeof zTimeframe>;
+
+export const zFrequency = z.object({
+ operator: z.enum(['gte', 'eq', 'lte']),
+ count: z.number().int().min(1),
+});
+
+export type Frequency = z.infer<typeof zFrequency>;
+
+export const zEventCriteria = z.object({
+ name: z.string().min(1).describe('The event name to match'),
+ filters: z
+ .array(zChartEventFilter)
+ .default([])
+ .describe('Filters applied to event properties'),
+ timeframe: zTimeframe.describe('When the event should have occurred'),
+ frequency: zFrequency
+ .optional()
+ .describe('How many times the event should have occurred'),
+});
+
+export type EventCriteria = z.infer<typeof zEventCriteria>;
+
+export const zEventBasedCohortDefinition = z.object({
+ type: z.literal('event'),
+ criteria: z.object({
+ operator: z.enum(['and', 'or']).describe('How to combine multiple events'),
+ events: z
+ .array(zEventCriteria)
+ .min(1)
+ .describe('Array of event criteria to match'),
+ }),
+});
+
+export type EventBasedCohortDefinition = z.infer<
+ typeof zEventBasedCohortDefinition
+>;
+
+export const zPropertyBasedCohortDefinition = z.object({
+ type: z.literal('property'),
+ criteria: z.object({
+ operator: z
+ .enum(['and', 'or'])
+ .describe('How to combine multiple properties'),
+ properties: z
+ .array(zChartEventFilter)
+ .min(1)
+ .describe('Array of profile property filters'),
+ }),
+});
+
+export type PropertyBasedCohortDefinition = z.infer<
+ typeof zPropertyBasedCohortDefinition
+>;
+
+export const zCohortDefinition = z.discriminatedUnion('type', [
+ zEventBasedCohortDefinition,
+ zPropertyBasedCohortDefinition,
+]);
+
+export type CohortDefinition = z.infer<typeof zCohortDefinition>;
+
+export const zCohortInput = z.object({
+ name: z
+ .string()
+ .min(1)
+ .max(255)
+ .describe('User-friendly name for the cohort'),
+ description: z
+ .string()
+ .max(1000)
+ .optional()
+ .describe('Optional description of the cohort'),
+ projectId: z.string().describe('The project this cohort belongs to'),
+ definition: zCohortDefinition.describe('The cohort criteria definition'),
+ isStatic: z
+ .boolean()
+ .default(false)
+ .describe(
+ 'Whether this cohort is frozen — if true, membership is computed once and not auto-refreshed'
+ ),
+});
+
+export type CohortInput = z.infer<typeof zCohortInput>;
+
+export const zCohortUpdate = z.object({
+ id: z.string().describe('The cohort ID to update'),
+ name: z.string().min(1).max(255).optional(),
+ description: z.string().max(1000).optional().nullable(),
+ definition: zCohortDefinition.optional(),
+ isStatic: z.boolean().optional(),
+});
+
+export type CohortUpdate = z.infer<typeof zCohortUpdate>;
+
+export const zCohortFilter = z.object({
+ cohortId: z.string().describe('The cohort ID to filter by'),
+ operator: z
+ .enum(['inCohort', 'notInCohort'])
+ .default('inCohort')
+ .describe('Whether to include or exclude cohort members'),
+});
+
+export type CohortFilter = z.infer<typeof zCohortFilter>;
diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.ts
new file mode 100644
index 000000000..a31f7d4ee
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts
@@ -0,0 +1,106 @@
+// Job/cron-fragment wiring — no ClickHouse/Postgres touched. The service
+// methods themselves are exercised in cohort.service.test.ts.
+
+import { expect, test } from 'bun:test';
+import type { JobCtx } from '../../context';
+import { createRecordingProducers } from '../../jobs/testing';
+import { queues } from '../../jobs.registry';
+import type { Logger } from '../../logger';
+import type { Services } from '../../services';
+import {
+ cohortCronJobs,
+ cohortCronSchedules,
+ cohortQueueJobs,
+} from './cohort.jobs';
+import type { CohortService } from './cohort.service';
+
+function stubLogger(): Logger {
+ const noop = () => undefined;
+ const logger: Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {
+ const services: Services = {
+ auth: {} as Services['auth'],
+ insight: {} as Services['insight'],
+ gsc: {} as Services['gsc'],
+ cohort: cohort as CohortService,
+ };
+ return {
+ db: {},
+ ch: {},
+ redis: {},
+ clients: {},
+ buffers: {},
+ logger: stubLogger(),
+ queues: createRecordingProducers(queues).queues,
+ services,
+ requestId: 'req_1',
+ job: { id: 'job_1', attempt: 0, queue: 'cohortCompute', name: jobName },
+ };
+}
+
+test('the cohortCompute jobs registry declares cohortCompute on the cohortCompute queue', () => {
+ expect(queues.cohortCompute.jobs.cohortCompute).toMatchObject({
+ queue: 'cohortCompute',
+ name: 'cohortCompute',
+ });
+});
+
+test('the cohort cron fragment is spread into the cron queue', () => {
+ expect(queues.cron.jobs.cohortRefresh).toBeDefined();
+});
+
+test('cohortCompute validates its payload', () => {
+ expect(() => cohortQueueJobs.cohortCompute.payload.parse({})).toThrow();
+ expect(
+ cohortQueueJobs.cohortCompute.payload.parse({ cohortId: 'c1' })
+ ).toEqual({ cohortId: 'c1' });
+});
+
+test('cohortCompute delegates to updateMembership', async () => {
+ const calls: unknown[] = [];
+ const ctx = stubJobCtx({
+ updateMembership: async (cohortId) => {
+ calls.push(cohortId);
+ },
+ });
+
+ await cohortQueueJobs.cohortCompute.handler({
+ payload: { cohortId: 'c1' },
+ ctx,
+ });
+
+ expect(calls).toEqual(['c1']);
+});
+
+test('cohortRefresh fans out one enqueueCompute per non-static cohort', async () => {
+ const calls: unknown[] = [];
+ const ctx = stubJobCtx({
+ listRefreshableCohortIds: async () => ['c1', 'c2'],
+ enqueueCompute: async (cohortId) => {
+ calls.push(cohortId);
+ },
+ });
+
+ await cohortCronJobs.cohortRefresh.handler({ payload: null, ctx });
+
+ expect(calls).toEqual(['c1', 'c2']);
+});
+
+// Byte-identity with the id/cadence schedulers.test.ts's golden snapshot pins
+// (apps/worker/src/boot-cron.ts).
+test('the cohort cron fragment carries V1 id and cadence unchanged', () => {
+ expect(cohortCronSchedules).toEqual([
+ { id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },
+ ]);
+});
diff --git a/packages/core/src/modules/cohort/cohort.jobs.ts b/packages/core/src/modules/cohort/cohort.jobs.ts
new file mode 100644
index 000000000..460543640
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.jobs.ts
@@ -0,0 +1,46 @@
+// Ported from apps/worker/src/jobs/cohort.compute.ts + cron.cohort-refresh.ts
+// (M5-003).
+//
+// `cohortCompute` is this module's own queue — registry key `cohortCompute`
+// EXACTLY (ADR-005's acceptance note: no env rename, so
+// COHORTCOMPUTE_CONCURRENCY keeps working). `cohortRefresh` is a cron
+// fragment: declared here, spread into the ONE `cron` queue by
+// jobs.registry.ts and into `CRON_SCHEDULES` by jobs/schedulers.ts. Scheduler
+// id and cadence are V1's, unchanged (apps/worker/src/boot-cron.ts).
+
+import { z } from 'zod';
+import { defineJob } from '../../jobs/define';
+import type { SchedulerDefinition } from '../../jobs/schedulers';
+
+const cohortComputePayload = z.object({ cohortId: z.string() });
+
+/** The `cohortCompute` queue's own job. */
+export const cohortQueueJobs = {
+ cohortCompute: defineJob({
+ payload: cohortComputePayload,
+ handler: async ({ payload, ctx }) => {
+ await ctx.services.cohort.updateMembership(payload.cohortId);
+ },
+ }),
+};
+
+/** This module's fragment of the `cron` queue's jobs. */
+export const cohortCronJobs = {
+ // Fans out one `cohortCompute` enqueue per non-static cohort. Matches V1
+ // (apps/worker/src/jobs/cron.cohort-refresh.ts).
+ cohortRefresh: defineJob({
+ payload: z.null(),
+ handler: async ({ ctx }) => {
+ const cohortIds = await ctx.services.cohort.listRefreshableCohortIds();
+
+ for (const cohortId of cohortIds) {
+ await ctx.services.cohort.enqueueCompute(cohortId);
+ }
+ },
+ }),
+};
+
+/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */
+export const cohortCronSchedules: readonly SchedulerDefinition[] = [
+ { id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },
+];
diff --git a/packages/core/src/modules/cohort/cohort.rpc.test.ts b/packages/core/src/modules/cohort/cohort.rpc.test.ts
new file mode 100644
index 000000000..80c93ccc8
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.rpc.test.ts
@@ -0,0 +1,136 @@
+// Only the "is anyone logged in" boundary is exercised here — no database.
+// The access-check + business logic ride on @openpanel/db (lazy-loaded, see
+// cohort.service.ts's header) and ctx.services.cohort; wiring this router
+// end-to-end against a real Postgres is P6's (protectedProcedure) job, not
+// this one's — see cohort.rpc.ts's header.
+
+import { expect, test } from 'bun:test';
+import { stubHttpCtx } from '../../../test/rpc-fixtures';
+import { makeTrpcContext } from '../../rpc/base';
+import type { CookieOptions } from '../../shared/cookie';
+import { cohortRouter } from './cohort.rpc';
+
+const COOKIE_OPTIONS: CookieOptions = {
+ domain: '.openpanel.dev',
+ secure: true,
+ sameSite: 'lax',
+ httpOnly: true,
+ path: '/',
+};
+
+// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —
+// `TrpcContext.session` is never literally `null`, only its `userId` is.
+const EMPTY_SESSION = { session: null, user: null, userId: null };
+
+async function anonCaller() {
+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);
+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {
+ cookieOptions: COOKIE_OPTIONS,
+ });
+ return cohortRouter.createCaller(trpcCtx);
+}
+
+test('list rejects an unauthenticated caller before touching a project', async () => {
+ const caller = await anonCaller();
+ await expect(caller.list({ projectId: 'proj_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('get rejects an unauthenticated caller before reading a cohort', async () => {
+ const caller = await anonCaller();
+ await expect(caller.get({ id: 'cohort_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('create rejects an unauthenticated caller before writing a cohort', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.create({
+ name: 'Power users',
+ projectId: 'proj_1',
+ definition: {
+ type: 'event',
+ criteria: {
+ operator: 'and',
+ events: [
+ {
+ name: 'purchase',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ },
+ ],
+ },
+ },
+ })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('update rejects an unauthenticated caller before writing a cohort', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.update({ id: 'cohort_1', name: 'Renamed' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('delete rejects an unauthenticated caller before deleting a cohort', async () => {
+ const caller = await anonCaller();
+ await expect(caller.delete({ id: 'cohort_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('listProfiles rejects an unauthenticated caller before reading membership', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.listProfiles({
+ projectId: 'proj_1',
+ cohortId: 'cohort_1',
+ take: 50,
+ })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('getCount rejects an unauthenticated caller before counting membership', async () => {
+ const caller = await anonCaller();
+ await expect(caller.getCount({ cohortId: 'cohort_1' })).rejects.toMatchObject(
+ { code: 'UNAUTHORIZED' }
+ );
+});
+
+test('preview rejects an unauthenticated caller before computing a sample', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.preview({
+ projectId: 'proj_1',
+ definition: {
+ type: 'event',
+ criteria: {
+ operator: 'and',
+ events: [
+ {
+ name: 'purchase',
+ filters: [],
+ timeframe: { type: 'relative', value: '30d' },
+ },
+ ],
+ },
+ },
+ })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('exportProfiles rejects an unauthenticated caller before reading membership', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.exportProfiles({ cohortId: 'cohort_1' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('refresh rejects an unauthenticated caller before enqueueing a recompute', async () => {
+ const caller = await anonCaller();
+ await expect(caller.refresh({ cohortId: 'cohort_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
diff --git a/packages/core/src/modules/cohort/cohort.rpc.ts b/packages/core/src/modules/cohort/cohort.rpc.ts
new file mode 100644
index 000000000..19d99e786
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.rpc.ts
@@ -0,0 +1,399 @@
+// Ported from packages/trpc/src/routers/cohort.ts (M5-003).
+//
+// V1's `protectedProcedure` — the logger/session-scope/rate-limit middleware
+// stack — lands in core with auth (rpc/base.ts: "those need the resolved
+// Session shape and the access rules, so they land with auth (P6)"). Until
+// then this router does its own minimal "is anyone logged in" check inline,
+// exactly like V1's `enforceUserIsAuthed`. V1 keeps serving the live route
+// through packages/trpc's own `protectedProcedure` (full stack included) and
+// delegates its handler bodies to core's cohort functions and
+// `ctx.services.cohort` (DELEGATE PATTERN), so nothing here is a live
+// regression.
+//
+// The per-project access ladder itself IS shared: `./src/access.ts` binds
+// core's shared/access.ts ladder to @openpanel/db's real lookups, the same
+// way packages/trpc/src/access.ts does for V1.
+//
+// CRUD (list/get/create/update/delete) reads/writes Prisma's `cohort` table
+// directly, matching V1's router — cohort.service.ts owns only the
+// compute-heavy and ClickHouse-touching operations, same split as V1.
+
+// The canonical zChartEventFilter, not cohort.constants.ts's private
+// TDZ-workaround copy — matches V1's router, which imports it from
+// @openpanel/validation's barrel rather than from cohort.validation.ts.
+import { zChartEventFilter } from '@openpanel/validation';
+import { z } from 'zod';
+import { createTRPCRouter, procedure } from '../../rpc/base';
+import { TRPCAccessError, TRPCNotFoundError } from '../../rpc/errors';
+import {
+ zCohortDefinition,
+ zCohortInput,
+ zCohortUpdate,
+} from './cohort.constants';
+import {
+ computeCohort,
+ countCohort,
+ deleteCohortMembership,
+ getCohortCount,
+ getCohortEventsPerDay,
+ getCohortMemberEvents,
+ getCohortMemberRoutes,
+ getCohortMembers,
+ listCohortMemberProfiles,
+} from './cohort.service';
+
+const EXPORT_PROFILES_MAX_LIMIT = 10_000;
+const EXPORT_PROFILES_DEFAULT_LIMIT = 10_000;
+
+function loadAccessChecks() {
+ return import('./src/access');
+}
+
+function loadDb() {
+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);
+}
+
+function requireLogin(userId: string | null | undefined): string {
+ if (!userId) {
+ throw new TRPCAccessError('Not authenticated');
+ }
+ return userId;
+}
+
+export const cohortRouter = createTRPCRouter({
+ list: procedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ includeCount: z.boolean().optional().default(false),
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ const db = await loadDb();
+ const cohorts = await db.cohort.findMany({
+ where: { projectId: input.projectId },
+ orderBy: { createdAt: 'desc' },
+ });
+
+ if (input.includeCount) {
+ return cohorts.map((cohort) => ({
+ ...cohort,
+ currentCount: cohort.profileCount ?? 0,
+ }));
+ }
+
+ return cohorts;
+ }),
+
+ get: procedure
+ .input(z.object({ id: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.id },
+ });
+
+ if (!cohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: cohort.projectId,
+ level: 'read',
+ });
+
+ return cohort;
+ }),
+
+ // No project-access check here, matching V1 (packages/trpc/src/routers/
+ // cohort.ts) exactly — protectedProcedure's session check is the only
+ // gate today. Preserved as found: ADR-011 invariant 1 forbids deleting an
+ // in-handler check, and this router does not serve live traffic yet.
+ create: procedure.input(zCohortInput).mutation(async ({ input, ctx }) => {
+ requireLogin(ctx.session.userId);
+
+ const db = await loadDb();
+ const cohort = await db.cohort.create({
+ data: {
+ name: input.name,
+ description: input.description,
+ projectId: input.projectId,
+ definition: input.definition,
+ isStatic: input.isStatic,
+ },
+ });
+
+ await ctx.services.cohort.enqueueCompute(cohort.id);
+
+ return cohort;
+ }),
+
+ update: procedure.input(zCohortUpdate).mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { id, ...data } = input;
+
+ const db = await loadDb();
+ const existingCohort = await db.cohort.findUnique({ where: { id } });
+
+ if (!existingCohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: existingCohort.projectId,
+ level: 'write',
+ });
+
+ const cohort = await db.cohort.update({
+ where: { id },
+ data: {
+ ...(data.name && { name: data.name }),
+ ...(data.description !== undefined && {
+ description: data.description,
+ }),
+ ...(data.definition && { definition: data.definition }),
+ ...(data.isStatic !== undefined && { isStatic: data.isStatic }),
+ },
+ });
+
+ if (data.definition) {
+ await ctx.services.cohort.enqueueCompute(cohort.id);
+ }
+
+ return cohort;
+ }),
+
+ delete: procedure
+ .input(z.object({ id: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.id },
+ });
+
+ if (!cohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: cohort.projectId,
+ level: 'write',
+ });
+
+ await db.cohort.delete({ where: { id: input.id } });
+
+ deleteCohortMembership(input.id, cohort.projectId).catch((err) => {
+ ctx.logger.error({ err }, 'Failed to cleanup cohort CH data');
+ });
+
+ return { success: true };
+ }),
+
+ listProfiles: procedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ cohortId: z.string(),
+ cursor: z.number().optional(),
+ take: z.number().default(50),
+ search: z.string().optional(),
+ filters: z.array(zChartEventFilter).default([]),
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ const { data, count } = await listCohortMemberProfiles(input);
+ return {
+ data,
+ meta: { count, pageCount: input.take },
+ };
+ }),
+
+ mostEvents: procedure
+ .input(z.object({ projectId: z.string(), cohortId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ return getCohortMemberEvents(input.projectId, input.cohortId);
+ }),
+
+ eventsPerDay: procedure
+ .input(z.object({ projectId: z.string(), cohortId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ return getCohortEventsPerDay(input.projectId, input.cohortId);
+ }),
+
+ popularRoutes: procedure
+ .input(z.object({ projectId: z.string(), cohortId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ return getCohortMemberRoutes(input.projectId, input.cohortId);
+ }),
+
+ getCount: procedure
+ .input(z.object({ cohortId: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: cohort.projectId,
+ level: 'read',
+ });
+
+ const count = await getCohortCount(input.cohortId, cohort.projectId);
+ return { count };
+ }),
+
+ preview: procedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ definition: zCohortDefinition,
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: input.projectId,
+ level: 'read',
+ });
+
+ const count = await countCohort(input.projectId, input.definition);
+ const sampleProfiles = await computeCohort(
+ input.projectId,
+ input.definition,
+ 10
+ );
+ return { count, sampleProfiles };
+ }),
+
+ exportProfiles: procedure
+ .input(
+ z.object({
+ cohortId: z.string(),
+ limit: z
+ .number()
+ .int()
+ .min(1)
+ .max(EXPORT_PROFILES_MAX_LIMIT)
+ .default(EXPORT_PROFILES_DEFAULT_LIMIT),
+ offset: z.number().int().min(0).default(0),
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: cohort.projectId,
+ level: 'read',
+ });
+
+ const result = await getCohortMembers(input.cohortId, cohort.projectId, {
+ limit: input.limit,
+ offset: input.offset,
+ });
+
+ return {
+ profileIds: result.profileIds,
+ total: result.total,
+ cohortName: cohort.name,
+ };
+ }),
+
+ refresh: procedure
+ .input(z.object({ cohortId: z.string() }))
+ .mutation(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ const db = await loadDb();
+ const cohort = await db.cohort.findUnique({
+ where: { id: input.cohortId },
+ });
+
+ if (!cohort) {
+ throw new TRPCNotFoundError('Cohort not found');
+ }
+
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({
+ userId,
+ projectId: cohort.projectId,
+ level: 'write',
+ });
+
+ if (cohort.isStatic) {
+ throw new Error(
+ 'Cannot refresh static cohorts — they are one-time snapshots'
+ );
+ }
+
+ await ctx.services.cohort.enqueueCompute(input.cohortId);
+
+ return { success: true };
+ }),
+});
diff --git a/packages/core/src/modules/cohort/cohort.service.test.ts b/packages/core/src/modules/cohort/cohort.service.test.ts
new file mode 100644
index 000000000..5f4ff3f53
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.service.test.ts
@@ -0,0 +1,219 @@
+// cohort.service.ts's db/ch access is lazy (`await import(...)` inside each
+// function — see the file's header), which is exactly what makes
+// `mock.module` work here with no import-time side effects to race: every
+// mock below is registered before the subject's first call, not before its
+// (side-effect-free) import.
+//
+// The pure SQL-shape builders (buildEventCriteriaQuery,
+// buildPropertyBasedCohortQuery, deriveCohortQuerySettings) have their own
+// file, src/cohort-sql.test.ts, and need none of this — they take no
+// ClickHouse connection.
+
+import { beforeAll, expect, mock, test } from 'bun:test';
+
+interface FakeCohort {
+ id: string;
+ projectId: string;
+ name: string;
+ definition: unknown;
+ isStatic: boolean;
+ profileCount: number;
+ lastComputedAt: Date | null;
+}
+
+const cohortStore = new Map<string, FakeCohort>();
+
+const cohort = {
+ findUnique: mock(
+ async ({ where: { id } }: { where: { id: string } }) =>
+ cohortStore.get(id) ?? null
+ ),
+ findMany: mock(async ({ where }: { where?: { isStatic?: boolean } } = {}) =>
+ [...cohortStore.values()].filter(
+ (c) => where?.isStatic === undefined || c.isStatic === where.isStatic
+ )
+ ),
+ update: mock(
+ async ({
+ where: { id },
+ data,
+ }: {
+ where: { id: string };
+ data: Partial<FakeCohort>;
+ }) => {
+ const existing = cohortStore.get(id);
+ if (!existing) {
+ throw new Error(`cohort ${id} not found`);
+ }
+ const updated = { ...existing, ...data };
+ cohortStore.set(id, updated);
+ return updated;
+ }
+ ),
+};
+
+mock.module('@openpanel/db/src/prisma-client', () => ({
+ db: { cohort },
+}));
+
+const chQuery = mock(async () => [] as unknown[]);
+const chInsert = mock(async (_args: { table: string }) => undefined);
+const chCommand = mock(async (_args: { query: string }) => undefined);
+const getReplicatedTableName = mock((table: string) => table);
+// `originalCh`/`TABLE_NAMES` are unused here but included because
+// `mock.module` replaces this specifier process-wide (bun runs every test
+// file in one shared module registry without `--isolate` — see AGENTS.md) —
+// gsc.service.test.ts and insight.service.test.ts mock the same path, so
+// every factory must be a superset of every consumer's needs, whichever one
+// ends up registered last.
+mock.module('@openpanel/db/src/clickhouse/client', () => ({
+ ch: { insert: chInsert, command: chCommand },
+ originalCh: {
+ query: mock(async () => ({ json: async () => [] as unknown[] })),
+ insert: mock(async () => undefined),
+ },
+ chQuery,
+ getReplicatedTableName,
+ TABLE_NAMES: { sessions: 'sessions' },
+}));
+
+let subject: typeof import('./cohort.service');
+beforeAll(async () => {
+ subject = await import('./cohort.service');
+});
+
+function seedCohort(overrides: Partial<FakeCohort> = {}): FakeCohort {
+ const seeded: FakeCohort = {
+ id: 'cohort_1',
+ projectId: 'proj_1',
+ name: 'Power users',
+ definition: {
+ type: 'property',
+ criteria: {
+ operator: 'and',
+ properties: [
+ { name: 'profile.email', operator: 'isNotNull', value: [] },
+ ],
+ },
+ },
+ isStatic: false,
+ profileCount: 0,
+ lastComputedAt: null,
+ ...overrides,
+ };
+ cohortStore.set(seeded.id, seeded);
+ return seeded;
+}
+
+test('updateCohortMembership computes, clears old membership, stores new membership', async () => {
+ cohortStore.clear();
+ chQuery.mockClear();
+ chInsert.mockClear();
+ chCommand.mockClear();
+ seedCohort();
+ chQuery.mockImplementationOnce(async () => [{ profile_id: 'p1' }]);
+
+ await subject.updateCohortMembership('cohort_1');
+
+ expect(chCommand).toHaveBeenCalledTimes(1);
+ expect(chCommand.mock.calls[0]?.[0]?.query).toContain('DELETE FROM');
+ expect(chInsert).toHaveBeenCalledTimes(2); // cohort_members + cohort_metadata
+ const updated = cohortStore.get('cohort_1');
+ expect(updated?.profileCount).toBe(1);
+ expect(updated?.lastComputedAt).toBeInstanceOf(Date);
+});
+
+test('updateCohortMembership does nothing when the cohort no longer exists', async () => {
+ cohortStore.clear();
+ chCommand.mockClear();
+
+ await subject.updateCohortMembership('missing');
+
+ expect(chCommand).not.toHaveBeenCalled();
+});
+
+test('deleteCohortMembership clears both cohort_members and cohort_metadata', async () => {
+ chCommand.mockClear();
+
+ await subject.deleteCohortMembership('cohort_1', 'proj_1');
+
+ expect(chCommand).toHaveBeenCalledTimes(2);
+});
+
+test('listRefreshableCohortIds returns only non-static cohorts', async () => {
+ cohortStore.clear();
+ seedCohort({ id: 'c1', isStatic: false });
+ seedCohort({ id: 'c2', isStatic: true });
+ seedCohort({ id: 'c3', isStatic: false });
+
+ const ids = await subject.listRefreshableCohortIds();
+
+ expect(ids.sort()).toEqual(['c1', 'c3']);
+});
+
+test('getCohortCount serves the cached profileCount within 15 minutes', async () => {
+ cohortStore.clear();
+ chQuery.mockClear();
+ seedCohort({
+ profileCount: 42,
+ lastComputedAt: new Date(Date.now() - 60_000),
+ });
+
+ const count = await subject.getCohortCount('cohort_1', 'proj_1');
+
+ expect(count).toBe(42);
+ expect(chQuery).not.toHaveBeenCalled();
+});
+
+test('getCohortCount re-queries ClickHouse once the cache is stale', async () => {
+ cohortStore.clear();
+ chQuery.mockClear();
+ seedCohort({
+ profileCount: 42,
+ lastComputedAt: new Date(Date.now() - 16 * 60 * 1000),
+ });
+ chQuery.mockImplementationOnce(async () => [{ count: 7 }]);
+
+ const count = await subject.getCohortCount('cohort_1', 'proj_1');
+
+ expect(count).toBe(7);
+ expect(chQuery).toHaveBeenCalledTimes(1);
+});
+
+test('createCohortService().enqueueCompute enqueues with the cohort-<id> deduplication key, not a jobId', async () => {
+ const addCalls: { payload: unknown; options: unknown }[] = [];
+ const deps = {
+ queues: {
+ cohortCompute: {
+ cohortCompute: {
+ add: async (payload: unknown, options?: unknown) => {
+ addCalls.push({ payload, options });
+ return 'job_1';
+ },
+ remove: async () => undefined,
+ },
+ },
+ },
+ } as unknown as Parameters<typeof subject.createCohortService>[0];
+
+ await subject.createCohortService(deps).enqueueCompute('cohort_1');
+
+ expect(addCalls).toEqual([
+ {
+ payload: { cohortId: 'cohort_1' },
+ options: { deduplicationId: 'cohort-cohort_1' },
+ },
+ ]);
+});
+
+test('createCohortService() binds updateMembership and listRefreshableCohortIds to the plain functions', () => {
+ const deps = {
+ queues: {},
+ } as unknown as Parameters<typeof subject.createCohortService>[0];
+ const service = subject.createCohortService(deps);
+
+ expect(service.updateMembership).toBe(subject.updateCohortMembership);
+ expect(service.listRefreshableCohortIds).toBe(
+ subject.listRefreshableCohortIds
+ );
+});
diff --git a/packages/db/src/services/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.ts
similarity index 69%
rename from packages/db/src/services/cohort.service.ts
rename to packages/core/src/modules/cohort/cohort.service.ts
index dacf352ad..ce89616ae 100644
--- a/packages/db/src/services/cohort.service.ts
+++ b/packages/core/src/modules/cohort/cohort.service.ts
@@ -1,29 +1,61 @@
+// Moved from packages/db/src/services/cohort.service.ts (M5-003). The
+// producer wrapper (`enqueueCohortCompute`) moves here too — db cannot hold
+// producers (ADR-007's discovery: "packages/db/src/services/cohort.service.ts:12
+// imports cohortComputeQueue from @openpanel/queue" was flagged as reaching
+// back into infrastructure it should not know about). packages/db loses this
+// file entirely: nothing outside trpc/worker/this module's own tests reached
+// it through @openpanel/db's barrel.
+//
+// db/ch access is LAZY (`load*` below), not a static top-level import — see
+// insight.service.ts's header for the full reasoning (jobs.registry.ts and
+// services.ts pull this module into the eager barrel chain nearly every core
+// test file reaches, and constructing @openpanel/db's clients at import time
+// would spawn a pino-pretty transport worker thread per test file).
+//
+// ClickHouse queries here still go through raw sqlstring-escaped strings, not
+// the `sql` tag: ADR-013 converts the analytics read path one query per P7
+// task, and this module's queries haven't been converted yet. Table names are
+// a local literal map (`TABLE`, below), not @openpanel/db's `TABLE_NAMES`: the
+// latter lives in the same module as `ch`/`chQuery` (clickhouse/client.ts,
+// which constructs a real pino logger at import time — the exact cost the lazy
+// loads elsewhere in this file exist to defer), and this module's SQL builders
+// are pure sync functions the P2 SQL-shape tests call with no ClickHouse
+// connection at all.
+//
+// V1's trpc router (packages/trpc/src/routers/cohort.ts) and worker cron job
+// (apps/worker/src/jobs/cron.cohort-refresh.ts) stay live (DELEGATE PATTERN)
+// and enqueue by calling @openpanel/queue's cohortComputeQueue directly,
+// same as gsc's V1 router/worker do for gscQueue (M5-002) — @openpanel/queue
+// itself imports @openpanel/core for its logger (packages/queue/src/queues.ts),
+// so core cannot import @openpanel/queue back without a real package cycle.
+// `CohortService.enqueueCompute` (below) is the canonical, ctx.queues-based
+// wrapper for callers that already have a Ctx (this module's own rpc
+// mutations and its cron fragment).
+
+import type { ClickHouseSettings } from '@clickhouse/client';
+import type { IServiceProfile } from '@openpanel/db/src/services/profile.service';
+import type { IChartEventFilter } from '@openpanel/validation';
import sqlstring from 'sqlstring';
+import type { ServiceDeps } from '../../services';
import type {
CohortDefinition,
EventBasedCohortDefinition,
EventCriteria,
Frequency,
- IChartEventFilter,
PropertyBasedCohortDefinition,
Timeframe,
-} from '@openpanel/validation';
-
-import { cohortComputeQueue } from '@openpanel/queue';
-import type { ClickHouseSettings } from '@clickhouse/client';
-import {
- TABLE_NAMES,
- ch,
- chQuery,
- getReplicatedTableName,
-} from '../clickhouse/client';
-import { db } from '../prisma-client';
-import { buildFilterWhere } from './filter-where.service';
-import {
- getProfiles,
- profileSearchSql,
- type IServiceProfile,
-} from './profile.service';
+} from './cohort.constants';
+
+// Physical ClickHouse table names this module reads/writes. Literal, not
+// imported from @openpanel/db's TABLE_NAMES — see the header comment.
+const TABLE = {
+ profiles: 'profiles',
+ events: 'events',
+ cohortMembers: 'cohort_members',
+ cohortMetadata: 'cohort_metadata',
+ eventProfileSummaryMv: 'event_profile_summary_mv',
+ eventPropertyProfileSummaryMv: 'event_property_profile_summary_mv',
+} as const;
// Max members materialized into cohort_members per compute. Cohorts larger
// than this are silently truncated to an arbitrary subset, so deployments
@@ -34,28 +66,20 @@ import {
// default. Number.parseInt would accept '5000junk' or '-1' (LIMIT -1 is a
// query error), and 0 is falsy at the `limit ? LIMIT ... : ''` call sites,
// which would silently remove the cap entirely.
-const COHORT_MATERIALIZE_LIMIT_RAW = process.env.COHORT_MATERIALIZE_LIMIT;
-const COHORT_MATERIALIZE_LIMIT_PARSED =
- COHORT_MATERIALIZE_LIMIT_RAW && /^\d+$/.test(COHORT_MATERIALIZE_LIMIT_RAW)
- ? Number(COHORT_MATERIALIZE_LIMIT_RAW)
- : Number.NaN;
-export const COHORT_MATERIALIZE_LIMIT =
- Number.isSafeInteger(COHORT_MATERIALIZE_LIMIT_PARSED) &&
- COHORT_MATERIALIZE_LIMIT_PARSED > 0
- ? COHORT_MATERIALIZE_LIMIT_PARSED
- : 10000;
-
-// Strictly a positive safe integer, or undefined — same validation rationale
-// as COHORT_MATERIALIZE_LIMIT above.
-function parsePositiveIntEnv(name: string): number | undefined {
- const raw = process.env[name];
- if (!raw || !/^\d+$/.test(raw)) {
+function parsePositiveInt(raw: string | undefined): number | undefined {
+ if (!(raw && /^\d+$/.test(raw))) {
return undefined;
}
const parsed = Number(raw);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined;
}
+const COHORT_MATERIALIZE_LIMIT_PARSED = parsePositiveInt(
+ process.env.COHORT_MATERIALIZE_LIMIT
+);
+export const COHORT_MATERIALIZE_LIMIT =
+ COHORT_MATERIALIZE_LIMIT_PARSED ?? 10_000;
+
// Property cohorts aggregate every profile row for the project, so they are
// the one cohort query that can outgrow the server's memory headroom. Two
// opt-in knobs bound them; with NEITHER set, no per-query settings are
@@ -74,30 +98,53 @@ function parsePositiveIntEnv(name: string): number | undefined {
// spilled is set by the data, not the threshold (measured on 8.3M profiles,
// ~281MB spilled whether the threshold was 300, 512 or 768MB, at
// 6.9s/6.7s/6.0s, while peak memory climbed 410/695/893MiB).
-const COHORT_QUERY_MEMORY_LIMIT_BYTES = parsePositiveIntEnv(
- 'COHORT_QUERY_MEMORY_LIMIT_BYTES',
-);
-const COHORT_QUERY_SPILL_BYTES_RAW = parsePositiveIntEnv(
- 'COHORT_QUERY_SPILL_BYTES',
-);
-const COHORT_QUERY_SPILL_BYTES =
- COHORT_QUERY_MEMORY_LIMIT_BYTES !== undefined &&
- (COHORT_QUERY_SPILL_BYTES_RAW === undefined ||
- COHORT_QUERY_SPILL_BYTES_RAW >= COHORT_QUERY_MEMORY_LIMIT_BYTES)
- ? // Clamped to 1: a (nonsensical) limit below 3 would derive 0, and
- // max_bytes_before_external_group_by = 0 means spilling DISABLED —
- // the exact inversion this derivation exists to prevent.
- Math.max(1, Math.floor(COHORT_QUERY_MEMORY_LIMIT_BYTES / 3))
- : COHORT_QUERY_SPILL_BYTES_RAW;
-
-export const PROFILE_COHORT_QUERY_SETTINGS: ClickHouseSettings = {
- ...(COHORT_QUERY_SPILL_BYTES !== undefined
- ? { max_bytes_before_external_group_by: String(COHORT_QUERY_SPILL_BYTES) }
- : {}),
- ...(COHORT_QUERY_MEMORY_LIMIT_BYTES !== undefined
- ? { max_memory_usage: String(COHORT_QUERY_MEMORY_LIMIT_BYTES) }
- : {}),
-};
+//
+// A standalone function of its raw env inputs (not a module-level read), so
+// a test can exercise every branch by calling it directly — bun:test shares
+// one module registry per file even under --isolate, so vitest's
+// vi.resetModules()-per-case porting has no equivalent (see AGENTS.md; this
+// was the one vi.resetModules site in the suite, ADR-010's tail table).
+export function deriveCohortQuerySettings({
+ memoryLimitBytesRaw,
+ spillBytesRaw,
+}: {
+ memoryLimitBytesRaw: string | undefined;
+ spillBytesRaw: string | undefined;
+}): ClickHouseSettings {
+ const memoryLimitBytes = parsePositiveInt(memoryLimitBytesRaw);
+ const spillBytesParsed = parsePositiveInt(spillBytesRaw);
+ const spillBytes =
+ memoryLimitBytes !== undefined &&
+ (spillBytesParsed === undefined || spillBytesParsed >= memoryLimitBytes)
+ ? // Clamped to 1: a (nonsensical) limit below 3 would derive 0, and
+ // max_bytes_before_external_group_by = 0 means spilling DISABLED —
+ // the exact inversion this derivation exists to prevent.
+ Math.max(1, Math.floor(memoryLimitBytes / 3))
+ : spillBytesParsed;
+
+ return {
+ ...(spillBytes !== undefined
+ ? { max_bytes_before_external_group_by: String(spillBytes) }
+ : {}),
+ ...(memoryLimitBytes !== undefined
+ ? { max_memory_usage: String(memoryLimitBytes) }
+ : {}),
+ };
+}
+
+export const PROFILE_COHORT_QUERY_SETTINGS: ClickHouseSettings =
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: process.env.COHORT_QUERY_MEMORY_LIMIT_BYTES,
+ spillBytesRaw: process.env.COHORT_QUERY_SPILL_BYTES,
+ });
+
+function loadDb() {
+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);
+}
+
+function loadChClient() {
+ return import('@openpanel/db/src/clickhouse/client');
+}
function buildTimeConstraint(timeframe: Timeframe): string {
if (timeframe.type === 'relative') {
@@ -131,19 +178,19 @@ function getFrequencyOperator(frequency: Frequency): string {
export function buildEventCriteriaQuery(
projectId: string,
- criteria: EventCriteria,
+ criteria: EventCriteria
): string {
const { name, filters, timeframe, frequency } = criteria;
const timeConstraint = buildTimeConstraint(timeframe);
const hasEventPropertyFilters = filters.some(
(f) =>
f.name.startsWith('properties.') &&
- !f.name.startsWith('profile.properties.'),
+ !f.name.startsWith('profile.properties.')
);
if (hasEventPropertyFilters) {
const propertyFilters = filters.filter((f) =>
- f.name.startsWith('properties.'),
+ f.name.startsWith('properties.')
);
const propertyConditions = propertyFilters
@@ -170,14 +217,14 @@ export function buildEventCriteriaQuery(
return `(property_key = ${sqlstring.escape(propertyKey)} AND (${value
.map(
(val) =>
- `property_value LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+ `property_value LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}))`;
case 'doesNotContain':
return `(property_key = ${sqlstring.escape(propertyKey)} AND (${value
.map(
(val) =>
- `property_value NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+ `property_value NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' AND ')}))`;
default:
@@ -192,7 +239,7 @@ export function buildEventCriteriaQuery(
const frequencyOp = getFrequencyOperator(frequency);
return `
SELECT profile_id
- FROM ${TABLE_NAMES.event_property_profile_summary_mv}
+ FROM ${TABLE.eventPropertyProfileSummaryMv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -204,7 +251,7 @@ export function buildEventCriteriaQuery(
return `
SELECT DISTINCT profile_id
- FROM ${TABLE_NAMES.event_property_profile_summary_mv}
+ FROM ${TABLE.eventPropertyProfileSummaryMv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -216,7 +263,7 @@ export function buildEventCriteriaQuery(
const frequencyOp = getFrequencyOperator(frequency);
return `
SELECT profile_id
- FROM ${TABLE_NAMES.event_profile_summary_mv}
+ FROM ${TABLE.eventProfileSummaryMv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -227,7 +274,7 @@ export function buildEventCriteriaQuery(
return `
SELECT DISTINCT profile_id
- FROM ${TABLE_NAMES.event_profile_summary_mv}
+ FROM ${TABLE.eventProfileSummaryMv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -248,7 +295,7 @@ function profileColumnAccess(name: string): string {
}
function buildProfileCohortHavingClause(
- definition: PropertyBasedCohortDefinition,
+ definition: PropertyBasedCohortDefinition
): string | null {
const { properties, operator } = definition.criteria;
@@ -264,7 +311,7 @@ function buildProfileCohortHavingClause(
// need a version tie AND a 64-bit collision between different rows — and
// even then every aggregate in the query still elects the same row.
const referencedColumns = Array.from(
- new Set(properties.map((f) => profileColumnAccess(f.name))),
+ new Set(properties.map((f) => profileColumnAccess(f.name)))
);
const latestRowKey = `tuple(last_seen_at, cityHash64(${referencedColumns.join(', ')}))`;
@@ -283,12 +330,12 @@ function buildProfileCohortHavingClause(
export function buildPropertyBasedCohortQuery(
projectId: string,
definition: PropertyBasedCohortDefinition,
- limit?: number,
+ limit?: number
): string {
const havingClause = buildProfileCohortHavingClause(definition);
if (!havingClause) {
- return `SELECT id as profile_id FROM ${TABLE_NAMES.profiles} WHERE 1=0`;
+ return `SELECT id as profile_id FROM ${TABLE.profiles} WHERE 1=0`;
}
// Resolve each profile's newest row with GROUP BY + argMax instead of
@@ -297,7 +344,7 @@ export function buildPropertyBasedCohortQuery(
// PROFILE_COHORT_QUERY_SETTINGS, and filters on aggregates move to HAVING.
return `
SELECT id as profile_id
- FROM ${TABLE_NAMES.profiles}
+ FROM ${TABLE.profiles}
WHERE project_id = ${sqlstring.escape(projectId)}
GROUP BY id
HAVING (${havingClause})
@@ -308,12 +355,13 @@ export function buildPropertyBasedCohortQuery(
export async function computeEventBasedCohort(
projectId: string,
definition: EventBasedCohortDefinition,
- limit?: number,
+ limit?: number
): Promise<string[]> {
const { events, operator } = definition.criteria;
+ const { chQuery } = await loadChClient();
const queries = events.map((eventCriteria) =>
- buildEventCriteriaQuery(projectId, eventCriteria),
+ buildEventCriteriaQuery(projectId, eventCriteria)
);
const combinedQuery =
@@ -329,12 +377,13 @@ export async function computeEventBasedCohort(
export async function countEventBasedCohort(
projectId: string,
- definition: EventBasedCohortDefinition,
+ definition: EventBasedCohortDefinition
): Promise<number> {
const { events, operator } = definition.criteria;
+ const { chQuery } = await loadChClient();
const queries = events.map((eventCriteria) =>
- buildEventCriteriaQuery(projectId, eventCriteria),
+ buildEventCriteriaQuery(projectId, eventCriteria)
);
const combinedQuery =
@@ -349,7 +398,7 @@ export async function countEventBasedCohort(
function getProfileFiltersWhereClause(
filters: IChartEventFilter[],
- { latestPerProfileKey }: { latestPerProfileKey?: string } = {},
+ { latestPerProfileKey }: { latestPerProfileKey?: string } = {}
): Record<string, string> {
const where: Record<string, string> = {};
@@ -384,7 +433,8 @@ function getProfileFiltersWhereClause(
switch (operator) {
case 'is': {
if (value.length === 1) {
- where[id] = `${columnAccess} = ${sqlstring.escape(String(value[0]).trim())}`;
+ where[id] =
+ `${columnAccess} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${columnAccess} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
@@ -394,7 +444,8 @@ function getProfileFiltersWhereClause(
}
case 'isNot': {
if (value.length === 1) {
- where[id] = `${columnAccess} != ${sqlstring.escape(String(value[0]).trim())}`;
+ where[id] =
+ `${columnAccess} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${columnAccess} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
@@ -406,7 +457,7 @@ function getProfileFiltersWhereClause(
where[id] = `(${value
.map(
(val) =>
- `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+ `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -415,7 +466,7 @@ function getProfileFiltersWhereClause(
where[id] = `(${value
.map(
(val) =>
- `${columnAccess} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+ `${columnAccess} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -424,7 +475,7 @@ function getProfileFiltersWhereClause(
where[id] = `(${value
.map(
(val) =>
- `${columnAccess} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
+ `${columnAccess} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -433,7 +484,7 @@ function getProfileFiltersWhereClause(
where[id] = `(${value
.map(
(val) =>
- `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
+ `${columnAccess} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
break;
@@ -479,30 +530,32 @@ function getProfileFiltersWhereClause(
export async function computePropertyBasedCohort(
projectId: string,
definition: PropertyBasedCohortDefinition,
- limit?: number,
+ limit?: number
): Promise<string[]> {
if (!buildProfileCohortHavingClause(definition)) {
return [];
}
+ const { chQuery } = await loadChClient();
const results = await chQuery<{ profile_id: string }>(
buildPropertyBasedCohortQuery(projectId, definition, limit),
- PROFILE_COHORT_QUERY_SETTINGS,
+ PROFILE_COHORT_QUERY_SETTINGS
);
return results.map((r) => r.profile_id);
}
export async function countPropertyBasedCohort(
projectId: string,
- definition: PropertyBasedCohortDefinition,
+ definition: PropertyBasedCohortDefinition
): Promise<number> {
if (!buildProfileCohortHavingClause(definition)) {
return 0;
}
+ const { chQuery } = await loadChClient();
const results = await chQuery<{ count: number }>(
`SELECT count() as count FROM (${buildPropertyBasedCohortQuery(projectId, definition)})`,
- PROFILE_COHORT_QUERY_SETTINGS,
+ PROFILE_COHORT_QUERY_SETTINGS
);
return results[0]?.count ?? 0;
}
@@ -511,8 +564,9 @@ export async function storeCohortMembership(
projectId: string,
cohortId: string,
profileIds: string[],
- version: number,
+ version: number
): Promise<void> {
+ const { ch } = await loadChClient();
const now = new Date().toISOString().slice(0, 19).replace('T', ' ');
if (profileIds.length > 0) {
@@ -526,7 +580,7 @@ export async function storeCohortMembership(
}));
await ch.insert({
- table: TABLE_NAMES.cohort_members,
+ table: TABLE.cohortMembers,
values: data,
format: 'JSONEachRow',
});
@@ -534,7 +588,7 @@ export async function storeCohortMembership(
const sampleProfiles = profileIds.slice(0, 10);
await ch.insert({
- table: TABLE_NAMES.cohort_metadata,
+ table: TABLE.cohortMetadata,
values: [
{
project_id: projectId,
@@ -552,8 +606,10 @@ export async function storeCohortMembership(
export async function getCohortMembers(
cohortId: string,
projectId: string,
- opts?: { limit?: number; offset?: number },
+ opts?: { limit?: number; offset?: number }
): Promise<{ profileIds: string[]; total: number }> {
+ const db = await loadDb();
+ const { chQuery } = await loadChClient();
const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
if (!cohort) {
@@ -564,7 +620,7 @@ export async function getCohortMembers(
SELECT
profile_id,
count() OVER() as total
- FROM ${TABLE_NAMES.cohort_members} FINAL
+ FROM ${TABLE.cohortMembers} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND cohort_id = ${sqlstring.escape(cohortId)}
ORDER BY matched_at DESC
@@ -579,10 +635,13 @@ export async function getCohortMembers(
};
}
+const COHORT_COUNT_CACHE_MS = 15 * 60 * 1000;
+
export async function getCohortCount(
cohortId: string,
- projectId: string,
+ projectId: string
): Promise<number> {
+ const db = await loadDb();
const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
if (!cohort) {
@@ -591,14 +650,15 @@ export async function getCohortCount(
if (cohort.lastComputedAt) {
const age = Date.now() - cohort.lastComputedAt.getTime();
- if (age < 15 * 60 * 1000) {
+ if (age < COHORT_COUNT_CACHE_MS) {
return cohort.profileCount;
}
}
+ const { chQuery } = await loadChClient();
const result = await chQuery<{ count: number }>(`
SELECT count() as count
- FROM ${TABLE_NAMES.cohort_members} FINAL
+ FROM ${TABLE.cohortMembers} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND cohort_id = ${sqlstring.escape(cohortId)}
`);
@@ -608,7 +668,7 @@ export async function getCohortCount(
export async function computeCohort(
projectId: string,
definition: CohortDefinition,
- limit?: number,
+ limit?: number
): Promise<string[]> {
if (definition.type === 'event') {
return computeEventBasedCohort(projectId, definition, limit);
@@ -621,7 +681,7 @@ export async function computeCohort(
export async function countCohort(
projectId: string,
- definition: CohortDefinition,
+ definition: CohortDefinition
): Promise<number> {
if (definition.type === 'event') {
return countEventBasedCohort(projectId, definition);
@@ -632,9 +692,9 @@ export async function countCohort(
return 0;
}
-export async function updateCohortMembership(
- cohortId: string,
-): Promise<void> {
+export async function updateCohortMembership(cohortId: string): Promise<void> {
+ const db = await loadDb();
+ const { ch, getReplicatedTableName } = await loadChClient();
const cohort = await db.cohort.findUnique({ where: { id: cohortId } });
if (!cohort) {
@@ -645,7 +705,7 @@ export async function updateCohortMembership(
const profileIds = await computeCohort(
cohort.projectId,
definition,
- COHORT_MATERIALIZE_LIMIT,
+ COHORT_MATERIALIZE_LIMIT
);
const version = Date.now();
@@ -654,18 +714,13 @@ export async function updateCohortMembership(
// (project_id, cohort_id, profile_id), so profiles that fell out of the
// cohort definition would otherwise linger forever. Clear them first.
await ch.command({
- query: `DELETE FROM ${getReplicatedTableName(TABLE_NAMES.cohort_members)} WHERE cohort_id = ${sqlstring.escape(cohort.id)} AND project_id = ${sqlstring.escape(cohort.projectId)}`,
+ query: `DELETE FROM ${getReplicatedTableName(TABLE.cohortMembers)} WHERE cohort_id = ${sqlstring.escape(cohort.id)} AND project_id = ${sqlstring.escape(cohort.projectId)}`,
clickhouse_settings: {
lightweight_deletes_sync: '1',
},
});
- await storeCohortMembership(
- cohort.projectId,
- cohort.id,
- profileIds,
- version,
- );
+ await storeCohortMembership(cohort.projectId, cohort.id, profileIds, version);
await db.cohort.update({
where: { id: cohortId },
@@ -678,10 +733,11 @@ export async function updateCohortMembership(
export async function deleteCohortMembership(
cohortId: string,
- projectId: string,
+ projectId: string
): Promise<void> {
+ const { ch, getReplicatedTableName } = await loadChClient();
const where = `cohort_id = ${sqlstring.escape(cohortId)} AND project_id = ${sqlstring.escape(projectId)}`;
- for (const table of [TABLE_NAMES.cohort_members, TABLE_NAMES.cohort_metadata]) {
+ for (const table of [TABLE.cohortMembers, TABLE.cohortMetadata]) {
await ch.command({
query: `DELETE FROM ${getReplicatedTableName(table)} WHERE ${where}`,
clickhouse_settings: {
@@ -693,34 +749,22 @@ export async function deleteCohortMembership(
export async function getProfilesInCohort(
cohortId: string,
- projectId: string,
+ projectId: string
): Promise<Set<string>> {
const { profileIds } = await getCohortMembers(cohortId, projectId, {
- limit: 100000,
+ limit: 100_000,
});
return new Set(profileIds);
}
-/**
- * Enqueue a recompute for a cohort.
- *
- * Uses `deduplication` rather than a fixed `jobId`. A fixed jobId makes BullMQ
- * short-circuit `add` for as long as *any* record for that id exists in Redis —
- * and `removeOnComplete: { age }` is not a TTL, it only trims on some other
- * job in the queue finishing. That deadlocks: nothing can be added because the
- * completed record is still there, and the record is never collected because
- * nothing gets added. The deduplication key, in contrast, is released by
- * `moveToFinished` on both completion and terminal failure, so it only collapses
- * a compute that is genuinely still in flight.
- */
-export async function enqueueCohortCompute(cohortId: string): Promise<void> {
- await cohortComputeQueue.add(
- 'cohortCompute',
- { cohortId },
- {
- deduplication: { id: `cohort-${cohortId}` },
- },
- );
+/** Every non-static cohort id, for the cohortRefresh cron fragment's fan-out. */
+export async function listRefreshableCohortIds(): Promise<string[]> {
+ const db = await loadDb();
+ const cohorts = await db.cohort.findMany({
+ where: { isStatic: false },
+ select: { id: true },
+ });
+ return cohorts.map((c) => c.id);
}
export async function listCohortMemberProfiles({
@@ -738,6 +782,14 @@ export async function listCohortMemberProfiles({
search?: string;
filters?: IChartEventFilter[];
}): Promise<{ data: IServiceProfile[]; count: number }> {
+ const { chQuery } = await loadChClient();
+ const { buildFilterWhere } = await import(
+ '@openpanel/db/src/services/filter-where.service'
+ );
+ const { getProfiles, profileSearchSql } = await import(
+ '@openpanel/db/src/services/profile.service'
+ );
+
const offset = Math.max(0, (cursor ?? 0) * take);
const searchClause = profileSearchSql(search);
const searchCondition = searchClause ? `AND ${searchClause}` : '';
@@ -748,7 +800,7 @@ export async function listCohortMemberProfiles({
selfTable: 'profiles',
profileIdExpr: 'id',
groupsExpr: 'groups',
- }),
+ })
)
: [];
const extraConditionSql = extraConditions.length
@@ -757,10 +809,10 @@ export async function listCohortMemberProfiles({
const rows = await chQuery<{ id: string; total_count: number }>(`
SELECT id, count() OVER () AS total_count
- FROM ${TABLE_NAMES.profiles} FINAL
+ FROM ${TABLE.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id IN (
- SELECT profile_id FROM ${TABLE_NAMES.cohort_members} FINAL
+ SELECT profile_id FROM ${TABLE.cohortMembers} FINAL
WHERE cohort_id = ${sqlstring.escape(cohortId)}
AND project_id = ${sqlstring.escape(projectId)}
)
@@ -772,7 +824,9 @@ export async function listCohortMemberProfiles({
const count = rows[0]?.total_count ?? 0;
const ids = rows.map((r) => r.id);
- if (ids.length === 0) return { data: [], count };
+ if (ids.length === 0) {
+ return { data: [], count };
+ }
const profiles = await getProfiles(ids, projectId);
const byId = new Map(profiles.map((p) => [p.id, p]));
@@ -785,14 +839,15 @@ export async function listCohortMemberProfiles({
export async function getCohortMemberEvents(
projectId: string,
cohortId: string,
- limit = 10,
+ limit = 10
): Promise<{ name: string; count: number }[]> {
+ const { chQuery } = await loadChClient();
return chQuery<{ name: string; count: number }>(`
SELECT name, count() AS count
- FROM ${TABLE_NAMES.events}
+ FROM ${TABLE.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND profile_id IN (
- SELECT profile_id FROM ${TABLE_NAMES.cohort_members} FINAL
+ SELECT profile_id FROM ${TABLE.cohortMembers} FINAL
WHERE cohort_id = ${sqlstring.escape(cohortId)}
AND project_id = ${sqlstring.escape(projectId)}
)
@@ -806,17 +861,18 @@ export async function getCohortMemberEvents(
export async function getCohortEventsPerDay(
projectId: string,
cohortId: string,
- days = 30,
+ days = 30
): Promise<{ date: string; count: number }[]> {
+ const { chQuery } = await loadChClient();
const rows = await chQuery<{ date: string; count: number }>(`
SELECT
toDate(created_at) AS date,
count() AS count
- FROM ${TABLE_NAMES.events}
+ FROM ${TABLE.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND created_at >= toDate(now() - INTERVAL ${days} DAY)
AND profile_id IN (
- SELECT profile_id FROM ${TABLE_NAMES.cohort_members} FINAL
+ SELECT profile_id FROM ${TABLE.cohortMembers} FINAL
WHERE cohort_id = ${sqlstring.escape(cohortId)}
AND project_id = ${sqlstring.escape(projectId)}
)
@@ -833,14 +889,15 @@ export async function getCohortEventsPerDay(
export async function getCohortMemberRoutes(
projectId: string,
cohortId: string,
- limit = 10,
+ limit = 10
): Promise<{ path: string; count: number }[]> {
+ const { chQuery } = await loadChClient();
return chQuery<{ path: string; count: number }>(`
SELECT path, count() AS count
- FROM ${TABLE_NAMES.events}
+ FROM ${TABLE.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND profile_id IN (
- SELECT profile_id FROM ${TABLE_NAMES.cohort_members} FINAL
+ SELECT profile_id FROM ${TABLE.cohortMembers} FINAL
WHERE cohort_id = ${sqlstring.escape(cohortId)}
AND project_id = ${sqlstring.escape(projectId)}
)
@@ -851,3 +908,35 @@ export async function getCohortMemberRoutes(
LIMIT ${limit}
`);
}
+
+export interface CohortService {
+ updateMembership(cohortId: string): Promise<void>;
+ listRefreshableCohortIds(): Promise<string[]>;
+ /**
+ * Enqueue a recompute for a cohort.
+ *
+ * Uses `deduplicationId` rather than `jobId`. A fixed jobId makes BullMQ
+ * short-circuit `add` for as long as *any* record for that id exists in
+ * Redis — and `removeOnComplete: { age }` is not a TTL, it only trims on
+ * some other job in the queue finishing. That deadlocks: nothing can be
+ * added because the completed record is still there, and the record is
+ * never collected because nothing gets added. `deduplicationId`, in
+ * contrast, is released by `moveToFinished` on both completion and
+ * terminal failure, so it only collapses a compute that is genuinely still
+ * in flight (ADR-005: "cohort must NOT be normalised onto jobId").
+ */
+ enqueueCompute(cohortId: string): Promise<void>;
+}
+
+export function createCohortService(deps: ServiceDeps): CohortService {
+ return {
+ updateMembership: updateCohortMembership,
+ listRefreshableCohortIds,
+ enqueueCompute: async (cohortId) => {
+ await deps.queues.cohortCompute.cohortCompute.add(
+ { cohortId },
+ { deduplicationId: `cohort-${cohortId}` }
+ );
+ },
+ };
+}
diff --git a/packages/core/src/modules/cohort/src/access.ts b/packages/core/src/modules/cohort/src/access.ts
new file mode 100644
index 000000000..de4a4168e
--- /dev/null
+++ b/packages/core/src/modules/cohort/src/access.ts
@@ -0,0 +1,21 @@
+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for
+// this module's own router — the same binding packages/trpc/src/access.ts
+// does for V1, self-contained here because the general core -> db binding
+// itself moves with auth (P6), not this wave.
+
+import {
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectAccess,
+} from '@openpanel/db/src/services/access.service';
+import { getProjectById } from '@openpanel/db/src/services/project.service';
+import { createAccessChecks } from '../../../shared/access';
+
+const checks = createAccessChecks({
+ getProjectAccess,
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectById,
+});
+
+export const requireProjectAccess = checks.requireProjectAccess;
diff --git a/packages/core/src/modules/cohort/src/cohort-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-sql.test.ts
new file mode 100644
index 000000000..637716e65
--- /dev/null
+++ b/packages/core/src/modules/cohort/src/cohort-sql.test.ts
@@ -0,0 +1,247 @@
+// SQL-shape tests for property-based cohort queries. Ported from
+// packages/db/src/services/cohort-property-sql.test.ts (M5-003) — minus its
+// itCH ClickHouse-reachability EXPLAIN check: `mock.module` substitutions
+// apply process-wide, not per file, so a sibling file's
+// `@openpanel/db/src/clickhouse/client` mock (cohort.service.test.ts) can
+// silently replace this file's "real" import too, making a reachability
+// check meaningless. No module in core's suite ports that pattern for the
+// same reason; the generated SQL was verified against local ClickHouse by
+// hand instead (see the M5-003 task summary).
+//
+// String assertions need no mocking at all: cohort.service.ts's db/ch access
+// is lazy, and buildPropertyBasedCohortQuery / deriveCohortQuerySettings
+// touch neither — see cohort.service.ts's header.
+//
+// V1's `PROFILE_COHORT_QUERY_SETTINGS` test used `vi.resetModules()` +
+// `vi.stubEnv()` to re-import the module under different env vars — the
+// suite's only `vi.resetModules` site (ADR-010's tail table). bun:test shares
+// one module registry per file even under `--isolate`, so re-importing would
+// not re-evaluate a module-level const. `deriveCohortQuerySettings` exists
+// so this ports as direct calls instead.
+
+import { expect, test } from 'bun:test';
+import type { PropertyBasedCohortDefinition } from '../cohort.constants';
+import {
+ buildPropertyBasedCohortQuery,
+ deriveCohortQuerySettings,
+} from '../cohort.service';
+
+const PROJECT_ID = 'test-sql-validation';
+
+function buildSql(
+ criteria: PropertyBasedCohortDefinition['criteria'],
+ limit?: number
+) {
+ return buildPropertyBasedCohortQuery(
+ PROJECT_ID,
+ { type: 'property', criteria } as PropertyBasedCohortDefinition,
+ limit
+ );
+}
+
+const mapFilter = {
+ id: 'a',
+ name: 'profile.properties.experiment',
+ operator: 'is' as const,
+ value: ['control'],
+};
+
+test('resolves the newest row per profile without FINAL', () => {
+ const sql = buildSql({ operator: 'and', properties: [mapFilter] });
+
+ // FINAL cannot spill to disk, so wide projects OOM on the dedup itself.
+ expect(sql).not.toContain('FINAL');
+ expect(sql).toContain('GROUP BY id');
+ expect(sql).toContain(
+ "argMax(profiles.properties['experiment'], tuple(last_seen_at, cityHash64(profiles.properties['experiment'])))"
+ );
+});
+
+test('orders every aggregate by ONE shared row key', () => {
+ // Per-column tie-breaking lets equal-version rows with conflicting
+ // fields each win a different column — an AND cohort could then match a
+ // synthetic combination no stored row contains. The shared key makes
+ // all aggregates read the same winning row.
+ const sql = buildSql({
+ operator: 'and',
+ properties: [
+ mapFilter,
+ {
+ id: 'b',
+ name: 'profile.email',
+ operator: 'is' as const,
+ value: ['[email protected]'],
+ },
+ ],
+ });
+
+ const sharedKey =
+ "tuple(last_seen_at, cityHash64(profiles.properties['experiment'], profiles.email))";
+ expect(sql).toContain(
+ `argMax(profiles.properties['experiment'], ${sharedKey})`
+ );
+ expect(sql).toContain(`argMax(profiles.email, ${sharedKey})`);
+});
+
+test('filters aggregates in HAVING, not WHERE', () => {
+ const sql = buildSql({ operator: 'and', properties: [mapFilter] });
+
+ const having = sql.indexOf('HAVING');
+ expect(having).toBeGreaterThan(-1);
+ expect(sql.indexOf('argMax')).toBeGreaterThan(having);
+});
+
+test('wraps plain columns too, so mixed cohorts stay on one scan', () => {
+ const sql = buildSql({
+ operator: 'or',
+ properties: [
+ mapFilter,
+ {
+ id: 'b',
+ name: 'profile.email',
+ operator: 'contains' as const,
+ value: ['@example.com'],
+ },
+ ],
+ });
+
+ expect(sql).toContain('argMax(profiles.email, tuple(last_seen_at,');
+ expect(sql).toContain(' OR ');
+ expect(sql).not.toContain('FINAL');
+});
+
+test('wraps numeric comparisons inside the cast', () => {
+ const sql = buildSql({
+ operator: 'and',
+ properties: [
+ {
+ id: 'n',
+ name: 'profile.properties.age',
+ operator: 'gt' as const,
+ value: ['30'],
+ },
+ ],
+ });
+
+ expect(sql).toContain(
+ "toFloat64OrNull(argMax(profiles.properties['age'], tuple(last_seen_at,"
+ );
+});
+
+test('escapes quotes in user-controlled property keys', () => {
+ const sql = buildSql({
+ operator: 'and',
+ properties: [
+ {
+ id: 'q',
+ name: "profile.properties.pl'an",
+ operator: 'is' as const,
+ value: ['x'],
+ },
+ ],
+ });
+
+ // The raw quote must never appear inside the literal unescaped.
+ expect(sql).toContain("profiles.properties['pl\\'an']");
+ expect(sql).not.toContain("properties['pl'an']");
+});
+
+test('applies the limit', () => {
+ const sql = buildSql({ operator: 'and', properties: [mapFilter] }, 10);
+ expect(sql).toContain('LIMIT 10');
+});
+
+test('matches nothing when every filter was dropped as empty', () => {
+ const sql = buildSql({
+ operator: 'and',
+ properties: [
+ {
+ id: 'a',
+ name: 'profile.properties.x',
+ operator: 'is' as const,
+ value: [],
+ },
+ ],
+ });
+
+ expect(sql).toContain('WHERE 1=0');
+ expect(sql).not.toContain('argMax');
+});
+
+test('deriveCohortQuerySettings applies NO settings when neither variable is set (upstream defaults govern)', () => {
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: undefined,
+ spillBytesRaw: undefined,
+ })
+ ).toEqual({});
+});
+
+test('deriveCohortQuerySettings derives the spill threshold as limit/3 when only the limit is set', () => {
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: '3000000000',
+ spillBytesRaw: undefined,
+ })
+ ).toEqual({
+ max_bytes_before_external_group_by: '1000000000',
+ max_memory_usage: '3000000000',
+ });
+});
+
+test('deriveCohortQuerySettings respects both values when both are set', () => {
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: '2000000000',
+ spillBytesRaw: '500000000',
+ })
+ ).toEqual({
+ max_bytes_before_external_group_by: '500000000',
+ max_memory_usage: '2000000000',
+ });
+});
+
+test('deriveCohortQuerySettings applies only the spill threshold when only it is set', () => {
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: undefined,
+ spillBytesRaw: '500000000',
+ })
+ ).toEqual({ max_bytes_before_external_group_by: '500000000' });
+});
+
+test('deriveCohortQuerySettings re-derives an inverted pair (spill >= limit would never spill)', () => {
+ // A GROUP BY only starts spilling once it crosses the threshold, so a
+ // threshold at/above the kill limit means the query dies before it ever
+ // writes to disk — the exact inversion ClickHouse Cloud ships.
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: '900000000',
+ spillBytesRaw: '900000000',
+ })
+ ).toEqual({
+ max_bytes_before_external_group_by: '300000000',
+ max_memory_usage: '900000000',
+ });
+});
+
+test('deriveCohortQuerySettings never derives a zero spill threshold (0 would DISABLE spilling)', () => {
+ for (const memoryLimitBytesRaw of ['1', '2', '3']) {
+ const settings = deriveCohortQuerySettings({
+ memoryLimitBytesRaw,
+ spillBytesRaw: undefined,
+ });
+ expect(
+ Number(settings.max_bytes_before_external_group_by)
+ ).toBeGreaterThanOrEqual(1);
+ }
+});
+
+test('deriveCohortQuerySettings ignores malformed values', () => {
+ expect(
+ deriveCohortQuerySettings({
+ memoryLimitBytesRaw: '2gb',
+ spillBytesRaw: '-1',
+ })
+ ).toEqual({});
+});
diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.ts
index 36ceeaccb..ce356a350 100644
--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts
+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts
@@ -28,6 +28,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {
const services: Services = {
auth: {} as Services['auth'],
insight: {} as Services['insight'],
+ cohort: {} as Services['cohort'],
gsc: gsc as GscService,
};
return {
@@ -118,6 +119,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', async
services: {
auth: {} as Services['auth'],
insight: {} as Services['insight'],
+ cohort: {} as Services['cohort'],
gsc: {
listConnectionsForSync: async () => [
{ projectId: 'p1' },
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.ts
index 3b775027d..e535f7743 100644
--- a/packages/core/src/modules/gsc/gsc.service.test.ts
+++ b/packages/core/src/modules/gsc/gsc.service.test.ts
@@ -95,15 +95,20 @@ const originalCh = {
query: mock(async () => ({ json: async () => [] as unknown[] })),
insert: mock(async () => undefined),
};
-// `ch` is unused here but included because `mock.module` replaces this
-// specifier process-wide (bun runs every test file in one shared module
-// registry without `--isolate` — see AGENTS.md) — insight.service.test.ts
-// mocks the same path, so both factories must be a superset of every
-// consumer's needs, whichever one ends up registered last.
+// `ch`/`getReplicatedTableName` are unused here but included because
+// `mock.module` replaces this specifier process-wide (bun runs every test
+// file in one shared module registry without `--isolate` — see AGENTS.md) —
+// insight.service.test.ts and cohort.service.test.ts mock the same path, so
+// every factory must be a superset of every consumer's needs, whichever one
+// ends up registered last.
mock.module('@openpanel/db/src/clickhouse/client', () => ({
- ch: {},
+ ch: {
+ insert: mock(async () => undefined),
+ command: mock(async () => undefined),
+ },
originalCh,
chQuery,
+ getReplicatedTableName: mock((table: string) => table),
TABLE_NAMES: { sessions: 'sessions' },
}));
diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.ts
index e02e69597..b47c44a5e 100644
--- a/packages/core/src/modules/insight/insight.jobs.test.ts
+++ b/packages/core/src/modules/insight/insight.jobs.test.ts
@@ -40,6 +40,7 @@ function stubJobCtx(
auth: {} as Services['auth'],
insight: insight as InsightService,
gsc: {} as Services['gsc'],
+ cohort: {} as Services['cohort'],
};
return {
db: {},
@@ -118,6 +119,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de
],
} as InsightService,
gsc: {} as Services['gsc'],
+ cohort: {} as Services['cohort'],
},
requestId: 'req_1',
job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },
diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.ts
index 2fb12340a..9eee88827 100644
--- a/packages/core/src/modules/insight/insight.service.test.ts
+++ b/packages/core/src/modules/insight/insight.service.test.ts
@@ -22,15 +22,23 @@ mock.module('@openpanel/db/src/prisma-client', () => ({
db: { $executeRaw },
}));
-// `originalCh`/`chQuery`/`TABLE_NAMES` are unused here but included because
-// `mock.module` replaces this specifier process-wide (bun runs every test
-// file in one shared module registry without `--isolate` — see AGENTS.md) —
-// gsc.service.test.ts mocks the same path, so both factories must be a
+// `originalCh`/`chQuery`/`TABLE_NAMES`/`getReplicatedTableName` are unused
+// here but included because `mock.module` replaces this specifier
+// process-wide (bun runs every test file in one shared module registry
+// without `--isolate` — see AGENTS.md) — gsc.service.test.ts and
+// cohort.service.test.ts mock the same path, so every factory must be a
// superset of every consumer's needs, whichever one ends up registered last.
mock.module('@openpanel/db/src/clickhouse/client', () => ({
- ch: {},
- originalCh: { query: mock(async () => ({ json: async () => [] })), insert: mock(async () => undefined) },
+ ch: {
+ insert: mock(async () => undefined),
+ command: mock(async () => undefined),
+ },
+ originalCh: {
+ query: mock(async () => ({ json: async () => [] })),
+ insert: mock(async () => undefined),
+ },
chQuery: mock(async () => []),
+ getReplicatedTableName: mock((table: string) => table),
TABLE_NAMES: { sessions: 'sessions' },
}));
diff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.ts
index 4c2a3ddb5..3e8801b8d 100644
--- a/packages/core/src/rpc.router.ts
+++ b/packages/core/src/rpc.router.ts
@@ -4,6 +4,7 @@
// (ADR-007 decision 19). Only `health` exists so far; the 28 real routers
// land with their module waves (P5-P8).
+import { cohortRouter } from './modules/cohort/cohort.rpc';
import { gscRouter } from './modules/gsc/gsc.rpc';
import { healthRouter } from './modules/health/health.rpc';
import { insightRouter } from './modules/insight/insight.rpc';
@@ -13,6 +14,7 @@ export const appRouter = createTRPCRouter({
health: healthRouter,
insight: insightRouter,
gsc: gscRouter,
+ cohort: cohortRouter,
});
export type AppRouter = typeof appRouter;
diff --git a/packages/core/src/services.ts b/packages/core/src/services.ts
index 5284c1dd1..73b4aa00f 100644
--- a/packages/core/src/services.ts
+++ b/packages/core/src/services.ts
@@ -7,6 +7,10 @@ import {
type AuthService,
createAuthService,
} from './modules/auth/auth.service';
+import {
+ type CohortService,
+ createCohortService,
+} from './modules/cohort/cohort.service';
import { createGscService, type GscService } from './modules/gsc/gsc.service';
import {
createInsightService,
@@ -32,6 +36,7 @@ export interface Services {
auth: AuthService;
insight: InsightService;
gsc: GscService;
+ cohort: CohortService;
}
export function createServices(deps: ServiceDeps): Services {
@@ -42,5 +47,6 @@ export function createServices(deps: ServiceDeps): Services {
auth: createAuthService(deps),
insight: createInsightService(deps),
gsc: createGscService(deps),
+ cohort: createCohortService(deps),
};
}
diff --git a/packages/db/index.ts b/packages/db/index.ts
index de55fe19f..1eb06c2f8 100644
--- a/packages/db/index.ts
+++ b/packages/db/index.ts
@@ -11,7 +11,6 @@ export * from './src/services/access.service';
export * from './src/services/auth-session.service';
export * from './src/services/chart.service';
export * from './src/services/clients.service';
-export * from './src/services/cohort.service';
export * from './src/services/conversation.service';
export * from './src/services/conversion.service';
export * from './src/services/dashboard.service';
diff --git a/packages/db/src/services/cohort-property-sql.test.ts b/packages/db/src/services/cohort-property-sql.test.ts
deleted file mode 100644
index 76c1f0d92..000000000
--- a/packages/db/src/services/cohort-property-sql.test.ts
+++ /dev/null
@@ -1,278 +0,0 @@
-/**
- * SQL-shape tests for property-based cohort queries.
- *
- * Same strategy as chart-sql.test.ts / funnel-sql.test.ts: string assertions
- * always run; `EXPLAIN` validation runs against a locally reachable
- * ClickHouse (`pnpm dock:up`) and skips otherwise.
- */
-import type { PropertyBasedCohortDefinition } from '@openpanel/validation';
-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
-
-vi.mock('../prisma-client', () => ({
- db: {},
-}));
-
-import { ch } from '../clickhouse/client';
-import {
- buildPropertyBasedCohortQuery,
- PROFILE_COHORT_QUERY_SETTINGS,
-} from './cohort.service';
-
-const PROJECT_ID = 'test-sql-validation';
-
-let chReachable = false;
-
-beforeAll(async () => {
- vi.spyOn(console, 'log').mockImplementation(() => {});
- try {
- await ch.command({ query: 'SELECT 1' });
- chReachable = true;
- } catch {
- chReachable = false;
- }
-});
-
-afterAll(() => {
- vi.restoreAllMocks();
-});
-
-const itCH = (name: string, fn: () => Promise<void>) =>
- it(name, async (ctx) => {
- if (!chReachable) {
- ctx.skip('ClickHouse not reachable at CLICKHOUSE_URL');
- }
- await fn();
- });
-
-function buildSql(
- criteria: PropertyBasedCohortDefinition['criteria'],
- limit?: number,
-) {
- return buildPropertyBasedCohortQuery(
- PROJECT_ID,
- { type: 'property', criteria } as PropertyBasedCohortDefinition,
- limit,
- );
-}
-
-const mapFilter = {
- id: 'a',
- name: 'profile.properties.experiment',
- operator: 'is' as const,
- value: ['control'],
-};
-
-describe('buildPropertyBasedCohortQuery', () => {
- it('resolves the newest row per profile without FINAL', () => {
- const sql = buildSql({ operator: 'and', properties: [mapFilter] });
-
- // FINAL cannot spill to disk, so wide projects OOM on the dedup itself.
- expect(sql).not.toContain('FINAL');
- expect(sql).toContain('GROUP BY id');
- expect(sql).toContain(
- "argMax(profiles.properties['experiment'], tuple(last_seen_at, cityHash64(profiles.properties['experiment'])))",
- );
- });
-
- it('orders every aggregate by ONE shared row key', () => {
- // Per-column tie-breaking lets equal-version rows with conflicting
- // fields each win a different column — an AND cohort could then match a
- // synthetic combination no stored row contains. The shared key makes
- // all aggregates read the same winning row.
- const sql = buildSql({
- operator: 'and',
- properties: [
- mapFilter,
- {
- id: 'b',
- name: 'profile.email',
- operator: 'is' as const,
- value: ['[email protected]'],
- },
- ],
- });
-
- const sharedKey =
- "tuple(last_seen_at, cityHash64(profiles.properties['experiment'], profiles.email))";
- expect(sql).toContain(
- `argMax(profiles.properties['experiment'], ${sharedKey})`,
- );
- expect(sql).toContain(`argMax(profiles.email, ${sharedKey})`);
- });
-
- it('filters aggregates in HAVING, not WHERE', () => {
- const sql = buildSql({ operator: 'and', properties: [mapFilter] });
-
- const having = sql.indexOf('HAVING');
- expect(having).toBeGreaterThan(-1);
- expect(sql.indexOf('argMax')).toBeGreaterThan(having);
- });
-
- it('wraps plain columns too, so mixed cohorts stay on one scan', () => {
- const sql = buildSql({
- operator: 'or',
- properties: [
- mapFilter,
- {
- id: 'b',
- name: 'profile.email',
- operator: 'contains' as const,
- value: ['@example.com'],
- },
- ],
- });
-
- expect(sql).toContain('argMax(profiles.email, tuple(last_seen_at,');
- expect(sql).toContain(' OR ');
- expect(sql).not.toContain('FINAL');
- });
-
- it('wraps numeric comparisons inside the cast', () => {
- const sql = buildSql({
- operator: 'and',
- properties: [
- {
- id: 'n',
- name: 'profile.properties.age',
- operator: 'gt' as const,
- value: ['30'],
- },
- ],
- });
-
- expect(sql).toContain(
- "toFloat64OrNull(argMax(profiles.properties['age'], tuple(last_seen_at,",
- );
- });
-
- it('escapes quotes in user-controlled property keys', () => {
- const sql = buildSql({
- operator: 'and',
- properties: [
- {
- id: 'q',
- name: "profile.properties.pl'an",
- operator: 'is' as const,
- value: ['x'],
- },
- ],
- });
-
- // The raw quote must never appear inside the literal unescaped.
- expect(sql).toContain("profiles.properties['pl\\'an']");
- expect(sql).not.toContain("properties['pl'an']");
- });
-
- it('applies the limit', () => {
- const sql = buildSql({ operator: 'and', properties: [mapFilter] }, 10);
- expect(sql).toContain('LIMIT 10');
- });
-
- it('matches nothing when every filter was dropped as empty', () => {
- const sql = buildSql({
- operator: 'and',
- properties: [
- {
- id: 'a',
- name: 'profile.properties.x',
- operator: 'is' as const,
- value: [],
- },
- ],
- });
-
- expect(sql).toContain('WHERE 1=0');
- expect(sql).not.toContain('argMax');
- });
-
- describe('PROFILE_COHORT_QUERY_SETTINGS', () => {
- const loadSettings = async (env: Record<string, string>) => {
- vi.resetModules();
- vi.stubEnv('COHORT_QUERY_MEMORY_LIMIT_BYTES', env.limit ?? '');
- vi.stubEnv('COHORT_QUERY_SPILL_BYTES', env.spill ?? '');
- const mod = await import('./cohort.service');
- return mod.PROFILE_COHORT_QUERY_SETTINGS;
- };
-
- afterAll(() => {
- vi.unstubAllEnvs();
- });
-
- it('applies NO settings when neither variable is set (upstream defaults govern)', async () => {
- expect(await loadSettings({})).toEqual({});
- });
-
- it('derives the spill threshold as limit/3 when only the limit is set', async () => {
- expect(await loadSettings({ limit: '3000000000' })).toEqual({
- max_bytes_before_external_group_by: '1000000000',
- max_memory_usage: '3000000000',
- });
- });
-
- it('respects both values when both are set', async () => {
- expect(
- await loadSettings({ limit: '2000000000', spill: '500000000' }),
- ).toEqual({
- max_bytes_before_external_group_by: '500000000',
- max_memory_usage: '2000000000',
- });
- });
-
- it('applies only the spill threshold when only it is set', async () => {
- expect(await loadSettings({ spill: '500000000' })).toEqual({
- max_bytes_before_external_group_by: '500000000',
- });
- });
-
- it('re-derives an inverted pair (spill >= limit would never spill)', async () => {
- // A GROUP BY only starts spilling once it crosses the threshold, so a
- // threshold at/above the kill limit means the query dies before it
- // ever writes to disk — the exact inversion ClickHouse Cloud ships.
- expect(
- await loadSettings({ limit: '900000000', spill: '900000000' }),
- ).toEqual({
- max_bytes_before_external_group_by: '300000000',
- max_memory_usage: '900000000',
- });
- });
-
- it('never derives a zero spill threshold (0 would DISABLE spilling)', async () => {
- for (const limit of ['1', '2', '3']) {
- const settings = await loadSettings({ limit });
- expect(
- Number(settings.max_bytes_before_external_group_by),
- ).toBeGreaterThanOrEqual(1);
- }
- });
-
- it('ignores malformed values', async () => {
- expect(await loadSettings({ limit: '2gb', spill: '-1' })).toEqual({});
- });
- });
-
- itCH('parses and resolves against ClickHouse', async () => {
- for (const sql of [
- buildSql({ operator: 'and', properties: [mapFilter] }, 10),
- buildSql({
- operator: 'or',
- properties: [
- mapFilter,
- {
- id: 'n',
- name: 'profile.properties.age',
- operator: 'gte' as const,
- value: ['30'],
- },
- {
- id: 'e',
- name: 'profile.email',
- operator: 'isNotNull' as const,
- value: [],
- },
- ],
- }),
- ]) {
- await ch.command({ query: `EXPLAIN ${sql}` });
- }
- });
-});
diff --git a/packages/trpc/src/routers/cohort.ts b/packages/trpc/src/routers/cohort.ts
index dd73fa49a..8a7f69464 100644
--- a/packages/trpc/src/routers/cohort.ts
+++ b/packages/trpc/src/routers/cohort.ts
@@ -1,16 +1,30 @@
+// Dissolved into @openpanel/core's cohort module (M5-003): the compute
+// functions, the ClickHouse read path and the producer wrapper
+// (`enqueueCohortCompute`) moved to packages/core/src/modules/cohort/cohort.service.ts
+// — db cannot hold producers (ADR-007). This router stays (DELEGATE
+// PATTERN): it keeps V1's protectedProcedure stack (session/access/logger/
+// rate-limit middleware) and its own Prisma cohort CRUD untouched, and
+// delegates every compute/ClickHouse call to the core service.
+//
+// The enqueue itself stays inline here, against @openpanel/queue's
+// cohortComputeQueue directly — same as gsc's router does for
+// `gscProjectBackfill` (M5-002). @openpanel/queue imports @openpanel/core for
+// its logger, so core cannot import @openpanel/queue back; the canonical,
+// ctx.queues-based wrapper is `CohortService.enqueueCompute`, used once this
+// router's core twin (cohort.rpc.ts) goes live.
import {
computeCohort,
countCohort,
- db,
deleteCohortMembership,
- enqueueCohortCompute,
getCohortCount,
getCohortEventsPerDay,
getCohortMemberEvents,
getCohortMemberRoutes,
getCohortMembers,
listCohortMemberProfiles,
-} from '@openpanel/db';
+} from '@openpanel/core';
+import { db } from '@openpanel/db';
+import { cohortComputeQueue } from '@openpanel/queue';
import {
type CohortDefinition,
zChartEventFilter,
@@ -23,6 +37,17 @@ import { getProjectAccess, requireProjectAccess } from '../access';
import { TRPCForbiddenError, TRPCNotFoundError } from '../errors';
import { createTRPCRouter, protectedProcedure } from '../trpc';
+// Same dedup scheme as core's CohortService.enqueueCompute
+// (packages/core/src/modules/cohort/cohort.service.ts) — deduplication, not
+// jobId, to avoid the retention-vs-dedup deadlock that file documents.
+async function enqueueCohortCompute(cohortId: string): Promise<void> {
+ await cohortComputeQueue.add(
+ 'cohortCompute',
+ { cohortId },
+ { deduplication: { id: `cohort-${cohortId}` } }
+ );
+}
+
export const cohortRouter = createTRPCRouter({
list: protectedProcedure
.input(
diff --git a/packages/validation/package.json b/packages/validation/package.json
index 7ba7c0a84..1fc089603 100644
--- a/packages/validation/package.json
+++ b/packages/validation/package.json
@@ -8,6 +8,7 @@
},
"dependencies": {
"@openpanel/constants": "workspace:*",
+ "@openpanel/core": "workspace:*",
"zod": "catalog:"
},
"devDependencies": {
diff --git a/packages/validation/src/cohort.validation.ts b/packages/validation/src/cohort.validation.ts
index cc79c6b85..779a9b4af 100644
--- a/packages/validation/src/cohort.validation.ts
+++ b/packages/validation/src/cohort.validation.ts
@@ -1,146 +1,8 @@
-import { operators } from '@openpanel/constants';
-import { z } from 'zod';
-
-// Defined locally to avoid a runtime TDZ from cohort.validation being
-// imported before index.ts has finished initializing zChartEventFilter.
-// Keep in sync with zChartEventFilter in ./index.ts.
-function objectToZodEnums<K extends string>(obj: Record<K, unknown>): [K, ...K[]] {
- const [firstKey, ...otherKeys] = Object.keys(obj) as K[];
- return [firstKey!, ...otherKeys];
-}
-
-const zChartEventFilter = z.object({
- id: z.string().optional().describe('Unique identifier for the filter'),
- name: z.string().describe('The property name to filter on'),
- operator: z
- .enum(objectToZodEnums(operators))
- .describe('The operator to use for the filter'),
- value: z
- .array(z.string().or(z.number()).or(z.boolean()).or(z.null()))
- .describe('The values to filter on'),
- cohortId: z
- .string()
- .optional()
- .describe('Cohort ID when using inCohort/notInCohort operators'),
-});
-
-export const zRelativeTimeframe = z.object({
- type: z.literal('relative'),
- value: z.enum(['7d', '30d', '90d', '180d', '365d']),
-});
-
-export const zAbsoluteTimeframe = z.object({
- type: z.literal('absolute'),
- start: z.string(),
- end: z.string().optional(),
-});
-
-export const zTimeframe = z.discriminatedUnion('type', [
- zRelativeTimeframe,
- zAbsoluteTimeframe,
-]);
-
-export type Timeframe = z.infer<typeof zTimeframe>;
-
-export const zFrequency = z.object({
- operator: z.enum(['gte', 'eq', 'lte']),
- count: z.number().int().min(1),
-});
-
-export type Frequency = z.infer<typeof zFrequency>;
-
-export const zEventCriteria = z.object({
- name: z.string().min(1).describe('The event name to match'),
- filters: z
- .array(zChartEventFilter)
- .default([])
- .describe('Filters applied to event properties'),
- timeframe: zTimeframe.describe('When the event should have occurred'),
- frequency: zFrequency
- .optional()
- .describe('How many times the event should have occurred'),
-});
-
-export type EventCriteria = z.infer<typeof zEventCriteria>;
-
-export const zEventBasedCohortDefinition = z.object({
- type: z.literal('event'),
- criteria: z.object({
- operator: z.enum(['and', 'or']).describe('How to combine multiple events'),
- events: z
- .array(zEventCriteria)
- .min(1)
- .describe('Array of event criteria to match'),
- }),
-});
-
-export type EventBasedCohortDefinition = z.infer<
- typeof zEventBasedCohortDefinition
->;
-
-export const zPropertyBasedCohortDefinition = z.object({
- type: z.literal('property'),
- criteria: z.object({
- operator: z
- .enum(['and', 'or'])
- .describe('How to combine multiple properties'),
- properties: z
- .array(zChartEventFilter)
- .min(1)
- .describe('Array of profile property filters'),
- }),
-});
-
-export type PropertyBasedCohortDefinition = z.infer<
- typeof zPropertyBasedCohortDefinition
->;
-
-export const zCohortDefinition = z.discriminatedUnion('type', [
- zEventBasedCohortDefinition,
- zPropertyBasedCohortDefinition,
-]);
-
-export type CohortDefinition = z.infer<typeof zCohortDefinition>;
-
-export const zCohortInput = z.object({
- name: z
- .string()
- .min(1)
- .max(255)
- .describe('User-friendly name for the cohort'),
- description: z
- .string()
- .max(1000)
- .optional()
- .describe('Optional description of the cohort'),
- projectId: z.string().describe('The project this cohort belongs to'),
- definition: zCohortDefinition.describe('The cohort criteria definition'),
- isStatic: z
- .boolean()
- .default(false)
- .describe(
- 'Whether this cohort is frozen — if true, membership is computed once and not auto-refreshed',
- ),
-});
-
-export type CohortInput = z.infer<typeof zCohortInput>;
-
-export const zCohortUpdate = z.object({
- id: z.string().describe('The cohort ID to update'),
- name: z.string().min(1).max(255).optional(),
- description: z.string().max(1000).optional().nullable(),
- definition: zCohortDefinition.optional(),
- isStatic: z.boolean().optional(),
-});
-
-export type CohortUpdate = z.infer<typeof zCohortUpdate>;
-
-export const zCohortFilter = z.object({
- cohortId: z.string().describe('The cohort ID to filter by'),
- operator: z
- .enum(['inCohort', 'notInCohort'])
- .default('inCohort')
- .describe('Whether to include or exclude cohort members'),
-});
-
-export type CohortFilter = z.infer<typeof zCohortFilter>;
+// Moved into @openpanel/core's cohort module (M5-003, ADR-008's module map:
+// cohort owns "C"). Re-exported here for existing @openpanel/validation
+// importers (apps/start's cohort UI, packages/db/src/types.ts) — same shape
+// as packages/db/src/gsc.ts since M5-002. The `./modules/cohort/cohort.constants`
+// subpath is a leaf, isomorphic file with no back-imports into
+// @openpanel/validation, so this re-export carries no runtime cycle despite
+// @openpanel/core already depending on @openpanel/validation elsewhere.
+export * from '@openpanel/core/modules/cohort/cohort.constants';
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ed9ddc735..41bc01939 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1177,6 +1177,9 @@ importers:
qrcode:
specifier: ^1.5.4
version: 1.5.4
+ sqlstring:
+ specifier: ^2.3.3
+ version: 2.3.3
superjson:
specifier: ^1.13.3
version: 1.13.3
@@ -1199,6 +1202,9 @@ importers:
'@types/qrcode':
specifier: ^1.5.6
version: 1.5.6
+ '@types/sqlstring':
+ specifier: ^2.3.2
+ version: 2.3.2
bun-types:
specifier: 1.4.0
version: 1.4.0
@@ -1839,6 +1845,9 @@ importers:
'@openpanel/constants':
specifier: workspace:*
version: link:../constants
+ '@openpanel/core':
+ specifier: workspace:*
+ version: link:../core
zod:
specifier: 'catalog:'
version: 4.3.6