OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M7-001): session module: service + context/consistency + trpc + sessions queue + reaper/vacuum crons into core
ralph3300f2a0071c
diff --git a/apps/worker/src/jobs/cron.session-reaper.test.ts b/apps/worker/src/jobs/cron.session-reaper.test.tsdeleted file mode 100644index 9d6423f40..000000000--- a/apps/worker/src/jobs/cron.session-reaper.test.ts+++ /dev/null@@ -1,119 +0,0 @@-/**- * Tests for the session reaper — the wall-clock deadman that closes idle- * sessions. Redis is mocked (we drive the wallclock ZSET + lock), sessionBuffer- * and enqueueSessionEndV2 are mocked so we assert decisions, not side effects.- */--import type { IClickhouseSession } from '@openpanel/db';-import { sessionBuffer } from '@openpanel/db';-import { beforeEach, describe, expect, it, vi } from 'vitest';-import { enqueueSessionEndV2 } from '@/utils/session-handler';-import { sessionReaperCronJob } from './cron.session-reaper';--const { redisMock } = vi.hoisted(() => ({- redisMock: {- smembers: vi.fn(),- set: vi.fn(),- del: vi.fn(),- zrangebyscore: vi.fn(),- zcard: vi.fn(),- srem: vi.fn(),- zrem: vi.fn(),- },-}));--vi.mock('@openpanel/redis', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/redis')>();- return { ...actual, getRedisCache: () => redisMock };-});--vi.mock('@openpanel/db', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/db')>();- return { ...actual, sessionBuffer: { getExistingSession: vi.fn() } };-});--vi.mock('@/utils/session-handler', () => ({- enqueueSessionEndV2: vi.fn().mockResolvedValue(undefined),-}));--// Concrete metric stubs — avoid loading the real metrics module (it registers-// prom gauges at import) and avoid a Proxy (its `then` trap confuses ESM interop).-vi.mock('@/metrics', () => {- const counter = () => ({ inc: vi.fn(), observe: vi.fn(), set: vi.fn() });- return {- sessionEndsEnqueued: counter(),- sessionsReaped: counter(),- sessionsReaperOrphans: counter(),- };-});--const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);--const session = (id: string, deviceId: string): IClickhouseSession =>- ({- id,- project_id: 'proj-1',- device_id: deviceId,- profile_id: deviceId,- ended_at: '2026-06-08 11:00:00',- duration: 1000,- groups: [],- }) as unknown as IClickhouseSession;--beforeEach(() => {- vi.clearAllMocks();- redisMock.smembers.mockResolvedValue(['proj-1']);- redisMock.set.mockResolvedValue('OK'); // lock acquired- redisMock.del.mockResolvedValue(1);- redisMock.zcard.mockResolvedValue(0); // project drained after reap- redisMock.srem.mockResolvedValue(1);- redisMock.zrem.mockResolvedValue(1);- redisMock.zrangebyscore.mockResolvedValue([]);-});--describe('sessionReaperCronJob', () => {- it('enqueues a session_end for an idle session blob', async () => {- redisMock.zrangebyscore.mockResolvedValue(['dev-1']);- getExistingSession.mockResolvedValue(session('sess-1', 'dev-1'));-- await sessionReaperCronJob();-- expect(enqueueSessionEndV2).toHaveBeenCalledTimes(1);- expect(- vi.mocked(enqueueSessionEndV2).mock.calls[0]![0].closedSession.id- ).toBe('sess-1');- // wallclock index emptied → project removed from the active set- expect(redisMock.srem).toHaveBeenCalledWith('session:projects', 'proj-1');- });-- it('ZREMs an orphan (wallclock entry with no blob) and does not enqueue', async () => {- redisMock.zrangebyscore.mockResolvedValue(['dev-orphan']);- getExistingSession.mockResolvedValue(null);-- await sessionReaperCronJob();-- expect(enqueueSessionEndV2).not.toHaveBeenCalled();- expect(redisMock.zrem).toHaveBeenCalledWith(- 'session:wallclock:proj-1',- 'dev-orphan'- );- });-- it('skips a project when the advisory lock is held', async () => {- redisMock.set.mockResolvedValue(null); // NX failed → another pod owns it-- await sessionReaperCronJob();-- expect(redisMock.zrangebyscore).not.toHaveBeenCalled();- expect(enqueueSessionEndV2).not.toHaveBeenCalled();- });-- it('is a no-op when disabled via SESSION_REAPER=0', async () => {- vi.stubEnv('SESSION_REAPER', '0');-- await sessionReaperCronJob();-- expect(redisMock.smembers).not.toHaveBeenCalled();- vi.unstubAllEnvs();- });-});diff --git a/apps/worker/src/jobs/cron.session-reaper.ts b/apps/worker/src/jobs/cron.session-reaper.tsindex 867b07db9..66c5171d6 100644--- a/apps/worker/src/jobs/cron.session-reaper.ts+++ b/apps/worker/src/jobs/cron.session-reaper.ts@@ -1,204 +1,18 @@-import { SESSION_TIMEOUT_MS, sessionBuffer } from '@openpanel/db';-import { getRedisCache } from '@openpanel/redis';+// Dissolved into @openpanel/core's session module (M7-001): the wall-clock+// deadman scan, per-project lock and orphan handling moved to+// packages/core/src/modules/session/src/session-reaper.ts. This file stays+// (DELEGATE PATTERN) — it is the `sessionReaper` case of `cron.ts`'s+// dispatcher, and enqueues through V1's BullMQ `sessionsQueue`.+import { loadSessionRuntime, reapIdleSessions } from '@openpanel/core';import { logger as baseLogger } from '@/utils/logger';-import {- sessionEndsEnqueued,- sessionsReaped,- sessionsReaperOrphans,-} from '@/metrics';import { enqueueSessionEndV2 } from '@/utils/session-handler';const logger = baseLogger.child({ job: 'session-reaper' });-const PROJECTS_SET_KEY = 'session:projects';-const REAPER_BATCH_SIZE = Number.parseInt(- process.env.SESSION_REAPER_BATCH_SIZE || '5000',- 10-);-// Wall-clock deadman: close sessions whose last *received* event is older-// than this. Single source of truth for "this session has ended", regardless-// of project traffic.-//-// Default 30min, matching SESSION_TIMEOUT_MS. With the 5-min reaper interval,-// max session_end latency is 30-35min after the last event.-//-// During a worker pause longer than this, sessions in flight will be split-// when the worker resumes. Bump via env if you expect long pauses, but-// remember that BullMQ jobId-dedup + the extension check in createSessionEnd-// already cover normal queue lag.-const WALLCLOCK_DEADMAN_MS = Number.parseInt(- process.env.SESSION_REAPER_WALLCLOCK_DEADMAN_MS ||- String(SESSION_TIMEOUT_MS),- 10-);-const LOCK_TTL_SECONDS = 60;--const wallclockSetKey = (projectId: string) =>- `session:wallclock:${projectId}`;-const lockKey = (projectId: string) => `session:reaper:lock:${projectId}`;--/**- * Cron: scan every project with active sessions, close any whose last event- * was received more than the deadman ago in wall-clock time.- *- * Idempotent — uses BullMQ jobId-dedup via enqueueSessionEndV2.- */export async function sessionReaperCronJob() {- if (process.env.SESSION_REAPER === '0') {- return;- }-- const redis = getRedisCache();- const projectIds = await redis.smembers(PROJECTS_SET_KEY);-- if (projectIds.length === 0) {- return;- }-- logger.debug({ projectCount: projectIds.length }, 'Reaper tick');-- let totalReaped = 0;- let totalErrors = 0;-- for (const projectId of projectIds) {- try {- const reaped = await reapProject(projectId);- totalReaped += reaped;- } catch (error) {- totalErrors++;- logger.error({ err: error, projectId }, 'Failed to reap project');- }- }-- if (totalReaped > 0 || totalErrors > 0) {- logger.info(- { reaped: totalReaped, errors: totalErrors, projects: projectIds.length },- 'Reaper tick complete',- );- }-}--async function reapProject(projectId: string): Promise<number> {- const redis = getRedisCache();-- // Per-project advisory lock keeps multiple worker pods from reaping the- // same project simultaneously. Set NX with a short TTL; the lock is best- // effort — if it expires mid-tick, BullMQ jobId-dedup is the real guard.- const locked = await redis.set(- lockKey(projectId),- '1',- 'EX',- LOCK_TTL_SECONDS,- 'NX'- );- if (locked === null) {- return 0;- }-- try {- const now = Date.now();- const cutoff = now - WALLCLOCK_DEADMAN_MS;-- const candidates = await redis.zrangebyscore(- wallclockSetKey(projectId),- 0,- cutoff,- 'LIMIT',- 0,- REAPER_BATCH_SIZE- );-- let reaped = 0;- for (const deviceId of candidates) {- if (await closeSession(projectId, deviceId)) {- reaped++;- }- }-- // If the project has no remaining sessions in the wallclock index,- // remove it from the projects set so we stop iterating it.- const remaining = await redis.zcard(wallclockSetKey(projectId));- if (remaining === 0) {- await redis.srem(PROJECTS_SET_KEY, projectId);- }-- return reaped;- } finally {- await redis.del(lockKey(projectId));- }-}--async function closeSession(- projectId: string,- deviceId: string-): Promise<boolean> {- const session = await sessionBuffer.getExistingSession({- projectId,- deviceId,+ await reapIdleSessions({+ ...(await loadSessionRuntime()),+ logger,+ enqueueSessionEnd: enqueueSessionEndV2,});-- if (!session) {- // Sorted-set entry exists but the blob is gone. Without a TTL on the- // blob this should be rare — only happens if cleanup() partially- // failed (worker crash mid-MULTI, network partition). Log + count it- // and ZREM the orphan entry.- sessionsReaperOrphans.inc({ reason: 'deadman' });- baseLogger.warn(- { projectId, deviceId },- 'Reaper found wallclock entry without blob — likely a partial cleanup failure',- );- await getRedisCache().zrem(wallclockSetKey(projectId), deviceId);- return false;- }-- try {- await enqueueSessionEndV2({- payload: {- projectId: session.project_id,- deviceId: session.device_id,- sessionId: session.id,- profileId: session.profile_id ?? '',- name: 'session_end',- properties: {},- groups: session.groups ?? [],- createdAt: new Date(session.ended_at),- duration: session.duration ?? 0,- sdkName: '',- sdkVersion: '',- city: session.city ?? '',- country: session.country ?? '',- region: session.region ?? '',- longitude: session.longitude ?? undefined,- latitude: session.latitude ?? undefined,- path: session.exit_path ?? '',- origin: session.exit_origin ?? '',- referrer: session.referrer ?? '',- referrerName: session.referrer_name ?? '',- referrerType: session.referrer_type ?? '',- os: session.os ?? '',- osVersion: session.os_version ?? '',- browser: session.browser ?? '',- browserVersion: session.browser_version ?? '',- device: session.device ?? '',- brand: session.brand ?? '',- model: session.model ?? '',- },- closedSession: session,- });-- sessionsReaped.inc({ reason: 'deadman' });- sessionEndsEnqueued.inc({ source: 'reaper' });-- logger.debug(- { sessionId: session.id, projectId, deviceId },- 'Enqueued session_end (reaped)',- );- return true;- } catch (error) {- logger.error(- { err: error, sessionId: session.id, projectId, deviceId },- 'Failed to enqueue session_end during reap',- );- return false;- }}diff --git a/apps/worker/src/jobs/cron.session-vacuum.test.ts b/apps/worker/src/jobs/cron.session-vacuum.test.tsdeleted file mode 100644index 419dcdfc9..000000000--- a/apps/worker/src/jobs/cron.session-vacuum.test.ts+++ /dev/null@@ -1,90 +0,0 @@-/**- * Tests for the daily vacuum — the backstop for sessions whose cleanup() leaked- * (worker crash mid-MULTI). For wallclock entries older than the stale threshold:- * a lingering blob → id-gated cleanup(); a missing blob → ZREM the orphan.- */--import type { IClickhouseSession } from '@openpanel/db';-import { sessionBuffer } from '@openpanel/db';-import { beforeEach, describe, expect, it, vi } from 'vitest';-import { sessionVacuumCronJob } from './cron.session-vacuum';--const { redisMock } = vi.hoisted(() => ({- redisMock: {- smembers: vi.fn(),- zrangebyscore: vi.fn(),- zrem: vi.fn(),- },-}));--vi.mock('@openpanel/redis', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/redis')>();- return { ...actual, getRedisCache: () => redisMock };-});--vi.mock('@openpanel/db', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/db')>();- return {- ...actual,- sessionBuffer: { getExistingSession: vi.fn(), cleanup: vi.fn() },- };-});--vi.mock('@/metrics', () => ({- sessionsVacuumed: { inc: vi.fn() },-}));--const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);-const cleanup = vi.mocked(sessionBuffer.cleanup);--beforeEach(() => {- vi.clearAllMocks();- redisMock.smembers.mockResolvedValue(['proj-1']);- redisMock.zrem.mockResolvedValue(1);- redisMock.zrangebyscore.mockResolvedValue([]);- cleanup.mockResolvedValue(undefined as never);-});--describe('sessionVacuumCronJob', () => {- it('id-gated cleanup() for a stale blob that lingered', async () => {- redisMock.zrangebyscore.mockResolvedValue(['dev-stale']);- getExistingSession.mockResolvedValue({- id: 'sess-stale',- project_id: 'proj-1',- device_id: 'dev-stale',- profile_id: 'dev-stale',- } as unknown as IClickhouseSession);-- await sessionVacuumCronJob();-- expect(cleanup).toHaveBeenCalledWith({- projectId: 'proj-1',- deviceId: 'dev-stale',- sessionId: 'sess-stale',- profileId: 'dev-stale',- });- expect(redisMock.zrem).not.toHaveBeenCalled();- });-- it('ZREMs an orphan wallclock entry when the blob is gone', async () => {- redisMock.zrangebyscore.mockResolvedValue(['dev-orphan']);- getExistingSession.mockResolvedValue(null);-- await sessionVacuumCronJob();-- expect(cleanup).not.toHaveBeenCalled();- expect(redisMock.zrem).toHaveBeenCalledWith(- 'session:wallclock:proj-1',- 'dev-orphan'- );- });-- it('is a no-op when disabled via SESSION_VACUUM=0', async () => {- vi.stubEnv('SESSION_VACUUM', '0');-- await sessionVacuumCronJob();-- expect(redisMock.smembers).not.toHaveBeenCalled();- vi.unstubAllEnvs();- });-});diff --git a/apps/worker/src/jobs/cron.session-vacuum.ts b/apps/worker/src/jobs/cron.session-vacuum.tsindex ea1911395..4c5d72c3f 100644--- a/apps/worker/src/jobs/cron.session-vacuum.ts+++ b/apps/worker/src/jobs/cron.session-vacuum.ts@@ -1,101 +1,12 @@-import { sessionBuffer } from '@openpanel/db';-import { getRedisCache } from '@openpanel/redis';+// Dissolved into @openpanel/core's session module (M7-001): the stale-blob /+// orphan sweep moved to packages/core/src/modules/session/src/session-vacuum.ts.+// This file stays (DELEGATE PATTERN) — it is the `sessionVacuum` case of+// `cron.ts`'s dispatcher.+import { loadSessionRuntime, vacuumStaleSessions } from '@openpanel/core';import { logger as baseLogger } from '@/utils/logger';-import { sessionsVacuumed } from '@/metrics';const logger = baseLogger.child({ job: 'session-vacuum' });-const PROJECTS_SET_KEY = 'session:projects';-const VACUUM_BATCH_SIZE = Number.parseInt(- process.env.SESSION_VACUUM_BATCH_SIZE || '1000',- 10-);-// Anything in the wallclock index that hasn't moved in this long is stale —-// either a leaked entry (cleanup() partially failed) or a session that's-// been forgotten by every code path. Much larger than the reaper deadman-// so it never races with normal reap timing.-const VACUUM_STALE_THRESHOLD_MS = Number.parseInt(- process.env.SESSION_VACUUM_STALE_THRESHOLD_MS ||- String(1000 * 60 * 60 * 24 * 7), // 7 days- 10-);--const wallclockSetKey = (projectId: string) =>- `session:wallclock:${projectId}`;--/**- * Daily backstop for the rare case where `cleanup()` fails to fully delete- * a session (worker crash mid-MULTI, Redis partition). Scans each project's- * wallclock index for entries older than the stale threshold and:- *- * - If the blob still exists → run cleanup() (id-gated, atomic via Lua)- * - If the blob is gone → ZREM the orphan entry- *- * Both paths increment `sessions_vacuumed_total` with a reason label, so- * persistent non-zero values surface deeper issues.- */export async function sessionVacuumCronJob() {- if (process.env.SESSION_VACUUM === '0') {- return;- }-- const redis = getRedisCache();- const projectIds = await redis.smembers(PROJECTS_SET_KEY);-- if (projectIds.length === 0) {- return;- }-- logger.info({ projectCount: projectIds.length }, 'Vacuum tick starting');-- const cutoff = Date.now() - VACUUM_STALE_THRESHOLD_MS;- let total = 0;- let staleBlobs = 0;- let missingBlobs = 0;-- for (const projectId of projectIds) {- try {- const candidates = await redis.zrangebyscore(- wallclockSetKey(projectId),- 0,- cutoff,- 'LIMIT',- 0,- VACUUM_BATCH_SIZE- );-- for (const deviceId of candidates) {- const session = await sessionBuffer.getExistingSession({- projectId,- deviceId,- });-- if (session) {- // Blob still exists but is ancient — cleanup leaked. Use the- // id-gated cleanup so we don't race with a fresh session that- // happens to occupy the same slot.- await sessionBuffer.cleanup({- projectId,- deviceId,- sessionId: session.id,- profileId: session.profile_id,- });- sessionsVacuumed.inc({ reason: 'stale_blob' });- staleBlobs++;- } else {- // Orphan: blob is gone, wallclock entry left behind.- await redis.zrem(wallclockSetKey(projectId), deviceId);- sessionsVacuumed.inc({ reason: 'missing_blob' });- missingBlobs++;- }- total++;- }- } catch (error) {- logger.error({ err: error, projectId }, 'Vacuum failed for project');- }- }-- if (total > 0) {- logger.info({ total, staleBlobs, missingBlobs }, 'Vacuum tick complete');- }+ await vacuumStaleSessions({ ...(await loadSessionRuntime()), logger });}diff --git a/apps/worker/src/jobs/events.create-session-end.test.ts b/apps/worker/src/jobs/events.create-session-end.test.tsdeleted file mode 100644index cdb71d1d2..000000000--- a/apps/worker/src/jobs/events.create-session-end.test.ts+++ /dev/null@@ -1,153 +0,0 @@-/**- * Tests for createSessionEnd — the close job. Covers the three-case logic- * (extended-after-enqueue skip, live vs snapshot, boundary), the first-writer- * idempotency claim, and that cleanup() is id-gated to the closed session.- */--import {- createEvent,- type IClickhouseSession,- type IServiceCreateEventPayload,- sessionBuffer,-} from '@openpanel/db';-import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';-import type { Job } from 'bullmq';-import { beforeEach, describe, expect, it, vi } from 'vitest';-import { createSessionEnd } from './events.create-session-end';--const { redisMock } = vi.hoisted(() => ({- redisMock: { set: vi.fn() },-}));--vi.mock('@openpanel/redis', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/redis')>();- return { ...actual, getRedisCache: () => redisMock };-});--vi.mock('@openpanel/db', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/db')>();- return {- ...actual,- createEvent: vi.fn().mockResolvedValue({ document: { id: 'evt-1' } }),- getNotificationRulesByProjectId: vi.fn().mockResolvedValue([]),- getHasFunnelRules: vi.fn().mockReturnValue(false),- transformEvent: vi.fn((e) => e),- sessionBuffer: { getExistingSession: vi.fn(), cleanup: vi.fn() },- };-});--vi.mock('@/metrics', () => {- const m = () => ({ inc: vi.fn(), observe: vi.fn(), set: vi.fn() });- return {- sessionDurationOnClose: m(),- sessionEndsEmitted: m(),- sessionEndsSkipped: m(),- sessionEventsOnClose: m(),- };-});--const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);-const cleanup = vi.mocked(sessionBuffer.cleanup);-const createEventMock = vi.mocked(createEvent);--const snapshot: IClickhouseSession = {- id: 'sess-1',- project_id: 'proj-1',- device_id: 'dev-1',- profile_id: 'dev-1',- created_at: '2026-06-08 10:30:00',- ended_at: '2026-06-08 11:00:00',- is_bounce: false,- duration: 1_800_000,- event_count: 3,- screen_view_count: 5,- exit_path: '/end',-} as unknown as IClickhouseSession;--const makeJob = (- snap: IClickhouseSession-): Job<EventsQueuePayloadCreateSessionEnd> =>- ({- id: 'job-1',- data: {- type: 'createSessionEnd',- payload: {- projectId: 'proj-1',- deviceId: 'dev-1',- profileId: 'dev-1',- name: 'session_end',- properties: {},- } as unknown as IServiceCreateEventPayload,- snapshot: snap,- },- }) as unknown as Job<EventsQueuePayloadCreateSessionEnd>;--beforeEach(() => {- vi.clearAllMocks();- redisMock.set.mockResolvedValue('OK'); // claim acquired by default- cleanup.mockResolvedValue(undefined as never);- createEventMock.mockResolvedValue({ document: { id: 'evt-1' } } as never);-});--describe('createSessionEnd', () => {- it('emits session_end and id-gated cleanup on the happy path', async () => {- getExistingSession.mockResolvedValue(snapshot); // live == snapshot, unchanged-- await createSessionEnd(makeJob(snapshot));-- expect(createEventMock).toHaveBeenCalledTimes(1);- expect(createEventMock.mock.calls[0]![0]).toMatchObject({- name: 'session_end',- sessionId: 'sess-1',- });- expect(cleanup).toHaveBeenCalledWith({- projectId: 'proj-1',- deviceId: 'dev-1',- sessionId: 'sess-1',- profileId: 'dev-1',- });- });-- it('skips when the session was extended after the close was enqueued', async () => {- getExistingSession.mockResolvedValue({- ...snapshot,- ended_at: '2026-06-08 11:05:00', // later than snapshot → still active- });-- const result = await createSessionEnd(makeJob(snapshot));-- expect(result).toBeNull();- expect(createEventMock).not.toHaveBeenCalled();- expect(cleanup).not.toHaveBeenCalled();- });-- it('skips when the idempotency claim is already taken', async () => {- getExistingSession.mockResolvedValue(snapshot);- redisMock.set.mockResolvedValue(null); // SET NX failed → already emitted-- const result = await createSessionEnd(makeJob(snapshot));-- expect(result).toBeNull();- expect(createEventMock).not.toHaveBeenCalled();- });-- it('uses the snapshot when a new session already owns the slot (boundary)', async () => {- // A boundary opened a fresh session under the same (project, device) slot.- getExistingSession.mockResolvedValue({- ...snapshot,- id: 'sess-2',- ended_at: '2026-06-08 12:00:00',- });-- await createSessionEnd(makeJob(snapshot));-- // Emits for the CLOSED session (snapshot), not the live one.- expect(createEventMock.mock.calls[0]![0]).toMatchObject({- sessionId: 'sess-1',- });- // cleanup is keyed on the closed id → id-gated Lua no-ops against sess-2.- expect(cleanup).toHaveBeenCalledWith(- expect.objectContaining({ sessionId: 'sess-1' })- );- });-});diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 460a40d76..d0539c15d 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -1,228 +1,23 @@+// Dissolved into @openpanel/core's session module (M7-001): the three-case+// live-vs-snapshot logic, the idempotency claim and the funnel-rule check+// moved to packages/core/src/modules/session/src/session-end.ts (and its+// ClickHouse query onto the `sql` tag). This file stays (DELEGATE PATTERN) —+// it adapts V1's BullMQ `{ type, payload, snapshot }` job to core's+// `{ event, snapshot }` and is what `sessions.ts` still calls.import {- checkNotificationRulesForSessionEnd,- convertClickhouseDateToJs,- createEvent,- formatClickhouseDate,- getEvents,- getHasFunnelRules,- getNotificationRulesByProjectId,- type IClickhouseSession,- type IServiceCreateEventPayload,- type IServiceEvent,- profileBackfillBuffer,- sessionBuffer,- TABLE_NAMES,- transformEvent,- transformSessionToEvent,-} from '@openpanel/db';+ createSessionEnd as createSessionEndCore,+ loadSessionEndDeps,+ loadSessionRuntime,+} from '@openpanel/core';import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';-import { getRedisCache } from '@openpanel/redis';import type { Job } from 'bullmq';import { logger as baseLogger } from '@/utils/logger';-import {- sessionDurationOnClose,- sessionEndsEmitted,- sessionEndsSkipped,- sessionEventsOnClose,-} from '@/metrics';--const MAX_SESSION_EVENTS = 500;-const SESSION_END_CLAIM_TTL = 60 * 60 * 2; // 2h, well past any plausible retry window--async function getSessionEvents({- session,- startAt,- endAt,-}: {- session: IClickhouseSession;- startAt: Date;- endAt: Date;-}): Promise<IServiceEvent[]> {- const sql = `- SELECT * FROM ${TABLE_NAMES.events}- WHERE- session_id = '${session.id}'- AND project_id = '${session.project_id}'- AND created_at BETWEEN '${formatClickhouseDate(new Date(startAt.getTime() - 1000))}' AND '${formatClickhouseDate(new Date(endAt.getTime() + 1000))}'- ORDER BY created_at DESC LIMIT ${MAX_SESSION_EVENTS};- `;-- const eventsInDb = await getEvents(sql);-- return [transformSessionToEvent(session), ...eventsInDb]- .sort(- (a, b) =>- new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()- );-}export async function createSessionEnd(job: Job<EventsQueuePayloadCreateSessionEnd>) {const { payload, snapshot } = job.data;- const logger = baseLogger.child({- payload,- jobId: job.id,- });-- logger.debug('Processing session end job');-- // Prefer the live blob — it reflects any late extensions that arrived after- // enqueue. Fall back to the snapshot if the blob expired (very long queue- // lag, retries hours later, etc.).- const live = await sessionBuffer.getExistingSession({- projectId: payload.projectId,- deviceId: payload.deviceId,- });-- // Distinguish three cases:- // - Same session, extended after enqueue → bail. Reaper will retry later.- // - Same session, no change → use live (snapshot equivalent).- // - Different session in slot (boundary) → use snapshot for emission;- // cleanup will be a no-op since the slot is owned by the new session.- // - No live blob at all (TTL expired) → use snapshot.- const sameSession = live && live.id === snapshot.id;- if (sameSession && live.ended_at > snapshot.ended_at) {- sessionEndsSkipped.inc({ reason: 'extended_after_enqueue' });- logger.info(- {- sessionId: live.id,- projectId: live.project_id,- snapshotEndedAt: snapshot.ended_at,- liveEndedAt: live.ended_at,- },- 'session was extended after close was enqueued, skipping',- );- return null;- }-- const session = sameSession ? live : snapshot;- if (!session) {- sessionEndsSkipped.inc({ reason: 'not_found' });- logger.warn(- { projectId: payload.projectId, deviceId: payload.deviceId },- 'No live session and no snapshot — skipping session_end',- );- return null;- }-- // Idempotency claim: first writer wins. Subsequent retries / reaper- // double-fires are no-ops.- const redis = getRedisCache();- const claimKey = `session:end:emitted:${session.project_id}:${session.device_id}:${session.id}`;- const claimed = await redis.set(claimKey, '1', 'EX', SESSION_END_CLAIM_TTL, 'NX');- if (claimed === null) {- sessionEndsSkipped.inc({ reason: 'already_emitted' });- logger.info(- { sessionId: session.id, projectId: session.project_id },- 'session_end already emitted, skipping',- );- return null;- }-- sessionEndsEmitted.inc();- sessionDurationOnClose.observe(Math.max(0, session.duration ?? 0));- sessionEventsOnClose.observe(- (session.event_count ?? 0) + (session.screen_view_count ?? 0)- );-- const profileId = session.profile_id || payload.profileId;-- if (- profileId !== session.device_id &&- process.env.EXPERIMENTAL_PROFILE_BACKFILL === '1'- ) {- const runOnProjects =- process.env.EXPERIMENTAL_PROFILE_BACKFILL_PROJECTS?.split(',').filter(- Boolean- ) ?? [];- if (- runOnProjects.length === 0 ||- runOnProjects.includes(payload.projectId)- ) {- await profileBackfillBuffer.add({- projectId: payload.projectId,- sessionId: session.id,- profileId,- });- }- }-- // Create session end event- const { document: sessionEndEvent } = await createEvent({- ...payload,- sessionId: session.id,- properties: {- ...payload.properties,- __bounce: session.is_bounce,- },- name: 'session_end',- duration: session.duration ?? 0,- path: session.exit_path ?? '',- createdAt: new Date(- convertClickhouseDateToJs(session.ended_at).getTime() + 1000- ),- profileId,- });-- try {- await handleSessionEndNotifications({- session,- payload,- sessionEndEvent: transformEvent(sessionEndEvent),- });- } catch (error) {- logger.error(- { err: error },- 'Creating notificatios for session end failed',- );- }-- // Clean up Redis state for this session. cleanup() is id-gated — if a new- // session has already taken over this (projectId, deviceId) slot via a- // boundary, the call is a no-op and the new session is preserved.- await sessionBuffer- .cleanup({- projectId: session.project_id,- deviceId: session.device_id,- sessionId: session.id,- profileId: session.profile_id,- })- .catch((error) => {- logger.error(- { err: error, sessionId: session.id },- 'Failed to cleanup session state after session_end',- );- });-- return sessionEndEvent;-}--async function handleSessionEndNotifications({- session,- payload,- sessionEndEvent,-}: {- session: IClickhouseSession;- payload: IServiceCreateEventPayload;- sessionEndEvent: IServiceEvent;-}) {- const notificationRules = await getNotificationRulesByProjectId(- payload.projectId- );- const hasFunnelRules = getHasFunnelRules(notificationRules);- const isEventCountReasonable =- session.event_count + session.screen_view_count < MAX_SESSION_EVENTS;-- if (hasFunnelRules && isEventCountReasonable) {- const events = await getSessionEvents({- session,- startAt: convertClickhouseDateToJs(session.created_at),- endAt: convertClickhouseDateToJs(session.ended_at),- });-- if (events.length > 0) {- await checkNotificationRulesForSessionEnd([...events, sessionEndEvent]);- }- }+ const logger = baseLogger.child({ payload, jobId: job.id });+ const deps = await loadSessionEndDeps(await loadSessionRuntime(), logger);+ return createSessionEndCore({ event: payload, snapshot }, deps);}diff --git a/apps/worker/src/jobs/sessions.ts b/apps/worker/src/jobs/sessions.tsindex 808d7ba51..2618094e1 100644--- a/apps/worker/src/jobs/sessions.ts+++ b/apps/worker/src/jobs/sessions.ts@@ -1,19 +1,13 @@-import type { Organization } from '@openpanel/db';-import {- db,- getOrganizationBillingEventsCount,- getProjectEventsCount,-} from '@openpanel/db';-import { sendEmail } from '@openpanel/email';+// Dissolved into @openpanel/core's session module (M7-001): the events-count+// refresh and the 80%/100% usage alerts moved to+// packages/core/src/modules/session/src/usage.ts. This file stays (DELEGATE+// PATTERN) — it is what `boot-workers.ts` hands V1's BullMQ sessions Worker.+import { updateEventsCount } from '@openpanel/core';import type { SessionsQueuePayload } from '@openpanel/queue';-import { cacheable } from '@openpanel/redis';import type { Job } from 'bullmq';import { createSessionEnd } from './events.create-session-end';import { logger } from '@/utils/logger';-const INT4_MAX = 2_147_483_647;-const USAGE_WARNING_THRESHOLD = 0.8;-export async function sessionsJob(job: Job<SessionsQueuePayload>) {const res = await createSessionEnd(job);try {@@ -23,206 +17,3 @@ export async function sessionsJob(job: Job<SessionsQueuePayload>) {}return res;}--const updateEventsCount = cacheable(async function updateEventsCount(- projectId: string-) {- const organization = await db.organization.findFirst({- where: {- projects: {- some: {- id: projectId,- },- },- },- include: {- projects: true,- },- });-- if (!organization) {- return;- }-- const organizationEventsCount =- await getOrganizationBillingEventsCount(organization);- const projectEventsCount = await getProjectEventsCount(projectId);-- if (projectEventsCount) {- await db.project.update({- where: {- id: projectId,- },- data: {- // Saturating counter: the column is INT4 and lifetime counts can- // exceed it. It's only used as a sort key and activity threshold,- // never for billing, so clamping is safe.- eventsCount: Math.min(projectEventsCount, INT4_MAX),- },- });- }-- if (organizationEventsCount) {- // Self-hosting has no billing/event limits. Never flag the org as- // exceeded, and clear any stale flag that was set before this guard- // existed (default limit is 0, which otherwise trips on the first event).- const isSelfHosted = process.env.SELF_HOSTED === 'true';-- await db.organization.update({- where: {- id: organization.id,- },- data: {- subscriptionPeriodEventsCount: Math.min(- organizationEventsCount,- INT4_MAX- ),- subscriptionPeriodEventsCountExceededAt: isSelfHosted- ? null- : organizationEventsCount >- organization.subscriptionPeriodEventsLimit &&- !organization.subscriptionPeriodEventsCountExceededAt- ? new Date()- : organizationEventsCount <=- organization.subscriptionPeriodEventsLimit- ? null- : organization.subscriptionPeriodEventsCountExceededAt,- },- });-- if (!isSelfHosted) {- try {- await sendUsageAlerts(organization, organizationEventsCount);- } catch (e) {- logger.error({ err: e }, 'Failed to send usage alert emails');- }- }- }-- return true;-}, 60 * 60);--/**- * One warning at 80% and one notice at 100% per billing cycle. The sent-at- * markers are cleared by the Polar webhook when a new cycle resets the usage- * counter (or the limit is raised), so each cycle can alert again.- */-async function sendUsageAlerts(organization: Organization, count: number) {- const limit = organization.subscriptionPeriodEventsLimit;- if (!limit || limit <= 0) {- return;- }-- const exceeded = count > limit && !organization.usageExceededSentAt;- const nearLimit =- !exceeded &&- count >= limit * USAGE_WARNING_THRESHOLD &&- count <= limit &&- !organization.usageWarningSentAt;-- if (!(exceeded || nearLimit)) {- return;- }-- // Claim the alert atomically BEFORE sending: session jobs for different- // projects of the same org can run concurrently, and both would otherwise- // read null markers and double-send. Marking the warning together with the- // exceeded notice keeps a both-thresholds-in-one-jump crossing from queueing- // a redundant warning afterwards. Rolled back if every send fails.- const claimedAt = new Date();- const claimed = await db.organization.updateMany({- where: {- id: organization.id,- ...(exceeded- ? { usageExceededSentAt: null }- : { usageWarningSentAt: null }),- },- data: exceeded- ? { usageExceededSentAt: claimedAt, usageWarningSentAt: claimedAt }- : { usageWarningSentAt: claimedAt },- });- if (claimed.count === 0) {- return;- }-- try {- const admins = await db.member.findMany({- where: {- organizationId: organization.id,- role: 'org:admin',- user: { deletedAt: null },- },- include: { user: { select: { email: true, firstName: true } } },- });-- const billingUrl = `${process.env.DASHBOARD_URL ?? 'https://dashboard.openpanel.dev'}/${organization.id}/billing`;- const recipients = new Map(- admins- .filter((member) => member.user?.email)- .map((member) => [- member.user!.email,- member.user!.firstName ?? undefined,- ])- );-- let failedRecipients = 0;- for (const [email, firstName] of recipients) {- // Per-recipient guard: one bad address must not abort the loop or roll- // back the claim — that would re-email the recipients that succeeded.- try {- if (exceeded) {- await sendEmail('usage-limit-exceeded', {- to: email,- data: {- firstName,- organizationName: organization.name,- billingUrl,- eventsLimit: limit,- },- });- } else {- await sendEmail('usage-near-limit', {- to: email,- data: {- firstName,- organizationName: organization.name,- billingUrl,- eventsCount: count,- eventsLimit: limit,- },- });- }- } catch (error) {- failedRecipients++;- logger.error(- { err: error, organizationId: organization.id, recipient: email },- 'Failed to send usage alert to recipient'- );- }- }-- logger.info(- {- organizationId: organization.id,- count,- limit,- kind: exceeded ? 'exceeded' : 'near-limit',- recipients: recipients.size,- failedRecipients,- },- 'Sent usage alert emails'- );- } catch (error) {- // Release the claim so the next usage update retries the alert.- await db.organization.updateMany({- where: { id: organization.id },- data: exceeded- ? {- usageExceededSentAt: null,- usageWarningSentAt: organization.usageWarningSentAt,- }- : { usageWarningSentAt: null },- });- throw error;- }-}diff --git a/apps/worker/src/utils/session-handler.ts b/apps/worker/src/utils/session-handler.tsindex ecf43dd9f..73f5708a5 100644--- a/apps/worker/src/utils/session-handler.ts+++ b/apps/worker/src/utils/session-handler.ts@@ -1,27 +1,22 @@-import type {- IClickhouseSession,- IServiceCreateEventPayload,-} from '@openpanel/db';+// The jobId/payload/retry rules live in @openpanel/core now (M7-001):+// packages/core/src/modules/session/src/session-end.ts. This file stays+// (DELEGATE PATTERN) because V1's BullMQ `sessionsQueue` wire shape is+// `{ type: 'createSessionEnd', payload, snapshot }` — core's+// `legacyCompat.sessions` reads exactly that — and core cannot import+// `@openpanel/queue` (cycle), so the `.add()` itself stays here.+import type { EnqueueSessionEndInput } from '@openpanel/core';+import {+ getSessionEndJobId,+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+} from '@openpanel/core';import { SESSION_TIMEOUT_MS } from '@openpanel/db';import { sessionsQueue } from '@openpanel/queue';-export { SESSION_TIMEOUT_MS };+export { SESSION_TIMEOUT_MS } from '@openpanel/db';export const SESSION_TIMEOUT = SESSION_TIMEOUT_MS;-/**- * Deterministic v2 jobId for a closed session.- *- * Keyed on the session's stable `id` so concurrent / retried closes for the- * same logical session dedupe in BullMQ. The `v2:` prefix lets the legacy- * drain script differentiate from pre-migration delayed jobs.- *- * Format constraint: BullMQ only accepts ':' in custom jobIds when splitting- * by ':' yields exactly 3 parts, so we keep the suffix as a single segment.- * Cross-project sessionInternalId collisions would be astronomical given the- * 128-bit hash used to generate them.- */-export const getSessionEndJobIdV2 = (sessionInternalId: string) =>- `sessionEnd:v2:${sessionInternalId}`;+export const getSessionEndJobIdV2 = getSessionEndJobId;/*** Enqueue a session_end job. Idempotent via jobId.@@ -29,35 +24,12 @@ export const getSessionEndJobIdV2 = (sessionInternalId: string) =>* Called when a boundary is detected during ingest (old session must close)* or by the reaper when a session has been idle past the timeout.*/-export async function enqueueSessionEndV2({- payload,- closedSession,-}: {- payload: IServiceCreateEventPayload;- closedSession: IClickhouseSession;-}) {- const jobId = getSessionEndJobIdV2(closedSession.id);+export async function enqueueSessionEndV2(input: EnqueueSessionEndInput) {+ const { event, snapshot } = sessionEndJobPayload(input);return sessionsQueue.add('session',- {- type: 'createSessionEnd',- payload: {- ...payload,- projectId: closedSession.project_id,- deviceId: closedSession.device_id,- sessionId: closedSession.id,- profileId: closedSession.profile_id || payload.profileId,- },- snapshot: closedSession,- },- {- jobId,- attempts: 3,- backoff: {- type: 'exponential',- delay: 200,- },- }+ { type: 'createSessionEnd', payload: event, snapshot },+ sessionEndEnqueueOptions(input.closedSession.id));}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 4c5b00df5..e8a95b01e 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -523,6 +523,53 @@ export {listReferences,updateReference,} from './modules/reference/reference.service';+// Dissolved from @openpanel/db's services/session.service.ts and+// session-context.ts, plus apps/worker's session-end job, reaper and vacuum+// (M7-001, ADR-008's module map: session owns "R,S,J") — packages/trpc's+// session router, the assistant/mcp tools and apps/worker's thin delegates+// call these directly, the same way V1 reaches every other dissolved service+// here. packages/db keeps re-export shims for both files. The `sessions`+// queue's own job and the reaper/vacuum cron fragments are registered in+// jobs.registry.ts, not exported.+export type {+ GetSessionListOptions,+ IClickhouseSession,+ IServiceSession,+ ISessionReplayChunkMeta,+ QuerySessionsInput,+ SessionDistinctField,+} from './modules/session/session.service';+export {+ getSessionById,+ getSessionDistinctValues,+ getSessionList,+ getSessionReplayChunksFrom,+ getSessionsCount,+ getSessionsCountCached,+ querySessionsCore,+ SESSION_DISTINCT_FIELDS,+ transformSession,+} from './modules/session/session.service';+export { loadSessionRuntime } from './modules/session/src/runtime';+export {+ als,+ getAlsSessionId,+ runWithAlsSession,+} from './modules/session/src/session-context';+export type {+ EnqueueSessionEndInput,+ SessionEndJobData,+} from './modules/session/src/session-end';+export {+ createSessionEnd,+ getSessionEndJobId,+ loadSessionEndDeps,+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+} from './modules/session/src/session-end';+export { reapIdleSessions } from './modules/session/src/session-reaper';+export { vacuumStaleSessions } from './modules/session/src/session-vacuum';+export { updateEventsCount } from './modules/session/src/usage';// Moved from packages/db/src/services/share.service.ts, plus the// query/mutation bodies packages/trpc/src/routers/share.ts held inline// (M6-004, ADR-008's module map: share owns "C") — packages/trpc's sharediff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex 082df1e9e..f2aede20b 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -89,6 +89,8 @@ test('the cron queue carries the insight module cron fragment', () => {'insightCleanup','insightsDaily','onboarding',+ 'sessionReaper',+ 'sessionVacuum','weeklyDigest',].sort());@@ -155,6 +157,25 @@ test('the cron queue carries the onboarding module cron fragment', () => {});});+// M7-001: the session module's own job (legacyCompat.sessions maps V1's+// `createSessionEnd` onto this exact name) and its reaper/vacuum cron+// fragment — the scheduler ids double as the job names.+test('the sessions queue carries the session module job', () => {+ expect(Object.keys(queues.sessions.jobs)).toEqual(['session']);+ expect(queues.sessions.jobs.session).toMatchObject({+ queue: 'sessions',+ name: 'session',+ });+});++test('the cron queue carries the session module cron fragment', () => {+ for (const name of ['sessionReaper', 'sessionVacuum']) {+ expect(+ queues.cron.jobs[name as keyof typeof queues.cron.jobs]+ ).toMatchObject({ queue: 'cron', name });+ }+});+// M5-004: the import module's own job. No cron fragment — imports are always// user-triggered (import.rpc.ts's create/retry), unlike gsc/cohort/insight.// legacyCompat.import already discriminates on this exact namediff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex 13ba6f32c..cbb990493 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -24,6 +24,10 @@ import {import { notificationQueueJobs } from './modules/notification/notification.jobs';import { onboardingCronJobs } from './modules/onboarding/onboarding.jobs';import { organizationCronJobs } from './modules/organization/organization.jobs';+import {+ sessionCronJobs,+ sessionQueueJobs,+} from './modules/session/session.jobs';const HOUR_IN_SECONDS = 3600;const DAY_IN_SECONDS = 86_400;@@ -46,7 +50,9 @@ const sessions = defineQueue('sessions', {defaults: { removeOnComplete: true },worker: { concurrency: CONCURRENCY.sessions },compat: legacyCompat.sessions,- jobs: {},+ jobs: {+ ...sessionQueueJobs,+ },});// One queue, one worker at concurrency 1, every scheduled job in the system.@@ -63,6 +69,7 @@ const cron = defineQueue('cron', {...cohortCronJobs,...organizationCronJobs,...onboardingCronJobs,+ ...sessionCronJobs,},});diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsindex 72733b703..4ecbccfe5 100644--- a/packages/core/src/jobs/schedulers.ts+++ b/packages/core/src/jobs/schedulers.ts@@ -10,6 +10,7 @@ import { gscCronSchedules } from '../modules/gsc/gsc.jobs';import { insightCronSchedules } from '../modules/insight/insight.jobs';import { onboardingCronSchedules } from '../modules/onboarding/onboarding.jobs';import { organizationCronSchedules } from '../modules/organization/organization.jobs';+import { sessionCronSchedules } from '../modules/session/session.jobs';import { wrap } from './envelope';export type RepeatSchedule = { pattern: string } | { every: number };@@ -77,9 +78,9 @@ export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [// cohortRefresh — owned by the cohort module, declared next to its jobs// (modules/cohort/cohort.jobs.ts)....cohortCronSchedules,- { id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },- // Daily 04:00 UTC — backstop for cleanup leaks.- { id: 'sessionVacuum', schedule: { pattern: '0 4 * * *' } },+ // sessionReaper / sessionVacuum — owned by the session module, declared+ // next to its jobs (modules/session/session.jobs.ts).+ ...sessionCronSchedules,// insightsDaily / insightCleanup / weeklyDigest — owned by the insight// module, declared next to its jobs (modules/insight/insight.jobs.ts)....insightCronSchedules,diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex 79091c719..7f3af1b8b 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -38,6 +38,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {cohort: cohort as CohortService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsindex ed72684bb..fff1b7051 100644--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -34,6 +34,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {gsc: gsc as GscService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},@@ -134,6 +135,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', async} as GscService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'gscSync' },diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.tsindex fd40f4801..8ce853768 100644--- a/packages/core/src/modules/import/import.jobs.test.ts+++ b/packages/core/src/modules/import/import.jobs.test.ts@@ -34,6 +34,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {import: importService as ImportService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex bbbdc84e4..a1f3c9f4e 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -45,6 +45,7 @@ function stubJobCtx(import: {} as Services['import'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},@@ -128,6 +129,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId deimport: {} as Services['import'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.tsindex a638f0279..29e993f99 100644--- a/packages/core/src/modules/notification/notification.jobs.test.ts+++ b/packages/core/src/modules/notification/notification.jobs.test.ts@@ -34,6 +34,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {import: {} as Services['import'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},@@ -45,7 +46,12 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {queues: createRecordingProducers(queues).queues,services,requestId: 'req_1',- job: { id: 'job_1', attempt: 0, queue: 'notification', name: 'sendNotification' },+ job: {+ id: 'job_1',+ attempt: 0,+ queue: 'notification',+ name: 'sendNotification',+ },};}diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.tsindex 805101b05..27cd1cac3 100644--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts@@ -37,6 +37,7 @@ function stubJobCtx(cohort: {} as Services['cohort'],organization: {} as Services['organization'],onboarding: onboarding as OnboardingService,+ session: {} as Services['session'],};return {db: {},diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.tsindex 60bbb4d23..11cac0848 100644--- a/packages/core/src/modules/organization/organization.jobs.test.ts+++ b/packages/core/src/modules/organization/organization.jobs.test.ts@@ -40,6 +40,7 @@ function stubJobCtx(cohort: {} as Services['cohort'],organization: organization as OrganizationService,onboarding: {} as Services['onboarding'],+ session: {} as Services['session'],};return {db: {},diff --git a/packages/core/src/modules/session/session.jobs.ts b/packages/core/src/modules/session/session.jobs.tsnew file mode 100644index 000000000..36ddc2639--- /dev/null+++ b/packages/core/src/modules/session/session.jobs.ts@@ -0,0 +1,97 @@+// Ported from apps/worker/src/jobs/sessions.ts (+ events.create-session-end.ts),+// cron.session-reaper.ts and cron.session-vacuum.ts (M7-001).+//+// `sessions` is this module's own queue — registry key and Redis name+// `sessions`, job name `session`, `removeOnComplete: true`, concurrency 1, all+// V1's (jobs.registry.ts). Its wire payload is `{ event, snapshot }`, which is+// what `legacyCompat.sessions` maps V1's `{type:'createSessionEnd', payload,+// snapshot}` onto. `sessionReaper` / `sessionVacuum` are cron fragments:+// declared here, spread into the ONE `cron` queue by jobs.registry.ts and into+// `CRON_SCHEDULES` by jobs/schedulers.ts, ids and cadences unchanged+// (apps/worker/src/boot-cron.ts).++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';+import { loadSessionRuntime } from './src/runtime';+import {+ createSessionEnd,+ loadSessionEndDeps,+ type SessionEndJobData,+} from './src/session-end';+import { reapIdleSessions } from './src/session-reaper';+import { vacuumStaleSessions } from './src/session-vacuum';+import { updateEventsCount } from './src/usage';++const MINUTE_MS = 60 * 1000;+const SESSION_REAPER_INTERVAL_MS = 5 * MINUTE_MS;+/** Daily 04:00 UTC — backstop for cleanup leaks. */+const SESSION_VACUUM_CRON = '0 4 * * *';++// Wire-shape check only — V1 never validated it either (see+// jobs/compat.test.ts). `event` is an IServiceCreateEventPayload after a JSON+// round-trip (its `createdAt` is a string; the handler replaces it anyway),+// `snapshot` an IClickhouseSession. Typed via `z.custom` so the producer side+// (`queues.sessions.session.add`) takes the real `SessionEndJobData`.+const sessionEndWireShape = z.object({+ event: z.object({ projectId: z.string(), deviceId: z.string() }),+ snapshot: z.object({+ id: z.string(),+ project_id: z.string(),+ device_id: z.string(),+ ended_at: z.string(),+ }),+});++const sessionEndPayload = z.custom<SessionEndJobData>(+ (value) => sessionEndWireShape.safeParse(value).success+);++/** The `sessions` queue's own job. */+export const sessionQueueJobs = {+ session: defineJob({+ payload: sessionEndPayload,+ handler: async ({ payload: data, ctx }) => {+ const logger = ctx.logger.child({ payload: data.event });+ const deps = await loadSessionEndDeps(await loadSessionRuntime(), logger);++ await createSessionEnd(data, deps);++ try {+ await updateEventsCount(data.event.projectId);+ } catch (error) {+ logger.error({ err: error }, 'Failed to update events count');+ }+ },+ }),+};++/** This module's fragment of the `cron` queue's jobs. */+export const sessionCronJobs = {+ sessionReaper: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await reapIdleSessions({+ ...(await loadSessionRuntime()),+ logger: ctx.logger.child({ job: 'session-reaper' }),+ enqueueSessionEnd: (input) =>+ ctx.services.session.enqueueSessionEnd(input),+ });+ },+ }),+ sessionVacuum: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await vacuumStaleSessions({+ ...(await loadSessionRuntime()),+ logger: ctx.logger.child({ job: 'session-vacuum' }),+ });+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — ids and cadences unchanged. */+export const sessionCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'sessionReaper', schedule: { every: SESSION_REAPER_INTERVAL_MS } },+ { id: 'sessionVacuum', schedule: { pattern: SESSION_VACUUM_CRON } },+];diff --git a/packages/core/src/modules/session/session.rpc.test.ts b/packages/core/src/modules/session/session.rpc.test.tsnew file mode 100644index 000000000..96cd09648--- /dev/null+++ b/packages/core/src/modules/session/session.rpc.test.ts@@ -0,0 +1,69 @@+// Only the "is anyone logged in" boundary is exercised here — no database.+// The access-check + query bodies ride on @openpanel/db (lazy-loaded, see+// session.service.ts's header); wiring this router end-to-end against a real+// ClickHouse is P6's (protectedProcedure) job — see session.rpc.ts's header.+// Same shape as realtime.rpc.test.ts.++import { expect, test } from 'bun:test';+import { stubHttpCtx } from '../../../test/rpc-fixtures';+import { makeTrpcContext } from '../../rpc/base';+import type { CookieOptions } from '../../shared/cookie';+import { sessionRouter } from './session.rpc';++const COOKIE_OPTIONS: CookieOptions = {+ domain: '.openpanel.dev',+ secure: true,+ sameSite: 'lax',+ httpOnly: true,+ path: '/',+};++// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —+// `TrpcContext.session` is never literally `null`, only its `userId` is.+const EMPTY_SESSION = { session: null, user: null, userId: null };++async function anonCaller() {+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {+ cookieOptions: COOKIE_OPTIONS,+ });+ return sessionRouter.createCaller(trpcCtx);+}++test('list rejects an unauthenticated caller before querying ClickHouse', async () => {+ const caller = await anonCaller();+ await expect(caller.list({ projectId: 'proj_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});++test('distinctValues rejects an unauthenticated caller before querying ClickHouse', async () => {+ const caller = await anonCaller();+ await expect(+ caller.distinctValues({ projectId: 'proj_1', field: 'country' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('distinctValues rejects a field outside SESSION_DISTINCT_FIELDS at the input boundary', async () => {+ const caller = await anonCaller();+ await expect(+ caller.distinctValues({+ projectId: 'proj_1',+ field: 'profile_id' as never,+ })+ ).rejects.toMatchObject({ code: 'BAD_REQUEST' });+});++test('byId rejects an unauthenticated caller before querying ClickHouse', async () => {+ const caller = await anonCaller();+ await expect(+ caller.byId({ projectId: 'proj_1', sessionId: 'sess_1' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('replayChunksFrom rejects an unauthenticated caller before querying ClickHouse', async () => {+ const caller = await anonCaller();+ await expect(+ caller.replayChunksFrom({ projectId: 'proj_1', sessionId: 'sess_1' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});diff --git a/packages/core/src/modules/session/session.rpc.ts b/packages/core/src/modules/session/session.rpc.tsnew file mode 100644index 000000000..5f8495ac9--- /dev/null+++ b/packages/core/src/modules/session/session.rpc.ts@@ -0,0 +1,109 @@+// Ported from packages/trpc/src/routers/session.ts (M7-001).+//+// Same arrangement as realtime.rpc.ts: V1's `protectedProcedure` stack lands+// with auth (P6), so until then each procedure does its own "is anyone logged+// in" + `requireProjectAccess({ level: 'read' })` off the `projectId` input,+// which is what V1's `enforceUserIsAuthed` + `enforceAccess` middleware pair+// does implicitly. packages/trpc's session router delegates its handler bodies+// onto `./session.service` while keeping V1's own `protectedProcedure` stack.+//+// V1's file also exported `encodeCursor` / `decodeCursor`; nothing imports+// them and `shared/pagination.ts` already has the generic pair, so they were+// not ported.++import { zChartEventFilter } from '@openpanel/validation';+import { z } from 'zod';+import { createTRPCRouter, procedure } from '../../rpc/base';+import { TRPCAccessError } from '../../rpc/errors';+import {+ getSessionById,+ getSessionDistinctValues,+ getSessionList,+ getSessionReplayChunksFrom,+ SESSION_DISTINCT_FIELDS,+} from './session.service';++const DEFAULT_LIST_TAKE = 50;++function loadAccessChecks() {+ return import('./src/access');+}++function requireLogin(userId: string | null | undefined): string {+ if (!userId) {+ throw new TRPCAccessError('Not authenticated');+ }+ return userId;+}++async function requireReadAccess(userId: string, projectId: string) {+ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({ userId, projectId, level: 'read' });+}++export const sessionRouter = createTRPCRouter({+ list: procedure+ .input(+ z.object({+ projectId: z.string(),+ profileId: z.string().optional(),+ cursor: z.string().nullish(),+ filters: z.array(zChartEventFilter).default([]),+ startDate: z.date().optional(),+ endDate: z.date().optional(),+ search: z.string().optional(),+ take: z.number().default(DEFAULT_LIST_TAKE),+ })+ )+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireReadAccess(userId, input.projectId);++ return getSessionList({+ ...input,+ cursor: input.cursor ? new Date(input.cursor) : undefined,+ });+ }),++ distinctValues: procedure+ .input(+ z.object({+ projectId: z.string(),+ field: z.enum(SESSION_DISTINCT_FIELDS),+ })+ )+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireReadAccess(userId, input.projectId);++ return getSessionDistinctValues(input.projectId, input.field);+ }),++ byId: procedure+ .input(z.object({ sessionId: z.string(), projectId: z.string() }))+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireReadAccess(userId, input.projectId);++ return getSessionById(input.sessionId, input.projectId);+ }),++ replayChunksFrom: procedure+ .input(+ z.object({+ sessionId: z.string(),+ projectId: z.string(),+ fromIndex: z.number().int().min(0).default(0),+ })+ )+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireReadAccess(userId, input.projectId);++ return getSessionReplayChunksFrom(+ input.sessionId,+ input.projectId,+ input.fromIndex+ );+ }),+});diff --git a/packages/core/src/modules/session/session.service.ts b/packages/core/src/modules/session/session.service.tsnew file mode 100644index 000000000..8b51e4be0--- /dev/null+++ b/packages/core/src/modules/session/session.service.ts@@ -0,0 +1,500 @@+// Ported from packages/db/src/services/session.service.ts (M7-001). That file+// is now a re-export shim over this one, so @openpanel/db importers+// (packages/trpc's session + event routers, apps/api's insights controller,+// apps/start's types) keep working while V1 runs (DELEGATE PATTERN).+//+// Every query here is a `sql` fragment (src/session.sql.ts) — the module's+// queries are converted, one at a time with a result-set proof each, per+// ADR-013. `buildFilterWhere` is NOT converted here: it is the shared filter+// compiler, out of this task's scope; src/filter-clauses.ts is the one bridge.+//+// db/ch access is LAZY (`load*` below) for the reason insight.service.ts's+// header gives: constructing @openpanel/db's clients at import time spawns a+// pino-pretty worker per `bun test --isolate` file. `@openpanel/redis`'s+// `cacheable` and the `sql` tag have no such side effect.++import type { IServiceProfile } from '@openpanel/db/src/services/profile.service';+import { cacheable } from '@openpanel/redis';+import type { IChartEventFilter } from '@openpanel/validation';+import type { ServiceDeps } from '../../services';+import { getSafeJson } from '../../shared/json';+import { convertClickhouseDateToJs } from './src/dates';+import {+ hasSessionListLookback,+ querySessionsQuery,+ type SessionDistinctField,+ sessionByIdQuery,+ sessionDistinctValuesQuery,+ sessionHasReplayQuery,+ sessionListQuery,+ sessionReplayChunksQuery,+ sessionsCountQuery,+} from './src/session.sql';+import {+ type EnqueueSessionEndInput,+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+} from './src/session-end';++export {+ SESSION_DISTINCT_FIELDS,+ type SessionDistinctField,+} from './src/session.sql';++function loadChClient() {+ return import('@openpanel/db/src/clickhouse/client');+}++function loadProfileService() {+ return import('@openpanel/db/src/services/profile.service');+}++function loadFilterCompiler() {+ return import('@openpanel/db/src/services/filter-where.service');+}++function loadLookback() {+ return import('@openpanel/db/src/services/lookback');+}++function loadDateService() {+ return import('@openpanel/db/src/services/date.service');+}++export interface IClickhouseSession {+ id: string;+ profile_id: string;+ event_count: number;+ screen_view_count: number;+ entry_path: string;+ entry_origin: string;+ exit_path: string;+ exit_origin: string;+ created_at: string;+ ended_at: string;+ referrer: string;+ referrer_name: string;+ referrer_type: string;+ os: string;+ os_version: string;+ browser: string;+ browser_version: string;+ device: string;+ brand: string;+ model: string;+ country: string;+ region: string;+ city: string;+ longitude: number | null;+ latitude: number | null;+ is_bounce: boolean;+ project_id: string;+ device_id: string;+ duration: number;+ utm_medium: string;+ utm_source: string;+ utm_campaign: string;+ utm_content: string;+ utm_term: string;+ revenue: number;+ sign: 1 | -1;+ version: number;+ // Dynamically added+ has_replay?: boolean;+ groups: string[];+}++export interface IServiceSession {+ id: string;+ profileId: string;+ eventCount: number;+ screenViewCount: number;+ entryPath: string;+ entryOrigin: string;+ exitPath: string;+ exitOrigin: string;+ createdAt: Date;+ endedAt: Date;+ referrer: string;+ referrerName: string;+ referrerType: string;+ os: string;+ osVersion: string;+ browser: string;+ browserVersion: string;+ device: string;+ brand: string;+ model: string;+ country: string;+ region: string;+ city: string;+ longitude: number | null;+ latitude: number | null;+ isBounce: boolean;+ projectId: string;+ deviceId: string;+ duration: number;+ utmMedium: string;+ utmSource: string;+ utmCampaign: string;+ utmContent: string;+ utmTerm: string;+ revenue: number;+ profile?: IServiceProfile;+ hasReplay?: boolean;+ groups: string[];+}++export interface GetSessionListOptions {+ projectId: string;+ profileId?: string;+ take: number;+ filters?: IChartEventFilter[];+ startDate?: Date;+ endDate?: Date;+ search?: string;+ cursor?: Date;+ dateIntervalInDays?: number;+}++/** The session list's filters compile against the sessions table itself. */+const SESSION_FILTER_TABLE = {+ selfTable: 'sessions',+ profileIdExpr: 'profile_id',+ groupsExpr: 'groups',+} as const;++const DEFAULT_LOOKBACK_DAYS = 0.5;+const SESSION_LIST_MAX_LOOKBACK_DAYS_DEFAULT = 365;+const SESSION_LIST_MAX_LOOKBACK_DAYS_ENV = 'SESSION_LIST_MAX_LOOKBACK_DAYS';++const SESSIONS_COUNT_CACHE_SECONDS = 60 * 10;+const REPLAY_CHUNKS_PAGE_SIZE = 40;+const DISTINCT_VALUES_DEFAULT_LIMIT = 200;+const QUERY_SESSIONS_DEFAULT_LIMIT = 20;++// clix always sent `session_timezone: 'UTC'`; the two queries converted from+// clix keep sending it so their result sets stay identical.+const CLIX_SESSION_TIMEZONE = { session_timezone: 'UTC' } as const;++export function transformSession(session: IClickhouseSession): IServiceSession {+ return {+ id: session.id,+ profileId: session.profile_id,+ eventCount: session.event_count,+ screenViewCount: session.screen_view_count,+ entryPath: session.entry_path,+ entryOrigin: session.entry_origin,+ exitPath: session.exit_path,+ exitOrigin: session.exit_origin,+ createdAt: convertClickhouseDateToJs(session.created_at),+ endedAt: convertClickhouseDateToJs(session.ended_at),+ referrer: session.referrer,+ referrerName: session.referrer_name,+ referrerType: session.referrer_type,+ os: session.os,+ osVersion: session.os_version,+ browser: session.browser,+ browserVersion: session.browser_version,+ device: session.device,+ brand: session.brand,+ model: session.model,+ country: session.country,+ region: session.region,+ city: session.city,+ longitude: session.longitude,+ latitude: session.latitude,+ isBounce: session.is_bounce,+ projectId: session.project_id,+ deviceId: session.device_id,+ duration: session.duration,+ utmMedium: session.utm_medium,+ utmSource: session.utm_source,+ utmCampaign: session.utm_campaign,+ utmContent: session.utm_content,+ utmTerm: session.utm_term,+ revenue: session.revenue,+ profile: undefined,+ hasReplay: session.has_replay,+ groups: session.groups,+ };+}++async function compileSessionFilters(+ filters: IChartEventFilter[] | undefined,+ projectId: string,+ range: { startDate?: Date; endDate?: Date }+) {+ if (!filters?.length) {+ return {};+ }+ const { buildFilterWhere } = await loadFilterCompiler();+ return buildFilterWhere(filters, projectId, {+ ...SESSION_FILTER_TABLE,+ ...range,+ });+}++function emptyProfile(profileId: string, projectId: string): IServiceProfile {+ return {+ id: profileId,+ email: '',+ avatar: '',+ firstName: '',+ lastName: '',+ createdAt: new Date(),+ lastSeenAt: new Date(),+ projectId,+ isExternal: false,+ properties: {},+ groups: [],+ };+}++export async function getSessionList(options: GetSessionListOptions) {+ const {+ cursor,+ take,+ projectId,+ profileId,+ filters,+ startDate,+ endDate,+ search,+ dateIntervalInDays = DEFAULT_LOOKBACK_DAYS,+ } = options;++ const { chQuery } = await loadChClient();+ const { resolveMaxLookbackDays } = await loadLookback();++ // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).+ const maxLookbackDays = resolveMaxLookbackDays(+ SESSION_LIST_MAX_LOOKBACK_DAYS_ENV,+ SESSION_LIST_MAX_LOOKBACK_DAYS_DEFAULT+ );+ const lookbackDays = Math.min(dateIntervalInDays, maxLookbackDays);++ const data = await chQuery<IClickhouseSession & { hasReplay: boolean }>(+ sessionListQuery({+ projectId,+ take,+ cursor,+ lookbackDays,+ startDate,+ endDate,+ profileId,+ search,+ filterClauses: await compileSessionFilters(filters, projectId, {+ startDate,+ endDate,+ }),+ })+ );++ // Empty window: double it and retry until the ceiling.+ if (+ data.length === 0 &&+ hasSessionListLookback({ cursor, startDate, endDate }) &&+ lookbackDays < maxLookbackDays+ ) {+ return getSessionList({+ ...options,+ dateIntervalInDays: dateIntervalInDays * 2,+ });+ }++ const { getProfilesCached } = await loadProfileService();+ const profileIds = data+ .filter((e) => e.device_id !== e.profile_id)+ .map((e) => e.profile_id);+ const profiles = await getProfilesCached(profileIds, projectId);+ const map = new Map<string, IServiceProfile>(profiles.map((p) => [p.id, p]));++ const items = data.map(transformSession).map((item) => ({+ ...item,+ profile: map.get(item.profileId) ?? emptyProfile(item.profileId, projectId),+ }));++ const last = items.at(-1);++ return {+ items,+ meta: { next: last ? last.createdAt.toISOString() : undefined },+ };+}++export async function getSessionsCount({+ projectId,+ profileId,+ filters,+ startDate,+ endDate,+ search,+}: Omit<GetSessionListOptions, 'take' | 'cursor'>) {+ const { chQuery } = await loadChClient();+ const result = await chQuery<{ count: number }>(+ sessionsCountQuery({+ projectId,+ profileId,+ startDate,+ endDate,+ search,+ filterClauses: await compileSessionFilters(filters, projectId, {+ startDate,+ endDate,+ }),+ })+ );+ return result[0]?.count ?? 0;+}++export const getSessionsCountCached = cacheable(+ getSessionsCount,+ SESSIONS_COUNT_CACHE_SECONDS+);++export interface ISessionReplayChunkMeta {+ chunk_index: number;+ started_at: string;+ ended_at: string;+ events_count: number;+ is_full_snapshot: boolean;+}++export async function getSessionReplayChunksFrom(+ sessionId: string,+ projectId: string,+ fromIndex: number+) {+ const { chQuery } = await loadChClient();+ const rows = await chQuery<{ chunk_index: number; payload: string }>(+ sessionReplayChunksQuery({+ sessionId,+ projectId,+ limit: REPLAY_CHUNKS_PAGE_SIZE + 1,+ offset: fromIndex,+ })+ );++ return {+ data: rows+ .slice(0, REPLAY_CHUNKS_PAGE_SIZE)+ .map((row, index) => {+ const events = getSafeJson<+ { type: number; data: unknown; timestamp: number }[]+ >(row.payload);+ if (!events) {+ return null;+ }+ return { chunkIndex: index + fromIndex, events };+ })+ .filter(Boolean),+ hasMore: rows.length > REPLAY_CHUNKS_PAGE_SIZE,+ };+}++export async function getSessionDistinctValues(+ projectId: string,+ field: SessionDistinctField,+ limit = DISTINCT_VALUES_DEFAULT_LIMIT+): Promise<string[]> {+ const { chQuery } = await loadChClient();+ const results = await chQuery<{ value: string }>(+ sessionDistinctValuesQuery({ projectId, field, limit })+ );+ return results.map((r) => r.value).filter(Boolean);+}++export async function getSessionById(sessionId: string, projectId: string) {+ const { chQuery } = await loadChClient();+ const [sessionRows, hasReplayRows] = await Promise.all([+ chQuery<IClickhouseSession>(+ sessionByIdQuery({ sessionId, projectId }),+ CLIX_SESSION_TIMEZONE+ ),+ chQuery<{ n: number }>(sessionHasReplayQuery({ sessionId, projectId })),+ ]);++ if (!sessionRows[0]) {+ throw new Error('Session not found');+ }++ return {+ ...transformSession(sessionRows[0]),+ hasReplay: hasReplayRows.length > 0,+ };+}++export interface QuerySessionsInput {+ projectId: string;+ startDate?: string;+ endDate?: string;+ country?: string;+ city?: string;+ device?: string;+ browser?: string;+ os?: string;+ referrer?: string;+ referrerName?: string;+ referrerType?: string;+ profileId?: string;+ filters?: IChartEventFilter[];+ limit?: number;+}++/** `clix.datetime`: any date input to a UTC `YYYY-MM-DD HH:mm:ss`. */+function toClixDatetime(date: string): string {+ return new Date(date).toISOString().slice(0, 19).replace('T', ' ');+}++export async function querySessionsCore(+ input: QuerySessionsInput+): Promise<IClickhouseSession[]> {+ const { chQuery } = await loadChClient();+ const { resolveDateRange } = await loadDateService();+ const { startDate, endDate } = resolveDateRange(+ input.startDate,+ input.endDate+ );++ return chQuery<IClickhouseSession>(+ querySessionsQuery({+ ...input,+ startDate: toClixDatetime(startDate),+ endDate: toClixDatetime(endDate),+ limit: input.limit ?? QUERY_SESSIONS_DEFAULT_LIMIT,+ filterClauses: await compileSessionFilters(+ input.filters,+ input.projectId,+ { startDate: new Date(startDate), endDate: new Date(endDate) }+ ),+ }),+ CLIX_SESSION_TIMEZONE+ );+}++export interface SessionService {+ byId(+ sessionId: string,+ projectId: string+ ): Promise<IServiceSession & { hasReplay: boolean }>;+ /**+ * Enqueue one `session_end` job, idempotent on the closed session's id.+ * The ctx.queues-based producer; V1's apps/worker keeps its own+ * @openpanel/queue producer (utils/session-handler.ts) because core cannot+ * import @openpanel/queue back (see cohort.service.ts's header).+ */+ enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<void>;+}++export function createSessionService(deps: ServiceDeps): SessionService {+ return {+ byId: getSessionById,+ enqueueSessionEnd: async (input) => {+ await deps.queues.sessions.session.add(+ sessionEndJobPayload(input),+ sessionEndEnqueueOptions(input.closedSession.id)+ );+ },+ };+}diff --git a/packages/core/src/modules/session/src/access.ts b/packages/core/src/modules/session/src/access.tsnew file mode 100644index 000000000..1b99a6d3e--- /dev/null+++ b/packages/core/src/modules/session/src/access.ts@@ -0,0 +1,22 @@+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for+// this module's own router — the same binding packages/trpc/src/access.ts+// does for V1, self-contained here because the general core -> db binding+// itself moves with auth (P6), not this wave. Same shape as+// realtime/src/access.ts.++import {+ canWriteProject,+ getOrganizationAccess,+ getProjectAccess,+} from '@openpanel/db/src/services/access.service';+import { getProjectById } from '@openpanel/db/src/services/project.service';+import { createAccessChecks } from '../../../shared/access';++const checks = createAccessChecks({+ getProjectAccess,+ canWriteProject,+ getOrganizationAccess,+ getProjectById,+});++export const requireProjectAccess = checks.requireProjectAccess;diff --git a/packages/core/src/modules/session/src/dates.ts b/packages/core/src/modules/session/src/dates.tsnew file mode 100644index 000000000..bcfcf8cb5--- /dev/null+++ b/packages/core/src/modules/session/src/dates.ts@@ -0,0 +1,18 @@+// @openpanel/db's clickhouse/client.ts owns these, but importing it constructs+// a client and a pino logger at import time (see insight.service.ts's header),+// and the fragment builders here must stay pure and synchronous. Same choice+// cohort.service.ts made for its table map.++const CLICKHOUSE_DATE_SUFFIX = /(\.\d{3})?Z+$/;++/** `YYYY-MM-DD HH:mm:ss` in UTC — what V1 sent as a date literal. */+export function formatClickhouseDate(date: Date | string): string {+ return new Date(date)+ .toISOString()+ .replace('T', ' ')+ .replace(CLICKHOUSE_DATE_SUFFIX, '');+}++export function convertClickhouseDateToJs(date: string): Date {+ return new Date(`${date.replace(' ', 'T')}Z`);+}diff --git a/packages/core/src/modules/session/src/filter-clauses.ts b/packages/core/src/modules/session/src/filter-clauses.tsnew file mode 100644index 000000000..778d80b5c--- /dev/null+++ b/packages/core/src/modules/session/src/filter-clauses.ts@@ -0,0 +1,27 @@+// The ONE place this module splices pre-compiled filter SQL.+//+// `buildFilterWhere` (packages/db/src/services/filter-where.service.ts) is+// V1's filter compiler and still emits sqlstring-escaped SQL text: ADR-013+// leaves the filter compilers as they are ("two behaviours, not two builders")+// until the shared filter compiler converts to fragments. Until then a+// `sql`-tag query that takes `IChartEventFilter[]` has to splice that text,+// and it does so here — through the same `SqlFragment` constructor `sql.id`+// uses — so the bridge is one grep away from deletion when the compiler moves,+// rather than a `sql.raw()` that would outlive it.++import { SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';++export type CompiledFilterClauses = Record<string, string>;++/** Each compiled clause is already parenthesised by the compiler. */+export function spliceCompiledFilters(+ clauses: CompiledFilterClauses+): SqlFragment {+ const fragments = Object.values(clauses).map(+ (clause) => new SqlFragment([clause], [])+ );+ if (fragments.length === 0) {+ return sql.empty;+ }+ return sql`AND ${sql.join(fragments, ' AND ')}`;+}diff --git a/packages/core/src/modules/session/src/runtime.ts b/packages/core/src/modules/session/src/runtime.tsnew file mode 100644index 000000000..026e3a4f9--- /dev/null+++ b/packages/core/src/modules/session/src/runtime.ts@@ -0,0 +1,59 @@+// What the session lifecycle (session_end emission, reaper, vacuum) needs from+// the outside world, as narrow structural interfaces. The job handlers bind+// them to the real @openpanel/db buffers and @openpanel/redis client through+// `loadSessionRuntime` (lazily — see session.service.ts's header); the tests+// hand in stubs, so no `mock.module` is needed and every assertion is on a+// call the code under test made.++import type { IClickhouseSession } from '../session.service';++export interface SessionStore {+ getExistingSession(input: {+ projectId: string;+ deviceId: string;+ }): Promise<IClickhouseSession | null>;+ /** Id-gated: a no-op when another session already owns the slot. */+ cleanup(input: {+ projectId: string;+ deviceId: string;+ sessionId: string;+ profileId?: string | null;+ }): Promise<void>;+}++/** The ioredis subset the lifecycle touches. */+export interface SessionRedis {+ set(+ key: string,+ value: string,+ expiryMode: 'EX',+ seconds: number,+ condition: 'NX'+ ): Promise<'OK' | null>;+ smembers(key: string): Promise<string[]>;+ srem(key: string, member: string): Promise<number>;+ zrangebyscore(+ key: string,+ min: number,+ max: number,+ limitToken: 'LIMIT',+ offset: number,+ count: number+ ): Promise<string[]>;+ zcard(key: string): Promise<number>;+ zrem(key: string, member: string): Promise<number>;+ del(key: string): Promise<number>;+}++export interface SessionRuntime {+ redis: SessionRedis;+ sessions: SessionStore;+}++export async function loadSessionRuntime(): Promise<SessionRuntime> {+ const [{ getRedisCache }, { sessionBuffer }] = await Promise.all([+ import('@openpanel/redis'),+ import('@openpanel/db/src/buffers'),+ ]);+ return { redis: getRedisCache(), sessions: sessionBuffer };+}diff --git a/packages/core/src/modules/session/src/session-context.ts b/packages/core/src/modules/session/src/session-context.tsnew file mode 100644index 000000000..2141e87eb--- /dev/null+++ b/packages/core/src/modules/session/src/session-context.ts@@ -0,0 +1,20 @@+// Moved from packages/db/src/session-context.ts (M7-001; module map: session+// owns session-context). The ALS scopes a request's dashboard session id so+// anything downstream — today only the Prisma read-replica consistency+// extension ADR-012 retires — can read it without threading a parameter.+// packages/db keeps a re-export shim for its V1 callers.++import { AsyncLocalStorage } from 'node:async_hooks';++interface SessionScope {+ sessionId: string | null;+}++export const als = new AsyncLocalStorage<SessionScope>();++export const runWithAlsSession = <T>(+ sessionId: string | null | undefined,+ fn: () => Promise<T>+) => als.run({ sessionId: sessionId || null }, fn);++export const getAlsSessionId = () => als.getStore()?.sessionId ?? null;diff --git a/packages/core/src/modules/session/src/session-end.test.ts b/packages/core/src/modules/session/src/session-end.test.tsnew file mode 100644index 000000000..080e13a85--- /dev/null+++ b/packages/core/src/modules/session/src/session-end.test.ts@@ -0,0 +1,169 @@+// Ported from apps/worker/src/jobs/events.create-session-end.test.ts (M7-001).+// Covers the three-case logic (extended-after-enqueue skip, live vs snapshot,+// boundary), the first-writer idempotency claim, and that cleanup() is+// id-gated to the closed session.++import { describe, expect, mock, test } from 'bun:test';+import type { IServiceCreateEventPayload } from '@openpanel/db/src/services/event.service';+import {+ createSessionEnd,+ getSessionEndJobId,+ type SessionEndDeps,+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+} from './session-end';+import {+ fixtureSession,+ stubLogger,+ stubRedis,+ stubStore,+} from './session-lifecycle.fixtures';++const snapshot = fixtureSession();++const payload = {+ projectId: 'proj-1',+ deviceId: 'dev-1',+ profileId: 'dev-1',+ name: 'session_end',+ properties: {},+} as unknown as IServiceCreateEventPayload;++function makeDeps(live = snapshot as ReturnType<typeof fixtureSession> | null) {+ const redis = stubRedis();+ const sessions = stubStore(live);+ const createEvent = mock<SessionEndDeps['createEvent']>(async () => ({+ document: { id: 'evt-1' } as never,+ }));+ const deps: SessionEndDeps = {+ redis,+ sessions,+ logger: stubLogger(),+ createEvent,+ transformEvent: (event) => event as never,+ transformSessionToEvent: () => ({}) as never,+ getEvents: mock(async () => []),+ profileBackfill: { add: mock(async () => undefined) },+ notifications: {+ getRules: mock(async () => []),+ hasFunnelRules: () => false,+ checkFunnelRules: mock(async () => undefined),+ },+ };+ return { deps, redis, sessions, createEvent };+}++describe('createSessionEnd', () => {+ test('emits session_end and id-gated cleanup on the happy path', async () => {+ // live == snapshot, unchanged+ const { deps, sessions, createEvent } = makeDeps(snapshot);++ const result = await createSessionEnd({ event: payload, snapshot }, deps);++ expect(result).toEqual({ id: 'evt-1' } as never);+ expect(createEvent).toHaveBeenCalledTimes(1);+ expect(createEvent.mock.calls[0]?.[0]).toMatchObject({+ name: 'session_end',+ sessionId: 'sess-1',+ duration: 1_800_000,+ path: '/end',+ // ended_at + 1s+ createdAt: new Date('2026-06-08T11:00:01.000Z'),+ });+ expect(sessions.cleanup).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'dev-1',+ sessionId: 'sess-1',+ profileId: 'dev-1',+ });+ });++ test('skips when the session was extended after the close was enqueued', async () => {+ const { deps, sessions, createEvent } = makeDeps(+ fixtureSession({ ended_at: '2026-06-08 11:05:00' })+ );++ const result = await createSessionEnd({ event: payload, snapshot }, deps);++ expect(result).toBeNull();+ expect(sessions.getExistingSession).toHaveBeenCalledTimes(1);+ expect(createEvent).not.toHaveBeenCalled();+ expect(sessions.cleanup).not.toHaveBeenCalled();+ });++ test('skips when the idempotency claim is already taken', async () => {+ const { deps, redis, createEvent } = makeDeps(snapshot);+ redis.set.mockResolvedValue(null); // SET NX failed → already emitted++ const result = await createSessionEnd({ event: payload, snapshot }, deps);++ expect(result).toBeNull();+ expect(redis.set).toHaveBeenCalledWith(+ 'session:end:emitted:proj-1:dev-1:sess-1',+ '1',+ 'EX',+ 7200,+ 'NX'+ );+ expect(createEvent).not.toHaveBeenCalled();+ });++ test('uses the snapshot when a new session already owns the slot (boundary)', async () => {+ // A boundary opened a fresh session under the same (project, device) slot.+ const { deps, sessions, createEvent } = makeDeps(+ fixtureSession({ id: 'sess-2', ended_at: '2026-06-08 12:00:00' })+ );++ await createSessionEnd({ event: payload, snapshot }, deps);++ // Emits for the CLOSED session (snapshot), not the live one.+ expect(createEvent.mock.calls[0]?.[0]).toMatchObject({+ sessionId: 'sess-1',+ });+ // cleanup is keyed on the closed id → id-gated Lua no-ops against sess-2.+ expect(sessions.cleanup).toHaveBeenCalledWith(+ expect.objectContaining({ sessionId: 'sess-1' })+ );+ });++ test('falls back to the snapshot when the blob is gone', async () => {+ const { deps, createEvent } = makeDeps(null);++ const result = await createSessionEnd({ event: payload, snapshot }, deps);++ expect(result).toEqual({ id: 'evt-1' } as never);+ expect(createEvent.mock.calls[0]?.[0]).toMatchObject({+ sessionId: 'sess-1',+ });+ });+});++describe('enqueue shape', () => {+ test('job id and options match V1 (sessionEnd:v2:<id>, 3 exponential retries)', () => {+ expect(getSessionEndJobId('sess-1')).toBe('sessionEnd:v2:sess-1');+ expect(sessionEndEnqueueOptions('sess-1')).toEqual({+ jobId: 'sessionEnd:v2:sess-1',+ attempts: 3,+ backoff: { type: 'exponential', delay: 200 },+ });+ });++ test('payload rewrites the event onto the closed session', () => {+ const closedSession = fixtureSession({+ id: 'sess-9',+ device_id: 'dev-9',+ profile_id: 'user-9',+ });++ expect(sessionEndJobPayload({ payload, closedSession })).toEqual({+ event: {+ ...payload,+ projectId: 'proj-1',+ deviceId: 'dev-9',+ sessionId: 'sess-9',+ profileId: 'user-9',+ },+ snapshot: closedSession,+ });+ });+});diff --git a/packages/core/src/modules/session/src/session-end.ts b/packages/core/src/modules/session/src/session-end.tsnew file mode 100644index 000000000..926728825--- /dev/null+++ b/packages/core/src/modules/session/src/session-end.ts@@ -0,0 +1,328 @@+// Ported from apps/worker/src/jobs/events.create-session-end.ts and+// utils/session-handler.ts (M7-001). The worker's copies are thin delegates+// onto this file until apps/worker dies (P9).++import type {+ IClickhouseEvent,+ IServiceCreateEventPayload,+ IServiceEvent,+} from '@openpanel/db/src/services/event.service';+import type { EnqueueOptions } from '../../../jobs/define';+import type { Logger } from '../../../logger';+import type { INotificationRuleCached } from '../../notification/notification.service';+import type { IClickhouseSession } from '../session.service';+import { convertClickhouseDateToJs } from './dates';+import type { SessionRuntime } from './runtime';+import {+ sessionDurationOnClose,+ sessionEndsEmitted,+ sessionEndsSkipped,+ sessionEventsOnClose,+} from './session.metrics';+import { sessionEventsQuery } from './session.sql';++const SESSION_END_JOB_ID_PREFIX = 'sessionEnd:v2';+const SESSION_END_ATTEMPTS = 3;+const SESSION_END_BACKOFF_MS = 200;++/** Funnel rules are not evaluated for sessions longer than this. */+const MAX_SESSION_EVENTS = 500;+/** 2h — well past any plausible retry window. */+const SESSION_END_CLAIM_TTL_SECONDS = 60 * 60 * 2;+/** The events query pads the session's own bounds by this much on each side. */+const SESSION_EVENTS_WINDOW_PADDING_MS = 1000;+/** The session_end event lands one second after the last event. */+const SESSION_END_OFFSET_MS = 1000;++const PROFILE_BACKFILL_FLAG_ENV = 'EXPERIMENTAL_PROFILE_BACKFILL';+const PROFILE_BACKFILL_PROJECTS_ENV = 'EXPERIMENTAL_PROFILE_BACKFILL_PROJECTS';++/**+ * Deterministic jobId for a closed session, so concurrent / retried closes+ * dedupe in BullMQ. BullMQ only accepts ':' in custom jobIds when splitting+ * by ':' yields exactly 3 parts, so the suffix stays a single segment.+ */+export function getSessionEndJobId(sessionId: string): string {+ return `${SESSION_END_JOB_ID_PREFIX}:${sessionId}`;+}++export function sessionEndEnqueueOptions(sessionId: string): EnqueueOptions {+ return {+ jobId: getSessionEndJobId(sessionId),+ attempts: SESSION_END_ATTEMPTS,+ backoff: { type: 'exponential', delay: SESSION_END_BACKOFF_MS },+ };+}++export interface EnqueueSessionEndInput {+ payload: IServiceCreateEventPayload;+ closedSession: IClickhouseSession;+}++/** What rides on the `sessions` queue (legacyCompat.sessions: `{ event, snapshot }`). */+export interface SessionEndJobData {+ event: IServiceCreateEventPayload;+ snapshot: IClickhouseSession;+}++export function sessionEndJobPayload({+ payload,+ closedSession,+}: EnqueueSessionEndInput): SessionEndJobData {+ return {+ event: {+ ...payload,+ projectId: closedSession.project_id,+ deviceId: closedSession.device_id,+ sessionId: closedSession.id,+ profileId: closedSession.profile_id || payload.profileId,+ },+ snapshot: closedSession,+ };+}++/** The bits of @openpanel/db the emission needs beyond the shared runtime. */+export interface SessionEndDeps extends SessionRuntime {+ logger: Logger;+ createEvent(+ payload: IServiceCreateEventPayload+ ): Promise<{ document: IClickhouseEvent }>;+ transformEvent(event: IClickhouseEvent): IServiceEvent;+ transformSessionToEvent(session: IClickhouseSession): IServiceEvent;+ getEvents(+ query: ReturnType<typeof sessionEventsQuery>+ ): Promise<IServiceEvent[]>;+ profileBackfill: {+ add(entry: {+ projectId: string;+ sessionId: string;+ profileId: string;+ }): Promise<unknown>;+ };+ notifications: {+ getRules(projectId: string): Promise<INotificationRuleCached[]>;+ hasFunnelRules(rules: INotificationRuleCached[]): boolean;+ /** Stays in @openpanel/db — it enqueues through @openpanel/queue. */+ checkFunnelRules(events: IServiceEvent[]): Promise<unknown>;+ };+}++export async function loadSessionEndDeps(+ runtime: SessionRuntime,+ logger: Logger+): Promise<SessionEndDeps> {+ const [+ { chQuery },+ eventService,+ { profileBackfillBuffer },+ notificationService,+ { checkNotificationRulesForSessionEnd },+ ] = await Promise.all([+ import('@openpanel/db/src/clickhouse/client'),+ import('@openpanel/db/src/services/event.service'),+ import('@openpanel/db/src/buffers'),+ import('../../notification/notification.service'),+ import('@openpanel/db/src/services/notification.service'),+ ]);+ return {+ ...runtime,+ logger,+ createEvent: eventService.createEvent,+ transformEvent: eventService.transformEvent,+ transformSessionToEvent: eventService.transformSessionToEvent,+ getEvents: async (query) =>+ (await chQuery<IClickhouseEvent>(query)).map(eventService.transformEvent),+ profileBackfill: profileBackfillBuffer,+ notifications: {+ getRules: notificationService.getNotificationRulesByProjectId,+ hasFunnelRules: notificationService.getHasFunnelRules,+ checkFunnelRules: checkNotificationRulesForSessionEnd,+ },+ };+}++function claimKey(session: IClickhouseSession): string {+ return `session:end:emitted:${session.project_id}:${session.device_id}:${session.id}`;+}++function isProfileBackfillEnabled(projectId: string): boolean {+ if (process.env[PROFILE_BACKFILL_FLAG_ENV] !== '1') {+ return false;+ }+ const runOnProjects =+ process.env[PROFILE_BACKFILL_PROJECTS_ENV]?.split(',').filter(Boolean) ??+ [];+ return runOnProjects.length === 0 || runOnProjects.includes(projectId);+}++/**+ * Emit the `session_end` event for a closed session. Returns the written+ * event, or null when nothing was emitted (extended, gone, or already done).+ */+export async function createSessionEnd(+ { event: payload, snapshot }: SessionEndJobData,+ deps: SessionEndDeps+): Promise<IClickhouseEvent | null> {+ const { logger } = deps;++ // Prefer the live blob — it reflects any late extensions that arrived after+ // enqueue. Fall back to the snapshot if the blob expired (very long queue+ // lag, retries hours later, etc.).+ const live = await deps.sessions.getExistingSession({+ projectId: payload.projectId,+ deviceId: payload.deviceId,+ });++ // Same session, extended after enqueue → bail, the reaper retries later.+ // Same session, unchanged → live. A different session in the slot (boundary)+ // or no blob at all → the snapshot; cleanup is then id-gated to a no-op.+ const sameSession = live && live.id === snapshot.id;+ if (sameSession && live.ended_at > snapshot.ended_at) {+ sessionEndsSkipped.inc({ reason: 'extended_after_enqueue' });+ logger.info(+ {+ sessionId: live.id,+ projectId: live.project_id,+ snapshotEndedAt: snapshot.ended_at,+ liveEndedAt: live.ended_at,+ },+ 'session was extended after close was enqueued, skipping'+ );+ return null;+ }++ const session = sameSession ? live : snapshot;+ if (!session) {+ sessionEndsSkipped.inc({ reason: 'not_found' });+ logger.warn(+ { projectId: payload.projectId, deviceId: payload.deviceId },+ 'No live session and no snapshot — skipping session_end'+ );+ return null;+ }++ // Idempotency claim: first writer wins; retries and reaper double-fires no-op.+ const claimed = await deps.redis.set(+ claimKey(session),+ '1',+ 'EX',+ SESSION_END_CLAIM_TTL_SECONDS,+ 'NX'+ );+ if (claimed === null) {+ sessionEndsSkipped.inc({ reason: 'already_emitted' });+ logger.info(+ { sessionId: session.id, projectId: session.project_id },+ 'session_end already emitted, skipping'+ );+ return null;+ }++ sessionEndsEmitted.inc();+ sessionDurationOnClose.observe(Math.max(0, session.duration ?? 0));+ sessionEventsOnClose.observe(+ (session.event_count ?? 0) + (session.screen_view_count ?? 0)+ );++ const profileId = session.profile_id || payload.profileId;++ if (+ profileId !== session.device_id &&+ isProfileBackfillEnabled(payload.projectId)+ ) {+ await deps.profileBackfill.add({+ projectId: payload.projectId,+ sessionId: session.id,+ profileId,+ });+ }++ const { document: sessionEndEvent } = await deps.createEvent({+ ...payload,+ sessionId: session.id,+ properties: {+ ...payload.properties,+ __bounce: session.is_bounce,+ },+ name: 'session_end',+ duration: session.duration ?? 0,+ path: session.exit_path ?? '',+ createdAt: new Date(+ convertClickhouseDateToJs(session.ended_at).getTime() ++ SESSION_END_OFFSET_MS+ ),+ profileId,+ });++ try {+ await notifyFunnelRules(+ session,+ payload.projectId,+ deps.transformEvent(sessionEndEvent),+ deps+ );+ } catch (error) {+ logger.error(+ { err: error },+ 'Creating notificatios for session end failed'+ );+ }++ await deps.sessions+ .cleanup({+ projectId: session.project_id,+ deviceId: session.device_id,+ sessionId: session.id,+ profileId: session.profile_id,+ })+ .catch((error) => {+ logger.error(+ { err: error, sessionId: session.id },+ 'Failed to cleanup session state after session_end'+ );+ });++ return sessionEndEvent;+}++async function notifyFunnelRules(+ session: IClickhouseSession,+ projectId: string,+ sessionEndEvent: IServiceEvent,+ deps: SessionEndDeps+): Promise<void> {+ const rules = await deps.notifications.getRules(projectId);+ const isEventCountReasonable =+ session.event_count + session.screen_view_count < MAX_SESSION_EVENTS;++ if (!(deps.notifications.hasFunnelRules(rules) && isEventCountReasonable)) {+ return;+ }++ const events = await getSessionEvents(session, deps);+ if (events.length > 0) {+ await deps.notifications.checkFunnelRules([...events, sessionEndEvent]);+ }+}++/** The session's events plus its own exit page, newest first. */+async function getSessionEvents(+ session: IClickhouseSession,+ deps: SessionEndDeps+): Promise<IServiceEvent[]> {+ const startAt = convertClickhouseDateToJs(session.created_at);+ const endAt = convertClickhouseDateToJs(session.ended_at);+ const eventsInDb = await deps.getEvents(+ sessionEventsQuery({+ sessionId: session.id,+ projectId: session.project_id,+ startAt: new Date(startAt.getTime() - SESSION_EVENTS_WINDOW_PADDING_MS),+ endAt: new Date(endAt.getTime() + SESSION_EVENTS_WINDOW_PADDING_MS),+ limit: MAX_SESSION_EVENTS,+ })+ );++ return [deps.transformSessionToEvent(session), ...eventsInDb].sort(+ (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()+ );+}diff --git a/packages/core/src/modules/session/src/session-keys.ts b/packages/core/src/modules/session/src/session-keys.tsnew file mode 100644index 000000000..9ae6394b3--- /dev/null+++ b/packages/core/src/modules/session/src/session-keys.ts@@ -0,0 +1,19 @@+// The Redis index the session buffer (packages/db/src/buffers/session-buffer.ts,+// moves at P8) maintains and the reaper/vacuum walk. Redefined here exactly as+// V1's reaper and vacuum redefine them — the buffer keeps its own copy and+// importing it would construct the ClickHouse client at import time.++const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;++/** Idle time after which a session is over; also the reaper's deadman default. */+export const SESSION_TIMEOUT_MS = Number.parseInt(+ process.env.SESSION_TIMEOUT_MS || String(DEFAULT_SESSION_TIMEOUT_MS),+ 10+);++/** Set of project ids with at least one live session. */+export const PROJECTS_SET_KEY = 'session:projects';++/** Per project: deviceId scored by wall-clock ms of the last received event. */+export const wallclockSetKey = (projectId: string) =>+ `session:wallclock:${projectId}`;diff --git a/packages/core/src/modules/session/src/session-lifecycle.fixtures.ts b/packages/core/src/modules/session/src/session-lifecycle.fixtures.tsnew file mode 100644index 000000000..4f0ddcdcf--- /dev/null+++ b/packages/core/src/modules/session/src/session-lifecycle.fixtures.ts@@ -0,0 +1,68 @@+// Stubs for the lifecycle tests (session-end, reaper, vacuum): typed against+// the structural deps in runtime.ts, so the code under test is exercised for+// real and every assertion is on a call it made — no `mock.module`.++import { mock } from 'bun:test';+import type { Logger } from '../../../logger';+import type { IClickhouseSession } from '../session.service';+import type { SessionRedis, SessionStore } from './runtime';++export function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++export type StubRedis = { [K in keyof SessionRedis]: ReturnType<typeof mock> };++/** Lock/claim acquired, one project, nothing idle — override per test. */+export function stubRedis(): StubRedis {+ return {+ set: mock(async () => 'OK'),+ smembers: mock(async () => ['proj-1']),+ srem: mock(async () => 1),+ zrangebyscore: mock(async () => []),+ zcard: mock(async () => 0),+ zrem: mock(async () => 1),+ del: mock(async () => 1),+ };+}++export type StubStore = {+ [K in keyof SessionStore]: ReturnType<typeof mock>;+};++export function stubStore(live: IClickhouseSession | null = null): StubStore {+ return {+ getExistingSession: mock(async () => live),+ cleanup: mock(async () => undefined),+ };+}++export function fixtureSession(+ overrides: Partial<IClickhouseSession> = {}+): IClickhouseSession {+ return {+ id: 'sess-1',+ project_id: 'proj-1',+ device_id: 'dev-1',+ profile_id: 'dev-1',+ created_at: '2026-06-08 10:30:00',+ ended_at: '2026-06-08 11:00:00',+ is_bounce: false,+ duration: 1_800_000,+ event_count: 3,+ screen_view_count: 5,+ exit_path: '/end',+ groups: [],+ ...overrides,+ } as unknown as IClickhouseSession;+}diff --git a/packages/core/src/modules/session/src/session-reaper.test.ts b/packages/core/src/modules/session/src/session-reaper.test.tsnew file mode 100644index 000000000..ba7245cb5--- /dev/null+++ b/packages/core/src/modules/session/src/session-reaper.test.ts@@ -0,0 +1,83 @@+// Ported from apps/worker/src/jobs/cron.session-reaper.test.ts (M7-001). The+// wall-clock deadman that closes idle sessions: stub Redis drives the+// wallclock ZSET + lock, so the assertions are on decisions, not side effects.++import { afterEach, describe, expect, mock, test } from 'bun:test';+import {+ fixtureSession,+ stubLogger,+ stubRedis,+ stubStore,+} from './session-lifecycle.fixtures';+import { reapIdleSessions, type SessionReaperDeps } from './session-reaper';++function makeDeps(live: ReturnType<typeof fixtureSession> | null) {+ const redis = stubRedis();+ const sessions = stubStore(live);+ const enqueueSessionEnd = mock<SessionReaperDeps['enqueueSessionEnd']>(+ async () => undefined+ );+ return {+ redis,+ sessions,+ enqueueSessionEnd,+ deps: { redis, sessions, enqueueSessionEnd, logger: stubLogger() },+ };+}++afterEach(() => {+ delete process.env.SESSION_REAPER;+});++describe('reapIdleSessions', () => {+ test('enqueues a session_end for an idle session blob', async () => {+ const { deps, redis, enqueueSessionEnd } = makeDeps(+ fixtureSession({ id: 'sess-1', device_id: 'dev-1' })+ );+ redis.zrangebyscore.mockResolvedValue(['dev-1']);++ await reapIdleSessions(deps);++ expect(enqueueSessionEnd).toHaveBeenCalledTimes(1);+ expect(enqueueSessionEnd.mock.calls[0]?.[0]).toMatchObject({+ closedSession: { id: 'sess-1' },+ payload: { projectId: 'proj-1', deviceId: 'dev-1', name: 'session_end' },+ });+ // wallclock index emptied → project removed from the active set+ expect(redis.srem).toHaveBeenCalledWith('session:projects', 'proj-1');+ expect(redis.del).toHaveBeenCalledWith('session:reaper:lock:proj-1');+ });++ test('ZREMs an orphan (wallclock entry with no blob) and does not enqueue', async () => {+ const { deps, redis, enqueueSessionEnd } = makeDeps(null);+ redis.zrangebyscore.mockResolvedValue(['dev-orphan']);++ await reapIdleSessions(deps);++ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ expect(redis.zrem).toHaveBeenCalledWith(+ 'session:wallclock:proj-1',+ 'dev-orphan'+ );+ });++ test('skips a project when the advisory lock is held', async () => {+ const { deps, redis, enqueueSessionEnd } = makeDeps(null);+ redis.set.mockResolvedValue(null); // NX failed → another pod owns it++ await reapIdleSessions(deps);++ expect(redis.set).toHaveBeenCalledTimes(1);+ expect(redis.zrangebyscore).not.toHaveBeenCalled();+ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ });++ test('is a no-op when disabled via SESSION_REAPER=0', async () => {+ process.env.SESSION_REAPER = '0';+ const { deps, redis } = makeDeps(null);++ await reapIdleSessions(deps);++ expect(redis.smembers).not.toHaveBeenCalled();+ });+});diff --git a/packages/core/src/modules/session/src/session-reaper.ts b/packages/core/src/modules/session/src/session-reaper.tsnew file mode 100644index 000000000..4d5c2b738--- /dev/null+++ b/packages/core/src/modules/session/src/session-reaper.ts@@ -0,0 +1,205 @@+// Ported from apps/worker/src/jobs/cron.session-reaper.ts (M7-001).++import type { IServiceCreateEventPayload } from '@openpanel/db/src/services/event.service';+import type { Logger } from '../../../logger';+import type { IClickhouseSession } from '../session.service';+import type { SessionRuntime } from './runtime';+import {+ sessionEndsEnqueued,+ sessionsReaped,+ sessionsReaperOrphans,+} from './session.metrics';+import type { EnqueueSessionEndInput } from './session-end';+import {+ PROJECTS_SET_KEY,+ SESSION_TIMEOUT_MS,+ wallclockSetKey,+} from './session-keys';++const REAPER_DISABLED_ENV = 'SESSION_REAPER';+const REAPER_BATCH_SIZE_ENV = 'SESSION_REAPER_BATCH_SIZE';+const REAPER_DEADMAN_ENV = 'SESSION_REAPER_WALLCLOCK_DEADMAN_MS';+const DEFAULT_REAPER_BATCH_SIZE = 5000;+const LOCK_TTL_SECONDS = 60;++const lockKey = (projectId: string) => `session:reaper:lock:${projectId}`;++function readIntEnv(name: string, fallback: number): number {+ return Number.parseInt(process.env[name] || String(fallback), 10);+}++export interface SessionReaperDeps extends SessionRuntime {+ logger: Logger;+ enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<unknown>;+}++/**+ * Cron: for every project with active sessions, close any whose last event+ * was *received* more than the wall-clock deadman ago. The deadman (default+ * SESSION_TIMEOUT_MS, 30min) is the single source of truth for "this session+ * has ended" regardless of project traffic; with the 5-min cadence, max+ * session_end latency is 30-35min after the last event. Idempotent through+ * the session_end jobId.+ */+export async function reapIdleSessions(deps: SessionReaperDeps): Promise<void> {+ if (process.env[REAPER_DISABLED_ENV] === '0') {+ return;+ }++ const projectIds = await deps.redis.smembers(PROJECTS_SET_KEY);+ if (projectIds.length === 0) {+ return;+ }++ deps.logger.debug({ projectCount: projectIds.length }, 'Reaper tick');++ let totalReaped = 0;+ let totalErrors = 0;++ for (const projectId of projectIds) {+ try {+ totalReaped += await reapProject(projectId, deps);+ } catch (error) {+ totalErrors++;+ deps.logger.error({ err: error, projectId }, 'Failed to reap project');+ }+ }++ if (totalReaped > 0 || totalErrors > 0) {+ deps.logger.info(+ { reaped: totalReaped, errors: totalErrors, projects: projectIds.length },+ 'Reaper tick complete'+ );+ }+}++async function reapProject(+ projectId: string,+ deps: SessionReaperDeps+): Promise<number> {+ const { redis } = deps;++ // Per-project advisory lock keeps multiple worker pods from reaping the+ // same project simultaneously. Best effort — if it expires mid-tick, the+ // jobId-dedup is the real guard.+ const locked = await redis.set(+ lockKey(projectId),+ '1',+ 'EX',+ LOCK_TTL_SECONDS,+ 'NX'+ );+ if (locked === null) {+ return 0;+ }++ try {+ const deadmanMs = readIntEnv(REAPER_DEADMAN_ENV, SESSION_TIMEOUT_MS);+ const cutoff = Date.now() - deadmanMs;+ const candidates = await redis.zrangebyscore(+ wallclockSetKey(projectId),+ 0,+ cutoff,+ 'LIMIT',+ 0,+ readIntEnv(REAPER_BATCH_SIZE_ENV, DEFAULT_REAPER_BATCH_SIZE)+ );++ let reaped = 0;+ for (const deviceId of candidates) {+ if (await closeSession(projectId, deviceId, deps)) {+ reaped++;+ }+ }++ // Nothing left in the wallclock index: stop iterating this project.+ const remaining = await redis.zcard(wallclockSetKey(projectId));+ if (remaining === 0) {+ await redis.srem(PROJECTS_SET_KEY, projectId);+ }++ return reaped;+ } finally {+ await redis.del(lockKey(projectId));+ }+}++async function closeSession(+ projectId: string,+ deviceId: string,+ deps: SessionReaperDeps+): Promise<boolean> {+ const session = await deps.sessions.getExistingSession({+ projectId,+ deviceId,+ });++ if (!session) {+ // Sorted-set entry without a blob: a partially failed cleanup() (worker+ // crash mid-MULTI, network partition). Count it and drop the orphan.+ sessionsReaperOrphans.inc({ reason: 'deadman' });+ deps.logger.warn(+ { projectId, deviceId },+ 'Reaper found wallclock entry without blob — likely a partial cleanup failure'+ );+ await deps.redis.zrem(wallclockSetKey(projectId), deviceId);+ return false;+ }++ try {+ await deps.enqueueSessionEnd({+ payload: sessionEndPayload(session),+ closedSession: session,+ });++ sessionsReaped.inc({ reason: 'deadman' });+ sessionEndsEnqueued.inc({ source: 'reaper' });++ deps.logger.debug(+ { sessionId: session.id, projectId, deviceId },+ 'Enqueued session_end (reaped)'+ );+ return true;+ } catch (error) {+ deps.logger.error(+ { err: error, sessionId: session.id, projectId, deviceId },+ 'Failed to enqueue session_end during reap'+ );+ return false;+ }+}++function sessionEndPayload(+ session: IClickhouseSession+): IServiceCreateEventPayload {+ return {+ projectId: session.project_id,+ deviceId: session.device_id,+ sessionId: session.id,+ profileId: session.profile_id ?? '',+ name: 'session_end',+ properties: {},+ groups: session.groups ?? [],+ createdAt: new Date(session.ended_at),+ duration: session.duration ?? 0,+ sdkName: '',+ sdkVersion: '',+ city: session.city ?? '',+ country: session.country ?? '',+ region: session.region ?? '',+ longitude: session.longitude ?? undefined,+ latitude: session.latitude ?? undefined,+ path: session.exit_path ?? '',+ origin: session.exit_origin ?? '',+ referrer: session.referrer ?? '',+ referrerName: session.referrer_name ?? '',+ referrerType: session.referrer_type ?? '',+ os: session.os ?? '',+ osVersion: session.os_version ?? '',+ browser: session.browser ?? '',+ browserVersion: session.browser_version ?? '',+ device: session.device ?? '',+ brand: session.brand ?? '',+ model: session.model ?? '',+ };+}diff --git a/packages/core/src/modules/session/src/session-vacuum.test.ts b/packages/core/src/modules/session/src/session-vacuum.test.tsnew file mode 100644index 000000000..75626cd12--- /dev/null+++ b/packages/core/src/modules/session/src/session-vacuum.test.ts@@ -0,0 +1,67 @@+// Ported from apps/worker/src/jobs/cron.session-vacuum.test.ts (M7-001). The+// daily backstop for sessions whose cleanup() leaked: a lingering blob →+// id-gated cleanup(); a missing blob → ZREM the orphan.++import { afterEach, describe, expect, test } from 'bun:test';+import {+ fixtureSession,+ stubLogger,+ stubRedis,+ stubStore,+} from './session-lifecycle.fixtures';+import { vacuumStaleSessions } from './session-vacuum';++function makeDeps(live: ReturnType<typeof fixtureSession> | null) {+ const redis = stubRedis();+ const sessions = stubStore(live);+ return { redis, sessions, deps: { redis, sessions, logger: stubLogger() } };+}++afterEach(() => {+ delete process.env.SESSION_VACUUM;+});++describe('vacuumStaleSessions', () => {+ test('id-gated cleanup() for a stale blob that lingered', async () => {+ const { deps, redis, sessions } = makeDeps(+ fixtureSession({+ id: 'sess-stale',+ device_id: 'dev-stale',+ profile_id: 'dev-stale',+ })+ );+ redis.zrangebyscore.mockResolvedValue(['dev-stale']);++ await vacuumStaleSessions(deps);++ expect(sessions.cleanup).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'dev-stale',+ sessionId: 'sess-stale',+ profileId: 'dev-stale',+ });+ expect(redis.zrem).not.toHaveBeenCalled();+ });++ test('ZREMs an orphan wallclock entry when the blob is gone', async () => {+ const { deps, redis, sessions } = makeDeps(null);+ redis.zrangebyscore.mockResolvedValue(['dev-orphan']);++ await vacuumStaleSessions(deps);++ expect(sessions.cleanup).not.toHaveBeenCalled();+ expect(redis.zrem).toHaveBeenCalledWith(+ 'session:wallclock:proj-1',+ 'dev-orphan'+ );+ });++ test('is a no-op when disabled via SESSION_VACUUM=0', async () => {+ process.env.SESSION_VACUUM = '0';+ const { deps, redis } = makeDeps(null);++ await vacuumStaleSessions(deps);++ expect(redis.smembers).not.toHaveBeenCalled();+ });+});diff --git a/packages/core/src/modules/session/src/session-vacuum.ts b/packages/core/src/modules/session/src/session-vacuum.tsnew file mode 100644index 000000000..d62f0adbe--- /dev/null+++ b/packages/core/src/modules/session/src/session-vacuum.ts@@ -0,0 +1,97 @@+// Ported from apps/worker/src/jobs/cron.session-vacuum.ts (M7-001).++import type { Logger } from '../../../logger';+import type { SessionRuntime } from './runtime';+import { sessionsVacuumed } from './session.metrics';+import { PROJECTS_SET_KEY, wallclockSetKey } from './session-keys';++const VACUUM_DISABLED_ENV = 'SESSION_VACUUM';+const VACUUM_BATCH_SIZE_ENV = 'SESSION_VACUUM_BATCH_SIZE';+const VACUUM_STALE_THRESHOLD_ENV = 'SESSION_VACUUM_STALE_THRESHOLD_MS';+const DEFAULT_VACUUM_BATCH_SIZE = 1000;+// Much larger than the reaper deadman, so it never races with normal reaping.+const DEFAULT_VACUUM_STALE_THRESHOLD_MS = 1000 * 60 * 60 * 24 * 7;++function readIntEnv(name: string, fallback: number): number {+ return Number.parseInt(process.env[name] || String(fallback), 10);+}++export interface SessionVacuumDeps extends SessionRuntime {+ logger: Logger;+}++/**+ * Daily backstop for the rare case where `cleanup()` did not fully delete a+ * session (worker crash mid-MULTI, Redis partition). Wallclock entries older+ * than the stale threshold are either cleaned up (blob still there, id-gated)+ * or dropped (blob gone). Both paths count in `sessions_vacuumed_total`, so a+ * persistent non-zero value surfaces a deeper issue.+ */+export async function vacuumStaleSessions(+ deps: SessionVacuumDeps+): Promise<void> {+ if (process.env[VACUUM_DISABLED_ENV] === '0') {+ return;+ }++ const { redis, sessions, logger } = deps;+ const projectIds = await redis.smembers(PROJECTS_SET_KEY);+ if (projectIds.length === 0) {+ return;+ }++ logger.info({ projectCount: projectIds.length }, 'Vacuum tick starting');++ const cutoff =+ Date.now() -+ readIntEnv(VACUUM_STALE_THRESHOLD_ENV, DEFAULT_VACUUM_STALE_THRESHOLD_MS);+ const batchSize = readIntEnv(+ VACUUM_BATCH_SIZE_ENV,+ DEFAULT_VACUUM_BATCH_SIZE+ );+ let total = 0;+ let staleBlobs = 0;+ let missingBlobs = 0;++ for (const projectId of projectIds) {+ try {+ const candidates = await redis.zrangebyscore(+ wallclockSetKey(projectId),+ 0,+ cutoff,+ 'LIMIT',+ 0,+ batchSize+ );++ for (const deviceId of candidates) {+ const session = await sessions.getExistingSession({+ projectId,+ deviceId,+ });++ if (session) {+ await sessions.cleanup({+ projectId,+ deviceId,+ sessionId: session.id,+ profileId: session.profile_id,+ });+ sessionsVacuumed.inc({ reason: 'stale_blob' });+ staleBlobs++;+ } else {+ await redis.zrem(wallclockSetKey(projectId), deviceId);+ sessionsVacuumed.inc({ reason: 'missing_blob' });+ missingBlobs++;+ }+ total++;+ }+ } catch (error) {+ logger.error({ err: error, projectId }, 'Vacuum failed for project');+ }+ }++ if (total > 0) {+ logger.info({ total, staleBlobs, missingBlobs }, 'Vacuum tick complete');+ }+}diff --git a/packages/core/src/modules/session/src/session.metrics.ts b/packages/core/src/modules/session/src/session.metrics.tsnew file mode 100644index 000000000..38b8050b4--- /dev/null+++ b/packages/core/src/modules/session/src/session.metrics.ts@@ -0,0 +1,77 @@+// The session lifecycle collectors, moved from apps/worker/src/metrics.ts+// (M7-001) onto core's one registry. Names, labels and buckets are V1's; the+// worker-side copies die with apps/worker (P9). The at-scrape gauges+// (`sessions_active_total`, `sessions_projects_active`, `sessions_hwm_lag_ms`)+// and the ingest-side `sessions_started_total` stay with the ingest path.++import client from 'prom-client';+import { registry } from '../../../metrics';++const SECOND_MS = 1000;+const MINUTE_MS = 60 * SECOND_MS;+const HOUR_MS = 60 * MINUTE_MS;+const DAY_MS = 24 * HOUR_MS;++export const sessionEndsEnqueued = new client.Counter({+ name: 'session_ends_enqueued_total',+ help: 'session_end jobs pushed onto the sessions queue, by trigger source',+ labelNames: ['source'], // 'boundary' | 'reaper'+ registers: [registry],+});++export const sessionEndsEmitted = new client.Counter({+ name: 'session_ends_emitted_total',+ help: 'session_end events actually written (post-idempotency claim)',+ registers: [registry],+});++export const sessionEndsSkipped = new client.Counter({+ name: 'session_ends_skipped_total',+ help: 'session_end jobs that ran but did not emit, by reason',+ labelNames: ['reason'], // 'not_found' | 'already_emitted' | 'extended_after_enqueue'+ registers: [registry],+});++export const sessionsReaped = new client.Counter({+ name: 'sessions_reaped_total',+ help: 'Sessions closed by the reaper, by trigger condition',+ labelNames: ['reason'], // 'deadman'+ registers: [registry],+});++export const sessionsReaperOrphans = new client.Counter({+ name: 'sessions_reaper_orphans_total',+ help: 'Reaper found a sorted-set entry whose session blob is missing. Non-zero usually means TTL mismatch.',+ labelNames: ['reason'], // 'deadman'+ registers: [registry],+});++export const sessionDurationOnClose = new client.Histogram({+ name: 'session_duration_ms_on_close',+ help: 'Duration of closed sessions (ms)',+ buckets: [+ SECOND_MS,+ 10 * SECOND_MS,+ MINUTE_MS,+ 5 * MINUTE_MS,+ 15 * MINUTE_MS,+ 30 * MINUTE_MS,+ HOUR_MS,+ DAY_MS,+ ],+ registers: [registry],+});++export const sessionEventsOnClose = new client.Histogram({+ name: 'session_events_on_close',+ help: 'Total events (event_count + screen_view_count) on session close',+ buckets: [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000],+ registers: [registry],+});++export const sessionsVacuumed = new client.Counter({+ name: 'sessions_vacuumed_total',+ help: 'Sessions removed by the daily vacuum cron (catches blobs that cleanup() missed)',+ labelNames: ['reason'], // 'stale_blob' | 'missing_blob'+ registers: [registry],+});diff --git a/packages/core/src/modules/session/src/session.sql.test.ts b/packages/core/src/modules/session/src/session.sql.test.tsnew file mode 100644index 000000000..dcf5a3004--- /dev/null+++ b/packages/core/src/modules/session/src/session.sql.test.ts@@ -0,0 +1,241 @@+// Shape tests for the session module's fragments: every user value binds as a+// `{pN:Type}` parameter (nothing is interpolated), the optional clauses toggle+// exactly on V1's conditions, and `sql.id` rejects a column outside the+// whitelist. Result-set equivalence against V1 was proven per query on the+// local prod-copy (M7-001 report), not here — these tests run offline.++import { describe, expect, test } from 'bun:test';+import {+ hasSessionListLookback,+ querySessionsQuery,+ SESSION_DISTINCT_FIELDS,+ sessionByIdQuery,+ sessionDistinctValuesQuery,+ sessionEventsQuery,+ sessionHasReplayQuery,+ sessionListQuery,+ sessionReplayChunksQuery,+ sessionsCountQuery,+} from './session.sql';++const PROJECT_ID = 'proj-1';++function collapse(text: string): string {+ return text.replace(/\s+/g, ' ').trim();+}++describe('sessionListQuery', () => {+ test('first page: lookback window anchored at now, no cursor clause', () => {+ const { query, query_params } = sessionListQuery({+ projectId: PROJECT_ID,+ take: 50,+ lookbackDays: 0.5,+ }).toStatement();+ const text = collapse(query);++ expect(text).toContain(+ "toBool(src.session_id != '') as hasReplay FROM sessions FINAL LEFT JOIN"+ );+ expect(text).toContain('started_at > now() - INTERVAL {p2:Float64} DAY');+ expect(text).toMatch(+ /AND created_at >= toDateTime64\(\{p4:String\}, 3\) - INTERVAL \{p5:Float64\} DAY ORDER BY created_at DESC LIMIT \{p6:UInt64\}/+ );+ expect(text).not.toContain('created_at <');+ expect(text).not.toContain('profile_id =');+ expect(text).not.toContain('ILIKE');+ expect(query_params).toMatchObject({+ p1: PROJECT_ID,+ p2: 0.5,+ p3: PROJECT_ID,+ p5: 0.5,+ p6: 50,+ });+ expect(query_params.p4).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/);+ });++ test('cursor page: lookback anchored at the cursor plus the `created_at <` boundary', () => {+ const cursor = new Date('2026-07-10T12:34:56.789Z');+ const { query, query_params } = sessionListQuery({+ projectId: PROJECT_ID,+ take: 50,+ cursor,+ lookbackDays: 1,+ }).toStatement();+ const text = collapse(query);++ expect(text).toContain(+ 'AND created_at >= toDateTime64({p4:String}, 3) - INTERVAL {p5:Float64} DAY AND created_at < {p6:String}'+ );+ expect(query_params.p4).toBe('2026-07-10 12:34:56');+ expect(query_params.p6).toBe('2026-07-10 12:34:56');+ });++ test('explicit date range: no lookback window, calendar-day BETWEEN, search and filters spliced', () => {+ const { query, query_params } = sessionListQuery({+ projectId: PROJECT_ID,+ take: 10,+ lookbackDays: 0.5,+ startDate: new Date('2026-07-01T00:00:00Z'),+ endDate: new Date('2026-07-31T23:59:59Z'),+ profileId: 'user-1',+ search: 'checkout',+ filterClauses: { f0: "country = 'SE'", f1: "device = 'mobile'" },+ }).toStatement();+ const text = collapse(query);++ expect(text).not.toContain('toDateTime64');+ expect(text).toContain(+ "AND toDate(created_at) BETWEEN toDate({p4:String}) AND toDate({p5:String}) AND profile_id = {p6:String} AND (entry_path ILIKE {p7:String} OR exit_path ILIKE {p8:String} OR referrer ILIKE {p9:String} OR referrer_name ILIKE {p10:String}) AND country = 'SE' AND device = 'mobile' ORDER BY"+ );+ expect(query_params).toMatchObject({+ p4: '2026-07-01 00:00:00',+ p5: '2026-07-31 23:59:59',+ p6: 'user-1',+ p7: '%checkout%',+ p10: '%checkout%',+ p11: 10,+ });+ });++ test('hasSessionListLookback mirrors V1: off only for a date-bounded first page', () => {+ const range = { startDate: new Date(), endDate: new Date() };+ expect(hasSessionListLookback({})).toBe(true);+ expect(hasSessionListLookback({ cursor: new Date(), ...range })).toBe(true);+ expect(hasSessionListLookback(range)).toBe(false);+ });+});++describe('sessionsCountQuery', () => {+ test('counts sign = 1 rows with the same optional clauses as the list', () => {+ const { query, query_params } = sessionsCountQuery({+ projectId: PROJECT_ID,+ search: 'x',+ }).toStatement();++ expect(collapse(query)).toBe(+ 'SELECT count(*) as count FROM sessions WHERE project_id = {p1:String} AND sign = 1 AND (entry_path ILIKE {p2:String} OR exit_path ILIKE {p3:String} OR referrer ILIKE {p4:String} OR referrer_name ILIKE {p5:String})'+ );+ expect(query_params).toEqual({+ p1: PROJECT_ID,+ p2: '%x%',+ p3: '%x%',+ p4: '%x%',+ p5: '%x%',+ });+ });+});++describe('replay and byId', () => {+ test('sessionReplayChunksQuery binds page bounds as UInt64', () => {+ const { query, query_params } = sessionReplayChunksQuery({+ sessionId: 'sess-1',+ projectId: PROJECT_ID,+ limit: 41,+ offset: 80,+ }).toStatement();++ expect(collapse(query)).toBe(+ 'SELECT chunk_index, payload FROM session_replay_chunks WHERE session_id = {p1:String} AND project_id = {p2:String} ORDER BY started_at, ended_at, chunk_index LIMIT {p3:UInt64} OFFSET {p4:UInt64}'+ );+ expect(query_params).toEqual({+ p1: 'sess-1',+ p2: PROJECT_ID,+ p3: 41,+ p4: 80,+ });+ });++ test('sessionByIdQuery and sessionHasReplayQuery keep V1 text', () => {+ const byId = sessionByIdQuery({+ sessionId: 'sess-1',+ projectId: PROJECT_ID,+ }).toStatement();+ expect(collapse(byId.query)).toBe(+ 'SELECT * FROM sessions FINAL WHERE id = {p1:String} AND project_id = {p2:String} AND sign = 1'+ );++ const hasReplay = sessionHasReplayQuery({+ sessionId: 'sess-1',+ projectId: PROJECT_ID,+ }).toStatement();+ expect(collapse(hasReplay.query)).toBe(+ 'SELECT 1 AS n FROM session_replay_chunks WHERE session_id = {p1:String} AND project_id = {p2:String} LIMIT 1'+ );+ });+});++describe('sessionDistinctValuesQuery', () => {+ test('whitelisted column is emitted as an identifier, limit bound', () => {+ const { query, query_params } = sessionDistinctValuesQuery({+ projectId: PROJECT_ID,+ field: 'referrer_name',+ limit: 200,+ }).toStatement();++ expect(collapse(query)).toBe(+ "SELECT referrer_name AS value, count() AS cnt FROM sessions WHERE project_id = {p1:String} AND referrer_name != '' AND sign = 1 AND created_at > now() - INTERVAL {p2:UInt64} DAY GROUP BY value ORDER BY cnt DESC LIMIT {p3:UInt64}"+ );+ expect(query_params).toEqual({ p1: PROJECT_ID, p2: 90, p3: 200 });+ });++ test('a column outside SESSION_DISTINCT_FIELDS throws instead of interpolating', () => {+ expect(SESSION_DISTINCT_FIELDS).not.toContain('profile_id');+ expect(() =>+ sessionDistinctValuesQuery({+ projectId: PROJECT_ID,+ field: 'profile_id' as never,+ limit: 200,+ })+ ).toThrow();+ });+});++describe('querySessionsQuery', () => {+ test('equality filters follow V1 clause order, dates bind as Strings', () => {+ const { query, query_params } = querySessionsQuery({+ projectId: PROJECT_ID,+ startDate: '2026-07-01 00:00:00',+ endDate: '2026-07-31 00:00:00',+ limit: 20,+ browser: 'Chrome',+ country: 'SE',+ profileId: 'user-1',+ }).toStatement();++ expect(collapse(query)).toBe(+ 'SELECT * FROM sessions WHERE project_id = {p1:String} AND sign = 1 AND profile_id = {p2:String} AND country = {p3:String} AND browser = {p4:String} AND created_at BETWEEN {p5:String} AND {p6:String} LIMIT {p7:UInt64}'+ );+ expect(query_params).toEqual({+ p1: PROJECT_ID,+ p2: 'user-1',+ p3: 'SE',+ p4: 'Chrome',+ p5: '2026-07-01 00:00:00',+ p6: '2026-07-31 00:00:00',+ p7: 20,+ });+ });+});++describe('sessionEventsQuery', () => {+ test('binds the padded window as ClickHouse date strings', () => {+ const { query, query_params } = sessionEventsQuery({+ sessionId: 'sess-1',+ projectId: PROJECT_ID,+ startAt: new Date('2026-06-08T10:29:59.000Z'),+ endAt: new Date('2026-06-08T11:00:01.000Z'),+ limit: 500,+ }).toStatement();++ expect(collapse(query)).toBe(+ 'SELECT * FROM events WHERE session_id = {p1:String} AND project_id = {p2:String} AND created_at BETWEEN {p3:String} AND {p4:String} ORDER BY created_at DESC LIMIT {p5:UInt64}'+ );+ expect(query_params).toEqual({+ p1: 'sess-1',+ p2: PROJECT_ID,+ p3: '2026-06-08 10:29:59',+ p4: '2026-06-08 11:00:01',+ p5: 500,+ });+ });+});diff --git a/packages/core/src/modules/session/src/session.sql.ts b/packages/core/src/modules/session/src/session.sql.tsnew file mode 100644index 000000000..01170548e--- /dev/null+++ b/packages/core/src/modules/session/src/session.sql.ts@@ -0,0 +1,319 @@+// Every ClickHouse query the session module runs, as pure `sql` fragments+// (ADR-013). Converted 1:1 from packages/db/src/services/session.service.ts+// and apps/worker/src/jobs/events.create-session-end.ts (M7-001): the SQL text+// is V1's, with every value bound as a `{pN:Type}` parameter instead of an+// escaped literal, and each conversion was proven byte-equivalent against the+// local prod-copy (see the task report).+//+// Dates bind as V1's own `YYYY-MM-DD HH:mm:ss` strings on purpose: a String+// param in a DateTime64 position is parsed exactly like the literal it+// replaces, so the result sets cannot drift by a millisecond truncation.+//+// Cluster note (docs/ENVIRONMENT.md): `sessions` and `session_replay_chunks`+// are Distributed on Cloud. The has_replay LEFT JOIN subquery and any+// `IN (SELECT ...)` a filter compiles to keep V1's exact shape and run under+// the client's `distributed_product_mode: 'allow'` as before — converting a+// query is not the place to change its cluster semantics.++import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import { formatClickhouseDate } from './dates';+import {+ type CompiledFilterClauses,+ spliceCompiledFilters,+} from './filter-clauses';++const TABLE = {+ sessions: 'sessions',+ sessionReplayChunks: 'session_replay_chunks',+ events: 'events',+} as const;++/** The list page's projection — `IClickhouseSession` minus what the table never renders. */+const SESSION_LIST_COLUMNS = [+ 'created_at',+ 'ended_at',+ 'id',+ 'profile_id',+ 'entry_path',+ 'exit_path',+ 'duration',+ 'is_bounce',+ 'referrer_name',+ 'referrer',+ 'country',+ 'city',+ 'os',+ 'browser',+ 'brand',+ 'model',+ 'device',+ 'screen_view_count',+ 'event_count',+ 'revenue',+ 'groups',+] as const;++export const SESSION_DISTINCT_FIELDS = [+ 'referrer_name',+ 'country',+ 'os',+ 'browser',+ 'device',+] as const;++export type SessionDistinctField = (typeof SESSION_DISTINCT_FIELDS)[number];++const DISTINCT_VALUES_LOOKBACK_DAYS = 90;++function searchCondition(search: string): SqlFragment {+ const pattern = sql.string(`%${search}%`);+ return sql`AND (entry_path ILIKE ${pattern} OR exit_path ILIKE ${pattern} OR referrer ILIKE ${pattern} OR referrer_name ILIKE ${pattern})`;+}++function calendarDayRange(startDate: Date, endDate: Date): SqlFragment {+ return sql`AND toDate(created_at) BETWEEN toDate(${sql.string(formatClickhouseDate(startDate))}) AND toDate(${sql.string(formatClickhouseDate(endDate))})`;+}++function optional(+ condition: boolean | string | undefined | null,+ fragment: () => SqlFragment+): SqlFragment {+ return condition ? fragment() : sql.empty;+}++export interface SessionListQuery {+ projectId: string;+ take: number;+ /** Anchors the lookback window and the `created_at <` page boundary. */+ cursor?: Date;+ /** Already capped by the caller (lookback.ts ceiling). */+ lookbackDays: number;+ startDate?: Date;+ endDate?: Date;+ profileId?: string;+ search?: string;+ filterClauses?: CompiledFilterClauses;+}++/** V1 only skips the lookback window when a page is bounded by explicit dates. */+export function hasSessionListLookback(query: {+ cursor?: Date;+ startDate?: Date;+ endDate?: Date;+}): boolean {+ return (+ query.cursor instanceof Date ||+ !(query.cursor || (query.startDate && query.endDate))+ );+}++export function sessionListQuery(query: SessionListQuery): SqlFragment {+ const {+ projectId,+ take,+ cursor,+ lookbackDays,+ startDate,+ endDate,+ profileId,+ search,+ filterClauses = {},+ } = query;+ const hasDateRange = Boolean(startDate && endDate);+ const lookbackAnchor = sql.string(formatClickhouseDate(cursor ?? new Date()));+ const lookback = sql.float64(lookbackDays);++ return sql`+ SELECT ${sql.join(+ SESSION_LIST_COLUMNS.map((column) => sql.id(column, SESSION_LIST_COLUMNS))+ )}, toBool(src.session_id != '') as hasReplay+ FROM ${sql.id(TABLE.sessions)} FINAL+ LEFT JOIN (+ SELECT DISTINCT session_id+ FROM ${sql.id(TABLE.sessionReplayChunks)}+ WHERE project_id = ${sql.string(projectId)}+ AND started_at > now() - INTERVAL ${lookback} DAY+ ) AS src ON src.session_id = id+ WHERE project_id = ${sql.string(projectId)}+ ${optional(+ hasSessionListLookback(query),+ () =>+ sql`AND created_at >= toDateTime64(${lookbackAnchor}, 3) - INTERVAL ${lookback} DAY`+ )}+ ${optional(+ cursor instanceof Date,+ () =>+ sql`AND created_at < ${sql.string(formatClickhouseDate(cursor as Date))}`+ )}+ ${optional(hasDateRange, () =>+ calendarDayRange(startDate as Date, endDate as Date)+ )}+ ${optional(profileId, () => sql`AND profile_id = ${sql.string(profileId as string)}`)}+ ${optional(search, () => searchCondition(search as string))}+ ${spliceCompiledFilters(filterClauses)}+ ORDER BY created_at DESC+ LIMIT ${sql.uint64(take)}+ `;+}++export interface SessionsCountQuery {+ projectId: string;+ profileId?: string;+ startDate?: Date;+ endDate?: Date;+ search?: string;+ filterClauses?: CompiledFilterClauses;+}++export function sessionsCountQuery(query: SessionsCountQuery): SqlFragment {+ const {+ projectId,+ profileId,+ startDate,+ endDate,+ search,+ filterClauses = {},+ } = query;+ const hasDateRange = Boolean(startDate && endDate);++ return sql`+ SELECT count(*) as count+ FROM ${sql.id(TABLE.sessions)}+ WHERE project_id = ${sql.string(projectId)}+ AND sign = 1+ ${optional(profileId, () => sql`AND profile_id = ${sql.string(profileId as string)}`)}+ ${optional(hasDateRange, () =>+ calendarDayRange(startDate as Date, endDate as Date)+ )}+ ${optional(search, () => searchCondition(search as string))}+ ${spliceCompiledFilters(filterClauses)}+ `;+}++export function sessionReplayChunksQuery(query: {+ sessionId: string;+ projectId: string;+ /** Page size + 1, so the caller can tell whether another page exists. */+ limit: number;+ offset: number;+}): SqlFragment {+ return sql`+ SELECT chunk_index, payload+ FROM ${sql.id(TABLE.sessionReplayChunks)}+ WHERE session_id = ${sql.string(query.sessionId)}+ AND project_id = ${sql.string(query.projectId)}+ ORDER BY started_at, ended_at, chunk_index+ LIMIT ${sql.uint64(query.limit)}+ OFFSET ${sql.uint64(query.offset)}+ `;+}++export function sessionDistinctValuesQuery(query: {+ projectId: string;+ field: SessionDistinctField;+ limit: number;+}): SqlFragment {+ const column = sql.id(query.field, SESSION_DISTINCT_FIELDS);+ return sql`+ SELECT ${column} AS value, count() AS cnt+ FROM ${sql.id(TABLE.sessions)}+ WHERE project_id = ${sql.string(query.projectId)}+ AND ${column} != ''+ AND sign = 1+ AND created_at > now() - INTERVAL ${sql.uint64(DISTINCT_VALUES_LOOKBACK_DAYS)} DAY+ GROUP BY value+ ORDER BY cnt DESC+ LIMIT ${sql.uint64(query.limit)}+ `;+}++export function sessionByIdQuery(query: {+ sessionId: string;+ projectId: string;+}): SqlFragment {+ return sql`+ SELECT *+ FROM ${sql.id(TABLE.sessions)} FINAL+ WHERE id = ${sql.string(query.sessionId)}+ AND project_id = ${sql.string(query.projectId)}+ AND sign = 1+ `;+}++export function sessionHasReplayQuery(query: {+ sessionId: string;+ projectId: string;+}): SqlFragment {+ return sql`+ SELECT 1 AS n+ FROM ${sql.id(TABLE.sessionReplayChunks)}+ WHERE session_id = ${sql.string(query.sessionId)}+ AND project_id = ${sql.string(query.projectId)}+ LIMIT 1+ `;+}++/** The equality filters `querySessionsCore` accepts, in V1's clause order. */+const QUERY_SESSIONS_EQUALITY_COLUMNS = [+ ['profileId', 'profile_id'],+ ['referrer', 'referrer'],+ ['referrerName', 'referrer_name'],+ ['referrerType', 'referrer_type'],+ ['device', 'device'],+ ['country', 'country'],+ ['city', 'city'],+ ['os', 'os'],+ ['browser', 'browser'],+] as const;++type QuerySessionsEqualityKey =+ (typeof QUERY_SESSIONS_EQUALITY_COLUMNS)[number][0];++export interface QuerySessionsQuery+ extends Partial<Record<QuerySessionsEqualityKey, string>> {+ projectId: string;+ /** `YYYY-MM-DD HH:mm:ss`, inclusive on both ends — what clix.datetime sent. */+ startDate: string;+ endDate: string;+ limit: number;+ filterClauses?: CompiledFilterClauses;+}++export function querySessionsQuery(query: QuerySessionsQuery): SqlFragment {+ const equalities = QUERY_SESSIONS_EQUALITY_COLUMNS.filter(+ ([key]) => query[key]+ ).map(+ ([key, column]) =>+ sql`AND ${sql.id(column)} = ${sql.string(query[key] as string)}`+ );++ return sql`+ SELECT *+ FROM ${sql.id(TABLE.sessions)}+ WHERE project_id = ${sql.string(query.projectId)}+ AND sign = 1+ ${sql.join(equalities, ' ')}+ AND created_at BETWEEN ${sql.string(query.startDate)} AND ${sql.string(query.endDate)}+ ${spliceCompiledFilters(query.filterClauses ?? {})}+ LIMIT ${sql.uint64(query.limit)}+ `;+}++/** A closed session's events, newest first, for funnel-rule matching. */+export function sessionEventsQuery(query: {+ sessionId: string;+ projectId: string;+ startAt: Date;+ endAt: Date;+ limit: number;+}): SqlFragment {+ return sql`+ SELECT *+ FROM ${sql.id(TABLE.events)}+ WHERE session_id = ${sql.string(query.sessionId)}+ AND project_id = ${sql.string(query.projectId)}+ AND created_at BETWEEN ${sql.string(formatClickhouseDate(query.startAt))} AND ${sql.string(formatClickhouseDate(query.endAt))}+ ORDER BY created_at DESC+ LIMIT ${sql.uint64(query.limit)}+ `;+}diff --git a/packages/core/src/modules/session/src/usage.ts b/packages/core/src/modules/session/src/usage.tsnew file mode 100644index 000000000..02869977f--- /dev/null+++ b/packages/core/src/modules/session/src/usage.ts@@ -0,0 +1,237 @@+// Ported from apps/worker/src/jobs/sessions.ts (M7-001): after each+// session_end the owning project's and organization's event counters are+// refreshed, at most once an hour per project, and usage alert emails go out+// at 80% / 100% of the billing limit.++import type { Organization } from '@openpanel/db/src/prisma-client';+import { cacheable } from '@openpanel/redis';+import { sendEmail } from '../../../clients/email';+import { createLogger, type ILogger } from '../../../clients/logger';+import type { Logger } from '../../../logger';+import { getOrganizationBillingEventsCount } from '../../organization/organization.service';+import { getProjectEventsCount } from '../../project/project.service';++const INT4_MAX = 2_147_483_647;+const USAGE_WARNING_THRESHOLD = 0.8;+const UPDATE_EVENTS_COUNT_CACHE_SECONDS = 60 * 60;+const DEFAULT_DASHBOARD_URL = 'https://dashboard.openpanel.dev';++function loadDb() {+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);+}++// `cacheable` keys on the function's arguments, so the per-run ctx.logger+// cannot travel with the projectId; a module logger, created on first use+// (same as import.service.ts), stands in for it.+let _logger: ILogger | undefined;+function getLogger(): Logger {+ _logger ??= createLogger({ name: 'core:session' });+ return _logger;+}++function isSelfHosted(): boolean {+ return process.env.SELF_HOSTED === 'true';+}++/** Saturating: the columns are INT4 and lifetime counts can exceed them. */+function clampToInt4(count: number): number {+ return Math.min(count, INT4_MAX);+}++function nextExceededAt(+ organization: Organization,+ count: number+): Date | null {+ // Self-hosting has no billing limits: never flag, and clear any stale flag+ // set before this guard existed (default limit 0 trips on the first event).+ if (isSelfHosted()) {+ return null;+ }+ const limit = organization.subscriptionPeriodEventsLimit;+ if (count > limit && !organization.subscriptionPeriodEventsCountExceededAt) {+ return new Date();+ }+ if (count <= limit) {+ return null;+ }+ return organization.subscriptionPeriodEventsCountExceededAt;+}++export const updateEventsCount = cacheable(async function updateEventsCount(+ projectId: string+) {+ const logger = getLogger();+ const db = await loadDb();+ const organization = await db.organization.findFirst({+ where: { projects: { some: { id: projectId } } },+ include: { projects: true },+ });++ if (!organization) {+ return;+ }++ const organizationEventsCount =+ await getOrganizationBillingEventsCount(organization);+ const projectEventsCount = await getProjectEventsCount(projectId);++ if (projectEventsCount) {+ // Only a sort key and activity threshold, never billing — clamping is safe.+ await db.project.update({+ where: { id: projectId },+ data: { eventsCount: clampToInt4(projectEventsCount) },+ });+ }++ if (organizationEventsCount) {+ await db.organization.update({+ where: { id: organization.id },+ data: {+ subscriptionPeriodEventsCount: clampToInt4(organizationEventsCount),+ subscriptionPeriodEventsCountExceededAt: nextExceededAt(+ organization,+ organizationEventsCount+ ),+ },+ });++ if (!isSelfHosted()) {+ try {+ await sendUsageAlerts(organization, organizationEventsCount, logger);+ } catch (error) {+ logger.error({ err: error }, 'Failed to send usage alert emails');+ }+ }+ }++ return true;+}, UPDATE_EVENTS_COUNT_CACHE_SECONDS);++/**+ * One warning at 80% and one notice at 100% per billing cycle. The sent-at+ * markers are cleared by the Polar webhook when a new cycle resets the usage+ * counter (or the limit is raised), so each cycle can alert again.+ */+async function sendUsageAlerts(+ organization: Organization,+ count: number,+ logger: Logger+) {+ const limit = organization.subscriptionPeriodEventsLimit;+ if (!limit || limit <= 0) {+ return;+ }++ const exceeded = count > limit && !organization.usageExceededSentAt;+ const nearLimit =+ !exceeded &&+ count >= limit * USAGE_WARNING_THRESHOLD &&+ count <= limit &&+ !organization.usageWarningSentAt;++ if (!(exceeded || nearLimit)) {+ return;+ }++ const db = await loadDb();++ // Claim the alert atomically BEFORE sending: session jobs for different+ // projects of the same org can run concurrently, and both would otherwise+ // read null markers and double-send. Marking the warning together with the+ // exceeded notice keeps a both-thresholds-in-one-jump crossing from queueing+ // a redundant warning afterwards. Rolled back if every send fails.+ const claimedAt = new Date();+ const claimed = await db.organization.updateMany({+ where: {+ id: organization.id,+ ...(exceeded+ ? { usageExceededSentAt: null }+ : { usageWarningSentAt: null }),+ },+ data: exceeded+ ? { usageExceededSentAt: claimedAt, usageWarningSentAt: claimedAt }+ : { usageWarningSentAt: claimedAt },+ });+ if (claimed.count === 0) {+ return;+ }++ try {+ const admins = await db.member.findMany({+ where: {+ organizationId: organization.id,+ role: 'org:admin',+ user: { deletedAt: null },+ },+ include: { user: { select: { email: true, firstName: true } } },+ });++ const billingUrl = `${process.env.DASHBOARD_URL ?? DEFAULT_DASHBOARD_URL}/${organization.id}/billing`;+ const recipients = new Map<string, string | undefined>();+ for (const member of admins) {+ if (member.user?.email) {+ recipients.set(member.user.email, member.user.firstName ?? undefined);+ }+ }++ let failedRecipients = 0;+ for (const [email, firstName] of recipients) {+ // One bad address must not abort the loop or roll back the claim —+ // that would re-email the recipients that succeeded.+ try {+ if (exceeded) {+ await sendEmail('usage-limit-exceeded', {+ to: email,+ data: {+ firstName,+ organizationName: organization.name,+ billingUrl,+ eventsLimit: limit,+ },+ });+ } else {+ await sendEmail('usage-near-limit', {+ to: email,+ data: {+ firstName,+ organizationName: organization.name,+ billingUrl,+ eventsCount: count,+ eventsLimit: limit,+ },+ });+ }+ } catch (error) {+ failedRecipients++;+ logger.error(+ { err: error, organizationId: organization.id, recipient: email },+ 'Failed to send usage alert to recipient'+ );+ }+ }++ logger.info(+ {+ organizationId: organization.id,+ count,+ limit,+ kind: exceeded ? 'exceeded' : 'near-limit',+ recipients: recipients.size,+ failedRecipients,+ },+ 'Sent usage alert emails'+ );+ } catch (error) {+ // Release the claim so the next usage update retries the alert.+ await db.organization.updateMany({+ where: { id: organization.id },+ data: exceeded+ ? {+ usageExceededSentAt: null,+ usageWarningSentAt: organization.usageWarningSentAt,+ }+ : { usageWarningSentAt: null },+ });+ throw error;+ }+}diff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.tsindex 5156a1fe0..5b9fc2225 100644--- a/packages/core/src/rpc.router.ts+++ b/packages/core/src/rpc.router.ts@@ -21,6 +21,7 @@ import { organizationRouter } from './modules/organization/organization.rpc';import { projectRouter } from './modules/project/project.rpc';import { realtimeRouter } from './modules/realtime/realtime.rpc';import { referenceRouter } from './modules/reference/reference.rpc';+import { sessionRouter } from './modules/session/session.rpc';import { shareRouter } from './modules/share/share.rpc';import { subscriptionRouter } from './modules/subscription/subscription.rpc';import { userRouter } from './modules/user/user.rpc';@@ -47,6 +48,7 @@ export const appRouter = createTRPCRouter({notification: notificationRouter,integration: integrationRouter,subscription: subscriptionRouter,+ session: sessionRouter,});export type AppRouter = typeof appRouter;diff --git a/packages/core/src/services.ts b/packages/core/src/services.tsindex bbd6c9351..b3dc6b17e 100644--- a/packages/core/src/services.ts+++ b/packages/core/src/services.ts@@ -32,6 +32,10 @@ import {createOrganizationService,type OrganizationService,} from './modules/organization/organization.service';+import {+ createSessionService,+ type SessionService,+} from './modules/session/session.service';/** What every service factory receives — derived from Ctx, so it cannot drift. */export type ServiceDeps = Pick<@@ -57,6 +61,7 @@ export interface Services {organization: OrganizationService;onboarding: OnboardingService;notification: NotificationService;+ session: SessionService;}export function createServices(deps: ServiceDeps): Services {@@ -72,5 +77,6 @@ export function createServices(deps: ServiceDeps): Services {organization: createOrganizationService(deps),onboarding: createOnboardingService(deps),notification: createNotificationService(deps),+ session: createSessionService(deps),};}diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex a8a2167af..8340129ef 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -1,554 +1,33 @@-import { getSafeJson } from '@openpanel/json';-import { cacheable } from '@openpanel/redis';-import type { IChartEventFilter } from '@openpanel/validation';-import sqlstring from 'sqlstring';-import {- ch,- chQuery,- convertClickhouseDateToJs,- formatClickhouseDate,- TABLE_NAMES,-} from '../clickhouse/client';-import { clix } from '../clickhouse/query-builder';-import { createSqlBuilder } from '../sql-builder';-import { resolveMaxLookbackDays } from './lookback';-import { buildFilterWhere } from './filter-where.service';-import { getProfilesCached, type IServiceProfile } from './profile.service';--export interface IClickhouseSession {- id: string;- profile_id: string;- event_count: number;- screen_view_count: number;- entry_path: string;- entry_origin: string;- exit_path: string;- exit_origin: string;- created_at: string;- ended_at: string;- referrer: string;- referrer_name: string;- referrer_type: string;- os: string;- os_version: string;- browser: string;- browser_version: string;- device: string;- brand: string;- model: string;- country: string;- region: string;- city: string;- longitude: number | null;- latitude: number | null;- is_bounce: boolean;- project_id: string;- device_id: string;- duration: number;- utm_medium: string;- utm_source: string;- utm_campaign: string;- utm_content: string;- utm_term: string;- revenue: number;- sign: 1 | -1;- version: number;- // Dynamically added- has_replay?: boolean;- groups: string[];-}--export interface IServiceSession {- id: string;- profileId: string;- eventCount: number;- screenViewCount: number;- entryPath: string;- entryOrigin: string;- exitPath: string;- exitOrigin: string;- createdAt: Date;- endedAt: Date;- referrer: string;- referrerName: string;- referrerType: string;- os: string;- osVersion: string;- browser: string;- browserVersion: string;- device: string;- brand: string;- model: string;- country: string;- region: string;- city: string;- longitude: number | null;- latitude: number | null;- isBounce: boolean;- projectId: string;- deviceId: string;- duration: number;- utmMedium: string;- utmSource: string;- utmCampaign: string;- utmContent: string;- utmTerm: string;- revenue: number;- profile?: IServiceProfile;- hasReplay?: boolean;- groups: string[];-}--export interface GetSessionListOptions {- projectId: string;- profileId?: string;- take: number;- filters?: IChartEventFilter[];- startDate?: Date;- endDate?: Date;- search?: string;- cursor?: Date;- dateIntervalInDays?: number;-}--export function transformSession(session: IClickhouseSession): IServiceSession {- return {- id: session.id,- profileId: session.profile_id,- eventCount: session.event_count,- screenViewCount: session.screen_view_count,- entryPath: session.entry_path,- entryOrigin: session.entry_origin,- exitPath: session.exit_path,- exitOrigin: session.exit_origin,- createdAt: convertClickhouseDateToJs(session.created_at),- endedAt: convertClickhouseDateToJs(session.ended_at),- referrer: session.referrer,- referrerName: session.referrer_name,- referrerType: session.referrer_type,- os: session.os,- osVersion: session.os_version,- browser: session.browser,- browserVersion: session.browser_version,- device: session.device,- brand: session.brand,- model: session.model,- country: session.country,- region: session.region,- city: session.city,- longitude: session.longitude,- latitude: session.latitude,- isBounce: session.is_bounce,- projectId: session.project_id,- deviceId: session.device_id,- duration: session.duration,- utmMedium: session.utm_medium,- utmSource: session.utm_source,- utmCampaign: session.utm_campaign,- utmContent: session.utm_content,- utmTerm: session.utm_term,- revenue: session.revenue,- profile: undefined,- hasReplay: session.has_replay,- groups: session.groups,- };-}--export async function getSessionList(options: GetSessionListOptions) {- const {- cursor,- take,- projectId,- profileId,- filters,- startDate,- endDate,- search,- dateIntervalInDays = 0.5,- } = options;-- const { sb, getSql } = createSqlBuilder();-- sb.from = `${TABLE_NAMES.sessions} FINAL`;- sb.limit = take;- sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;-- // Deployment-tunable ceiling for the empty-result lookback (see lookback.ts).- const MAX_DATE_INTERVAL_IN_DAYS = resolveMaxLookbackDays(- 'SESSION_LIST_MAX_LOOKBACK_DAYS',- 365,- );- // Cap the date interval to prevent infinity- const safeDateIntervalInDays = Math.min(- dateIntervalInDays,- MAX_DATE_INTERVAL_IN_DAYS- );-- if (cursor instanceof Date) {- sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(cursor))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;- sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;- }-- if (!(cursor || (startDate && endDate))) {- sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;- }-- if (startDate && endDate) {- sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;- }-- sb.orderBy.created_at = 'created_at DESC';-- if (profileId) {- sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;- }- if (search) {- const s = sqlstring.escape(`%${search}%`);- sb.where.search = `(entry_path ILIKE ${s} OR exit_path ILIKE ${s} OR referrer ILIKE ${s} OR referrer_name ILIKE ${s})`;- }- if (filters?.length) {- Object.assign(- sb.where,- buildFilterWhere(filters, projectId, {- selfTable: 'sessions',- profileIdExpr: 'profile_id',- groupsExpr: 'groups',- startDate,- endDate,- })- );- }-- const columns = [- 'created_at',- 'ended_at',- 'id',- 'profile_id',- 'entry_path',- 'exit_path',- 'duration',- 'is_bounce',- 'referrer_name',- 'referrer',- 'country',- 'city',- 'os',- 'browser',- 'brand',- 'model',- 'device',- 'screen_view_count',- 'event_count',- 'revenue',- 'groups',- ];-- columns.forEach((column) => {- sb.select[column] = column;- });-- sb.select.has_replay = `toBool(src.session_id != '') as hasReplay`;- sb.joins.has_replay = `LEFT JOIN (SELECT DISTINCT session_id FROM ${TABLE_NAMES.session_replay_chunks} WHERE project_id = ${sqlstring.escape(projectId)} AND started_at > now() - INTERVAL ${safeDateIntervalInDays} DAY) AS src ON src.session_id = id`;-- const sql = getSql();- const data = await chQuery<- IClickhouseSession & {- latestCreatedAt: string;- hasReplay: boolean;- }- >(sql);-- // If no results and we haven't reached the max window, retry with a larger interval- if (- data.length === 0 &&- sb.where.cursorWindow &&- safeDateIntervalInDays < MAX_DATE_INTERVAL_IN_DAYS- ) {- return getSessionList({- ...options,- dateIntervalInDays: dateIntervalInDays * 2,- });- }-- // Profile hydration (unchanged)- const profileIds = data- .filter((e) => e.device_id !== e.profile_id)- .map((e) => e.profile_id);- const profiles = await getProfilesCached(profileIds, projectId);- const map = new Map<string, IServiceProfile>(profiles.map((p) => [p.id, p]));-- const items = data.map(transformSession).map((item) => ({- ...item,- profile: map.get(item.profileId) ?? {- id: item.profileId,- email: '',- avatar: '',- firstName: '',- lastName: '',- createdAt: new Date(),- lastSeenAt: new Date(),- projectId,- isExternal: false,- properties: {},- groups: [],- },- }));-- // Compute cursors from page edges- const last = items.at(-1);-- const meta = {- next: last ? last.createdAt.toISOString() : undefined,- };-- return { items, meta };-}--export async function getSessionsCount({- projectId,- profileId,- filters,- startDate,- endDate,- search,-}: Omit<GetSessionListOptions, 'take' | 'cursor'>) {- const { sb, getSql } = createSqlBuilder();-- sb.select.count = 'count(*) as count';- sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;- sb.where.sign = 'sign = 1';-- if (profileId) {- sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;- }-- if (startDate && endDate) {- sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;- }-- if (search) {- const s = sqlstring.escape(`%${search}%`);- sb.where.search = `(entry_path ILIKE ${s} OR exit_path ILIKE ${s} OR referrer ILIKE ${s} OR referrer_name ILIKE ${s})`;- }-- if (filters && filters.length > 0) {- Object.assign(- sb.where,- buildFilterWhere(filters, projectId, {- selfTable: 'sessions',- profileIdExpr: 'profile_id',- groupsExpr: 'groups',- startDate,- endDate,- })- );- }-- sb.from = TABLE_NAMES.sessions;-- const result = await chQuery<{ count: number }>(getSql());- return result[0]?.count ?? 0;-}--export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10);--export interface ISessionReplayChunkMeta {- chunk_index: number;- started_at: string;- ended_at: string;- events_count: number;- is_full_snapshot: boolean;-}--const REPLAY_CHUNKS_PAGE_SIZE = 40;--export async function getSessionReplayChunksFrom(- sessionId: string,- projectId: string,- fromIndex: number-) {- const rows = await chQuery<{ chunk_index: number; payload: string }>(- `SELECT chunk_index, payload- FROM ${TABLE_NAMES.session_replay_chunks}- WHERE session_id = ${sqlstring.escape(sessionId)}- AND project_id = ${sqlstring.escape(projectId)}- ORDER BY started_at, ended_at, chunk_index- LIMIT ${REPLAY_CHUNKS_PAGE_SIZE + 1}- OFFSET ${fromIndex}`- );-- return {- data: rows- .slice(0, REPLAY_CHUNKS_PAGE_SIZE)- .map((row, index) => {- const events = getSafeJson<- { type: number; data: unknown; timestamp: number }[]- >(row.payload);- if (!events) {- return null;- }- return { chunkIndex: index + fromIndex, events };- })- .filter(Boolean),- hasMore: rows.length > REPLAY_CHUNKS_PAGE_SIZE,- };-}--export const SESSION_DISTINCT_FIELDS = [- 'referrer_name',- 'country',- 'os',- 'browser',- 'device',-] as const;--export type SessionDistinctField = (typeof SESSION_DISTINCT_FIELDS)[number];--export async function getSessionDistinctValues(- projectId: string,- field: SessionDistinctField,- limit = 200-): Promise<string[]> {- const sql = `- SELECT ${field} AS value, count() AS cnt- FROM ${TABLE_NAMES.sessions}- WHERE project_id = ${sqlstring.escape(projectId)}- AND ${field} != ''- AND sign = 1- AND created_at > now() - INTERVAL 90 DAY- GROUP BY value- ORDER BY cnt DESC- LIMIT ${limit}- `;- const results = await chQuery<{ value: string }>(sql);- return results.map((r) => r.value).filter(Boolean);-}--class SessionService {- private readonly client: typeof ch;- constructor(client: typeof ch) {- this.client = client;- }-- async byId(sessionId: string, projectId: string) {- const [sessionRows, hasReplayRows] = await Promise.all([- clix(this.client)- .select<IClickhouseSession>(['*'])- .from(TABLE_NAMES.sessions, true)- .where('id', '=', sessionId)- .where('project_id', '=', projectId)- .where('sign', '=', 1)- .execute(),- chQuery<{ n: number }>(- `SELECT 1 AS n- FROM ${TABLE_NAMES.session_replay_chunks}- WHERE session_id = ${sqlstring.escape(sessionId)}- AND project_id = ${sqlstring.escape(projectId)}- LIMIT 1`- ),- ]);-- if (!sessionRows[0]) {- throw new Error('Session not found');- }-- const session = transformSession(sessionRows[0]);-- return {- ...session,- hasReplay: hasReplayRows.length > 0,- };- }-}--export const sessionService = new SessionService(ch);--import { resolveDateRange } from './date.service';--export interface QuerySessionsInput {- projectId: string;- startDate?: string;- endDate?: string;- country?: string;- city?: string;- device?: string;- browser?: string;- os?: string;- referrer?: string;- referrerName?: string;- referrerType?: string;- profileId?: string;- filters?: IChartEventFilter[];- limit?: number;-}--export async function querySessionsCore(- input: QuerySessionsInput-): Promise<IClickhouseSession[]> {- const builder = clix(ch)- .select<IClickhouseSession>([])- .from(TABLE_NAMES.sessions)- .where('project_id', '=', input.projectId)- .where('sign', '=', 1);-- if (input.profileId) {- builder.where('profile_id', '=', input.profileId);- }-- if (input.referrer) {- builder.where('referrer', '=', input.referrer);- }-- if (input.referrerName) {- builder.where('referrer_name', '=', input.referrerName);- }-- if (input.referrerType) {- builder.where('referrer_type', '=', input.referrerType);- }-- if (input.device) {- builder.where('device', '=', input.device);- }-- if (input.country) {- builder.where('country', '=', input.country);- }-- if (input.city) {- builder.where('city', '=', input.city);- }-- if (input.os) {- builder.where('os', '=', input.os);- }-- if (input.browser) {- builder.where('browser', '=', input.browser);- }-- const { startDate: start, endDate: end } = resolveDateRange(- input.startDate,- input.endDate- );-- builder.where('created_at', 'BETWEEN', [- clix.datetime(start),- clix.datetime(end),- ]);-- if (input.filters?.length) {- const filterClauses = buildFilterWhere(input.filters, input.projectId, {- selfTable: 'sessions',- profileIdExpr: 'profile_id',- groupsExpr: 'groups',- startDate: new Date(start),- endDate: new Date(end),- });- for (const clause of Object.values(filterClauses)) {- builder.rawWhere(clause);- }- }-- return builder.limit(input.limit ?? 20).execute();-}+// The session service lives in @openpanel/core now (M7-001):+// packages/core/src/modules/session/session.service.ts, with every ClickHouse+// query rewritten onto the `sql` tag (packages/core/src/modules/session/src/+// session.sql.ts). Re-exported here for existing `@openpanel/db` importers+// (packages/trpc's event router, the assistant/mcp tools, session-buffer's+// IClickhouseSession type) — same shape as share.service.ts since M6-004.+export {+ type GetSessionListOptions,+ getSessionById,+ getSessionDistinctValues,+ getSessionList,+ getSessionReplayChunksFrom,+ getSessionsCount,+ getSessionsCountCached,+ type IClickhouseSession,+ type IServiceSession,+ type ISessionReplayChunkMeta,+ type QuerySessionsInput,+ querySessionsCore,+ SESSION_DISTINCT_FIELDS,+ type SessionDistinctField,+ transformSession,+} from '@openpanel/core';++import { getSessionById } from '@openpanel/core';++// V1's `sessionService.byId(sessionId, projectId)` shape, kept for the+// callers that still reach it through `@openpanel/db`. Resolved at call time:+// core ↔ db is an import cycle and this module may evaluate first.+export const sessionService = {+ byId: (sessionId: string, projectId: string) =>+ getSessionById(sessionId, projectId),+};diff --git a/packages/db/src/session-context.ts b/packages/db/src/session-context.tsindex 275287bb3..21f0ef87f 100644--- a/packages/db/src/session-context.ts+++ b/packages/db/src/session-context.ts@@ -1,12 +1,6 @@-import { AsyncLocalStorage } from 'node:async_hooks';--type Ctx = { sessionId: string | null };--export const als = new AsyncLocalStorage<Ctx>();--export const runWithAlsSession = <T>(- sid: string | null | undefined,- fn: () => Promise<T>,-) => als.run({ sessionId: sid || null }, fn);--export const getAlsSessionId = () => als.getStore()?.sessionId ?? null;+// The ALS session scope lives in @openpanel/core now (M7-001):+// packages/core/src/modules/session/src/session-context.ts. Re-exported here+// so V1 (`packages/trpc`, `apps/api`, session-consistency.ts) and core share+// the ONE AsyncLocalStorage instance — two stores would never see each+// other's session id.+export { als, getAlsSessionId, runWithAlsSession } from '@openpanel/core';diff --git a/packages/trpc/src/routers/session.ts b/packages/trpc/src/routers/session.tsindex 0f94450c7..ec1a5fead 100644--- a/packages/trpc/src/routers/session.ts+++ b/packages/trpc/src/routers/session.ts@@ -1,37 +1,21 @@+// Dissolved into @openpanel/core's session module (M7-001): the ClickHouse+// queries moved to packages/core/src/modules/session/session.service.ts.+// This router stays (DELEGATE PATTERN) — it keeps V1's protectedProcedure+// stack and delegates every handler body to core's session functions, same+// as realtime.ts/notification.ts. V1's unused `encodeCursor`/`decodeCursor`+// went with it (packages/core/src/shared/pagination.ts owns the generic pair).+import {+ getSessionById,getSessionDistinctValues,getSessionList,getSessionReplayChunksFrom,SESSION_DISTINCT_FIELDS,- sessionService,-} from '@openpanel/db';+} from '@openpanel/core';import { zChartEventFilter } from '@openpanel/validation';import { z } from 'zod';import { createTRPCRouter, protectedProcedure } from '../trpc';-export function encodeCursor(cursor: {- createdAt: string;- id: string;-}): string {- const json = JSON.stringify(cursor);- return Buffer.from(json, 'utf8').toString('base64url'); // URL-safe-}--export function decodeCursor(- encoded: string-): { createdAt: string; id: string } | null {- try {- const json = Buffer.from(encoded, 'base64url').toString('utf8');- const obj = JSON.parse(json);- if (typeof obj.createdAt === 'string' && typeof obj.id === 'string') {- return obj;- }- return null;- } catch {- return null;- }-}-export const sessionRouter = createTRPCRouter({list: protectedProcedure.input(@@ -67,7 +51,7 @@ export const sessionRouter = createTRPCRouter({byId: protectedProcedure.input(z.object({ sessionId: z.string(), projectId: z.string() })).query(({ input: { sessionId, projectId } }) => {- return sessionService.byId(sessionId, projectId);+ return getSessionById(sessionId, projectId);}),replayChunksFrom: protectedProcedure