OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M8-004): salt module + session-context/consistency + cron flush handlers into core
ralphd8781743bb77
diff --git a/apps/worker/src/jobs/cron.salt.ts b/apps/worker/src/jobs/cron.salt.tsindex 16236db86..5e10b4d71 100644--- a/apps/worker/src/jobs/cron.salt.ts+++ b/apps/worker/src/jobs/cron.salt.ts@@ -1,59 +1,10 @@-import { generateSalt } from '@openpanel/core';-import { db, getSalts } from '@openpanel/db';--async function generateNewSalt() {- const newSalt = await db.$transaction(async (tx) => {- const existingSalts = await tx.salt.findMany({- orderBy: {- createdAt: 'desc',- },- take: 2,- });-- const created = await tx.salt.create({- data: {- salt: generateSalt(),- },- });-- // Keep the new salt + the previous newest (if exists)- const previousNewest = existingSalts[0];- const saltsToKeep = previousNewest- ? [created.salt, previousNewest.salt]- : [created.salt];-- await tx.salt.deleteMany({- where: {- salt: {- notIn: saltsToKeep,- },- },- });-- return created;- });-- await getSalts.clear();-- return newSalt;-}+// Dissolved into @openpanel/core's salt module (M8-004): rotation logic+// moved to packages/core/src/modules/salt/salt.service.ts#rotateSalt. This+// file stays (DELEGATE PATTERN) — it is `cron.ts`'s dispatcher `salt` case,+// a thin wrapper around the core function, same shape as `cron.delete.ts`+// (M6-001).+import { rotateSalt } from '@openpanel/core';export async function salt() {- const ALLOWED_RETRIES = 5;- const BASE_DELAY = 1000;- const generateNewSaltWithRetry = async (retryCount = 0) => {- try {- return await generateNewSalt();- } catch (error) {- if (retryCount < ALLOWED_RETRIES) {- await new Promise((resolve) =>- setTimeout(resolve, BASE_DELAY * 2 ** retryCount)- );- return generateNewSaltWithRetry(retryCount + 1);- }- throw error;- }- };-- return await generateNewSaltWithRetry();+ return await rotateSalt();}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 82177897a..057ab4bdf 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1221,6 +1221,16 @@ export {insertPingRecord,runPingCron,} from './modules/misc/misc.service';+// Moved from packages/db/src/services/salt.service.ts ++// apps/worker/src/jobs/cron.salt.ts (M8-004) — apps/worker's boot+// (createInitialSalts) and cron dispatch (rotateSalt) call these directly,+// the same way V1 reaches every other dissolved service here.+export type { Salts } from './modules/salt/salt.service';+export {+ createInitialSalts,+ getSalts,+ rotateSalt,+} from './modules/salt/salt.service';// New module (M7-008) — apps/api's tools controller calls these directly,// the same way V1 reaches every other dissolved service here.export type {diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex e8fdaa74a..482371196 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -84,16 +84,25 @@ test('the insights queue carries the insight module job', () => {// its scheduler is added by `startSchedulers` only when SELF_HOSTED &&// production (jobs/schedulers.ts's `PING_SCHEDULE`), but the job HANDLER is// always registered here so a scheduled run always has somewhere to land.+// `salt` (salt.jobs.ts) and the six `flush*` jobs (event/profile/session/+// group.jobs.ts) join the list at M8-004.test('the cron queue carries the insight module cron fragment', () => {expect(Object.keys(queues.cron.jobs).sort()).toEqual(['cohortRefresh','delete',+ 'flushEvents',+ 'flushGroups',+ 'flushProfileBackfill',+ 'flushProfiles',+ 'flushReplay',+ 'flushSessions','gscSync','insightCleanup','insightsDaily','onboarding','ping',+ 'salt','sessionReaper','sessionVacuum','weeklyDigest',diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex 1242d9384..18944bd60 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -15,6 +15,8 @@ import type { Producers } from './jobs/define';import { defineQueue } from './jobs/define';import type { ProducerHandle } from './jobs/producers';import { cohortCronJobs, cohortQueueJobs } from './modules/cohort/cohort.jobs';+import { eventCronJobs } from './modules/event/event.jobs';+import { groupCronJobs } from './modules/group/group.jobs';import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';import { importQueueJobs } from './modules/import/import.jobs';import {@@ -25,6 +27,8 @@ import { miscCronJobs } from './modules/misc/misc.jobs';import { notificationQueueJobs } from './modules/notification/notification.jobs';import { onboardingCronJobs } from './modules/onboarding/onboarding.jobs';import { organizationCronJobs } from './modules/organization/organization.jobs';+import { profileCronJobs } from './modules/profile/profile.jobs';+import { saltCronJobs } from './modules/salt/salt.jobs';import {sessionCronJobs,sessionQueueJobs,@@ -65,6 +69,10 @@ const cron = defineQueue('cron', {compat: legacyCompat.cron,// Every module's cron fragment spreads in here (ADR-005: "no cron module").jobs: {+ ...saltCronJobs,+ ...eventCronJobs,+ ...profileCronJobs,+ ...groupCronJobs,...insightCronJobs,...gscCronJobs,...cohortCronJobs,diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsindex 4ecbccfe5..3a9e77ac7 100644--- a/packages/core/src/jobs/schedulers.ts+++ b/packages/core/src/jobs/schedulers.ts@@ -6,10 +6,14 @@import type { Logger } from '../logger';import { cohortCronSchedules } from '../modules/cohort/cohort.jobs';+import { eventCronSchedules } from '../modules/event/event.jobs';+import { groupCronSchedules } from '../modules/group/group.jobs';import { gscCronSchedules } from '../modules/gsc/gsc.jobs';import { insightCronSchedules } from '../modules/insight/insight.jobs';import { onboardingCronSchedules } from '../modules/onboarding/onboarding.jobs';import { organizationCronSchedules } from '../modules/organization/organization.jobs';+import { profileCronSchedules } from '../modules/profile/profile.jobs';+import { saltCronSchedules } from '../modules/salt/salt.jobs';import { sessionCronSchedules } from '../modules/session/session.jobs';import { wrap } from './envelope';@@ -59,16 +63,23 @@ const MINUTE_MS = 60_000;// (apps/worker/src/boot-cron.ts). `ping` is the 20th and is conditional —// see `PING_SCHEDULE` and `startSchedulers`.export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [- { id: 'salt', schedule: { pattern: '0 0 * * *' } },+ // salt — owned by the salt module, declared next to its jobs+ // (modules/salt/salt.jobs.ts).+ ...saltCronSchedules,// delete — owned by the organization module, declared next to its jobs// (modules/organization/organization.jobs.ts)....organizationCronSchedules,- { id: 'flushEvents', schedule: { every: 10_000 } },- { id: 'flushProfiles', schedule: { every: 10_000 } },- { id: 'flushSessions', schedule: { every: 10_000 } },- { id: 'flushProfileBackfill', schedule: { every: 30_000 } },- { id: 'flushReplay', schedule: { every: 10_000 } },- { id: 'flushGroups', schedule: { every: 10_000 } },+ // flushEvents — owned by the event module, declared next to its jobs+ // (modules/event/event.jobs.ts).+ ...eventCronSchedules,+ // flushProfiles / flushProfileBackfill — owned by the profile module,+ // declared next to its jobs (modules/profile/profile.jobs.ts).+ ...profileCronSchedules,+ // flushSessions / flushReplay — owned by the session module, spread in+ // below alongside sessionReaper / sessionVacuum.+ // flushGroups — owned by the group module, declared next to its jobs+ // (modules/group/group.jobs.ts).+ ...groupCronSchedules,// onboarding — owned by the onboarding module, declared next to its jobs// (modules/onboarding/onboarding.jobs.ts)....onboardingCronSchedules,@@ -78,8 +89,9 @@ export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [// cohortRefresh — owned by the cohort module, declared next to its jobs// (modules/cohort/cohort.jobs.ts)....cohortCronSchedules,- // sessionReaper / sessionVacuum — owned by the session module, declared- // next to its jobs (modules/session/session.jobs.ts).+ // flushSessions / flushReplay / sessionReaper / sessionVacuum — owned by+ // the session module, declared next to its jobs+ // (modules/session/session.jobs.ts)....sessionCronSchedules,// insightsDaily / insightCleanup / weeklyDigest — owned by the insight// module, declared next to its jobs (modules/insight/insight.jobs.ts).diff --git a/packages/core/src/modules/event/event.jobs.ts b/packages/core/src/modules/event/event.jobs.tsnew file mode 100644index 000000000..34d495944--- /dev/null+++ b/packages/core/src/modules/event/event.jobs.ts@@ -0,0 +1,25 @@+// Ported from apps/worker/src/jobs/cron.ts's `flushEvents` case ++// boot-cron.ts's `flush`/`flushEvents` schedule (M8-004). The event buffer+// (Redis → ClickHouse `events`) is this module's domain, so its flush cron+// fragment lives here rather than on the ingest module that merely writes to it.++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';++const FLUSH_EVENTS_INTERVAL_MS = 10_000;++/** This module's fragment of the `cron` queue's jobs. */+export const eventCronJobs = {+ flushEvents: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.event.tryFlush({ trigger: 'cron' });+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */+export const eventCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'flushEvents', schedule: { every: FLUSH_EVENTS_INTERVAL_MS } },+];diff --git a/packages/core/src/modules/group/group.jobs.ts b/packages/core/src/modules/group/group.jobs.tsnew file mode 100644index 000000000..8eae2a5ba--- /dev/null+++ b/packages/core/src/modules/group/group.jobs.ts@@ -0,0 +1,24 @@+// Ported from apps/worker/src/jobs/cron.ts's `flushGroups` case ++// boot-cron.ts's `flush`/`flushGroups` schedule (M8-004). The group buffer+// (Redis → ClickHouse `groups`) is this module's domain.++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';++const FLUSH_GROUPS_INTERVAL_MS = 10_000;++/** This module's fragment of the `cron` queue's jobs. */+export const groupCronJobs = {+ flushGroups: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.group.tryFlush({ trigger: 'cron' });+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */+export const groupCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'flushGroups', schedule: { every: FLUSH_GROUPS_INTERVAL_MS } },+];diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsindex 3eb73260d..8cfc37971 100644--- a/packages/core/src/modules/ingest/ingest.service.test.ts+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -15,7 +15,15 @@* `mock.module` (unhoisted, hence the `await import` in `beforeAll`).*/-import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';+import {+ afterAll,+ beforeAll,+ beforeEach,+ describe,+ expect,+ it,+ mock,+} from 'bun:test';import type { Buffers } from '../../buffers/create-buffers';import type { GeoLocation } from '../../clients/geo';import type { IReplayPayload, ITrackHandlerPayload } from './ingest.constants';@@ -43,18 +51,20 @@ const upsertProfile = mock(async () => undefined);const getProfileById = mock(async () => null);mock.module('../../clients/geo', () => ({ getGeoLocation, getAsnInfo }));-mock.module('@openpanel/db/src/services/salt.service', () => ({ getSalts }));let getOverrideDeviceId: typeof import('./ingest.service').getOverrideDeviceId;let handleReplay: typeof import('./ingest.service').handleReplay;let ingestTrack: typeof import('./ingest.service').ingestTrack;-// The profile module is spread and then overridden by name: event.service-// imports more of it than ingest.service does, so a factory returning only-// the three overrides makes the rest vanish under `--isolate`. Overriding-// `identifyProfile` (not just `upsertProfile`) is what keeps the identify-// path off Postgres and the profile buffer — profile.service calls its own-// local `upsertProfile`, not the exported binding.+// The profile and salt modules are spread and then overridden by name:+// event.service imports more of profile.service than ingest.service does+// (and salt.service exports fetchSalts/createInitialSalts/rotateSalt besides+// getSalts), so a factory returning only the overrides makes the rest vanish+// under `--isolate`. Overriding `identifyProfile` (not just `upsertProfile`)+// is what keeps the identify path off Postgres and the profile buffer —+// profile.service calls its own local `upsertProfile`, not the exported+// binding.+let realSaltService: typeof import('../salt/salt.service');beforeAll(async () => {const profile = await import('../profile/profile.service');mock.module('../profile/profile.service', () => ({@@ -63,11 +73,23 @@ beforeAll(async () => {upsertProfile,getProfileById,}));+ realSaltService = { ...(await import('../salt/salt.service')) };+ mock.module('../salt/salt.service', () => ({ ...realSaltService, getSalts }));({ getOverrideDeviceId, handleReplay, ingestTrack } = await import('./ingest.service'));});+// salt.service.ts's own rotateSalt calls the exported `getSalts` internally+// (to invalidate its cache), so leaving this file's fake `getSalts` — which+// has no `.clear()` — in the module registry after these tests finish breaks+// salt.service.test.ts's own `rotateSalt` assertions under a bare (non+// `--isolate`) `bun test` run, which shares one module registry across every+// file in the run.+afterAll(() => {+ mock.module('../salt/salt.service', () => realSaltService);+});+const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>({type: 'track',diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex 06b8a0651..555b77d25 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -35,6 +35,7 @@ import {identifyProfile,upsertProfile,} from '../profile/profile.service';+import { getSalts } from '../salt/salt.service';import { convertClickhouseDateToJs } from '../session/src/dates';import type {IAssignGroupPayload,@@ -99,10 +100,6 @@ const ONE_MINUTE_MS = 60 * 1000;const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;const FALLBACK_USER_AGENT = 'unknown/1.0';-function loadSalts() {- return import('@openpanel/db/src/services/salt.service');-}-/** The whitelisted subset that ships with the queue payload. */export function getStringHeaders(headers: IngestHeaders@@ -250,7 +247,6 @@ async function buildContext(// Get geo location (needed for track and identify) + ASN (bot detection).// Both hit the same MaxMind readers keyed on the same IP and are cached, so// resolving them together adds no meaningful latency.- const { getSalts } = await loadSalts();const [geo, asnInfo, salts] = await Promise.all([getGeoLocation(ip),getAsnInfo(ip),@@ -540,7 +536,6 @@ export async function fetchDeviceIdentity(buffers: Pick<IngestBuffers, 'session'>,logger: Logger): Promise<DeviceIdentity> {- const { getSalts } = await loadSalts();const salts = await getSalts();const projectId = request.projectId;if (!projectId) {diff --git a/packages/core/src/modules/profile/profile.jobs.ts b/packages/core/src/modules/profile/profile.jobs.tsnew file mode 100644index 000000000..b111364e6--- /dev/null+++ b/packages/core/src/modules/profile/profile.jobs.ts@@ -0,0 +1,38 @@+// Ported from apps/worker/src/jobs/cron.ts's `flushProfiles` /+// `flushProfileBackfill` cases + boot-cron.ts's `flush` schedules (M8-004).+// Both buffers are this module's domain: the profile buffer+// (Redis → ClickHouse `profiles`) and the profile-backfill buffer, which+// patches a session's `profile_id` once a device is identified after the+// session row was already written.++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';++const FLUSH_PROFILES_INTERVAL_MS = 10_000;+const FLUSH_PROFILE_BACKFILL_INTERVAL_MS = 30_000;++/** This module's fragment of the `cron` queue's jobs. */+export const profileCronJobs = {+ flushProfiles: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.profile.tryFlush({ trigger: 'cron' });+ },+ }),+ flushProfileBackfill: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.profileBackfill.tryFlush({ trigger: 'cron' });+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — ids and cadences unchanged. */+export const profileCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'flushProfiles', schedule: { every: FLUSH_PROFILES_INTERVAL_MS } },+ {+ id: 'flushProfileBackfill',+ schedule: { every: FLUSH_PROFILE_BACKFILL_INTERVAL_MS },+ },+];diff --git a/packages/core/src/modules/salt/salt.jobs.ts b/packages/core/src/modules/salt/salt.jobs.tsnew file mode 100644index 000000000..3bd52647e--- /dev/null+++ b/packages/core/src/modules/salt/salt.jobs.ts@@ -0,0 +1,27 @@+// Ported from apps/worker/src/jobs/cron.salt.ts + boot-cron.ts's `salt`+// schedule (M8-004).+//+// `salt` is this module's fragment of the ONE `cron` queue's jobs: declared+// here, spread into jobs.registry.ts and into `CRON_SCHEDULES` by+// jobs/schedulers.ts. Scheduler id and cadence are V1's, unchanged+// (apps/worker/src/boot-cron.ts).++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';+import { rotateSalt } from './salt.service';++/** This module's fragment of the `cron` queue's jobs. */+export const saltCronJobs = {+ salt: defineJob({+ payload: z.null(),+ handler: async () => {+ await rotateSalt();+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */+export const saltCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },+];diff --git a/packages/core/src/modules/salt/salt.service.test.ts b/packages/core/src/modules/salt/salt.service.test.tsnew file mode 100644index 000000000..a9b58838d--- /dev/null+++ b/packages/core/src/modules/salt/salt.service.test.ts@@ -0,0 +1,172 @@+// salt.service.ts's db access is lazy (`await import(...)` inside each+// function — see the file's header), which is exactly what makes+// `mock.module` work here with no import-time side effects to race: the+// mock is registered before the subject's first call, not before its+// (side-effect-free) import.++import { afterAll, beforeAll, beforeEach, expect, mock, test } from 'bun:test';++interface FakeSalt {+ salt: string;+ createdAt: Date;+}++let saltStore: FakeSalt[] = [];++function resetStore(initial: FakeSalt[] = []): void {+ saltStore = [...initial];+}++function sortedByCreatedAtDesc(rows: FakeSalt[]): FakeSalt[] {+ return [...rows].sort(+ (a, b) => b.createdAt.getTime() - a.createdAt.getTime()+ );+}++const findMany = mock(+ async ({ take }: { orderBy?: unknown; take?: number } = {}) => {+ const sorted = sortedByCreatedAtDesc(saltStore);+ return typeof take === 'number' ? sorted.slice(0, take) : sorted;+ }+);++const create = mock(+ async ({ data }: { data: { salt: string; createdAt?: Date } }) => {+ const row: FakeSalt = {+ salt: data.salt,+ createdAt: data.createdAt ?? new Date(),+ };+ saltStore.push(row);+ return row;+ }+);++const deleteMany = mock(+ async ({ where }: { where: { salt: { notIn: string[] } } }) => {+ const keep = new Set(where.salt.notIn);+ const before = saltStore.length;+ saltStore = saltStore.filter((row) => keep.has(row.salt));+ return { count: before - saltStore.length };+ }+);++const saltDelegate = { findMany, create, deleteMany };++const $transaction = mock(+ async (fn: (tx: { salt: typeof saltDelegate }) => Promise<unknown>) =>+ fn({ salt: saltDelegate })+);++const actualPrismaClient = await import('@openpanel/db/src/prisma-client');+mock.module('@openpanel/db/src/prisma-client', () => ({+ ...actualPrismaClient,+ db: { salt: saltDelegate, $transaction },+}));++// Bypasses the Redis cache-aside entirely, same as organization.service.test.ts's+// `cacheableStub` — this module's own logic is exercised directly, caching is+// @openpanel/redis's concern. Only the `cacheable(name, fn, ttl)` overload+// salt.service.ts uses needs covering.+function cacheableStub<T extends (...args: unknown[]) => unknown>(+ _name: string,+ fn: T,+ _ttl: number+) {+ return Object.assign(fn, {+ getKey: () => '',+ clear: async () => 0,+ set: () => async () => 'OK' as const,+ });+}++const realRedis = { ...(await import('@openpanel/redis')) };+mock.module('@openpanel/redis', () => ({+ ...realRedis,+ cacheable: cacheableStub,+}));++afterAll(() => {+ mock.module('@openpanel/redis', () => realRedis);+});++let subject: typeof import('./salt.service');+beforeAll(async () => {+ subject = await import('./salt.service');+});++beforeEach(() => {+ resetStore();+});++test('createInitialSalts creates two salts when none exist, backdating the older one', async () => {+ await subject.createInitialSalts();++ expect(saltStore).toHaveLength(2);+ const [older, newer] = sortedByCreatedAtDesc(saltStore).reverse();+ expect(newer?.createdAt.getTime()).toBeGreaterThan(+ older?.createdAt.getTime() ?? Number.NaN+ );+});++test('createInitialSalts is a no-op once a salt already exists', async () => {+ resetStore([{ salt: 'existing', createdAt: new Date() }]);++ await subject.createInitialSalts();++ expect(saltStore).toHaveLength(1);+ expect(saltStore[0]?.salt).toBe('existing');+});++// `fetchSalts` is the uncached read `getSalts` wraps — asserted directly+// rather than through `getSalts` itself, because `getSalts`'s `cacheable(...)`+// wrapping is bound at this module's first-ever evaluation, which a bare+// (non `--isolate`) `bun test` run does not let this file control (see the+// module header). `createInitialSalts`'s own tests above already exercise+// the same "does a salt exist" branch through the production call path.+test('fetchSalts returns current + previous, newest first', async () => {+ resetStore([+ { salt: 'older', createdAt: new Date('2024-01-01T00:00:00Z') },+ { salt: 'newer', createdAt: new Date('2024-01-02T00:00:00Z') },+ ]);++ await expect(subject.fetchSalts()).resolves.toEqual({+ current: 'newer',+ previous: 'older',+ });+});++test('fetchSalts falls back previous to current when only one salt exists', async () => {+ resetStore([{ salt: 'only-one', createdAt: new Date() }]);++ await expect(subject.fetchSalts()).resolves.toEqual({+ current: 'only-one',+ previous: 'only-one',+ });+});++test('fetchSalts throws when the table is empty', async () => {+ resetStore([]);++ await expect(subject.fetchSalts()).rejects.toThrow('No salt found');+});++test('rotateSalt adds a salt and prunes everything except it and the previous newest', async () => {+ resetStore([+ { salt: 'oldest', createdAt: new Date('2024-01-01T00:00:00Z') },+ { salt: 'newest', createdAt: new Date('2024-01-02T00:00:00Z') },+ ]);++ const created = await subject.rotateSalt();++ expect(saltStore.map((row) => row.salt).sort()).toEqual(+ [created.salt, 'newest'].sort()+ );+});++test('rotateSalt keeps only the new salt when none existed before', async () => {+ resetStore([]);++ const created = await subject.rotateSalt();++ expect(saltStore.map((row) => row.salt)).toEqual([created.salt]);+});diff --git a/packages/core/src/modules/salt/salt.service.ts b/packages/core/src/modules/salt/salt.service.tsnew file mode 100644index 000000000..475d02d75--- /dev/null+++ b/packages/core/src/modules/salt/salt.service.ts@@ -0,0 +1,137 @@+// Moved from packages/db/src/services/salt.service.ts ++// apps/worker/src/jobs/cron.salt.ts (M8-004). V1's boot (createInitialSalts)+// and cron dispatch (rotateSalt, exported there as `salt`) stay the LIVE call+// sites (DELEGATE PATTERN); packages/db keeps a re-export shim.+//+// db access is LAZY (`loadDb` below), narrowed to `@openpanel/db/src/prisma-client`+// — this module is pulled into the eager barrel via ingest.service.ts, which+// resolves salts on every /track call, and a static import of the full+// @openpanel/db barrel here would spawn a pino-pretty transport worker per+// `bun test --isolate` file (see organization.service.ts's header for the+// fuller reasoning).+//+// `getSalts` wraps `fetchSalts` in `cacheable(...)` — called once, eagerly,+// at this module's first evaluation. `createInitialSalts` deliberately calls+// the raw `fetchSalts` rather than `getSalts`: under a bare (non `--isolate`)+// `bun test` run this module's first evaluation can be forced by an unrelated+// earlier test file (anything eagerly reaching ingest.service.ts or the+// jobs registry), which permanently binds `getSalts`'s caching to whatever+// `@openpanel/redis` was at that moment — the same reason+// organization.service.test.ts never exercises its own `cacheable`-wrapped+// exports. `fetchSalts` never touches `@openpanel/redis`, so it stays+// deterministic regardless of evaluation order.++import { cacheable } from '@openpanel/redis';+import { generateSalt } from '../../shared/crypto';++const SALT_CACHE_NAME = 'op:salt';+const SALT_CACHE_TTL_SECONDS = 60 * 5;+const SALT_HISTORY_SIZE = 2;+const MAX_RETRIES = 5;+const BASE_RETRY_DELAY_MS = 1000;+const NO_SALT_FOUND_MESSAGE = 'No salt found';+// The very first boot needs `current` and `previous` to already differ, so+// the bootstrap's older salt is backdated a day rather than created at the+// same instant as the newer one.+const INITIAL_PREVIOUS_SALT_BACKDATE_MS = 24 * 60 * 60 * 1000;++export interface Salts {+ current: string;+ previous: string;+}++function loadDb() {+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);+}++function sleep(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++/** The uncached read `getSalts` wraps — also boot's own existence check. */+export async function fetchSalts(): Promise<Salts> {+ const db = await loadDb();+ const [curr, prev] = await db.salt.findMany({+ orderBy: { createdAt: 'desc' },+ take: SALT_HISTORY_SIZE,+ });++ if (!curr) {+ throw new Error(NO_SALT_FOUND_MESSAGE);+ }++ return { current: curr.salt, previous: prev?.salt ?? curr.salt };+}++export const getSalts = cacheable(+ SALT_CACHE_NAME,+ fetchSalts,+ SALT_CACHE_TTL_SECONDS+);++/** Boot bootstrap: creates the first two salts the first time this ever runs. */+export async function createInitialSalts(retryCount = 0): Promise<void> {+ try {+ // Uncached: boot has nothing worth reusing a stale answer for.+ await fetchSalts();+ } catch (error) {+ if (!(error instanceof Error && error.message === NO_SALT_FOUND_MESSAGE)) {+ if (retryCount >= MAX_RETRIES) {+ throw new Error(`Failed to create salts after ${MAX_RETRIES} attempts`);+ }+ await sleep(BASE_RETRY_DELAY_MS * 2 ** retryCount);+ await createInitialSalts(retryCount + 1);+ return;+ }++ const db = await loadDb();+ await db.salt.create({+ data: {+ salt: generateSalt(),+ createdAt: new Date(Date.now() - INITIAL_PREVIOUS_SALT_BACKDATE_MS),+ },+ });+ await db.salt.create({ data: { salt: generateSalt() } });+ }+}++async function generateNewSalt() {+ const db = await loadDb();+ const created = await db.$transaction(async (tx) => {+ const existingSalts = await tx.salt.findMany({+ orderBy: { createdAt: 'desc' },+ take: SALT_HISTORY_SIZE,+ });++ const newSalt = await tx.salt.create({ data: { salt: generateSalt() } });++ // Keep the new salt + the previous newest (if any).+ const previousNewest = existingSalts[0];+ const saltsToKeep = previousNewest+ ? [newSalt.salt, previousNewest.salt]+ : [newSalt.salt];++ await tx.salt.deleteMany({ where: { salt: { notIn: saltsToKeep } } });++ return newSalt;+ });++ await getSalts.clear();++ return created;+}++/** Daily rotation: the `salt` cron job's body. */+export async function rotateSalt(+ retryCount = 0+): ReturnType<typeof generateNewSalt> {+ try {+ return await generateNewSalt();+ } catch (error) {+ if (retryCount >= MAX_RETRIES) {+ throw error;+ }+ await sleep(BASE_RETRY_DELAY_MS * 2 ** retryCount);+ return rotateSalt(retryCount + 1);+ }+}diff --git a/packages/core/src/modules/session/session.jobs.ts b/packages/core/src/modules/session/session.jobs.tsindex 36ddc2639..ca4b4dde9 100644--- a/packages/core/src/modules/session/session.jobs.ts+++ b/packages/core/src/modules/session/session.jobs.ts@@ -1,14 +1,18 @@// Ported from apps/worker/src/jobs/sessions.ts (+ events.create-session-end.ts),-// cron.session-reaper.ts and cron.session-vacuum.ts (M7-001).+// cron.session-reaper.ts and cron.session-vacuum.ts (M7-001); `flushSessions`+// and `flushReplay` (apps/worker/src/jobs/cron.ts) join them here at M8-004.//// `sessions` is this module's own queue — registry key and Redis name// `sessions`, job name `session`, `removeOnComplete: true`, concurrency 1, all// V1's (jobs.registry.ts). Its wire payload is `{ event, snapshot }`, which is// what `legacyCompat.sessions` maps V1's `{type:'createSessionEnd', payload,-// snapshot}` onto. `sessionReaper` / `sessionVacuum` are cron fragments:-// declared here, spread into the ONE `cron` queue by jobs.registry.ts and into-// `CRON_SCHEDULES` by jobs/schedulers.ts, ids and cadences unchanged-// (apps/worker/src/boot-cron.ts).+// snapshot}` onto. `flushSessions` / `flushReplay` / `sessionReaper` /+// `sessionVacuum` are cron fragments: declared here, spread into the ONE+// `cron` queue by jobs.registry.ts and into `CRON_SCHEDULES` by+// jobs/schedulers.ts, ids and cadences unchanged (apps/worker/src/boot-cron.ts).+// `flushSessions`/`flushReplay` read the buffers off `ctx.buffers` — the+// AppDeps boot singleton (ADR-007) — never a module-level import of+// @openpanel/db's buffer shim.import { z } from 'zod';import { defineJob } from '../../jobs/define';@@ -27,6 +31,8 @@ const MINUTE_MS = 60 * 1000;const SESSION_REAPER_INTERVAL_MS = 5 * MINUTE_MS;/** Daily 04:00 UTC — backstop for cleanup leaks. */const SESSION_VACUUM_CRON = '0 4 * * *';+const FLUSH_SESSIONS_INTERVAL_MS = 10_000;+const FLUSH_REPLAY_INTERVAL_MS = 10_000;// Wire-shape check only — V1 never validated it either (see// jobs/compat.test.ts). `event` is an IServiceCreateEventPayload after a JSON@@ -68,6 +74,21 @@ export const sessionQueueJobs = {/** This module's fragment of the `cron` queue's jobs. */export const sessionCronJobs = {+ flushSessions: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.session.tryFlush({ trigger: 'cron' });+ },+ }),+ // Session replay chunks are ingested onto this module's session id, so the+ // replay buffer (Redis → ClickHouse `session_replay_chunks`) flushes here+ // too, next to the sessions it belongs to.+ flushReplay: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.buffers.replay.tryFlush({ trigger: 'cron' });+ },+ }),sessionReaper: defineJob({payload: z.null(),handler: async ({ ctx }) => {@@ -92,6 +113,8 @@ export const sessionCronJobs = {/** This module's fragment of `CRON_SCHEDULES` — ids and cadences unchanged. */export const sessionCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'flushSessions', schedule: { every: FLUSH_SESSIONS_INTERVAL_MS } },+ { id: 'flushReplay', schedule: { every: FLUSH_REPLAY_INTERVAL_MS } },{ id: 'sessionReaper', schedule: { every: SESSION_REAPER_INTERVAL_MS } },{ id: 'sessionVacuum', schedule: { pattern: SESSION_VACUUM_CRON } },];diff --git a/packages/db/package.json b/packages/db/package.jsonindex 5b8a25c15..63bf51640 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -26,7 +26,6 @@"@openpanel/redis": "workspace:*","@openpanel/validation": "workspace:*","@prisma/client": "^6.14.0",- "@prisma/extension-read-replicas": "^0.4.1","fast-deep-equal": "^3.1.3","jiti": "^2.4.1","mathjs": "^12.3.2",diff --git a/packages/db/src/services/salt.service.ts b/packages/db/src/services/salt.service.tsindex 6668e263b..6a0e79a51 100644--- a/packages/db/src/services/salt.service.ts+++ b/packages/db/src/services/salt.service.ts@@ -1,65 +1,7 @@-import { generateSalt } from '@openpanel/core';-import { cacheable } from '@openpanel/redis';-import { db } from '../prisma-client';--export const getSalts = cacheable(- 'op:salt',- async () => {- const [curr, prev] = await db.salt.findMany({- orderBy: {- createdAt: 'desc',- },- take: 2,- });-- if (!curr) {- throw new Error('No salt found');- }-- const salts = {- current: curr.salt,- previous: prev?.salt ?? curr.salt,- };-- return salts;- },- 60 * 5-);--export async function createInitialSalts() {- const MAX_RETRIES = 5;- const BASE_DELAY = 1000; // 1 second- const createSaltsWithRetry = async (retryCount = 0): Promise<void> => {- try {- await getSalts();- } catch (error) {- if (error instanceof Error && error.message === 'No salt found') {- console.log('Creating salts for the first time');- await db.salt.create({- data: {- salt: generateSalt(),- createdAt: new Date(new Date().getTime() - 1000 * 60 * 60 * 24),- },- });- await db.salt.create({- data: {- salt: generateSalt(),- },- });- } else {- console.log('Error getting salts', error);- if (retryCount < MAX_RETRIES) {- const delay = BASE_DELAY * 2 ** retryCount;- console.log(- `Retrying in ${delay}ms... (Attempt ${retryCount + 1}/${MAX_RETRIES})`- );- await new Promise((resolve) => setTimeout(resolve, delay));- return createSaltsWithRetry(retryCount + 1);- }- throw new Error(`Failed to create salts after ${MAX_RETRIES} attempts`);- }- }- };-- await createSaltsWithRetry();-}+// Moved to @openpanel/core (M8-004): packages/core/src/modules/salt/salt.service.ts.+// Re-exported here so V1's apps/worker (createInitialSalts at boot) and+// apps/api's event.controller.ts (getSalts) keep working unchanged+// (DELEGATE PATTERN) — V1 is not deleted before P9. `rotateSalt` is V1's+// `cron.salt.ts`'s `salt()`, re-exported under its core name for the one+// caller that still imports this file directly rather than the barrel.+export { createInitialSalts, getSalts, rotateSalt } from '@openpanel/core';diff --git a/packages/db/src/session-consistency.ts b/packages/db/src/session-consistency.tsdeleted file mode 100644index d00054b0e..000000000--- a/packages/db/src/session-consistency.ts+++ /dev/null@@ -1,241 +0,0 @@-import { getRedisCache } from '@openpanel/redis';-import type { Operation } from '@prisma/client/runtime/client';-import { Prisma, type PrismaClient } from './generated/prisma/client';-import { logger } from './logger';-import { getAlsSessionId } from './session-context';--type BarePrismaClient = {- $queryRaw: <T>(query: TemplateStringsArray, ...args: unknown[]) => Promise<T>;-};--// WAL LSN tracking for read-after-write consistency-const LSN_CACHE_PREFIX = 'db:wal_lsn:';-const LSN_CACHE_TTL = 5;-const MAX_RETRY_ATTEMPTS = 5;-const INITIAL_RETRY_DELAY_MS = 10;--const READ_OPERATIONS: Operation[] = [- 'findUnique',- 'findUniqueOrThrow',- 'findFirst',- 'findFirstOrThrow',- 'findMany',- 'aggregate',- 'groupBy',- 'count',-];--const WRITE_OPERATIONS: Operation[] = [- 'create',- 'update',- 'delete',- 'createMany',- 'createManyAndReturn',- 'updateMany',- 'deleteMany',- 'upsert',-];--const isWriteOperation = (operation: string) =>- WRITE_OPERATIONS.includes(operation as Operation);--const isReadOperation = (operation: string) =>- READ_OPERATIONS.includes(operation as Operation);--async function getCurrentWalLsn(- prismaClient: BarePrismaClient,-): Promise<string | null> {- try {- const result = await prismaClient.$queryRaw<[{ lsn: string }]>`- SELECT pg_current_wal_lsn()::text AS lsn- `;- return result[0]?.lsn || null;- } catch (error) {- logger.error({ err: error }, 'Failed to get WAL LSN');- return null;- }-}--async function cacheWalLsnForSession(- sessionId: string,- lsn: string,-): Promise<void> {- try {- const redis = getRedisCache();- await redis.setex(`${LSN_CACHE_PREFIX}${sessionId}`, LSN_CACHE_TTL, lsn);- } catch (error) {- logger.error({ err: error, sessionId }, 'Failed to cache WAL LSN');- }-}--async function getCachedWalLsn(sessionId: string): Promise<string | null> {- try {- const redis = getRedisCache();- return await redis.get(`${LSN_CACHE_PREFIX}${sessionId}`);- } catch (error) {- logger.error({ err: error, sessionId }, 'Failed to get cached WAL LSN');- return null;- }-}--function compareWalLsn(lsn1: string, lsn2: string): number {- const [x1, y1] = lsn1.split('/').map((x) => BigInt(`0x${x}`));- const [x2, y2] = lsn2.split('/').map((x) => BigInt(`0x${x}`));-- const v1 = ((x1 ?? 0n) << 32n) + (y1 ?? 0n);- const v2 = ((x2 ?? 0n) << 32n) + (y2 ?? 0n);-- if (v1 < v2) return -1;- if (v1 > v2) return 1;- return 0;-}--async function sleep(ms: number): Promise<void> {- return new Promise((resolve) => setTimeout(resolve, ms));-}--// Method not used for now,-// Need a way to check LSN on the actual replica that will be used for the read.-async function waitForReplicaCatchup(- prismaClient: BarePrismaClient,- sessionId: string,-): Promise<boolean> {- const expectedLsn = await getCachedWalLsn(sessionId);-- if (!expectedLsn) {- return true;- }-- for (let attempt = 0; attempt < MAX_RETRY_ATTEMPTS; attempt++) {- const currentLsn = await getCurrentWalLsn(prismaClient);- if (!currentLsn) {- return true;- }-- // Check if replica has caught up (current >= expected)- if (compareWalLsn(currentLsn, expectedLsn) >= 0) {- logger.debug(- {- attempt: attempt + 1,- currentLsn,- expectedLsn,- sessionId,- },- 'Replica caught up',- );- return true;- }-- // Exponential backoff- if (attempt < MAX_RETRY_ATTEMPTS - 1) {- const delayMs = INITIAL_RETRY_DELAY_MS * 2 ** attempt;- logger.debug(- {- attempt: attempt + 1,- delayMs,- currentLsn,- expectedLsn,- sessionId,- },- 'Waiting for replica to catch up',- );- await sleep(delayMs);- }- }-- logger.warn(- { sessionId, expectedLsn },- 'Replica did not catch up after max retries, falling back to primary',- );- return false;-}--/**- * Prisma extension for session-based read-after-write consistency.- *- * This extension tracks WAL LSN positions after writes and ensures that- * subsequent reads within the same session see those writes, even when- * using read replicas.- *- * How it works:- * 1. After any write operation with a session ID, it captures the WAL LSN- * 2. Before read operations with a session ID, it checks if the replica has caught up- * 3. If the replica hasn't caught up after retries, it forces the read to the primary- *- */-export function sessionConsistency() {- return Prisma.defineExtension((client) =>- client.$extends({- name: 'session-consistency',- query: {- $allOperations: async ({- operation,- model,- args,- query,- // This is a hack to force reads to primary when replica hasn't caught up.- // The readReplicas extension routes queries to primary when in a transaction,- // so we set __internalParams.transaction = true to achieve this.- // @ts-expect-error - __internalParams is not in the types- __internalParams,- }) => {- const sessionId = getAlsSessionId();-- // For write operations with session: cache WAL LSN after write- if (isWriteOperation(operation)) {- // logger.info('Prisma operation', {- // operation,- // args,- // model,- // });-- const result = await query(args);-- if (sessionId) {- // Get current WAL LSN and cache it for this session- const lsn = await getCurrentWalLsn(client);- if (lsn) {- await cacheWalLsnForSession(sessionId, lsn);- logger.debug(- { sessionId, lsn, operation, model },- 'Cached WAL LSN after write',- );- }- }-- return result;- }-- // For now, we just force the read to the primary without checking the replica- // Since the check probably goes to the primary anyways it will always be true,- // Not sure how to check LSN on the actual replica that will be used for the read.- if (- model !== 'Session' &&- isReadOperation(operation) &&- sessionId &&- (await getCachedWalLsn(sessionId))- ) {- const MAX_RETRIES = 3;- const INITIAL_RETRY_DELAY_MS = 50;-- for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {- const result = await query(args);-- if (result !== null) {- return result;- }-- // If not the last attempt, wait with exponential backoff before retrying- if (attempt < MAX_RETRIES - 1) {- const delayMs = INITIAL_RETRY_DELAY_MS * 2 ** attempt;- await sleep(delayMs);- }- }- }-- return query(args);- },- },- }),- );-}diff --git a/packages/db/src/session-context.ts b/packages/db/src/session-context.tsindex 21f0ef87f..745983d86 100644--- a/packages/db/src/session-context.ts+++ b/packages/db/src/session-context.ts@@ -1,6 +1,8 @@// The ALS session scope lives in @openpanel/core now (M7-001):// packages/core/src/modules/session/src/session-context.ts. Re-exported here-// so V1 (`packages/trpc`, `apps/api`, session-consistency.ts) and core share-// the ONE AsyncLocalStorage instance — two stores would never see each-// other's session id.+// so V1 (`packages/trpc`, `apps/api`) and core share the ONE+// AsyncLocalStorage instance — two stores would never see each other's+// session id. `session-consistency.ts`, its one other V1 reader, is deleted+// (M8-004, ADR-012: the read-replica extension it backed is never wired into+// the Prisma client, and Carl ruled both out — docs/ANSWERS.md §1.5).export { als, getAlsSessionId, runWithAlsSession } from '@openpanel/core';diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d1e8bc1f1..4819409cc 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1299,9 +1299,6 @@ importers:'@prisma/client':specifier: ^6.14.0- '@prisma/extension-read-replicas':- specifier: ^0.4.1- version: 0.4.1(@prisma/[email protected]([email protected]([email protected])([email protected]))([email protected]))fast-deep-equal:specifier: ^3.1.3version: 3.1.3@@ -6848,11 +6845,6 @@ packages:'@prisma/[email protected]':resolution: {integrity: sha512-LhJjqsALFEcoAtF07nSaOkVguaxw/ZsgfROIYZ8bAZDobe7y8Wy+PkYQaPOK1iLSsFgV2MhCO/eNrI1gdSOj6w==}- '@prisma/[email protected]':- resolution: {integrity: sha512-mCMDloqUKUwx2o5uedTs1FHX3Nxdt1GdRMoeyp1JggjiwOALmIYWhxfIN08M2BZ0w8SKwvJqicJZMjkQYkkijw==}- peerDependencies:- '@prisma/client': ^6.5.0-'@prisma/[email protected]':resolution: {integrity: sha512-l0goQOMcNVOJs1kAcwqpKq3ylvkD9F04Ioe1oJoCqmz05mw22bNAKKGWuDd3zTUoUZr97va0c/UfLNru+PDmNA==}@@ -26134,10 +26126,6 @@ snapshots:'@prisma/fetch-engine': 6.14.0'@prisma/get-platform': 6.14.0- '@prisma/[email protected](@prisma/[email protected]([email protected]([email protected])([email protected]))([email protected]))':- dependencies:- '@prisma/client': 6.14.0([email protected]([email protected])([email protected]))([email protected])-'@prisma/[email protected]':dependencies:'@prisma/debug': 5.9.1