OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P2-004): jobs/ layer: envelope, naming, define, producers, workers, testing - legacy-shape tests FIRST
ralph4f4044150da0
diff --git a/packages/core/package.json b/packages/core/package.jsonindex 64156c693..dcb4aaf09 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -15,6 +15,7 @@"@elysiajs/cors": "1.4.2","@elysiajs/openapi": "1.4.15","@trpc/server": "^11.17.0",+ "bullmq": "6.3.3","elysia": "1.4.30","nanoid": "^5.1.6","superjson": "^1.13.3",diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.tsindex a76734bd2..a5412517f 100644--- a/packages/core/src/context.test.ts+++ b/packages/core/src/context.test.ts@@ -1,5 +1,9 @@import { beforeAll, expect, mock, test } from 'bun:test';-import type { AppDeps, Ctx, ProducerScope } from './context';+import type { AppDeps, Ctx } from './context';+import type { JobMeta } from './jobs/envelope';+import { createRecordingProducers } from './jobs/testing';+import type { QueueProducers } from './jobs.registry';+import { queues } from './jobs.registry';import type { Logger } from './logger';import type { ServiceDeps, Services } from './services';@@ -36,10 +40,18 @@ function stubLogger(): Logger {return logger;}-function stubDeps(): { deps: AppDeps; scopes: ProducerScope[] } {- const scopes: ProducerScope[] = [];+function stubDeps(): {+ deps: AppDeps;+ scopes: JobMeta[];+ scoped: QueueProducers[];+} {+ const scopes: JobMeta[] = [];+ const scoped: QueueProducers[] = [];+ const producers = createRecordingProducers(queues);+return {scopes,+ scoped,deps: {db: {},ch: {},@@ -47,9 +59,12 @@ function stubDeps(): { deps: AppDeps; scopes: ProducerScope[] } {clients: {},buffers: {},producers: {+ ...producers,scope(meta) {scopes.push(meta);- return { scopedFor: meta.requestId };+ const surface = producers.scope(meta);+ scoped.push(surface);+ return surface;},},logger: stubLogger(),@@ -58,13 +73,19 @@ function stubDeps(): { deps: AppDeps; scopes: ProducerScope[] } {};}-function build(): { ctx: Ctx; deps: AppDeps; scopes: ProducerScope[] } {- const { deps, scopes } = stubDeps();+function build(): {+ ctx: Ctx;+ deps: AppDeps;+ scopes: JobMeta[];+ scoped: QueueProducers[];+} {+ const { deps, scopes, scoped } = stubDeps();const logger = deps.logger.child({ requestId: REQUEST_ID });return {ctx: createCtx(deps, { requestId: REQUEST_ID, logger }),deps,scopes,+ scoped,};}@@ -103,11 +124,11 @@ test('the services graph is built from the scoped ctx, not from boot deps', () =});test('the requestId reaches the producer scope and the ctx', () => {- const { ctx, scopes } = build();+ const { ctx, scopes, scoped } = build();expect(scopes).toEqual([{ requestId: REQUEST_ID }]);expect(ctx.requestId).toBe(REQUEST_ID);- expect(ctx.queues).toEqual({ scopedFor: REQUEST_ID });+ expect(ctx.queues).toBe(scoped[0] as QueueProducers);});test('the logger is the caller-bound child, unwrapped', () => {diff --git a/packages/core/src/context.ts b/packages/core/src/context.tsindex 04e771c28..c3328ca25 100644--- a/packages/core/src/context.ts+++ b/packages/core/src/context.ts@@ -4,6 +4,7 @@// edge reaches the query, the enqueue and the job that enqueue causes// (ADR-007 decision 18, ADR-018 R1).+import type { QueueProducerHandle, QueueProducers } from './jobs.registry';import type { Logger } from './logger';import { createServices, type Services } from './services';import type { CookieOptions } from './shared/cookie';@@ -19,19 +20,6 @@ export type RedisClient = unknown;export type ServiceClients = unknown;export type Buffers = unknown;-// Replaced by jobs.registry.ts (P2-004). `scope` is real here because-// createCtx calls it.-export type QueueProducers = unknown;--/** The envelope meta a scoped producer stamps on everything it enqueues. */-export interface ProducerScope {- requestId: string;-}--export interface ProducerHandle {- scope(meta: ProducerScope): QueueProducers;-}-// Elysia's cookie primitive, wrapped (P2-007), and the resolved session// (P6 auth).export type CookieJar = unknown;@@ -50,7 +38,7 @@ export interface AppDeps {redis: RedisClient;clients: ServiceClients;buffers: Buffers;- producers: ProducerHandle;+ producers: QueueProducerHandle;logger: Logger;config: RuntimeFlags;}diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsnew file mode 100644index 000000000..722a0eb25--- /dev/null+++ b/packages/core/src/jobs.registry.test.ts@@ -0,0 +1,67 @@+import { expect, test } from 'bun:test';+import { queues } from './jobs.registry';++// V1's `defaultJobOptions`, verbatim from packages/queue/src/queues.ts.+// ADR-005's acceptance note: retry and retention values port as-is, defects+// included — five queues that never retry, four with unbounded failed sets.+const V1_DEFAULTS = {+ sessions: { removeOnComplete: true },+ cron: { removeOnComplete: 10 },+ notification: { removeOnComplete: 10 },+ import: { removeOnComplete: 10, removeOnFail: 50 },+ insights: { removeOnComplete: 100 },+ gsc: { removeOnComplete: 50, removeOnFail: 100 },+ cohortCompute: {+ attempts: 3,+ backoff: { type: 'exponential', delay: 5000 },+ removeOnComplete: { age: 3600, count: 100 },+ removeOnFail: { age: 86_400, count: 100 },+ },+} as const;++// apps/worker/src/boot-workers.ts:209-277 — `getConcurrencyFor(name, default)`.+const V1_CONCURRENCY = {+ sessions: 1,+ cron: 1,+ notification: 1,+ import: 1,+ insights: 5,+ gsc: 5,+ cohortCompute: 2,+} as const;++test('the registry declares the seven queues, keyed by their Redis names', () => {+ expect(Object.keys(queues).sort()).toEqual([+ 'cohortCompute',+ 'cron',+ 'gsc',+ 'import',+ 'insights',+ 'notification',+ 'sessions',+ ]);++ for (const [key, definition] of Object.entries(queues)) {+ expect(definition.name).toBe(key);+ }+});++test('every queue carries V1 retry and retention values unchanged', () => {+ for (const [key, defaults] of Object.entries(V1_DEFAULTS)) {+ expect(queues[key as keyof typeof queues].defaults).toEqual(defaults);+ }+});++test('every queue carries its V1 default concurrency', () => {+ for (const [key, concurrency] of Object.entries(V1_CONCURRENCY)) {+ expect(queues[key as keyof typeof queues].worker?.concurrency).toBe(+ concurrency+ );+ }+});++test('every queue has a compat hook — the cutover has no drain step', () => {+ for (const definition of Object.values(queues)) {+ expect(typeof definition.compat).toBe('function');+ }+});diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsnew file mode 100644index 000000000..b3585fbb7--- /dev/null+++ b/packages/core/src/jobs.registry.ts@@ -0,0 +1,110 @@+// The seven BullMQ queues, statically composed. Adding a module's jobs is one+// spread into the queue that owns them; there is no auto-discovery, because+// static composition is what keeps `QueueProducers` typed (ADR-007).+//+// Registry key === Redis name for all seven, `cohortCompute` included: ADR-005's+// acceptance note refuses the rename, so `COHORTCOMPUTE_CONCURRENCY` keeps+// working. `queueKey` decides what actually reaches Redis.+//+// Retry and retention values are V1's, verbatim, defects included — five+// queues that never retry, four whose failed sets are unbounded. Changing them+// is a behaviour change and a separate decision (ADR-005 acceptance note).++import { legacyCompat } from './jobs/compat';+import type { Producers } from './jobs/define';+import { defineQueue } from './jobs/define';+import type { ProducerHandle } from './jobs/producers';++const HOUR_IN_SECONDS = 3600;+const DAY_IN_SECONDS = 86_400;+const COHORT_BACKOFF_MS = 5000;+const COHORT_ATTEMPTS = 3;++// Defaults from apps/worker/src/boot-workers.ts; `<KEY>_CONCURRENCY`+// overrides one at boot.+const CONCURRENCY = {+ sessions: 1,+ cron: 1,+ notification: 1,+ import: 1,+ insights: 5,+ gsc: 5,+ cohortCompute: 2,+} as const;++const sessions = defineQueue('sessions', {+ defaults: { removeOnComplete: true },+ worker: { concurrency: CONCURRENCY.sessions },+ compat: legacyCompat.sessions,+ jobs: {},+});++// One queue, one worker at concurrency 1, every scheduled job in the system.+// Pausing it from bull-board halts all buffer flushing — preserved+// deliberately (docs/ANSWERS.md §3: "known!").+const cron = defineQueue('cron', {+ defaults: { removeOnComplete: 10 },+ worker: { concurrency: CONCURRENCY.cron },+ compat: legacyCompat.cron,+ jobs: {},+});++const notification = defineQueue('notification', {+ defaults: { removeOnComplete: 10 },+ worker: { concurrency: CONCURRENCY.notification },+ compat: legacyCompat.notification,+ jobs: {},+});++const importQueue = defineQueue('import', {+ defaults: { removeOnComplete: 10, removeOnFail: 50 },+ worker: { concurrency: CONCURRENCY.import },+ compat: legacyCompat.import,+ jobs: {},+});++const insights = defineQueue('insights', {+ defaults: { removeOnComplete: 100 },+ worker: { concurrency: CONCURRENCY.insights },+ compat: legacyCompat.insights,+ jobs: {},+});++const gsc = defineQueue('gsc', {+ defaults: { removeOnComplete: 50, removeOnFail: 100 },+ worker: { concurrency: CONCURRENCY.gsc },+ compat: legacyCompat.gsc,+ jobs: {},+});++const cohortCompute = defineQueue('cohortCompute', {+ defaults: {+ attempts: COHORT_ATTEMPTS,+ backoff: { type: 'exponential', delay: COHORT_BACKOFF_MS },+ // `age` alone only trims when another job in this queue finishes, so it is+ // paired with a count bound.+ removeOnComplete: { age: HOUR_IN_SECONDS, count: 100 },+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },+ },+ worker: { concurrency: CONCURRENCY.cohortCompute },+ compat: legacyCompat.cohortCompute,+ jobs: {},+});++export const queues = {+ sessions,+ cron,+ notification,+ import: importQueue,+ insights,+ gsc,+ cohortCompute,+};++export type Queues = typeof queues;++/** What `ctx.queues` is: `ctx.queues.import.run.add({ importId })`. */+export type QueueProducers = Producers<Queues>;++/** What `AppDeps.producers` is — built once at boot, scoped per unit of work. */+export type QueueProducerHandle = ProducerHandle<Queues>;diff --git a/packages/core/src/jobs/compat.test.ts b/packages/core/src/jobs/compat.test.tsnew file mode 100644index 000000000..a097d938c--- /dev/null+++ b/packages/core/src/jobs/compat.test.ts@@ -0,0 +1,191 @@+import { describe, expect, test } from 'bun:test';+import { queues } from '../jobs.registry';+import { resolveJob } from './compat';+import { wrap } from './envelope';++// One test per legacy shape, from ADR-005's table. The data is exactly what a+// V1 producer left in Redis; the assertion is the V2 (job name, payload) pair+// a worker must see. Fixtures are shape-only — nothing here validates a+// payload, and nothing in V1 did either.++describe('sessions', () => {+ const payload = {+ name: 'session_end',+ projectId: 'proj_1',+ deviceId: 'dev_1',+ sessionId: 'ses_1',+ profileId: 'prof_1',+ };+ const snapshot = {+ id: 'ses_1',+ project_id: 'proj_1',+ device_id: 'dev_1',+ profile_id: 'prof_1',+ entry_path: '/',+ };++ test('{type:createSessionEnd, payload, snapshot} lands as {event, snapshot}', () => {+ expect(+ resolveJob(queues.sessions, {+ name: 'session',+ data: { type: 'createSessionEnd', payload, snapshot },+ })+ ).toEqual({+ job: 'session',+ // `snapshot` sat BESIDE `payload` on the wire, not inside it — the+ // envelope wraps that, it does not clean it up.+ payload: { event: payload, snapshot },+ meta: {},+ });+ });+});++describe('cron', () => {+ // `payload: undefined` disappears through JSON, so what is actually stored+ // is `{"type":"salt"}` — the hook must resolve both spellings, and it+ // resolves them to the same `null` a V2-enqueued cron job carries.+ test('{type, payload:undefined} keys the job on the type', () => {+ expect(+ resolveJob(queues.cron, { name: 'salt', data: { type: 'salt' } })+ ).toEqual({ job: 'salt', payload: null, meta: {} });++ expect(+ resolveJob(queues.cron, {+ name: 'flushSessions',+ data: { type: 'flushSessions', payload: undefined },+ })+ ).toEqual({ job: 'flushSessions', payload: null, meta: {} });+ });+});++describe('notification', () => {+ test('{type:sendNotification, payload} keeps the wire job name', () => {+ const notification = { projectId: 'proj_1', payload: {} };++ expect(+ resolveJob(queues.notification, {+ name: 'sendNotification',+ data: { type: 'sendNotification', payload: { notification } },+ })+ ).toEqual({+ job: 'sendNotification',+ payload: { notification },+ meta: {},+ });+ });+});++describe('import', () => {+ test('{type:import, payload:{importId}}', () => {+ expect(+ resolveJob(queues.import, {+ name: 'import',+ data: { type: 'import', payload: { importId: 'imp_1' } },+ })+ ).toEqual({ job: 'import', payload: { importId: 'imp_1' }, meta: {} });+ });+});++describe('insights', () => {+ test('{type:insightsProject, payload:{projectId,date}}', () => {+ expect(+ resolveJob(queues.insights, {+ name: 'insightsProject',+ data: {+ type: 'insightsProject',+ payload: { projectId: 'proj_1', date: '2026-09-01' },+ },+ })+ ).toEqual({+ job: 'insightsProject',+ payload: { projectId: 'proj_1', date: '2026-09-01' },+ meta: {},+ });+ });+});++describe('gsc', () => {+ test('both job names live on one queue and stay distinct', () => {+ expect(+ resolveJob(queues.gsc, {+ name: 'gscProjectSync',+ data: { type: 'gscProjectSync', payload: { projectId: 'proj_1' } },+ })+ ).toEqual({+ job: 'gscProjectSync',+ payload: { projectId: 'proj_1' },+ meta: {},+ });++ expect(+ resolveJob(queues.gsc, {+ name: 'gscProjectBackfill',+ data: { type: 'gscProjectBackfill', payload: { projectId: 'proj_1' } },+ })+ ).toEqual({+ job: 'gscProjectBackfill',+ payload: { projectId: 'proj_1' },+ meta: {},+ });+ });+});++describe('cohortCompute', () => {+ // The only V1 payload with no `type` discriminant, and the reason the+ // envelope test is `'payload' in data && 'meta' in data`.+ test('a bare {cohortId} resolves to the cohortCompute job', () => {+ expect(+ resolveJob(queues.cohortCompute, {+ name: 'cohortCompute',+ data: { cohortId: 'coh_1' },+ })+ ).toEqual({+ job: 'cohortCompute',+ payload: { cohortId: 'coh_1' },+ meta: {},+ });+ });+});++describe('the envelope path', () => {+ test('an envelope is taken as-is and keeps the BullMQ job name', () => {+ expect(+ resolveJob(queues.import, {+ name: 'import',+ data: wrap({ importId: 'imp_1' }, { requestId: 'req_1' }),+ })+ ).toEqual({+ job: 'import',+ payload: { importId: 'imp_1' },+ meta: { requestId: 'req_1' },+ });+ });++ test('the compat hook is not consulted for an envelope', () => {+ // A shape no hook recognises, wrapped: it resolves anyway.+ expect(+ resolveJob(queues.sessions, {+ name: 'session',+ data: wrap({ anything: true }, {}),+ })+ ).toEqual({ job: 'session', payload: { anything: true }, meta: {} });+ });+});++describe('unresolvable jobs', () => {+ // V1's cron switch had no `default`: an unknown type completed having done+ // nothing. V2 throws instead (ADR-005 risk 2).+ test.each([+ ['sessions' as const, { nope: true }],+ ['cron' as const, { payload: { a: 1 } }],+ ['notification' as const, 'a string'],+ ['import' as const, null],+ ['insights' as const, { type: 'somethingElse', payload: {} }],+ ['gsc' as const, { type: 'gscSomethingElse', payload: {} }],+ ['cohortCompute' as const, { notACohortId: 'x' }],+ ])('%s throws rather than silently doing nothing', (key, data) => {+ expect(() => resolveJob(queues[key], { name: 'whatever', data })).toThrow(+ /cannot resolve/+ );+ });+});diff --git a/packages/core/src/jobs/compat.ts b/packages/core/src/jobs/compat.tsnew file mode 100644index 000000000..4a7b80cc8--- /dev/null+++ b/packages/core/src/jobs/compat.ts@@ -0,0 +1,99 @@+// The whole of the P9 cutover. There is no dual-write window, no drain and no+// queue rename: a job enqueued by V1 is read back by V2 through these hooks+// (ADR-005). A wrong one is silent job loss, which is why the tests next to+// this file were written before it.++import type { CompatHook } from './define';+import { isEnvelope, type JobMeta } from './envelope';++export interface ResolvedJob {+ /** The V2 job name — a key in the queue definition's `jobs`. */+ job: string;+ payload: unknown;+ meta: JobMeta;+}++const NO_META: JobMeta = {};++/**+ * Envelope or legacy, one answer. Throws on anything neither path recognises:+ * V1's cron `switch` had no `default` and completed an unknown type having+ * done nothing, which is the failure mode this replaces (ADR-005 risk 2).+ */+export function resolveJob(+ definition: { name: string; compat?: CompatHook },+ job: { name: string; data: unknown }+): ResolvedJob {+ if (isEnvelope(job.data)) {+ return { job: job.name, payload: job.data.payload, meta: job.data.meta };+ }++ const resolved = definition.compat?.(job.data);+ if (!resolved) {+ throw new Error(+ `Queue '${definition.name}' cannot resolve job '${job.name}': not an envelope, and no compat hook matched its data`+ );+ }++ return { ...resolved, meta: NO_META };+}++function asRecord(data: unknown): Record<string, unknown> | undefined {+ return typeof data === 'object' && data !== null && !Array.isArray(data)+ ? (data as Record<string, unknown>)+ : undefined;+}++/** The four queues whose legacy data is `{type, payload}` and whose wire job name IS the type. */+function discriminatedByType(types: readonly string[]): CompatHook {+ return (data) => {+ const record = asRecord(data);+ const type = record?.type;+ if (typeof type !== 'string' || !types.includes(type)) {+ return undefined;+ }+ return { job: type, payload: record?.payload };+ };+}++/**+ * One hook per queue, keyed by registry key. Each is deletable on its own —+ * but not until no legacy-shaped job can still be replayed, and four queues+ * have no `removeOnFail` at all, so their failed sets never lapse+ * (ADR-005 risk 1).+ */+export const legacyCompat = {+ // Job name `'session'`, payload type `'createSessionEnd'` — the one queue+ // where the two genuinely differ. `snapshot` sat beside `payload` on the+ // wire; it moves inside, unchanged.+ sessions: ((data) => {+ const record = asRecord(data);+ if (record?.type !== 'createSessionEnd') {+ return undefined;+ }+ return {+ job: 'session',+ payload: { event: record.payload, snapshot: record.snapshot },+ };+ }) satisfies CompatHook,++ // The scheduler id, the payload type and the BullMQ job name are all the+ // same string. `payload` was always undefined and JSON dropped it.+ cron: ((data) => {+ const type = asRecord(data)?.type;+ return typeof type === 'string' ? { job: type, payload: null } : undefined;+ }) satisfies CompatHook,++ notification: discriminatedByType(['sendNotification']),+ import: discriminatedByType(['import']),+ insights: discriminatedByType(['insightsProject']),+ gsc: discriminatedByType(['gscProjectSync', 'gscProjectBackfill']),++ // The only V1 payload with no discriminant at all.+ cohortCompute: ((data) => {+ const record = asRecord(data);+ return typeof record?.cohortId === 'string'+ ? { job: 'cohortCompute', payload: record }+ : undefined;+ }) satisfies CompatHook,+} as const;diff --git a/packages/core/src/jobs/define.ts b/packages/core/src/jobs/define.tsnew file mode 100644index 000000000..2bb8e5551--- /dev/null+++ b/packages/core/src/jobs/define.ts@@ -0,0 +1,179 @@+import type { z } from 'zod';+import type { JobCtx } from '../context';++/**+ * What a caller may set on a single enqueue.+ *+ * Deliberately narrower than BullMQ's `JobsOptions`: this type hangs off `Ctx`+ * and so lands in the type graph a frontend's `tsc` walks to resolve+ * `AppRouter`. Widen it by adding the field you need; never by re-exporting+ * BullMQ's.+ */+export interface EnqueueOptions {+ /** Milliseconds to hold the job before a worker may take it. */+ delay?: number;+ priority?: number;+ attempts?: number;+ backoff?: JobBackoff;+ /**+ * BullMQ short-circuits `add` while ANY record for this id exists, which is+ * why it is not a general idempotency key — paired with a retention `age` it+ * deadlocks (see `deduplicationId`). `sessions` and `insights` use it because+ * their records are trimmed by count.+ */+ jobId?: string;+ /**+ * Collapses repeat enqueues while a job holding this key is unfinished.+ *+ * Released by `moveToFinished` on completion and on terminal failure, so a+ * later replay enqueues normally. `cohortCompute` must keep this rather than+ * be normalised onto `jobId`.+ */+ deduplicationId?: string;+}++export interface JobBackoff {+ type: 'fixed' | 'exponential';+ /** Milliseconds; the base that `exponential` doubles from. */+ delay: number;+}++/** `age` in seconds, `count` in jobs. `age` alone only trims when another job finishes. */+export type JobRetention = { count: number } | { age: number; count?: number };++/** Applied to every job on a queue unless a single enqueue overrides it. */+export interface QueueDefaults {+ attempts?: number;+ backoff?: JobBackoff;+ /** `true` drops it immediately, a number keeps the last N. Redis memory is finite. */+ removeOnComplete?: number | boolean | JobRetention;+ removeOnFail?: number | boolean | JobRetention;+}++export interface WorkerSettings {+ /** Jobs this queue's worker runs at once, per process. */+ concurrency?: number;+ /**+ * Ceiling on jobs *started* per window, shared by every worker on the queue.+ * This is the knob for a third party's rate limit; `concurrency` is the knob+ * for our own CPU.+ */+ limiter?: { max: number; duration: number };+}++export interface JobDefinition<TPayload extends z.ZodType> {+ /**+ * Must survive a JSON round-trip: BullMQ stores job data as JSON, so a+ * `z.date()` comes back out as a string and fails its own schema. Use+ * `z.iso.datetime()` for times, and ids rather than objects for entities.+ */+ payload: TPayload;+ handler: (args: {+ payload: z.output<TPayload>;+ ctx: JobCtx;+ }) => Promise<void>;+ /** Defaults for this job alone, over the queue's. */+ options?: EnqueueOptions;+}++/**+ * A job with its payload type erased, for the machinery that handles all of+ * them. `never` as the handler's parameter is what makes every concrete+ * handler assignable to it.+ */+export interface AnyJob {+ payload: z.ZodType;+ handler: (args: never) => Promise<void>;+ options?: EnqueueOptions;+}++/** Stamped on by `defineQueue`, so a job value alone is enough to enqueue it. */+export interface JobBinding {+ readonly queue: string;+ readonly name: string;+}++export type AnyBoundJob = AnyJob & JobBinding;++/** What an enqueuer passes: the schema's *input*, before any transform it declares. */+export type PayloadOf<TJob> = TJob extends { payload: infer TSchema }+ ? TSchema extends z.ZodType+ ? z.input<TSchema>+ : never+ : never;++/**+ * Translates one V1 job's raw data into the (job name, payload) pair V2 speaks.+ *+ * Returns `undefined` for a shape it does not recognise, which `resolveJob`+ * turns into a throw. The BullMQ job *name* on the wire is not authoritative —+ * V1 dispatched on `data.type`, and `sessions` is named `'session'` while its+ * type is `'createSessionEnd'` — so the hook decides the name too.+ */+export type CompatHook = (+ data: unknown+) => { job: string; payload: unknown } | undefined;++export interface QueueDefinition<+ TJobs extends Record<string, AnyJob> = Record<string, AnyJob>,+> {+ name: string;+ defaults?: QueueDefaults;+ worker?: WorkerSettings;+ /** Deletable once no legacy-shaped job can still be replayed — see ADR-005 risk 1. */+ compat?: CompatHook;+ jobs: { [K in keyof TJobs]: TJobs[K] & JobBinding };+}++export type QueueMap = Record<string, QueueDefinition>;++/** The typed enqueue surface derived from the registry — `queues.import.import.add(...)`. */+export type Producers<TQueues extends QueueMap> = {+ [Q in keyof TQueues]: {+ [J in keyof TQueues[Q]['jobs']]: {+ add(+ payload: PayloadOf<TQueues[Q]['jobs'][J]>,+ options?: EnqueueOptions+ ): Promise<string>;+ /** Job ids are queue-scoped, not job-scoped: this removes from the whole queue. */+ remove(jobId: string): Promise<void>;+ };+ };+};++export function defineJob<TPayload extends z.ZodType>(+ definition: JobDefinition<TPayload>+): JobDefinition<TPayload> {+ return definition;+}++/**+ * Declares a queue and the jobs on it. Constructs nothing: no Redis connection+ * exists until `createProducers` or `startWorkers` is handed the registry,+ * which is what keeps importing a module free of side effects and lets one+ * declaration serve a producer-only process and a worker one.+ */+export function defineQueue<TJobs extends Record<string, AnyJob>>(+ name: string,+ config: {+ defaults?: QueueDefaults;+ worker?: WorkerSettings;+ compat?: CompatHook;+ jobs: TJobs;+ }+): QueueDefinition<TJobs> {+ const jobs = Object.fromEntries(+ Object.entries(config.jobs).map(([jobName, job]) => [+ jobName,+ { ...job, queue: name, name: jobName },+ ])+ ) as QueueDefinition<TJobs>['jobs'];++ return {+ name,+ defaults: config.defaults,+ worker: config.worker,+ compat: config.compat,+ jobs,+ };+}diff --git a/packages/core/src/jobs/envelope.test.ts b/packages/core/src/jobs/envelope.test.tsnew file mode 100644index 000000000..ab76c1d8c--- /dev/null+++ b/packages/core/src/jobs/envelope.test.ts@@ -0,0 +1,47 @@+import { expect, test } from 'bun:test';+import { isEnvelope, unwrap, wrap } from './envelope';++test('an envelope round-trips through wrap and unwrap', () => {+ const envelope = wrap({ importId: 'imp_1' }, { requestId: 'req_1' });++ expect(envelope).toEqual({+ payload: { importId: 'imp_1' },+ meta: { requestId: 'req_1' },+ });+ expect(unwrap(envelope)).toEqual(envelope);+ // It has to survive Redis, which stores it as JSON.+ expect(unwrap(JSON.parse(JSON.stringify(envelope)))).toEqual(envelope);+});++// A payload-less job (cron) still has to survive JSON, which drops an+// undefined value and with it the very key `isEnvelope` looks for.+test('a payload-less job round-trips as null, not as a missing key', () => {+ const envelope = wrap(undefined, {});++ expect(envelope).toEqual({ payload: null, meta: {} });+ expect(isEnvelope(JSON.parse(JSON.stringify(envelope)))).toBe(true);+ expect(unwrap(JSON.parse(JSON.stringify(envelope))).payload).toBeNull();+});++// The detection rule is `'payload' in data && 'meta' in data`, and the reason+// for the second half is right here: `cohortCompute` is a bare `{cohortId}`+// and the other five legacy shapes carry a `payload` of their own.+test('a bare {cohortId} is not mistaken for an envelope', () => {+ expect(isEnvelope({ cohortId: 'coh_1' })).toBe(false);+ expect(() => unwrap({ cohortId: 'coh_1' })).toThrow(/not a job envelope/);+});++test('a legacy {type, payload} is not mistaken for an envelope', () => {+ expect(isEnvelope({ type: 'import', payload: { importId: 'imp_1' } })).toBe(+ false+ );+ expect(isEnvelope({ type: 'salt' })).toBe(false);+ expect(isEnvelope({ meta: { requestId: 'req_1' } })).toBe(false);+});++test('nothing that is not an object is an envelope', () => {+ expect(isEnvelope(null)).toBe(false);+ expect(isEnvelope(undefined)).toBe(false);+ expect(isEnvelope('payload')).toBe(false);+ expect(() => unwrap(null)).toThrow(/not a job envelope/);+});diff --git a/packages/core/src/jobs/envelope.ts b/packages/core/src/jobs/envelope.tsnew file mode 100644index 000000000..2790b3ee1--- /dev/null+++ b/packages/core/src/jobs/envelope.ts@@ -0,0 +1,58 @@+/**+ * What actually gets stored as a job's data.+ *+ * The payload is nested rather than merged with the metadata so a module's+ * schema stays exactly the shape it declared — nothing added here can collide+ * with a field a payload wanted for itself.+ */+export interface JobEnvelope<TPayload = unknown> {+ payload: TPayload;+ meta: JobMeta;+}++/**+ * Travels with the job so a worker's logs join the request that caused them,+ * which is otherwise unrecoverable once the response has gone out. A job+ * enqueued from another job inherits its parent's.+ */+export interface JobMeta {+ requestId?: string;+}++/**+ * `undefined` becomes `null` because BullMQ stores job data as JSON, which+ * drops an undefined value and with it the `payload` key that `isEnvelope`+ * looks for. A payload-less job (cron) therefore carries `null` on both the+ * envelope and the compat path, so one schema covers both.+ */+export function wrap<TPayload>(+ payload: TPayload,+ meta: JobMeta+): JobEnvelope<TPayload extends undefined ? null : TPayload> {+ return {+ payload: (payload ?? null) as TPayload extends undefined ? null : TPayload,+ meta,+ };+}++/**+ * Both keys, not just `payload`: five of the seven V1 shapes carry a `payload`+ * of their own, so `meta` is the half that tells an envelope from a legacy job.+ */+export function isEnvelope(data: unknown): data is JobEnvelope {+ return (+ typeof data === 'object' &&+ data !== null &&+ 'payload' in data &&+ 'meta' in data+ );+}++/** Loud about data that never went through `wrap`, rather than handing a handler `undefined`. */+export function unwrap(data: unknown): JobEnvelope {+ if (!isEnvelope(data)) {+ throw new Error('Job data is not a job envelope');+ }++ return data;+}diff --git a/packages/core/src/jobs/jobs.test.ts b/packages/core/src/jobs/jobs.test.tsnew file mode 100644index 000000000..947727456--- /dev/null+++ b/packages/core/src/jobs/jobs.test.ts@@ -0,0 +1,102 @@+import { expect, test } from 'bun:test';+import { z } from 'zod';+import { defineJob, defineQueue } from './define';+import { toJobsOptions } from './producers';+import { createRecordingProducers } from './testing';++const testQueue = defineQueue('test', {+ jobs: {+ greet: defineJob({+ payload: z.object({ name: z.string() }),+ handler: async () => undefined,+ }),+ },+});++const registry = { test: testQueue };++test('defineQueue binds each job to its queue and name', () => {+ expect(testQueue.jobs.greet).toMatchObject({ queue: 'test', name: 'greet' });+});++test('the payload is validated at the call site, not in a worker later', async () => {+ const producers = createRecordingProducers(registry);++ await expect(+ // @ts-expect-error the same mistake the schema catches at runtime+ producers.queues.test.greet.add({ name: 42 })+ ).rejects.toThrow();+ expect(producers.recorded).toHaveLength(0);+});++test('a scope stamps its meta onto everything enqueued through it', async () => {+ const producers = createRecordingProducers(registry);++ await producers.queues.test.greet.add({ name: 'unscoped' });+ await producers.scope({ requestId: 'req_1' }).test.greet.add({+ name: 'scoped',+ });++ expect(producers.recorded).toEqual([+ { queue: 'test', job: 'greet', payload: { name: 'unscoped' }, meta: {} },+ {+ queue: 'test',+ job: 'greet',+ payload: { name: 'scoped' },+ meta: { requestId: 'req_1' },+ },+ ]);+});++test('remove is recorded too, so the import retry flow is testable', async () => {+ const producers = createRecordingProducers(registry);++ await producers.queues.test.greet.remove('job_1');++ expect(producers.removed).toEqual([{ queue: 'test', jobId: 'job_1' }]);+});++test('queue defaults carry retention as {age, count}, which cohortCompute needs', () => {+ expect(+ toJobsOptions({+ attempts: 3,+ backoff: { type: 'exponential', delay: 5000 },+ removeOnComplete: { age: 3600, count: 100 },+ removeOnFail: { age: 86_400, count: 100 },+ })+ ).toEqual({+ attempts: 3,+ backoff: { type: 'exponential', delay: 5000 },+ removeOnComplete: { age: 3600, count: 100 },+ removeOnFail: { age: 86_400, count: 100 },+ });++ expect(toJobsOptions({ removeOnComplete: true })).toEqual({+ removeOnComplete: true,+ });+});++test('all three V1 idempotency conventions survive the option mapping', () => {+ // sessions and insights key on jobId; cohort keys on deduplication and must+ // NOT be normalised onto jobId (cohort.service.ts records the deadlock).+ expect(toJobsOptions(undefined, { jobId: 'sessionEnd:v2:ses_1' })).toEqual({+ jobId: 'sessionEnd:v2:ses_1',+ });+ expect(toJobsOptions(undefined, { deduplicationId: 'cohort-coh_1' })).toEqual(+ { deduplication: { id: 'cohort-coh_1' } }+ );+ expect(toJobsOptions(undefined, {})).toEqual({});+});++test('a single enqueue overrides the queue default it names, and nothing else', () => {+ expect(+ toJobsOptions(+ { attempts: 3, backoff: { type: 'exponential', delay: 5000 } },+ { attempts: 1, delay: 250 }+ )+ ).toEqual({+ attempts: 1,+ backoff: { type: 'exponential', delay: 5000 },+ delay: 250,+ });+});diff --git a/packages/core/src/jobs/naming.test.ts b/packages/core/src/jobs/naming.test.tsnew file mode 100644index 000000000..2d71efc38--- /dev/null+++ b/packages/core/src/jobs/naming.test.ts@@ -0,0 +1,71 @@+import { expect, test } from 'bun:test';+import { queues } from '../jobs.registry';+import { queueKey } from './naming';++// Byte-identity snapshot, mirroring+// `verification/golden/queue-keys/keys.json` — which the controller's+// `check.sh` independently recomputes from V1's `getQueueName`. Both sides pin+// the same 14 strings; a one-character drift orphans every queue in Redis+// (ADR-005 risk 4).+const GOLDEN_KEYS = [+ { name: 'cohortCompute', queueCluster: false, key: 'cohortCompute' },+ { name: 'cohortCompute', queueCluster: true, key: '{cohortCompute}' },+ { name: 'cron', queueCluster: false, key: 'cron' },+ { name: 'cron', queueCluster: true, key: '{cron}' },+ { name: 'gsc', queueCluster: false, key: 'gsc' },+ { name: 'gsc', queueCluster: true, key: '{gsc}' },+ { name: 'import', queueCluster: false, key: 'import' },+ { name: 'import', queueCluster: true, key: '{import}' },+ { name: 'insights', queueCluster: false, key: 'insights' },+ { name: 'insights', queueCluster: true, key: '{insights}' },+ { name: 'notification', queueCluster: false, key: 'notification' },+ { name: 'notification', queueCluster: true, key: '{notification}' },+ { name: 'sessions', queueCluster: false, key: 'sessions' },+ { name: 'sessions', queueCluster: true, key: '{sessions}' },+] as const;++test('every golden key is reproduced byte-for-byte', () => {+ const produced = GOLDEN_KEYS.map((row) => ({+ ...row,+ produced: queueKey(row.name, { cluster: row.queueCluster }),+ }));++ expect(produced.map((row) => row.produced)).toEqual(+ GOLDEN_KEYS.map((row) => row.key)+ );+});++test('the golden table covers exactly the queues the registry declares', () => {+ const golden = [...new Set(GOLDEN_KEYS.map((row) => row.name))].sort();++ expect(Object.keys(queues).sort()).toEqual(golden);+ // Registry key === Redis name for all seven, `cohortCompute` included+ // (ADR-005 acceptance note: no env rename).+ expect(+ Object.values(queues)+ .map((queue) => queue.name)+ .sort()+ ).toEqual(golden);+});++test('braces appear only under QUEUE_CLUSTER — the divergence from fika', () => {+ expect(queueKey('cron')).toBe('cron');+ expect(queueKey('cron', {})).toBe('cron');+ expect(queueKey('cron', { cluster: false })).toBe('cron');+ expect(queueKey('cron', { cluster: true })).toBe('{cron}');+});++// The `QUEUE_CLUSTER=off, namespace=set` cell is the one the plan left+// unspecified; ADR-005 says the snapshot is where it gets pinned. Braces are a+// Redis Cluster hash-tag concern, the namespace is an isolation concern, so+// they are independent: the namespace always applies, the braces only cluster.+test('the namespace always applies; only the braces are cluster-conditional', () => {+ expect(queueKey('cron', { namespace: 'wt3' })).toBe('cron-wt3');+ expect(queueKey('cron', { cluster: false, namespace: 'wt3' })).toBe(+ 'cron-wt3'+ );+ expect(queueKey('cron', { cluster: true, namespace: 'wt3' })).toBe(+ '{cron-wt3}'+ );+ expect(queueKey('cron', { cluster: true, namespace: '' })).toBe('{cron}');+});diff --git a/packages/core/src/jobs/naming.ts b/packages/core/src/jobs/naming.tsnew file mode 100644index 000000000..87443e3e9--- /dev/null+++ b/packages/core/src/jobs/naming.ts@@ -0,0 +1,25 @@+/** How a queue's name is namespaced and where Redis Cluster hashes it. */+export interface QueueKeyOptions {+ /**+ * Redis Cluster picks a slot from the substring inside `{}`, and BullMQ's+ * Lua scripts touch several of a queue's keys in one call — which the+ * cluster rejects unless they all hash to the same slot.+ */+ cluster?: boolean;+ /** Isolates two worktrees, or a developer and staging, sharing one Redis. */+ namespace?: string;+}++/**+ * The Redis key a queue lives under.+ *+ * Deliberate divergence from the fika template this is ported from: fika+ * braces unconditionally, we brace only under `QUEUE_CLUSTER`, because V1's+ * `getQueueName` did and every existing queue in every deployment is named+ * that way (ADR-005). `-` joins the namespace because BullMQ rejects a `:`+ * in a queue name.+ */+export function queueKey(name: string, options?: QueueKeyOptions): string {+ const scoped = options?.namespace ? `${name}-${options.namespace}` : name;+ return options?.cluster ? `{${scoped}}` : scoped;+}diff --git a/packages/core/src/jobs/producers.ts b/packages/core/src/jobs/producers.tsnew file mode 100644index 000000000..2e65302f9--- /dev/null+++ b/packages/core/src/jobs/producers.ts@@ -0,0 +1,179 @@+import {+ Queue as BullQueue,+ type ConnectionOptions,+ type JobsOptions,+} from 'bullmq';+import type {+ AnyBoundJob,+ EnqueueOptions,+ Producers,+ QueueDefaults,+ QueueDefinition,+ QueueMap,+} from './define';+import { type JobMeta, wrap } from './envelope';+import { queueKey } from './naming';++export type EnqueueFn = (+ job: AnyBoundJob,+ payload: unknown,+ options?: EnqueueOptions,+ meta?: JobMeta+) => Promise<string>;++export type RemoveFn = (queue: string, jobId: string) => Promise<void>;++export interface ProducerHandle<TQueues extends QueueMap> {+ /** Enqueues with no request attached — a boot-time or scheduled enqueue. */+ queues: Producers<TQueues>;+ /** The same surface, stamping `meta` onto everything enqueued through it. */+ scope(meta: JobMeta): Producers<TQueues>;+ /** The untyped seam the worker runtime enqueues through. Prefer `queues`. */+ enqueue: EnqueueFn;+ remove: RemoveFn;+ /**+ * The live BullMQ queues, for the four call sites that speak BullMQ directly+ * rather than enqueueing: the buffers' `cronQueue.isPaused()`, import's+ * `getJob`, shutdown's `getActiveCount`, and bull-board. Nothing that+ * enqueues should reach for these: they take unvalidated data and no+ * metadata, which is the whole of what `queues` adds.+ */+ bullQueues: BullQueue[];+ close(): Promise<void>;+}++export interface CreateProducersOptions {+ /** Shared: producing never blocks, so one client serves every queue. */+ connection: ConnectionOptions;+ /** `QUEUE_CLUSTER`. Off in every deployment that exists today — see `queueKey`. */+ cluster?: boolean;+ namespace?: string;+}++/**+ * Turns the registry into something that can enqueue.+ *+ * Every process needs this, including one that runs no workers — an RPC+ * procedure enqueues. The Redis client is the caller's to close; closing it+ * here would take down a connection the workers may still be using.+ */+export function createProducers<TQueues extends QueueMap>(+ definitions: TQueues,+ { connection, cluster, namespace }: CreateProducersOptions+): ProducerHandle<TQueues> {+ const bullQueues = new Map<string, BullQueue>();++ for (const definition of Object.values(definitions)) {+ bullQueues.set(+ definition.name,+ new BullQueue(queueKey(definition.name, { cluster, namespace }), {+ connection,+ defaultJobOptions: toJobsOptions(definition.defaults),+ })+ );+ }++ const queueFor = (name: string): BullQueue => {+ const queue = bullQueues.get(name);+ if (!queue) {+ throw new Error(`Queue '${name}' is not in the registry`);+ }+ return queue;+ };++ const enqueue: EnqueueFn = async (job, payload, options, meta) => {+ // Validated here as well as in the worker, so a bad payload fails at the+ // call site — with that caller's stack — instead of surfacing later as a+ // failed job nobody is watching.+ const parsed = job.payload.parse(payload);++ const added = await queueFor(job.queue).add(+ job.name,+ wrap(parsed, meta ?? {}),+ toJobsOptions(undefined, { ...job.options, ...options })+ );++ if (!added.id) {+ throw new Error(+ `Enqueued '${job.queue}.${job.name}' but Redis returned no job id`+ );+ }+ return added.id;+ };++ const remove: RemoveFn = async (queue, jobId) => {+ await queueFor(queue).remove(jobId);+ };++ return {+ queues: buildProducers(definitions, enqueue, remove, {}),+ scope: (meta) => buildProducers(definitions, enqueue, remove, meta),+ enqueue,+ remove,+ bullQueues: [...bullQueues.values()],+ close: async () => {+ await Promise.all([...bullQueues.values()].map((queue) => queue.close()));+ },+ };+}++/** Shared with the test double in `jobs/testing.ts`, which swaps only the two seams. */+export function buildProducers<TQueues extends QueueMap>(+ definitions: TQueues,+ enqueue: EnqueueFn,+ remove: RemoveFn,+ meta: JobMeta+): Producers<TQueues> {+ const producers: Record<string, Record<string, unknown>> = {};++ for (const [key, definition] of Object.entries(definitions)) {+ producers[key] = Object.fromEntries(+ Object.entries((definition as QueueDefinition).jobs).map(+ ([jobName, job]) => [+ jobName,+ {+ add: (payload: unknown, options?: EnqueueOptions) =>+ enqueue(job, payload, options, meta),+ remove: (jobId: string) => remove(job.queue, jobId),+ },+ ]+ )+ );+ }++ return producers as Producers<TQueues>;+}++/** Exported for the option-mapping tests; the two call sites above are the real ones. */+export function toJobsOptions(+ defaults?: QueueDefaults,+ options?: EnqueueOptions+): JobsOptions {+ return {+ ...(defaults?.removeOnComplete !== undefined && {+ removeOnComplete: defaults.removeOnComplete,+ }),+ ...(defaults?.removeOnFail !== undefined && {+ removeOnFail: defaults.removeOnFail,+ }),+ ...(options?.delay !== undefined && { delay: options.delay }),+ ...(options?.priority !== undefined && { priority: options.priority }),+ ...(options?.jobId !== undefined && { jobId: options.jobId }),+ ...(options?.deduplicationId !== undefined && {+ // No `ttl`: a ttl-less key clears when the job finishes, which is what+ // lets the same key be enqueued again later.+ deduplication: { id: options.deduplicationId },+ }),+ ...pickRetry(defaults, options),+ };+}++function pickRetry(defaults?: QueueDefaults, options?: EnqueueOptions) {+ const attempts = options?.attempts ?? defaults?.attempts;+ const backoff = options?.backoff ?? defaults?.backoff;++ return {+ ...(attempts !== undefined && { attempts }),+ ...(backoff !== undefined && { backoff }),+ };+}diff --git a/packages/core/src/jobs/testing.ts b/packages/core/src/jobs/testing.tsnew file mode 100644index 000000000..9b7f00561--- /dev/null+++ b/packages/core/src/jobs/testing.ts@@ -0,0 +1,70 @@+import type { QueueMap } from './define';+import type { JobMeta } from './envelope';+import {+ buildProducers,+ type EnqueueFn,+ type ProducerHandle,+ type RemoveFn,+} from './producers';++export interface RecordedJob {+ queue: string;+ job: string;+ payload: unknown;+ meta: JobMeta;+}++export interface RemovedJob {+ queue: string;+ jobId: string;+}++export interface RecordingProducerHandle<TQueues extends QueueMap>+ extends ProducerHandle<TQueues> {+ recorded: RecordedJob[];+ removed: RemovedJob[];+}++/**+ * Producers that record instead of enqueueing, so a test can assert what a+ * procedure would have queued without a Redis anywhere near it.+ *+ * Payloads are still validated against their schemas — the point is to keep+ * the contract, not to bypass it. Handlers need none of this: a handler is a+ * plain function of `{ payload, ctx }` and is called directly.+ */+export function createRecordingProducers<TQueues extends QueueMap>(+ definitions: TQueues+): RecordingProducerHandle<TQueues> {+ const recorded: RecordedJob[] = [];+ const removed: RemovedJob[] = [];+ let nextId = 1;++ // async, not a returned Promise: `parse` throws, and a caller asserting on+ // a rejection must not get a synchronous throw instead.+ const enqueue: EnqueueFn = async (job, payload, _options, meta) => {+ recorded.push({+ queue: job.queue,+ job: job.name,+ payload: job.payload.parse(payload),+ meta: meta ?? {},+ });+ return await Promise.resolve(String(nextId++));+ };++ const remove: RemoveFn = async (queue, jobId) => {+ removed.push({ queue, jobId });+ await Promise.resolve();+ };++ return {+ recorded,+ removed,+ queues: buildProducers(definitions, enqueue, remove, {}),+ scope: (meta) => buildProducers(definitions, enqueue, remove, meta),+ enqueue,+ remove,+ bullQueues: [],+ close: () => Promise.resolve(),+ };+}diff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.tsnew file mode 100644index 000000000..3d2eca95d--- /dev/null+++ b/packages/core/src/jobs/workers.test.ts@@ -0,0 +1,113 @@+import { expect, test } from 'bun:test';+import { z } from 'zod';+import type { AppDeps, JobCtx } from '../context';+import { queues } from '../jobs.registry';+import type { Logger } from '../logger';+import { defineJob, defineQueue } from './define';+import { wrap } from './envelope';+import { createRecordingProducers } from './testing';+import { runJob } from './workers';++function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++function stubDeps(): AppDeps {+ return {+ db: {},+ ch: {},+ redis: {},+ clients: {},+ buffers: {},+ producers: createRecordingProducers(queues),+ logger: stubLogger(),+ config: { selfHosted: true },+ };+}++const seen: { payload: unknown; ctx: JobCtx }[] = [];++const queue = defineQueue('import', {+ jobs: {+ import: defineJob({+ payload: z.object({ importId: z.string() }),+ handler: async (args) => {+ seen.push(args);+ },+ }),+ },+ compat: (data) =>+ typeof data === 'object' &&+ data !== null &&+ (data as { type?: unknown }).type === 'import'+ ? { job: 'import', payload: (data as { payload: unknown }).payload }+ : undefined,+});++function bullJob(name: string, data: unknown) {+ return { id: 'job_1', name, data, attemptsMade: 0 };+}++test('an enveloped job reaches its handler with the requestId that caused it', async () => {+ seen.length = 0;++ await runJob(+ queue,+ bullJob('import', wrap({ importId: 'imp_1' }, { requestId: 'req_1' })),+ stubDeps()+ );++ expect(seen).toHaveLength(1);+ expect(seen[0]?.payload).toEqual({ importId: 'imp_1' });+ expect(seen[0]?.ctx.requestId).toBe('req_1');+ expect(seen[0]?.ctx.job).toEqual({+ id: 'job_1',+ attempt: 1,+ queue: 'import',+ name: 'import',+ });+});++test('a legacy job reaches the same handler through the compat hook', async () => {+ seen.length = 0;++ await runJob(+ queue,+ bullJob('import', { type: 'import', payload: { importId: 'imp_1' } }),+ stubDeps()+ );++ expect(seen).toHaveLength(1);+ expect(seen[0]?.payload).toEqual({ importId: 'imp_1' });+ // No request caused it, so one is minted rather than left absent — a job's+ // own logs and its follow-up enqueues still correlate (ADR-018 R2).+ expect(seen[0]?.ctx.requestId).toMatch(/^[\w-]{21}$/);+});++test('a job the registry does not declare throws, never silently completes', async () => {+ await expect(+ runJob(queue, bullJob('nope', wrap({ importId: 'imp_1' }, {})), stubDeps())+ ).rejects.toThrow(/has no job named 'nope'/);+});++test('an unresolvable legacy shape throws', async () => {+ await expect(+ runJob(queue, bullJob('import', { surprise: true }), stubDeps())+ ).rejects.toThrow(/cannot resolve/);+});++test('the payload is re-parsed in the worker, not trusted', async () => {+ await expect(+ runJob(queue, bullJob('import', wrap({ importId: 42 }, {})), stubDeps())+ ).rejects.toThrow();+});diff --git a/packages/core/src/jobs/workers.ts b/packages/core/src/jobs/workers.tsnew file mode 100644index 000000000..826a67617--- /dev/null+++ b/packages/core/src/jobs/workers.ts@@ -0,0 +1,185 @@+import {+ type Job as BullJob,+ Worker as BullWorker,+ type ConnectionOptions,+} from 'bullmq';+import { type AppDeps, createCtx, extendCtx, type JobCtx } from '../context';+import { REQUEST_ID_LENGTH, REQUEST_ID_LOG_FIELD } from '../logger';+import { generateId } from '../shared/id';+import { resolveJob } from './compat';+import type { QueueDefinition, QueueMap } from './define';+import { queueKey } from './naming';++export interface TerminalFailure {+ queue: string;+ job: string;+ jobId?: string;+ attempts: number;+ error: Error;+ data: unknown;+}++export interface StartWorkersOptions<TQueues extends QueueMap> {+ /** Already filtered to the queues this role consumes. */+ definitions: TQueues;+ deps: AppDeps;+ /**+ * Called once per queue: a worker blocks its connection, so it cannot share+ * one. The caller keeps what it hands over and closes it.+ */+ createConnection: (name: string) => ConnectionOptions;+ cluster?: boolean;+ namespace?: string;+ /**+ * A job that has exhausted its attempts. Five of the seven queues never+ * retry, so for those this fires on the first throw.+ */+ onTerminalFailure?: (failure: TerminalFailure) => void;+}++export interface WorkerHandle {+ /**+ * Stops taking new jobs and waits for the running ones. Whatever supervises+ * the process has to allow longer than the slowest job takes, or the drain+ * is killed halfway and every interrupted job comes back as a stalled retry.+ *+ * Closes the workers, not the connections they were handed.+ */+ close(): Promise<void>;+}++/**+ * Starts one worker per queue handed in. Only a process whose role includes+ * consuming calls this; an API-only process builds producers and stops there.+ */+export function startWorkers<TQueues extends QueueMap>({+ definitions,+ deps,+ createConnection,+ cluster,+ namespace,+ onTerminalFailure,+}: StartWorkersOptions<TQueues>): WorkerHandle {+ const started = Object.values(definitions).map((definition) => {+ const key = queueKey(definition.name, { cluster, namespace });+ const logger = deps.logger.child({ queue: definition.name });++ const worker = new BullWorker(key, (job) => runJob(definition, job, deps), {+ connection: createConnection(`worker:${definition.name}`),+ ...(definition.worker?.concurrency !== undefined && {+ concurrency: definition.worker.concurrency,+ }),+ ...(definition.worker?.limiter !== undefined && {+ limiter: definition.worker.limiter,+ }),+ });++ // Fires on the connection itself, not on any one job. V1 lost the api+ // process roughly daily to unlistened ioredis ECONNRESETs re-emitted here.+ worker.on('error', (error) => logger.error({ err: error }, 'worker error'));++ // A stall means the job was taken and then its lock lapsed: the process+ // died mid-job, or the handler blocked the event loop past the lock.+ worker.on('stalled', (jobId) =>+ logger.warn({ jobId }, 'job stalled and will be retried')+ );++ worker.on('failed', (job, error) => {+ // `failed` fires on every attempt; only a finished one is out of retries.+ if (!job?.finishedOn) {+ return;+ }++ logger.error(+ {+ err: error,+ jobId: job.id,+ job: job.name,+ attempts: job.attemptsMade,+ },+ 'job failed terminally'+ );+ onTerminalFailure?.({+ queue: definition.name,+ job: job.name,+ jobId: job.id,+ attempts: job.attemptsMade,+ error,+ data: job.data,+ });+ });++ logger.info({ key }, 'worker started');+ return worker;+ });++ return {+ close: async () => {+ await Promise.all(started.map((worker) => worker.close()));+ },+ };+}++/** The shape of a BullMQ job this needs, so a test does not have to build one. */+export interface RunnableJob {+ id?: string;+ name: string;+ data: unknown;+ attemptsMade: number;+}++/**+ * Exported for the worker tests, and it is what the `BullWorker` processor+ * above calls. Throws on a job it cannot resolve or does not declare — V1+ * completed those having done nothing (ADR-005 risk 2).+ */+export async function runJob(+ definition: QueueDefinition,+ job: RunnableJob | BullJob,+ deps: AppDeps+): Promise<void> {+ const resolved = resolveJob(definition, { name: job.name, data: job.data });++ const jobDefinition = definition.jobs[resolved.job];+ if (!jobDefinition) {+ throw new Error(+ `Queue '${definition.name}' has no job named '${resolved.job}'`+ );+ }++ // Parsed rather than trusted: this job may have been enqueued by the deploy+ // this one is replacing, under the previous shape of the schema.+ const payload = jobDefinition.payload.parse(resolved.payload);++ const attempt = job.attemptsMade + 1;+ // A scheduled or legacy job has no originating request; minting one keeps+ // its own logs and its follow-up enqueues correlated (ADR-018 R2).+ const requestId =+ resolved.meta.requestId ?? generateId(undefined, REQUEST_ID_LENGTH);++ const logger = deps.logger.child({+ queue: definition.name,+ job: resolved.job,+ jobId: job.id,+ attempt,+ [REQUEST_ID_LOG_FIELD]: requestId,+ });++ const ctx: JobCtx = extendCtx(createCtx(deps, { requestId, logger }), {+ job: {+ id: job.id ?? '',+ attempt,+ queue: definition.name,+ name: resolved.job,+ },+ });++ // The registry erases each handler's payload type so the machinery can hold+ // all of them (see `AnyJob`); the schema above is what re-establishes it.+ const handler = jobDefinition.handler as (args: {+ payload: unknown;+ ctx: JobCtx;+ }) => Promise<void>;++ await handler({ payload, ctx });+}diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.tsindex 96a7e6bdf..1df55465b 100644--- a/packages/core/src/logger.ts+++ b/packages/core/src/logger.ts@@ -25,6 +25,10 @@ export interface Logger {// V1's `reqId`. Exported here to kill the four-file literal coupling.export const REQUEST_ID_LOG_FIELD = 'requestId' as const;+// 126 bits of nanoid (ADR-018 R2). A correlation id, not a secret, and the+// hot path pays for it 5000x/s — hence `nanoid/non-secure` behind generateId.+export const REQUEST_ID_LENGTH = 21;+// The wire header stays `request-id` byte-for-byte — it is caller-visible// and NON_GOALS forbids changing endpoint contracts, even though the log// field it feeds is renamed.diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex dff7a0bac..50ec58aef 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1207,6 +1207,9 @@ importers:'@trpc/server':specifier: ^11.17.0version: 11.17.0([email protected])+ bullmq:+ specifier: 6.3.3+ version: 6.3.3([email protected]([email protected]))elysia:specifier: 1.4.30version: 1.4.30(@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])@@ -5291,31 +5294,61 @@ packages:cpu: [arm64]os: [darwin]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==}+ cpu: [arm64]+ os: [darwin]+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==}cpu: [x64]os: [darwin]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==}+ cpu: [x64]+ os: [darwin]+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==}cpu: [arm64]os: [linux]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==}+ cpu: [arm64]+ os: [linux]+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==}cpu: [arm]os: [linux]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==}+ cpu: [arm]+ os: [linux]+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==}cpu: [x64]os: [linux]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==}+ cpu: [x64]+ os: [linux]+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==}cpu: [x64]os: [win32]+ '@msgpackr-extract/[email protected]':+ resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==}+ cpu: [x64]+ os: [win32]+'@napi-rs/[email protected]':resolution: {integrity: sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==}@@ -10901,6 +10934,24 @@ packages:resolution: {integrity: sha512-HT1iM3Jt4bZeg3Ru/MxrOy2iIItxcl1Pz5Ync1Vrot70jBpVguMxFEiSaDU57BwYwR4iwnObDnzct2lirKkX5A==}+ resolution: {integrity: sha512-ie2/80FlcAF2pZM6NU90adIPPpvuqlPSqbRRW3t5ktgJMTrXRJ7D0e8gjTXgST4skpJeTitM0GKNUBEgrmBq3A==}+ engines: {node: '>=14.17.0'}+ peerDependencies:+ bullmq-otel: '>=2.0.0'+ ioredis: '>=5.0.0'+ pg: '>=8.0.0'+ redis: '>=5.0.0'+ peerDependenciesMeta:+ bullmq-otel:+ optional: true+ ioredis:+ optional: true+ pg:+ optional: true+ redis:+ optional: true+resolution: {integrity: sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q==}@@ -11443,6 +11494,10 @@ packages:engines: {node: '>=12.0.0'}deprecated: v4 is no longer maintained, upgrade to v5+ resolution: {integrity: sha512-izNAxJyRWUP8ljBoDSub5WyrVOUlT4SLGShswE7eoRBpp6QUsSycYxLBMJlbshgPBMcPT/nrfgjNY2918ayv2A==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-p9nwwR4qyT5W996vBZhdvBCnMhicY5ytZkR4D1Xj0wuTDEiMnjwR57Q3RXYY/s0EpX6Ay3vgIcfaR+ewGHsi+g==}engines: {node: '>=18.0'}@@ -15397,9 +15452,16 @@ packages:resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==}hasBin: true+ resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==}+ hasBin: true+resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==}+ resolution: {integrity: sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==}+resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==}@@ -15589,6 +15651,10 @@ packages:resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==}hasBin: true+ resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==}+ hasBin: true+resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}hasBin: true@@ -17515,6 +17581,11 @@ packages:engines: {node: '>=10'}hasBin: true+ resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==}+ engines: {node: '>=10'}+ hasBin: true+resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}engines: {node: '>= 0.8.0'}@@ -24824,21 +24895,39 @@ snapshots:'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@msgpackr-extract/[email protected]':optional: true+ '@msgpackr-extract/[email protected]':+ optional: true+'@napi-rs/[email protected]':dependencies:'@emnapi/core': 1.3.1@@ -31660,6 +31749,16 @@ snapshots:transitivePeerDependencies:- supports-color+ dependencies:+ cron-parser: 5.10.0+ msgpackr: 2.1.0+ node-abort-controller: 3.1.1+ semver: 7.8.5+ tslib: 2.8.1+ optionalDependencies:+ ioredis: 5.8.2([email protected])+dependencies:'@types/node': 20.19.24@@ -32253,6 +32352,10 @@ snapshots:dependencies:luxon: 3.7.2+ dependencies:+ luxon: 3.7.2+@@ -35966,7 +36069,7 @@ snapshots:lodash.isstring: 4.0.1lodash.once: 4.1.1ms: 2.1.3- semver: 7.6.0+ semver: 7.7.4dependencies:@@ -37252,7 +37355,7 @@ snapshots:pkg-types: 2.3.0postcss: 8.5.6postcss-nested: 7.0.2([email protected])- semver: 7.7.3+ semver: 7.7.4tinyglobby: 0.2.16optionalDependencies:typescript: 5.9.3@@ -37350,10 +37453,26 @@ snapshots:'@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2optional: true+ dependencies:+ node-gyp-build-optional-packages: 5.2.2+ optionalDependencies:+ '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4+ '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4+ '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4+ '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4+ '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4+ '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4+ optional: true+optionalDependencies:msgpackr-extract: 3.0.2+ optionalDependencies:+ msgpackr-extract: 3.0.4+@@ -37557,7 +37676,7 @@ snapshots:rollup: 4.52.5rollup-plugin-visualizer: 6.0.5([email protected](@emnapi/[email protected])(@emnapi/[email protected]))([email protected])scule: 1.3.0- semver: 7.7.2+ semver: 7.7.4serve-placeholder: 2.0.2serve-static: 2.2.0([email protected])source-map: 0.7.6@@ -37733,6 +37852,11 @@ snapshots:optional: true+ dependencies:+ detect-libc: 2.1.2+ optional: true+@@ -40167,6 +40291,8 @@ snapshots:+ [email protected]: {}+dependencies:debug: 2.6.9([email protected])@@ -40345,7 +40471,7 @@ snapshots:dependencies:'@img/colour': 1.0.0detect-libc: 2.1.2- semver: 7.7.3+ semver: 7.7.4optionalDependencies:'@img/sharp-darwin-arm64': 0.34.5'@img/sharp-darwin-x64': 0.34.5