Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

6.4 KB
Raw Blame
import { generateId } from '@openpanel/shared';
import {
  type Job as BullJob,
  Worker as BullWorker,
  type ConnectionOptions,
} from 'bullmq';
import { type AppDeps, createCtx, extendCtx, type JobCtx } from '../context';
import { REQUEST_ID_LENGTH, REQUEST_ID_LOG_FIELD } from '../logger';
import { resolveJob } from './compat';
import type { QueueDefinition, QueueMap } from './define';
import { isEnvelope } from './envelope';
import { jobDurationMs } from './jobs.metrics';
import { queueKey } from './naming';

export interface TerminalFailure {
  queue: string;
  job: string;
  jobId?: string;
  attempts: number;
  error: Error;
  data: unknown;
}

export interface StartWorkersOptions<TQueues extends QueueMap> {
  /** Already filtered to the queues this role consumes. */
  definitions: TQueues;
  deps: AppDeps;
  /**
   * Called once per queue: a worker blocks its connection, so it cannot share
   * one. The caller keeps what it hands over and closes it.
   */
  createConnection: (name: string) => ConnectionOptions;
  cluster?: boolean;
  namespace?: string;
  /**
   * A job that has exhausted its attempts. Five of the seven queues never
   * retry, so for those this fires on the first throw.
   */
  onTerminalFailure?: (failure: TerminalFailure) => void;
}

export interface WorkerHandle {
  /**
   * Stops taking new jobs and waits for the running ones. Whatever supervises
   * the process has to allow longer than the slowest job takes, or the drain
   * is killed halfway and every interrupted job comes back as a stalled retry.
   *
   * Closes the workers, not the connections they were handed.
   */
  close(): Promise<void>;
}

/**
 * Starts one worker per queue handed in. Only a process whose role includes
 * consuming calls this; an API-only process builds producers and stops there.
 */
export function startWorkers<TQueues extends QueueMap>({
  definitions,
  deps,
  createConnection,
  cluster,
  namespace,
  onTerminalFailure,
}: StartWorkersOptions<TQueues>): WorkerHandle {
  const started = Object.values(definitions).map((definition) => {
    const key = queueKey(definition.name, { cluster, namespace });
    const logger = deps.logger.child({ queue: definition.name });

    const worker = new BullWorker(key, (job) => runJob(definition, job, deps), {
      connection: createConnection(`worker:${definition.name}`),
      ...(definition.worker?.concurrency !== undefined && {
        concurrency: definition.worker.concurrency,
      }),
      ...(definition.worker?.limiter !== undefined && {
        limiter: definition.worker.limiter,
      }),
    });

    // Fires on the connection itself, not on any one job — an unlistened
    // 'error' event on this emitter crashes the process (ioredis ECONNRESET
    // etc.), so it must always have a listener.
    worker.on('error', (error) => logger.error({ err: error }, 'worker error'));

    // A stall means the job was taken and then its lock lapsed: the process
    // died mid-job, or the handler blocked the event loop past the lock.
    worker.on('stalled', (jobId) =>
      logger.warn({ jobId }, 'job stalled and will be retried')
    );

    worker.on('failed', (job, error) => {
      // `failed` fires on every attempt; only a finished one is out of retries.
      if (!job?.finishedOn) {
        return;
      }

      // FAILURE-ONLY, and the label is the Redis key. See jobs.metrics.ts for
      // why a success timing may not join this series.
      if (job.processedOn) {
        jobDurationMs.observe(
          { name: key, status: 'failed' },
          job.finishedOn - job.processedOn
        );
      }

      logger.error(
        {
          err: error,
          jobId: job.id,
          job: job.name,
          attempts: job.attemptsMade,
        },
        'job failed terminally'
      );
      onTerminalFailure?.({
        queue: definition.name,
        job: job.name,
        jobId: job.id,
        attempts: job.attemptsMade,
        error,
        data: job.data,
      });
    });

    logger.info({ key }, 'worker started');
    return worker;
  });

  return {
    close: async () => {
      await Promise.all(started.map((worker) => worker.close()));
    },
  };
}

/** The shape of a BullMQ job this needs, so a test does not have to build one. */
export interface RunnableJob {
  id?: string;
  name: string;
  data: unknown;
  attemptsMade: number;
}

/**
 * Exported for the worker tests, and it is what the `BullWorker` processor
 * above calls. Throws on a job it cannot resolve or does not declare, rather
 * than completing it having done nothing.
 */
export async function runJob(
  definition: QueueDefinition,
  job: RunnableJob | BullJob,
  deps: AppDeps
): Promise<void> {
  const resolved = resolveJob(definition, { name: job.name, data: job.data });

  const jobDefinition = definition.jobs[resolved.job];
  if (!jobDefinition) {
    throw new Error(
      `Queue '${definition.name}' has no job named '${resolved.job}'`
    );
  }

  // Parsed rather than trusted: this job may have been enqueued by the deploy
  // this one is replacing, under the previous shape of the schema.
  const payload = jobDefinition.payload.parse(resolved.payload);

  const attempt = job.attemptsMade + 1;
  // A scheduled or legacy job has no originating request; minting one keeps its
  // own logs and its follow-up enqueues correlated.
  const requestId =
    resolved.meta.requestId ?? generateId(undefined, REQUEST_ID_LENGTH);

  const logger = deps.logger.child({
    queue: definition.name,
    job: resolved.job,
    jobId: job.id,
    attempt,
    [REQUEST_ID_LOG_FIELD]: requestId,
  });

  const ctx: JobCtx = extendCtx(createCtx(deps, { requestId, logger }), {
    job: {
      id: job.id ?? '',
      attempt,
      queue: definition.name,
      name: resolved.job,
    },
  });

  // The registry erases each handler's payload type so the machinery can hold
  // all of them (see `AnyJob`); the schema above is what re-establishes it.
  const handler = jobDefinition.handler as (args: {
    payload: unknown;
    ctx: JobCtx;
  }) => Promise<void>;

  // `debug`, not `info`: `sessions` runs one of these per session close.
  // `legacy` tracks whether this job predates the envelope format — while it
  // is ever true, the compat hooks that translate it cannot be removed.
  const legacy = !isEnvelope(job.data);
  logger.debug({ legacy }, 'job started');
  const startedAt = performance.now();

  await handler({ payload, ctx });

  logger.debug(
    { legacy, durationMs: Math.round(performance.now() - startedAt) },
    'job completed'
  );
}