Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
18.2 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {randomUUID} from 'node:crypto';
import {Logger} from '@app/api/Logger';
import type {WorkerLaneDefinition} from '@app/api/worker/WorkerLaneConfig';
import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
import {
	AckPolicy,
	type ConsumerInfo,
	type ConsumerUpdateConfig,
	DeliverPolicy,
	DiscardPolicy,
	JetStreamApiError,
	type JetStreamManager,
	ReplayPolicy,
	RetentionPolicy,
	StorageType,
	type StreamConfig,
} from '@nats-io/jetstream';
import {millis, nanos} from '@nats-io/transport-node';
import type {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';

const STREAM_NAME = 'JOBS';
const SUBJECT_PREFIX = 'jobs.';
export const JOBS_STREAM_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
const LEGACY_CONSUMER_NAME = 'workers';
const DLQ_STREAM_NAME = 'JOBS_DLQ';
const DLQ_SUBJECT_PREFIX = 'dlq.';
const DLQ_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
const STREAM_MAX_MSGS = 2_000_000;
const STREAM_MAX_BYTES = 8 * 1024 * 1024 * 1024;
const STREAM_MIN_BYTES = 64 * 1024 * 1024;
const STREAM_MAX_MSGS_PER_SUBJECT = 250_000;
const DLQ_MAX_BYTES = 64 * 1024 * 1024;
const DLQ_MIN_BYTES = 8 * 1024 * 1024;
const DLQ_PUBLISH_TIMEOUT_MS = 5000;
const DUPLICATE_WINDOW_MS = 2 * 60 * 1000;
const STREAM_FULL_ERR_CODES = new Set([10023, 10077]);
const STREAM_NO_STORAGE_ERR_CODE = 10047;
const STREAM_NAME_IN_USE_ERR_CODE = 10058;
const STREAM_NOT_FOUND_ERR_CODE = 10059;
const CONSUMER_NOT_FOUND_ERR_CODE = 10014;
const CONSUMER_EXISTS_ERR_CODES = new Set([10013, 10148]);

interface WorkerStreamDefinition {
	name: string;
	subject: string;
	retention: RetentionPolicy;
	maxAgeMs: number;
	minBytes: number;
	maxMessages: number;
	maxMessagesPerSubject: number;
	discard: DiscardPolicy;
	discardNewPerSubject: boolean;
}

const JOBS_STREAM: WorkerStreamDefinition = {
	name: STREAM_NAME,
	subject: `${SUBJECT_PREFIX}>`,
	retention: RetentionPolicy.Workqueue,
	maxAgeMs: JOBS_STREAM_MAX_AGE_MS,
	minBytes: STREAM_MIN_BYTES,
	maxMessages: STREAM_MAX_MSGS,
	maxMessagesPerSubject: STREAM_MAX_MSGS_PER_SUBJECT,
	discard: DiscardPolicy.New,
	discardNewPerSubject: true,
};

const DEAD_LETTER_STREAM: WorkerStreamDefinition = {
	name: DLQ_STREAM_NAME,
	subject: `${DLQ_SUBJECT_PREFIX}>`,
	retention: RetentionPolicy.Limits,
	maxAgeMs: DLQ_MAX_AGE_MS,
	minBytes: DLQ_MIN_BYTES,
	maxMessages: -1,
	maxMessagesPerSubject: -1,
	discard: DiscardPolicy.Old,
	discardNewPerSubject: false,
};

export interface WorkerDeadLetterMetadata {
	messageId: string;
	originalSeq: number;
	errorMessage: string;
	deliveryCount: number;
	lane: string;
	runAt?: string;
}

function jsErrorCode(error: unknown): number | null {
	return error instanceof JetStreamApiError ? error.code : null;
}

function describeStreamRejection(error: unknown): string | null {
	if (!(error instanceof JetStreamApiError) || !STREAM_FULL_ERR_CODES.has(error.code)) {
		return null;
	}
	return error.apiError().description || 'stream rejected the publish';
}

function isSameInstant(a: string, b: string): boolean {
	if (a === b) return true;
	const parsedA = Date.parse(a);
	return Number.isFinite(parsedA) && parsedA === Date.parse(b);
}

export class JetStreamWorkerQueue {
	private readonly connectionManager: JetStreamConnectionManager;
	private streamReady = false;
	private dlqStreamReady = false;
	private consumersReady = false;
	private streamSetup: Promise<void> | null = null;
	private dlqStreamSetup: Promise<void> | null = null;
	private jobsStreamMaxAgeMs = JOBS_STREAM_MAX_AGE_MS;

	constructor(connectionManager: JetStreamConnectionManager) {
		this.connectionManager = connectionManager;
	}

	async ensureStream(): Promise<void> {
		if (this.streamReady) return;
		this.streamSetup ??= this.initializeStream().finally(() => {
			this.streamSetup = null;
		});
		await this.streamSetup;
	}

	private async initializeStream(): Promise<void> {
		const jsm = await this.connectionManager.getJetStreamManager();
		const existingConfig = await this.readStreamConfig(jsm, JOBS_STREAM);
		if (existingConfig === null) {
			await this.addStream(jsm);
		} else {
			this.jobsStreamMaxAgeMs = millis(existingConfig.max_age);
			await this.applyStreamLimits(jsm, existingConfig);
		}
		this.streamReady = true;
	}

	getJobsStreamMaxAgeMs(): number {
		return this.jobsStreamMaxAgeMs;
	}

	private async oversizedSubjects(jsm: JetStreamManager): Promise<Array<[string, number]> | null> {
		try {
			const info = await jsm.streams.info(STREAM_NAME, {subjects_filter: JOBS_STREAM.subject});
			const subjects = info.state.subjects;
			if (subjects === undefined) return [];
			return Object.entries(subjects).filter(([, count]) => count > STREAM_MAX_MSGS_PER_SUBJECT);
		} catch (error) {
			Logger.warn({err: error, stream: STREAM_NAME}, 'Could not read jobs stream subject counts, leaving limits alone');
			return null;
		}
	}

	private async applyStreamLimits(jsm: JetStreamManager, existing: StreamConfig): Promise<void> {
		const unmanaged = this.unmanagedStreamSettings(existing, JOBS_STREAM);
		if (unmanaged.length > 0) {
			Logger.warn(
				{stream: STREAM_NAME, unmanaged},
				'Jobs stream has settings this service does not own, change them explicitly with publishers and workers stopped',
			);
		}
		const drifted = this.diffStreamLimits(existing, JOBS_STREAM);
		if (drifted.length === 0) return;
		const oversized = await this.oversizedSubjects(jsm);
		if (oversized === null) return;
		if (oversized.length > 0) {
			Logger.error(
				{stream: STREAM_NAME, drifted, oversized, limit: STREAM_MAX_MSGS_PER_SUBJECT},
				'Jobs stream limits are out of date but tightening them would drop queued jobs, drain these subjects then migrate the stream explicitly',
			);
			return;
		}
		const startBytes = existing.max_bytes > 0 ? existing.max_bytes : STREAM_MAX_BYTES;
		try {
			const maxBytes = await this.fitToStorageBudget(startBytes, STREAM_MIN_BYTES, (bytes) =>
				jsm.streams.update(STREAM_NAME, {
					max_msgs: JOBS_STREAM.maxMessages,
					max_bytes: bytes,
					max_msgs_per_subject: JOBS_STREAM.maxMessagesPerSubject,
					discard: JOBS_STREAM.discard,
					discard_new_per_subject: JOBS_STREAM.discardNewPerSubject,
				}),
			);
			Logger.info({stream: STREAM_NAME, drifted, max_bytes: maxBytes}, 'Applied jobs stream limits');
		} catch (error) {
			Logger.error(
				{err: error, stream: STREAM_NAME, drifted},
				'Could not apply jobs stream limits, continuing with the limits the stream already has',
			);
		}
	}

	private async readStreamConfig(jsm: JetStreamManager, stream: WorkerStreamDefinition): Promise<StreamConfig | null> {
		try {
			const {config} = await jsm.streams.info(stream.name);
			this.assertStreamIdentity(config, stream);
			return config;
		} catch (error) {
			if (jsErrorCode(error) === STREAM_NOT_FOUND_ERR_CODE) return null;
			throw error;
		}
	}

	private assertStreamIdentity(config: StreamConfig, stream: WorkerStreamDefinition): void {
		const incompatible: Array<string> = [];
		if (config.name !== stream.name) incompatible.push('name');
		if (config.subjects?.length !== 1 || config.subjects[0] !== stream.subject) incompatible.push('subjects');
		if (config.retention !== stream.retention) incompatible.push('retention');
		if (config.storage !== StorageType.File) incompatible.push('storage');
		for (const field of ['sealed', 'no_ack', 'mirror', 'republish', 'subject_transform'] as const) {
			if (config[field]) incompatible.push(field);
		}
		if (config.sources?.length) incompatible.push('sources');
		if (incompatible.length > 0) {
			throw new Error(
				`Worker stream ${stream.name} has incompatible ${incompatible.join(', ')} configuration, stop publishers and workers and migrate it explicitly before startup`,
			);
		}
	}

	private diffStreamLimits(config: StreamConfig, stream: WorkerStreamDefinition): Array<string> {
		const drifted: Array<string> = [];
		if (!Number.isSafeInteger(config.max_bytes) || config.max_bytes < stream.minBytes) drifted.push('max_bytes');
		if (config.max_msgs !== stream.maxMessages) drifted.push('max_msgs');
		if (config.max_msgs_per_subject !== stream.maxMessagesPerSubject) drifted.push('max_msgs_per_subject');
		if (config.discard !== stream.discard) drifted.push('discard');
		if ((config.discard_new_per_subject ?? false) !== stream.discardNewPerSubject) {
			drifted.push('discard_new_per_subject');
		}
		return drifted;
	}

	private unmanagedStreamSettings(config: StreamConfig, stream: WorkerStreamDefinition): Array<string> {
		const unmanaged: Array<string> = [];
		if (config.max_age !== nanos(stream.maxAgeMs)) unmanaged.push('max_age');
		if (config.duplicate_window !== nanos(DUPLICATE_WINDOW_MS)) unmanaged.push('duplicate_window');
		if (config.max_msg_size !== undefined && config.max_msg_size !== -1) unmanaged.push('max_msg_size');
		if (config.persist_mode !== undefined && config.persist_mode !== 'default') unmanaged.push('persist_mode');
		for (const field of ['allow_rollup_hdrs', 'allow_msg_ttl', 'allow_msg_counter'] as const) {
			if (config[field]) unmanaged.push(field);
		}
		return unmanaged;
	}

	private async fitToStorageBudget(
		startBytes: number,
		minBytes: number,
		apply: (maxBytes: number) => Promise<unknown>,
	): Promise<number> {
		let maxBytes = startBytes;
		for (;;) {
			try {
				await apply(maxBytes);
				return maxBytes;
			} catch (error) {
				if (jsErrorCode(error) !== STREAM_NO_STORAGE_ERR_CODE || maxBytes <= minBytes) {
					throw error;
				}
				maxBytes = Math.floor(maxBytes / 2);
			}
		}
	}

	private async adoptConcurrentStream(
		jsm: JetStreamManager,
		stream: WorkerStreamDefinition,
		error: unknown,
	): Promise<StreamConfig | null> {
		if (jsErrorCode(error) !== STREAM_NAME_IN_USE_ERR_CODE) {
			return null;
		}
		const config = await this.readStreamConfig(jsm, stream);
		if (config !== null) {
			Logger.info({stream: stream.name, max_bytes: config.max_bytes}, 'Stream was created concurrently, adopting it');
		}
		return config;
	}

	private async createStream(jsm: JetStreamManager, stream: WorkerStreamDefinition, maxBytes: number): Promise<void> {
		const {config} = await jsm.streams.add({
			name: stream.name,
			subjects: [stream.subject],
			retention: stream.retention,
			storage: StorageType.File,
			max_age: nanos(stream.maxAgeMs),
			duplicate_window: nanos(DUPLICATE_WINDOW_MS),
			num_replicas: 1,
			max_msgs: stream.maxMessages,
			max_bytes: maxBytes,
			max_msgs_per_subject: stream.maxMessagesPerSubject,
			discard: stream.discard,
			discard_new_per_subject: stream.discardNewPerSubject,
		});
		this.assertStreamIdentity(config, stream);
	}

	private async addStream(jsm: JetStreamManager): Promise<void> {
		let maxBytes: number;
		try {
			maxBytes = await this.fitToStorageBudget(STREAM_MAX_BYTES, STREAM_MIN_BYTES, (bytes) =>
				this.createStream(jsm, JOBS_STREAM, bytes),
			);
		} catch (error) {
			const adopted = await this.adoptConcurrentStream(jsm, JOBS_STREAM, error);
			if (adopted !== null) {
				return;
			}
			if (jsErrorCode(error) === STREAM_NO_STORAGE_ERR_CODE) {
				Logger.error(
					{err: error, stream: STREAM_NAME, max_bytes: STREAM_MIN_BYTES},
					'Jobs stream does not fit the JetStream storage budget at its smallest size, free disk space on the NATS store',
				);
			}
			throw error;
		}
		if (maxBytes !== STREAM_MAX_BYTES) {
			Logger.warn(
				{stream: STREAM_NAME, max_bytes: maxBytes},
				'Created the jobs stream below its target size to fit the JetStream storage budget',
			);
		}
	}

	async ensureDlqStream(): Promise<void> {
		if (this.dlqStreamReady) return;
		this.dlqStreamSetup ??= this.initializeDlqStream().finally(() => {
			this.dlqStreamSetup = null;
		});
		await this.dlqStreamSetup;
	}

	private async initializeDlqStream(): Promise<void> {
		const jsm = await this.connectionManager.getJetStreamManager();
		if ((await this.readStreamConfig(jsm, DEAD_LETTER_STREAM)) === null) {
			if (!(await this.addDlqStream(jsm))) {
				return;
			}
		}
		this.dlqStreamReady = true;
	}

	private async addDlqStream(jsm: JetStreamManager): Promise<boolean> {
		try {
			const maxBytes = await this.fitToStorageBudget(DLQ_MAX_BYTES, DLQ_MIN_BYTES, (bytes) =>
				this.createStream(jsm, DEAD_LETTER_STREAM, bytes),
			);
			Logger.info({stream: DLQ_STREAM_NAME, max_bytes: maxBytes}, 'Dead-letter stream created');
			return true;
		} catch (error) {
			if ((await this.adoptConcurrentStream(jsm, DEAD_LETTER_STREAM, error)) !== null) {
				return true;
			}
			if (jsErrorCode(error) !== STREAM_NO_STORAGE_ERR_CODE) {
				throw error;
			}
			Logger.warn(
				{err: error, stream: DLQ_STREAM_NAME},
				'JetStream has no room for the dead-letter stream, failed jobs stay in the jobs stream until they expire',
			);
			return false;
		}
	}

	async ensureConsumers(lanes: ReadonlyArray<WorkerLaneDefinition>): Promise<void> {
		if (this.consumersReady) {
			return;
		}
		const jsm = await this.connectionManager.getJetStreamManager();
		for (const lane of lanes) {
			await this.ensureConsumer(jsm, lane);
		}
		this.consumersReady = true;
	}

	private async readConsumer(jsm: JetStreamManager, name: string): Promise<ConsumerInfo | null> {
		try {
			return await jsm.consumers.info(STREAM_NAME, name);
		} catch (error) {
			if (jsErrorCode(error) === CONSUMER_NOT_FOUND_ERR_CODE) {
				return null;
			}
			throw error;
		}
	}

	private async ensureConsumer(jsm: JetStreamManager, lane: WorkerLaneDefinition): Promise<void> {
		const config = {
			max_deliver: lane.maxDeliver,
			ack_wait: nanos(lane.ackWaitMs),
			max_ack_pending: lane.maxAckPending,
			filter_subject: undefined,
			filter_subjects: [...lane.taskTypes, ...lane.retiredTaskTypes].map((task) => `${SUBJECT_PREFIX}${task}`),
		} satisfies ConsumerUpdateConfig;
		let existing = await this.readConsumer(jsm, lane.consumerName);
		if (existing === null) {
			try {
				const created = await jsm.consumers.add(STREAM_NAME, {
					...config,
					durable_name: lane.consumerName,
					ack_policy: AckPolicy.Explicit,
					deliver_policy: DeliverPolicy.All,
					replay_policy: ReplayPolicy.Instant,
				});
				this.requireConsumerConfiguration(created, lane.consumerName);
				Logger.info({lane: lane.name, consumer: lane.consumerName}, 'Consumer created');
				return;
			} catch (error) {
				if (!CONSUMER_EXISTS_ERR_CODES.has(jsErrorCode(error) ?? 0)) {
					throw error;
				}
				existing = await this.readConsumer(jsm, lane.consumerName);
				if (existing === null) {
					throw error;
				}
			}
		}
		this.requireConsumerConfiguration(existing, lane.consumerName);
		const updated = await jsm.consumers.update(STREAM_NAME, lane.consumerName, config);
		this.requireConsumerConfiguration(updated, lane.consumerName);
		const current = await this.readConsumer(jsm, lane.consumerName);
		if (current === null) {
			throw new Error(`Worker consumer ${lane.consumerName} was deleted during startup`);
		}
		if (!isSameInstant(current.created, existing.created)) {
			this.requireConsumerConfiguration(current, lane.consumerName);
			this.requireConsumerConfiguration(
				await jsm.consumers.update(STREAM_NAME, lane.consumerName, config),
				lane.consumerName,
			);
			Logger.warn(
				{lane: lane.name, consumer: lane.consumerName, previous: existing.created, current: current.created},
				'Consumer was recreated by another worker during startup, using the current one',
			);
			return;
		}
		Logger.info({lane: lane.name, consumer: lane.consumerName}, 'Consumer updated without resetting delivery state');
	}

	private requireConsumerConfiguration(consumer: ConsumerInfo, name: string): void {
		const current = consumer.config;
		if (
			current.durable_name !== name ||
			current.ack_policy !== AckPolicy.Explicit ||
			current.deliver_policy !== DeliverPolicy.All ||
			current.replay_policy !== ReplayPolicy.Instant ||
			current.deliver_subject ||
			current.headers_only ||
			(current.inactive_threshold ?? 0) > 0 ||
			(current.backoff?.length ?? 0) > 0
		) {
			throw new Error(
				`Worker consumer ${name} has an incompatible configuration, migrate it explicitly with workers stopped`,
			);
		}
	}

	private async migrateLegacyConsumer(): Promise<void> {
		const jsm = await this.connectionManager.getJetStreamManager();
		if ((await this.readConsumer(jsm, LEGACY_CONSUMER_NAME)) === null) return;
		try {
			await jsm.consumers.delete(STREAM_NAME, LEGACY_CONSUMER_NAME);
			Logger.info({consumer: LEGACY_CONSUMER_NAME}, 'Legacy consumer deleted, lane consumers will handle its messages');
		} catch (error) {
			Logger.error(
				{err: error, consumer: LEGACY_CONSUMER_NAME},
				'Could not delete the legacy worker consumer, delete it manually once legacy workers are stopped',
			);
		}
	}

	async ensureInfrastructure(lanes: ReadonlyArray<WorkerLaneDefinition>): Promise<void> {
		await this.ensureStream();
		await this.ensureDlqStream();
		await this.migrateLegacyConsumer();
		await this.ensureConsumers(lanes);
	}

	async enqueue(
		taskType: string,
		payload: WorkerJobPayload,
		options?: {
			runAt?: Date;
			jobKey?: string;
		},
	): Promise<{seq: string; duplicate: boolean}> {
		const js = this.connectionManager.getJetStreamClient();
		const subject = `${SUBJECT_PREFIX}${taskType}`;
		const body = JSON.stringify({
			payload,
			run_at: options?.runAt?.toISOString(),
			created_at: new Date().toISOString(),
		});
		const msgID = options?.jobKey ? `${taskType}:${options.jobKey}` : randomUUID();
		try {
			const ack = await js.publish(subject, body, {
				msgID,
			});
			return {seq: `${ack.seq}`, duplicate: ack.duplicate === true};
		} catch (error) {
			const rejection = describeStreamRejection(error);
			if (rejection === null) {
				throw error;
			}
			throw new WorkerQueueOverflowError(taskType, rejection);
		}
	}

	async publishToDlq(
		taskType: string,
		originalPayload: Record<string, unknown>,
		meta: WorkerDeadLetterMetadata,
	): Promise<void> {
		const js = this.connectionManager.getJetStreamClient();
		const subject = `${DLQ_SUBJECT_PREFIX}${taskType}`;
		const body = JSON.stringify({
			original_subject: `${SUBJECT_PREFIX}${taskType}`,
			original_seq: meta.originalSeq,
			payload: originalPayload,
			error_message: meta.errorMessage,
			delivery_count: meta.deliveryCount,
			lane: meta.lane,
			run_at: meta.runAt,
			failed_at: new Date().toISOString(),
		});
		await js.publish(subject, body, {
			msgID: meta.messageId,
			timeout: DLQ_PUBLISH_TIMEOUT_MS,
		});
	}

	getStreamName(): string {
		return STREAM_NAME;
	}

	getConnectionManager(): JetStreamConnectionManager {
		return this.connectionManager;
	}
}