Skip to content

Code-first video & image, rendered natively on WebGPU.

4.3 KB
Raw Blame
import { ApiError } from "./errors.js";
import { storage } from "./storage.js";
import { gatewayJson } from "./transport.js";
import type {
	CompletedQueueStatus,
	InQueueQueueStatus,
	ModelData,
	QueueStatus,
	RequestOptions,
	Result,
	StatusOptions,
	SubmitOptions,
	SubscribeOptions,
} from "./types.js";

/** Statuses worth asking again for: rate limits and a gateway or model service hiccup. */
const TRANSIENT = new Set([429, 502, 503, 504]);
const READ_TRIES = 4;

export interface Queue {
	/** Queues a job and returns at once with its `request_id`. Billed when it runs. */
	submit<Input = ModelData>(
		model: string,
		options: SubmitOptions<Input>,
	): Promise<InQueueQueueStatus>;
	status(model: string, options: StatusOptions): Promise<QueueStatus>;
	/** The output of a finished job. Fetching it again does not run or bill the model again. */
	result<Output = ModelData>(
		model: string,
		options: RequestOptions,
	): Promise<Result<Output>>;
	cancel(model: string, options: RequestOptions): Promise<void>;
	/** Resolves when the job is `COMPLETED`, calling `onQueueUpdate` after each check. */
	subscribeToStatus(
		model: string,
		options: WaitOptions,
	): Promise<CompletedQueueStatus>;
}

export type WaitOptions = StatusOptions &
	Pick<SubscribeOptions, "pollInterval" | "onQueueUpdate">;

const sleep = (ms: number, signal?: AbortSignal) =>
	new Promise<void>((resolve, reject) => {
		signal?.throwIfAborted();
		const timer = setTimeout(resolve, ms);
		signal?.addEventListener(
			"abort",
			() => {
				clearTimeout(timer);
				reject(signal.reason);
			},
			{ once: true },
		);
	});

/** Reads (status, result) are free and idempotent, so transient failures are retried. */
async function read<T>(
	run: () => Promise<T>,
	signal?: AbortSignal,
): Promise<T> {
	for (let attempt = 1; ; attempt++) {
		try {
			return await run();
		} catch (err) {
			const transient = err instanceof ApiError && TRANSIENT.has(err.status);
			if (!transient || attempt >= READ_TRIES) throw err;
			await sleep(500 * 2 ** attempt, signal);
		}
	}
}

const requestPath = (model: string, requestId: string, suffix = "") =>
	`${model}/requests/${encodeURIComponent(requestId)}${suffix}`;

async function submit<Input>(
	model: string,
	{ input, abortSignal }: SubmitOptions<Input>,
): Promise<InQueueQueueStatus> {
	const { data } = await gatewayJson<InQueueQueueStatus>({
		service: "queue",
		path: model,
		body: (await storage.transformInput(input)) ?? {},
		signal: abortSignal,
	});
	return data;
}

function status(
	model: string,
	{ requestId, logs, abortSignal }: StatusOptions,
): Promise<QueueStatus> {
	return read(async () => {
		const { data } = await gatewayJson<QueueStatus>({
			service: "queue",
			path: requestPath(model, requestId, "/status"),
			query: { logs: logs ? "1" : undefined },
			signal: abortSignal,
		});
		return data;
	}, abortSignal);
}

function result<Output>(
	model: string,
	{ requestId, abortSignal }: RequestOptions,
): Promise<Result<Output>> {
	return read(async () => {
		const { data } = await gatewayJson<Output>({
			service: "queue",
			path: requestPath(model, requestId),
			signal: abortSignal,
		});
		return { data, requestId };
	}, abortSignal);
}

async function cancel(
	model: string,
	{ requestId, abortSignal }: RequestOptions,
): Promise<void> {
	await gatewayJson<unknown>({
		service: "queue",
		path: requestPath(model, requestId, "/cancel"),
		method: "PUT",
		signal: abortSignal,
	});
}

async function subscribeToStatus(
	model: string,
	options: WaitOptions,
): Promise<CompletedQueueStatus> {
	const interval = options.pollInterval ?? 500;
	for (;;) {
		const current = await status(model, options);
		options.onQueueUpdate?.(current);
		if (current.status === "COMPLETED") return current;
		await sleep(interval, options.abortSignal);
	}
}

export const queue: Queue = {
	submit,
	status,
	result,
	cancel,
	subscribeToStatus,
};

/** Queues the job, waits for it, and returns its output. */
export async function subscribe<Output = ModelData, Input = ModelData>(
	model: string,
	options: SubscribeOptions<Input>,
): Promise<Result<Output>> {
	const { request_id: requestId } = await submit(model, options);
	options.onEnqueue?.(requestId);
	await subscribeToStatus(model, { ...options, requestId });
	return result<Output>(model, {
		requestId,
		abortSignal: options.abortSignal,
	});
}