Skip to content

Email for professionals and teams

16.0 KB
Raw Blame
import { and, desc, eq, inArray, lt } from "drizzle-orm";
import { getDb } from "@/db";
import { importJobs, mailboxes } from "@/db/schema";
import { newId } from "@/lib/ids";
import { getImportMessageUserId, parseImportDestination } from "./destination";
import { fetchImapMessages } from "./imap";
import { getImapChunkSize, growImapChunkSize, isMissingImapFolderError, isPermanentImapError } from "./imap-utils";
import { deleteStagedImport, importBatchKey, stageImportMessages, unpackImportMessages } from "./staging";
import { importMessagesToMailbox } from "./service";
import type {
	FileImportCursor,
	FileImportSpec,
	ImapImportCursor,
	ImapImportSource,
	ImapImportSpec,
	ImportJobMessage,
	ImportJobSummary,
} from "./jobs-types";
import type { ImportMailboxResult, ImportMessageInput } from "./types";

const ACTIVE = ["queued", "running"] as const;
const MAX_STORED_ERRORS = 20;
/** IMAP messages fetched per queue message; the connection is reopened for each. */
const IMAP_CHUNK_SIZE = 50;
/** Consecutive failed IMAP chunks, each retried smaller, before the job gives up. */
const MAX_IMAP_FAILURES = 4;
const STALE_AFTER_MS = 30 * 60 * 1000;
const KEEP_FINISHED_MS = 7 * 24 * 60 * 60 * 1000;

type JobRow = typeof importJobs.$inferSelect;

type Chunk = {
	result: ImportMailboxResult;
	cursor: FileImportCursor | ImapImportCursor;
	totalDelta: number;
	done: boolean;
	/** Staged object that is safe to delete once the chunk is recorded. */
	consumedKey?: string;
};

function parseJson<T>(value: string, fallback: T): T {
	try {
		return JSON.parse(value) as T;
	} catch {
		return fallback;
	}
}

async function enqueue(env: CloudflareEnv, message: ImportJobMessage): Promise<void> {
	await env.OUTBOUND_QUEUE.send(message);
}

export function toImportJobSummary(row: JobRow): ImportJobSummary {
	let currentSource: string | null = null;
	if (row.kind === "imap" && row.status === "running") {
		const spec = parseJson<ImapImportSpec | null>(row.spec, null);
		const cursor = parseJson<ImapImportCursor>(row.cursor, { source: 0, done: 0, counted: false });
		currentSource = spec?.sources[cursor.source]?.folder ?? null;
	}
	return {
		id: row.id,
		mailboxId: row.mailboxId,
		kind: row.kind,
		label: row.label,
		status: row.status,
		total: row.total,
		processed: row.processed,
		imported: row.imported,
		skipped: row.skipped,
		errors: parseJson<string[]>(row.errors, []),
		errorCount: row.errorCount,
		error: row.error,
		currentSource,
		createdAt: row.createdAt.toISOString(),
		startedAt: row.startedAt?.toISOString() ?? null,
		finishedAt: row.finishedAt?.toISOString() ?? null,
	};
}

export async function listImportJobs(env: CloudflareEnv, userId: string, limit = 20): Promise<ImportJobSummary[]> {
	const rows = await getDb(env)
		.select()
		.from(importJobs)
		.where(eq(importJobs.userId, userId))
		.orderBy(desc(importJobs.createdAt))
		.limit(limit);
	return rows.map(toImportJobSummary);
}

export async function createFileImportJob(
	env: CloudflareEnv,
	input: { userId: string; mailboxId: string; label: string; destination: string; messages: ImportMessageInput[] },
): Promise<ImportJobSummary> {
	const db = getDb(env);
	const id = newId("imp");
	const batches = await stageImportMessages(env, id, input.messages);
	const spec: FileImportSpec = { destination: input.destination, batches };
	const cursor: FileImportCursor = { batch: 0 };
	try {
		const [row] = await db
			.insert(importJobs)
			.values({
				id,
				userId: input.userId,
				mailboxId: input.mailboxId,
				kind: "file",
				label: input.label,
				total: input.messages.length,
				spec: JSON.stringify(spec),
				cursor: JSON.stringify(cursor),
			})
			.returning();
		await enqueue(env, { kind: "import.run", jobId: id });
		return toImportJobSummary(row);
	} catch (error) {
		await deleteStagedImport(env, id, batches);
		await db.delete(importJobs).where(eq(importJobs.id, id));
		throw error;
	}
}

export async function createImapImportJob(
	env: CloudflareEnv,
	input: {
		userId: string;
		mailboxId: string;
		password: string;
		spec: ImapImportSpec;
	},
): Promise<ImportJobSummary> {
	const db = getDb(env);
	const id = newId("imp");
	const cursor: ImapImportCursor = { source: 0, done: 0, counted: false };
	try {
		const [row] = await db
			.insert(importJobs)
			.values({
				id,
				userId: input.userId,
				mailboxId: input.mailboxId,
				kind: "imap",
				label: `${input.spec.username} @ ${input.spec.host}`,
				spec: JSON.stringify(input.spec),
				cursor: JSON.stringify(cursor),
			})
			.returning();
		await enqueue(env, { kind: "import.run", jobId: id, imapPassword: input.password });
		return toImportJobSummary(row);
	} catch (error) {
		await db.delete(importJobs).where(eq(importJobs.id, id));
		throw error;
	}
}

async function cleanupJob(env: CloudflareEnv, row: JobRow): Promise<void> {
	if (row.kind !== "file") return;
	const spec = parseJson<FileImportSpec>(row.spec, { destination: "", batches: 0 });
	await deleteStagedImport(env, row.id, spec.batches);
}

/** Cancels a running job, or removes the record of a finished one. Returns false when it is not the user's. */
export async function cancelOrDismissImportJob(env: CloudflareEnv, userId: string, jobId: string): Promise<boolean> {
	const db = getDb(env);
	const [row] = await db
		.select()
		.from(importJobs)
		.where(and(eq(importJobs.id, jobId), eq(importJobs.userId, userId)))
		.limit(1);
	if (!row) return false;
	if (row.status === "queued" || row.status === "running") {
		const now = new Date();
		await db
			.update(importJobs)
			.set({ status: "cancelled", finishedAt: now, updatedAt: now })
			.where(and(eq(importJobs.id, jobId), inArray(importJobs.status, [...ACTIVE])));
		await cleanupJob(env, row);
	} else {
		await db.delete(importJobs).where(eq(importJobs.id, jobId));
		await cleanupJob(env, row);
	}
	return true;
}

export type ResumeImportResult =
	| { ok: true; job: ImportJobSummary }
	| { ok: false; status: 400 | 404 | 409; error: string };

/**
 * Continues a failed job from its saved cursor. IMAP passwords are never stored, so the caller supplies
 * it again; file jobs continue from their staged batches when those still exist.
 */
export async function resumeImportJob(
	env: CloudflareEnv,
	userId: string,
	jobId: string,
	password?: string,
): Promise<ResumeImportResult> {
	const db = getDb(env);
	const [row] = await db
		.select()
		.from(importJobs)
		.where(and(eq(importJobs.id, jobId), eq(importJobs.userId, userId)))
		.limit(1);
	if (!row) return { ok: false, status: 404, error: "Import not found" };
	if (row.status !== "failed") return { ok: false, status: 409, error: "Only a failed import can be resumed" };

	let cursor = row.cursor;
	if (row.kind === "imap") {
		if (!password) return { ok: false, status: 400, error: "Enter the IMAP password to resume" };
		const parsed = parseJson<ImapImportCursor>(row.cursor, { source: 0, done: 0, counted: false });
		cursor = JSON.stringify({ ...parsed, failures: undefined });
	} else {
		const parsed = parseJson<FileImportCursor>(row.cursor, { batch: 0 });
		if (!(await env.BUCKET.head(importBatchKey(row.id, parsed.batch)))) {
			return { ok: false, status: 409, error: "Uploaded files are no longer available; upload them again" };
		}
	}

	const now = new Date();
	const [claimed] = await db
		.update(importJobs)
		.set({ status: "queued", error: null, finishedAt: null, cursor, updatedAt: now })
		.where(and(eq(importJobs.id, row.id), eq(importJobs.status, "failed")))
		.returning();
	if (!claimed) return { ok: false, status: 409, error: "Only a failed import can be resumed" };
	await enqueue(env, { kind: "import.run", jobId: row.id, ...(row.kind === "imap" ? { imapPassword: password } : {}) });
	return { ok: true, job: toImportJobSummary(claimed) };
}

/** Runs one chunk of a job and queues the next. Never throws for import failures; they land on the job. */
export async function processImportJob(env: CloudflareEnv, delivered: ImportJobMessage): Promise<void> {
	const { attempt, ...message } = delivered;
	const db = getDb(env);
	const [job] = await db.select().from(importJobs).where(eq(importJobs.id, message.jobId)).limit(1);
	if (!job || (job.status !== "queued" && job.status !== "running")) return;

	if (job.status === "queued") {
		await db
			.update(importJobs)
			.set({ status: "running", startedAt: new Date(), updatedAt: new Date() })
			.where(and(eq(importJobs.id, job.id), eq(importJobs.status, "queued")));
	}

	let chunk: Chunk;
	try {
		chunk = job.kind === "file" ? await runFileChunk(env, job) : await runImapChunk(env, job, message.imapPassword, attempt);
	} catch (error) {
		const reason = error instanceof Error ? error.message : "Import failed";
		if (job.kind === "imap" && (await retryImapChunkSmaller(env, job, message, reason))) return;
		const now = new Date();
		await db
			.update(importJobs)
			.set({
				status: "failed",
				error: reason,
				finishedAt: now,
				updatedAt: now,
			})
			.where(and(eq(importJobs.id, job.id), inArray(importJobs.status, [...ACTIVE])));
		// Staged uploads stay so the job can be resumed; dismissing or pruning it deletes them.
		return;
	}

	const now = new Date();
	const errors = [...parseJson<string[]>(job.errors, []), ...chunk.result.errors].slice(0, MAX_STORED_ERRORS);
	const recorded = await db
		.update(importJobs)
		.set({
			total: job.total + chunk.totalDelta,
			processed: job.processed + chunk.result.imported + chunk.result.skipped,
			imported: job.imported + chunk.result.imported,
			skipped: job.skipped + chunk.result.skipped,
			errors: JSON.stringify(errors),
			errorCount: job.errorCount + chunk.result.errors.length,
			cursor: JSON.stringify(chunk.cursor),
			updatedAt: now,
			...(chunk.done ? { status: "completed" as const, finishedAt: now } : {}),
		})
		.where(and(eq(importJobs.id, job.id), inArray(importJobs.status, [...ACTIVE])))
		.returning({ id: importJobs.id });

	// Cancelled while the chunk ran: the cancel already cleaned up.
	if (!recorded.length) return;
	if (chunk.consumedKey) await env.BUCKET.delete(chunk.consumedKey).catch(() => undefined);
	if (!chunk.done) await enqueue(env, message);
}

/** Records the failure, halves the chunk size and queues the same chunk again. False when the job should fail instead. */
async function retryImapChunkSmaller(
	env: CloudflareEnv,
	job: JobRow,
	message: ImportJobMessage,
	reason: string,
): Promise<boolean> {
	if (isPermanentImapError(reason)) return false;
	const cursor = parseJson<ImapImportCursor>(job.cursor, { source: 0, done: 0, counted: false });
	const failures = (cursor.failures ?? 0) + 1;
	if (failures >= MAX_IMAP_FAILURES) return false;
	const spec = parseJson<ImapImportSpec | null>(job.spec, null);
	const base = spec ? imapBaseChunkSize(spec, cursor) : IMAP_CHUNK_SIZE;
	const next: ImapImportCursor = { ...cursor, failures, size: Math.max(1, Math.floor(getImapChunkSize(base, cursor.size) / 2)) };
	const recorded = await getDb(env)
		.update(importJobs)
		.set({ cursor: JSON.stringify(next), updatedAt: new Date() })
		.where(and(eq(importJobs.id, job.id), inArray(importJobs.status, [...ACTIVE])))
		.returning({ id: importJobs.id });
	if (recorded.length) await enqueue(env, message);
	return true;
}

function imapBaseChunkSize(spec: ImapImportSpec, cursor: ImapImportCursor): number {
	return Math.min(IMAP_CHUNK_SIZE, spec.importAll ? spec.limit : spec.limit - cursor.done);
}

async function getImportUserId(env: CloudflareEnv, job: JobRow, destinationValue: string): Promise<string> {
	const [mailbox] = await getDb(env)
		.select({ userId: mailboxes.userId })
		.from(mailboxes)
		.where(eq(mailboxes.id, job.mailboxId))
		.limit(1);
	if (!mailbox) throw new Error("Mailbox no longer exists");
	return getImportMessageUserId(parseImportDestination(destinationValue), job.userId, mailbox.userId);
}

async function runFileChunk(env: CloudflareEnv, job: JobRow): Promise<Chunk> {
	const spec = parseJson<FileImportSpec>(job.spec, { destination: "", batches: 0 });
	const cursor = parseJson<FileImportCursor>(job.cursor, { batch: 0 });
	const key = importBatchKey(job.id, cursor.batch);
	const object = await env.BUCKET.get(key);
	if (!object) throw new Error("Uploaded files are no longer available; upload them again");
	const messages = unpackImportMessages(new Uint8Array(await object.arrayBuffer()));
	const result = await importMessagesToMailbox(env, {
		userId: await getImportUserId(env, job, spec.destination),
		mailboxId: job.mailboxId,
		destination: parseImportDestination(spec.destination),
		messages,
	});
	const next = cursor.batch + 1;
	return { result, cursor: { batch: next }, totalDelta: 0, done: next >= spec.batches, consumedKey: key };
}

async function runImapChunk(
	env: CloudflareEnv,
	job: JobRow,
	password: string | undefined,
	attempt?: number,
): Promise<Chunk> {
	if (!password) throw new Error("IMAP credentials were lost; start the import again");
	const spec = parseJson<ImapImportSpec | null>(job.spec, null);
	if (!spec) throw new Error("Import settings are unreadable");
	const cursor = parseJson<ImapImportCursor>(job.cursor, { source: 0, done: 0, counted: false });
	const source: ImapImportSource | undefined = spec.sources[cursor.source];
	const empty: ImportMailboxResult = { imported: 0, skipped: 0, errors: [] };
	if (!source) return { result: empty, cursor, totalDelta: 0, done: true };

	const base = imapBaseChunkSize(spec, cursor);
	const batchSize = base > 0 ? getImapChunkSize(base, cursor.size, attempt) : 0;
	let fetched: Awaited<ReturnType<typeof fetchImapMessages>> = { messages: [], total: 0, nextOffset: null };
	if (batchSize > 0) {
		try {
			fetched = await fetchImapMessages({
				host: spec.host,
				port: spec.port,
				secure: spec.secure,
				username: spec.username,
				password,
				folder: source.folder,
				limit: batchSize,
				offset: cursor.done,
			});
		} catch (error) {
			const reason = error instanceof Error ? error.message : "";
			if (!isMissingImapFolderError(reason)) throw error;
			// A folder that cannot be opened (Gmail's `[Gmail]` parent) must not sink the whole import.
			const nextSource = cursor.source + 1;
			return {
				result: { imported: 0, skipped: 0, errors: [`${source.folder}: ${reason}`] },
				cursor: { source: nextSource, done: 0, counted: false },
				totalDelta: 0,
				done: nextSource >= spec.sources.length,
			};
		}
	}

	const result = fetched.messages.length
		? await importMessagesToMailbox(env, {
				userId: await getImportUserId(env, job, source.destination),
				mailboxId: job.mailboxId,
				destination: parseImportDestination(source.destination),
				messages: fetched.messages,
			})
		: empty;

	const done = cursor.done + fetched.messages.length;
	const grown = batchSize > 0 ? growImapChunkSize(IMAP_CHUNK_SIZE, batchSize) : undefined;
	const totalDelta = cursor.counted ? 0 : spec.importAll ? fetched.total : Math.min(spec.limit, fetched.total);
	const sourceFinished = fetched.nextOffset === null || fetched.messages.length === 0 || (!spec.importAll && done >= spec.limit);
	if (!sourceFinished) {
		return { result, cursor: { source: cursor.source, done, counted: true, size: grown }, totalDelta, done: false };
	}
	const nextSource = cursor.source + 1;
	return {
		result,
		cursor: { source: nextSource, done: 0, counted: false, size: grown },
		totalDelta,
		done: nextSource >= spec.sources.length,
	};
}

/** Fails jobs whose queue messages were lost, and drops old finished records. Safe to run on every cron tick. */
export async function sweepImportJobs(env: CloudflareEnv, now = new Date()): Promise<void> {
	const db = getDb(env);
	const stale = await db
		.select()
		.from(importJobs)
		.where(and(inArray(importJobs.status, [...ACTIVE]), lt(importJobs.updatedAt, new Date(now.getTime() - STALE_AFTER_MS))));
	for (const row of stale) {
		await db
			.update(importJobs)
			.set({ status: "failed", error: "Import was interrupted", finishedAt: now, updatedAt: now })
			.where(and(eq(importJobs.id, row.id), inArray(importJobs.status, [...ACTIVE])));
	}
	const expired = and(
		inArray(importJobs.status, ["completed", "failed", "cancelled"]),
		lt(importJobs.finishedAt, new Date(now.getTime() - KEEP_FINISHED_MS)),
	);
	const old = await db.select().from(importJobs).where(expired);
	for (const row of old) await cleanupJob(env, row);
	await db.delete(importJobs).where(expired);
}