Email for professionals and teams
Allow resume importing
Hieu Nguyen (Jack)b4d00d181d76
diff --git a/CLAUDE.md b/CLAUDE.mdindex eab803f..db740fd 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -157,7 +157,7 @@ Optional tables are detected with `isMissingTableError` (`src/lib/database-error### Imports are background jobs-`/api/import/messages` (uploads) and `/api/import/imap` only validate and create a row in `import_jobs`, then answer 202 with the job; nothing is imported inside the request. `processImportJob` (`src/lib/import/jobs.ts`) handles one chunk per `import.run` message on `OUTBOUND_QUEUE` (like webhook retries, so no extra binding), records counters and the resume `cursor` on the row, and enqueues the next chunk itself. Uploaded files are split and staged in the bucket as `import-jobs/<id>/<n>.bin` batches (`staging.ts`), deleted as consumed. IMAP jobs read the folder list from `spec`, but the password never touches the database: it rides along in each queue message, so a lost message fails the job instead of resuming. Cancelling flips the status; every chunk's final write is conditional on the job still being active. `sweepImportJobs` (cron / Node scheduler) fails jobs idle for 30 minutes and prunes finished records after 7 days. The import page polls `GET /api/import/jobs`; `DELETE /api/import/jobs/[id]` cancels an active job or dismisses a finished one. IMAP chunks pipeline their FETCHes (sizes first via `RFC822.SIZE`, then groups bounded by `planImapFetchGroups`). Chunk size is adaptive: `cursor.size` halves when a chunk throws (retried up to 4 times, except credential errors) or when the queue redelivers the message (`attempt` from `msg.attempts`, e.g. a resource-limit 503), and climbs back after clean chunks. `import_jobs` is in `INTERNAL_TABLES`, so backups skip it.+`/api/import/messages` (uploads) and `/api/import/imap` only validate and create a row in `import_jobs`, then answer 202 with the job; nothing is imported inside the request. `processImportJob` (`src/lib/import/jobs.ts`) handles one chunk per `import.run` message on `OUTBOUND_QUEUE` (like webhook retries, so no extra binding), records counters and the resume `cursor` on the row, and enqueues the next chunk itself. Uploaded files are split and staged in the bucket as `import-jobs/<id>/<n>.bin` batches (`staging.ts`), deleted as consumed. IMAP jobs read the folder list from `spec`, but the password never touches the database: it rides along in each queue message, so a lost message fails the job instead of resuming. Cancelling flips the status; every chunk's final write is conditional on the job still being active. `sweepImportJobs` (cron / Node scheduler) fails jobs idle for 30 minutes and prunes finished records after 7 days. The import page polls `GET /api/import/jobs`; `DELETE /api/import/jobs/[id]` cancels an active job or dismisses a finished one. IMAP chunks pipeline their FETCHes (sizes first via `RFC822.SIZE`, then groups bounded by `planImapFetchGroups`). Chunk size is adaptive: `cursor.size` halves when a chunk throws (retried up to 4 times, except credential errors) or when the queue redelivers the message (`attempt` from `msg.attempts`, e.g. a resource-limit 503), and climbs back after clean chunks. A failed job can be resumed (`resumeImportJob`, `POST /api/import/jobs/[id]/resume`): it re-queues from the saved cursor, IMAP asks for the password again, and file jobs keep their staged batches until dismissed or pruned. An IMAP source the server cannot open (`isMissingImapFolderError`, e.g. Gmail's `[Gmail]` parent) is skipped with a recorded error instead of failing the job. `import_jobs` is in `INTERNAL_TABLES`, so backups skip it.### Search is an FTS5 index kept by triggersdiff --git a/src/app/(settings)/settings/import/utils.ts b/src/app/(settings)/settings/import/utils.tsindex aa32f69..eb1ea67 100644--- a/src/app/(settings)/settings/import/utils.ts+++ b/src/app/(settings)/settings/import/utils.ts@@ -131,6 +131,19 @@ export async function removeImportJob(id: string): Promise<void> {if (!response.ok) throw new Error("Unable to update the import");}+/** Continues a failed import. IMAP imports need the password again, since it is never stored. */+export async function resumeImportJob(id: string, password?: string): Promise<void> {+ const response = await authFetch(`/api/import/jobs/${encodeURIComponent(id)}/resume`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ password }),+ });+ if (!response.ok) {+ const data = (await response.json().catch(() => null)) as { error?: string } | null;+ throw new Error(data?.error || "Unable to resume the import");+ }+}+async function readImportJson<T extends object>(response: Response): Promise<T> {const text = await response.text();try {diff --git a/src/app/api/import/jobs/[id]/resume/route.ts b/src/app/api/import/jobs/[id]/resume/route.tsnew file mode 100644index 0000000..0ca8562--- /dev/null+++ b/src/app/api/import/jobs/[id]/resume/route.ts@@ -0,0 +1,20 @@+import { NextResponse } from "next/server";+import { requireSessionUser } from "@/lib/api/auth";+import { getEnv } from "@/lib/cloudflare";+import { readJsonBody } from "@/lib/http/request";+import { resumeImportJob } from "@/lib/import/jobs";++type Params = { params: Promise<{ id: string }> };++/** Continues a failed import from where it stopped. IMAP imports need the password again. */+export async function POST(request: Request, { params }: Params) {+ const env = getEnv();+ const { user, error } = await requireSessionUser(env, request);+ if (error) return error;+ const { id } = await params;+ const body = await readJsonBody<{ password?: unknown }>(request, 8 * 1024).catch(() => ({}) as { password?: unknown });+ const password = typeof body.password === "string" && body.password ? body.password : undefined;+ const result = await resumeImportJob(env, user.id, id, password);+ if (!result.ok) return NextResponse.json({ error: result.error }, { status: result.status });+ return NextResponse.json({ job: result.job }, { status: 202 });+}diff --git a/src/components/settings/import-jobs.tsx b/src/components/settings/import-jobs.tsxindex b0192c2..a52cc95 100644--- a/src/components/settings/import-jobs.tsx+++ b/src/components/settings/import-jobs.tsx@@ -4,8 +4,9 @@ import { useCallback, useEffect, useRef, useState } from "react";import { Loader2 } from "@/components/icons";import { useLanguage } from "@/components/language-provider";import { Button } from "@/components/ui/button";+import { Input } from "@/components/ui/input";import type { ImportJobSummary } from "@/lib/import/jobs-types";-import { fetchImportJobs, removeImportJob } from "@/app/(settings)/settings/import/utils";+import { fetchImportJobs, removeImportJob, resumeImportJob } from "@/app/(settings)/settings/import/utils";import { getImportJobPercent, IMPORT_JOBS_POLL_MS, isImportJobActive } from "./import-jobs-utils";const STATUS_STYLE: Record<ImportJobSummary["status"], string> = {@@ -21,6 +22,9 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {const { t } = useLanguage();const [jobs, setJobs] = useState<ImportJobSummary[]>([]);const [loaded, setLoaded] = useState(false);+ const [resuming, setResuming] = useState<string | null>(null);+ const [password, setPassword] = useState("");+ const [resumeError, setResumeError] = useState<string | null>(null);const previous = useRef<Map<string, ImportJobSummary["status"]>>(new Map());const load = useCallback(async () => {@@ -60,6 +64,25 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {}}+ async function onResume(job: ImportJobSummary) {+ if (job.kind === "imap" && resuming !== job.id) {+ setResuming(job.id);+ setPassword("");+ setResumeError(null);+ return;+ }+ try {+ await resumeImportJob(job.id, job.kind === "imap" ? password : undefined);+ setResuming(null);+ setPassword("");+ setResumeError(null);+ } catch (error) {+ setResumeError(error instanceof Error ? error.message : t("importJobs.resumeFailed"));+ } finally {+ await load();+ }+ }+if (loaded && jobs.length === 0) return null;return (@@ -88,6 +111,11 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {<span className={`rounded-full px-2 py-0.5 text-xs font-medium ${STATUS_STYLE[job.status]}`}>{t(`importJobs.status.${job.status}`)}</span>+ {job.status === "failed" && (+ <Button type="button" size="sm" onClick={() => void onResume(job)}>+ {t("importJobs.resume")}+ </Button>+ )}<Button type="button" variant="outline" size="sm" onClick={() => void onRemove(job)}>{active ? t("importJobs.cancel") : t("importJobs.dismiss")}</Button>@@ -128,6 +156,28 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {}))}</p>+ {resuming === job.id && job.status === "failed" && (+ <form+ className="flex items-center gap-2"+ onSubmit={(event) => {+ event.preventDefault();+ void onResume(job);+ }}+ >+ <Input+ type="password"+ autoComplete="off"+ value={password}+ onChange={(event) => setPassword(event.target.value)}+ placeholder={t("importJobs.resumePassword")}+ aria-label={t("importJobs.resumePassword")}+ />+ <Button type="submit" size="sm" disabled={!password}>+ {t("importJobs.resume")}+ </Button>+ </form>+ )}+ {resuming === job.id && resumeError && <p className="text-xs text-red-700">{resumeError}</p>}{job.error && (<p className="rounded-lg border border-red-100 bg-red-50 px-3 py-2 text-xs text-red-700">{job.error}</p>)}diff --git a/src/lib/i18n/en.json b/src/lib/i18n/en.jsonindex a377b4b..4e5a68f 100644--- a/src/lib/i18n/en.json+++ b/src/lib/i18n/en.json@@ -2036,6 +2036,9 @@"importJobs.counting": "Counting messages…","importJobs.cancel": "Cancel","importJobs.dismiss": "Dismiss",+ "importJobs.resume": "Resume",+ "importJobs.resumePassword": "IMAP password",+ "importJobs.resumeFailed": "Unable to resume the import","importJobs.status.queued": "Queued","importJobs.status.running": "Running","importJobs.status.completed": "Completed",diff --git a/src/lib/import/imap-utils.ts b/src/lib/import/imap-utils.tsindex 7383662..c956088 100644--- a/src/lib/import/imap-utils.ts+++ b/src/lib/import/imap-utils.ts@@ -49,6 +49,11 @@ export function isPermanentImapError(message: string): boolean {return /login|auth|credential|not allowed|unreadable|no longer exists/i.test(message);}+/** A folder the server says does not exist or cannot be selected, such as Gmail's `[Gmail]` parent. */+export function isMissingImapFolderError(message: string): boolean {+ return /NONEXISTENT|unknown mailbox|mailbox (?:does not|doesn't) exist|nonexistent namespace|NOSELECT/i.test(message);+}+/** FETCH commands sent back to back, and the raw mail one group may hold before the next is requested. */const IMAP_PIPELINE_COUNT = 10;const IMAP_PIPELINE_BYTES = 4 * 1024 * 1024;diff --git a/src/lib/import/jobs.ts b/src/lib/import/jobs.tsindex 2f6a443..e50d87e 100644--- a/src/lib/import/jobs.ts+++ b/src/lib/import/jobs.ts@@ -4,7 +4,7 @@ import { importJobs, mailboxes } from "@/db/schema";import { newId } from "@/lib/ids";import { getImportMessageUserId, parseImportDestination } from "./destination";import { fetchImapMessages } from "./imap";-import { getImapChunkSize, growImapChunkSize, isPermanentImapError } from "./imap-utils";+import { getImapChunkSize, growImapChunkSize, isMissingImapFolderError, isPermanentImapError } from "./imap-utils";import { deleteStagedImport, importBatchKey, stageImportMessages, unpackImportMessages } from "./staging";import { importMessagesToMailbox } from "./service";import type {@@ -176,10 +176,57 @@ export async function cancelOrDismissImportJob(env: CloudflareEnv, userId: strinawait cleanupJob(env, row);} else {await db.delete(importJobs).where(eq(importJobs.id, jobId));+ await cleanupJob(env, row);}return true;}+export type ResumeImportResult =+ | { ok: true; job: ImportJobSummary }+ | { ok: false; status: 400 | 404 | 409; error: string };++/**+ * Continues a failed job from its saved cursor. IMAP passwords are never stored, so the caller supplies+ * it again; file jobs continue from their staged batches when those still exist.+ */+export async function resumeImportJob(+ env: CloudflareEnv,+ userId: string,+ jobId: string,+ password?: string,+): Promise<ResumeImportResult> {+ const db = getDb(env);+ const [row] = await db+ .select()+ .from(importJobs)+ .where(and(eq(importJobs.id, jobId), eq(importJobs.userId, userId)))+ .limit(1);+ if (!row) return { ok: false, status: 404, error: "Import not found" };+ if (row.status !== "failed") return { ok: false, status: 409, error: "Only a failed import can be resumed" };++ let cursor = row.cursor;+ if (row.kind === "imap") {+ if (!password) return { ok: false, status: 400, error: "Enter the IMAP password to resume" };+ const parsed = parseJson<ImapImportCursor>(row.cursor, { source: 0, done: 0, counted: false });+ cursor = JSON.stringify({ ...parsed, failures: undefined });+ } else {+ const parsed = parseJson<FileImportCursor>(row.cursor, { batch: 0 });+ if (!(await env.BUCKET.head(importBatchKey(row.id, parsed.batch)))) {+ return { ok: false, status: 409, error: "Uploaded files are no longer available; upload them again" };+ }+ }++ const now = new Date();+ const [claimed] = await db+ .update(importJobs)+ .set({ status: "queued", error: null, finishedAt: null, cursor, updatedAt: now })+ .where(and(eq(importJobs.id, row.id), eq(importJobs.status, "failed")))+ .returning();+ if (!claimed) return { ok: false, status: 409, error: "Only a failed import can be resumed" };+ await enqueue(env, { kind: "import.run", jobId: row.id, ...(row.kind === "imap" ? { imapPassword: password } : {}) });+ return { ok: true, job: toImportJobSummary(claimed) };+}+/** Runs one chunk of a job and queues the next. Never throws for import failures; they land on the job. */export async function processImportJob(env: CloudflareEnv, delivered: ImportJobMessage): Promise<void> {const { attempt, ...message } = delivered;@@ -210,7 +257,7 @@ export async function processImportJob(env: CloudflareEnv, delivered: ImportJobMupdatedAt: now,}).where(and(eq(importJobs.id, job.id), inArray(importJobs.status, [...ACTIVE])));- await cleanupJob(env, job);+ // Staged uploads stay so the job can be resumed; dismissing or pruning it deletes them.return;}@@ -308,19 +355,32 @@ async function runImapChunk(const base = imapBaseChunkSize(spec, cursor);const batchSize = base > 0 ? getImapChunkSize(base, cursor.size, attempt) : 0;- const fetched =- batchSize > 0- ? await fetchImapMessages({- host: spec.host,- port: spec.port,- secure: spec.secure,- username: spec.username,- password,- folder: source.folder,- limit: batchSize,- offset: cursor.done,- })- : { messages: [], total: 0, nextOffset: null };+ let fetched: Awaited<ReturnType<typeof fetchImapMessages>> = { messages: [], total: 0, nextOffset: null };+ if (batchSize > 0) {+ try {+ fetched = await fetchImapMessages({+ host: spec.host,+ port: spec.port,+ secure: spec.secure,+ username: spec.username,+ password,+ folder: source.folder,+ limit: batchSize,+ offset: cursor.done,+ });+ } catch (error) {+ const reason = error instanceof Error ? error.message : "";+ if (!isMissingImapFolderError(reason)) throw error;+ // A folder that cannot be opened (Gmail's `[Gmail]` parent) must not sink the whole import.+ const nextSource = cursor.source + 1;+ return {+ result: { imported: 0, skipped: 0, errors: [`${source.folder}: ${reason}`] },+ cursor: { source: nextSource, done: 0, counted: false },+ totalDelta: 0,+ done: nextSource >= spec.sources.length,+ };+ }+ }const result = fetched.messages.length? await importMessagesToMailbox(env, {@@ -359,14 +419,12 @@ export async function sweepImportJobs(env: CloudflareEnv, now = new Date()): Pro.update(importJobs).set({ status: "failed", error: "Import was interrupted", finishedAt: now, updatedAt: now }).where(and(eq(importJobs.id, row.id), inArray(importJobs.status, [...ACTIVE])));- await cleanupJob(env, row);}- await db- .delete(importJobs)- .where(- and(- inArray(importJobs.status, ["completed", "failed", "cancelled"]),- lt(importJobs.finishedAt, new Date(now.getTime() - KEEP_FINISHED_MS)),- ),- );+ const expired = and(+ inArray(importJobs.status, ["completed", "failed", "cancelled"]),+ lt(importJobs.finishedAt, new Date(now.getTime() - KEEP_FINISHED_MS)),+ );+ const old = await db.select().from(importJobs).where(expired);+ for (const row of old) await cleanupJob(env, row);+ await db.delete(importJobs).where(expired);}diff --git a/tests/imap-chunk-size.test.mjs b/tests/imap-chunk-size.test.mjsindex a6cc6a8..5a91f79 100644--- a/tests/imap-chunk-size.test.mjs+++ b/tests/imap-chunk-size.test.mjs@@ -1,7 +1,7 @@import assert from "node:assert/strict";import test from "node:test";-import { getImapChunkSize, planImapFetchGroups, growImapChunkSize, isPermanentImapError } from "../src/lib/import/imap-utils.ts";+import { getImapChunkSize, planImapFetchGroups, growImapChunkSize, isPermanentImapError, isMissingImapFolderError } from "../src/lib/import/imap-utils.ts";test("chunk size defaults to the base and never exceeds it", () => {assert.equal(getImapChunkSize(50, undefined), 50);@@ -35,3 +35,8 @@ test("fetch groups pipeline small mail, split big mail and drop the tail past thassert.deepEqual(planImapFetchGroups(uids, big).flat(), ["5", "4", "3", "2"]);assert.deepEqual(planImapFetchGroups(["2", "1"], new Map([["1", 5]])), [["2"]]);});++test("a folder the server cannot open is detected so the import can skip it", () => {+ assert.equal(isMissingImapFolderError("IMAP command failed: A0002 NO [NONEXISTENT] Unknown Mailbox: [Gmail] (Failure)"), true);+ assert.equal(isMissingImapFolderError("IMAP connection closed unexpectedly"), false);+});