Skip to content

Email for professionals and teams

Allow resume importing

Hieu Nguyen (Jack)b4d00d181d76
diff --git a/CLAUDE.md b/CLAUDE.md
index eab803f..db740fd 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -157,7 +157,7 @@ Optional tables are detected with `isMissingTableError` (`src/lib/database-error
### Imports are background jobs
-`/api/import/messages` (uploads) and `/api/import/imap` only validate and create a row in `import_jobs`, then answer 202 with the job; nothing is imported inside the request. `processImportJob` (`src/lib/import/jobs.ts`) handles one chunk per `import.run` message on `OUTBOUND_QUEUE` (like webhook retries, so no extra binding), records counters and the resume `cursor` on the row, and enqueues the next chunk itself. Uploaded files are split and staged in the bucket as `import-jobs/<id>/<n>.bin` batches (`staging.ts`), deleted as consumed. IMAP jobs read the folder list from `spec`, but the password never touches the database: it rides along in each queue message, so a lost message fails the job instead of resuming. Cancelling flips the status; every chunk's final write is conditional on the job still being active. `sweepImportJobs` (cron / Node scheduler) fails jobs idle for 30 minutes and prunes finished records after 7 days. The import page polls `GET /api/import/jobs`; `DELETE /api/import/jobs/[id]` cancels an active job or dismisses a finished one. IMAP chunks pipeline their FETCHes (sizes first via `RFC822.SIZE`, then groups bounded by `planImapFetchGroups`). Chunk size is adaptive: `cursor.size` halves when a chunk throws (retried up to 4 times, except credential errors) or when the queue redelivers the message (`attempt` from `msg.attempts`, e.g. a resource-limit 503), and climbs back after clean chunks. `import_jobs` is in `INTERNAL_TABLES`, so backups skip it.
+`/api/import/messages` (uploads) and `/api/import/imap` only validate and create a row in `import_jobs`, then answer 202 with the job; nothing is imported inside the request. `processImportJob` (`src/lib/import/jobs.ts`) handles one chunk per `import.run` message on `OUTBOUND_QUEUE` (like webhook retries, so no extra binding), records counters and the resume `cursor` on the row, and enqueues the next chunk itself. Uploaded files are split and staged in the bucket as `import-jobs/<id>/<n>.bin` batches (`staging.ts`), deleted as consumed. IMAP jobs read the folder list from `spec`, but the password never touches the database: it rides along in each queue message, so a lost message fails the job instead of resuming. Cancelling flips the status; every chunk's final write is conditional on the job still being active. `sweepImportJobs` (cron / Node scheduler) fails jobs idle for 30 minutes and prunes finished records after 7 days. The import page polls `GET /api/import/jobs`; `DELETE /api/import/jobs/[id]` cancels an active job or dismisses a finished one. IMAP chunks pipeline their FETCHes (sizes first via `RFC822.SIZE`, then groups bounded by `planImapFetchGroups`). Chunk size is adaptive: `cursor.size` halves when a chunk throws (retried up to 4 times, except credential errors) or when the queue redelivers the message (`attempt` from `msg.attempts`, e.g. a resource-limit 503), and climbs back after clean chunks. A failed job can be resumed (`resumeImportJob`, `POST /api/import/jobs/[id]/resume`): it re-queues from the saved cursor, IMAP asks for the password again, and file jobs keep their staged batches until dismissed or pruned. An IMAP source the server cannot open (`isMissingImapFolderError`, e.g. Gmail's `[Gmail]` parent) is skipped with a recorded error instead of failing the job. `import_jobs` is in `INTERNAL_TABLES`, so backups skip it.
### Search is an FTS5 index kept by triggers
diff --git a/src/app/(settings)/settings/import/utils.ts b/src/app/(settings)/settings/import/utils.ts
index aa32f69..eb1ea67 100644
--- a/src/app/(settings)/settings/import/utils.ts
+++ b/src/app/(settings)/settings/import/utils.ts
@@ -131,6 +131,19 @@ export async function removeImportJob(id: string): Promise<void> {
if (!response.ok) throw new Error("Unable to update the import");
}
+/** Continues a failed import. IMAP imports need the password again, since it is never stored. */
+export async function resumeImportJob(id: string, password?: string): Promise<void> {
+ const response = await authFetch(`/api/import/jobs/${encodeURIComponent(id)}/resume`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ password }),
+ });
+ if (!response.ok) {
+ const data = (await response.json().catch(() => null)) as { error?: string } | null;
+ throw new Error(data?.error || "Unable to resume the import");
+ }
+}
+
async function readImportJson<T extends object>(response: Response): Promise<T> {
const text = await response.text();
try {
diff --git a/src/app/api/import/jobs/[id]/resume/route.ts b/src/app/api/import/jobs/[id]/resume/route.ts
new file mode 100644
index 0000000..0ca8562
--- /dev/null
+++ b/src/app/api/import/jobs/[id]/resume/route.ts
@@ -0,0 +1,20 @@
+import { NextResponse } from "next/server";
+import { requireSessionUser } from "@/lib/api/auth";
+import { getEnv } from "@/lib/cloudflare";
+import { readJsonBody } from "@/lib/http/request";
+import { resumeImportJob } from "@/lib/import/jobs";
+
+type Params = { params: Promise<{ id: string }> };
+
+/** Continues a failed import from where it stopped. IMAP imports need the password again. */
+export async function POST(request: Request, { params }: Params) {
+ const env = getEnv();
+ const { user, error } = await requireSessionUser(env, request);
+ if (error) return error;
+ const { id } = await params;
+ const body = await readJsonBody<{ password?: unknown }>(request, 8 * 1024).catch(() => ({}) as { password?: unknown });
+ const password = typeof body.password === "string" && body.password ? body.password : undefined;
+ const result = await resumeImportJob(env, user.id, id, password);
+ if (!result.ok) return NextResponse.json({ error: result.error }, { status: result.status });
+ return NextResponse.json({ job: result.job }, { status: 202 });
+}
diff --git a/src/components/settings/import-jobs.tsx b/src/components/settings/import-jobs.tsx
index b0192c2..a52cc95 100644
--- a/src/components/settings/import-jobs.tsx
+++ b/src/components/settings/import-jobs.tsx
@@ -4,8 +4,9 @@ import { useCallback, useEffect, useRef, useState } from "react";
import { Loader2 } from "@/components/icons";
import { useLanguage } from "@/components/language-provider";
import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
import type { ImportJobSummary } from "@/lib/import/jobs-types";
-import { fetchImportJobs, removeImportJob } from "@/app/(settings)/settings/import/utils";
+import { fetchImportJobs, removeImportJob, resumeImportJob } from "@/app/(settings)/settings/import/utils";
import { getImportJobPercent, IMPORT_JOBS_POLL_MS, isImportJobActive } from "./import-jobs-utils";
const STATUS_STYLE: Record<ImportJobSummary["status"], string> = {
@@ -21,6 +22,9 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {
const { t } = useLanguage();
const [jobs, setJobs] = useState<ImportJobSummary[]>([]);
const [loaded, setLoaded] = useState(false);
+ const [resuming, setResuming] = useState<string | null>(null);
+ const [password, setPassword] = useState("");
+ const [resumeError, setResumeError] = useState<string | null>(null);
const previous = useRef<Map<string, ImportJobSummary["status"]>>(new Map());
const load = useCallback(async () => {
@@ -60,6 +64,25 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {
}
}
+ async function onResume(job: ImportJobSummary) {
+ if (job.kind === "imap" && resuming !== job.id) {
+ setResuming(job.id);
+ setPassword("");
+ setResumeError(null);
+ return;
+ }
+ try {
+ await resumeImportJob(job.id, job.kind === "imap" ? password : undefined);
+ setResuming(null);
+ setPassword("");
+ setResumeError(null);
+ } catch (error) {
+ setResumeError(error instanceof Error ? error.message : t("importJobs.resumeFailed"));
+ } finally {
+ await load();
+ }
+ }
+
if (loaded && jobs.length === 0) return null;
return (
@@ -88,6 +111,11 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {
<span className={`rounded-full px-2 py-0.5 text-xs font-medium ${STATUS_STYLE[job.status]}`}>
{t(`importJobs.status.${job.status}`)}
</span>
+ {job.status === "failed" && (
+ <Button type="button" size="sm" onClick={() => void onResume(job)}>
+ {t("importJobs.resume")}
+ </Button>
+ )}
<Button type="button" variant="outline" size="sm" onClick={() => void onRemove(job)}>
{active ? t("importJobs.cancel") : t("importJobs.dismiss")}
</Button>
@@ -128,6 +156,28 @@ export function ImportJobs({ refreshKey }: { refreshKey: number }) {
})
)}
</p>
+ {resuming === job.id && job.status === "failed" && (
+ <form
+ className="flex items-center gap-2"
+ onSubmit={(event) => {
+ event.preventDefault();
+ void onResume(job);
+ }}
+ >
+ <Input
+ type="password"
+ autoComplete="off"
+ value={password}
+ onChange={(event) => setPassword(event.target.value)}
+ placeholder={t("importJobs.resumePassword")}
+ aria-label={t("importJobs.resumePassword")}
+ />
+ <Button type="submit" size="sm" disabled={!password}>
+ {t("importJobs.resume")}
+ </Button>
+ </form>
+ )}
+ {resuming === job.id && resumeError && <p className="text-xs text-red-700">{resumeError}</p>}
{job.error && (
<p className="rounded-lg border border-red-100 bg-red-50 px-3 py-2 text-xs text-red-700">{job.error}</p>
)}
diff --git a/src/lib/i18n/en.json b/src/lib/i18n/en.json
index a377b4b..4e5a68f 100644
--- a/src/lib/i18n/en.json
+++ b/src/lib/i18n/en.json
@@ -2036,6 +2036,9 @@
"importJobs.counting": "Counting messages…",
"importJobs.cancel": "Cancel",
"importJobs.dismiss": "Dismiss",
+ "importJobs.resume": "Resume",
+ "importJobs.resumePassword": "IMAP password",
+ "importJobs.resumeFailed": "Unable to resume the import",
"importJobs.status.queued": "Queued",
"importJobs.status.running": "Running",
"importJobs.status.completed": "Completed",
diff --git a/src/lib/import/imap-utils.ts b/src/lib/import/imap-utils.ts
index 7383662..c956088 100644
--- a/src/lib/import/imap-utils.ts
+++ b/src/lib/import/imap-utils.ts
@@ -49,6 +49,11 @@ export function isPermanentImapError(message: string): boolean {
return /login|auth|credential|not allowed|unreadable|no longer exists/i.test(message);
}
+/** A folder the server says does not exist or cannot be selected, such as Gmail's `[Gmail]` parent. */
+export function isMissingImapFolderError(message: string): boolean {
+ return /NONEXISTENT|unknown mailbox|mailbox (?:does not|doesn't) exist|nonexistent namespace|NOSELECT/i.test(message);
+}
+
/** FETCH commands sent back to back, and the raw mail one group may hold before the next is requested. */
const IMAP_PIPELINE_COUNT = 10;
const IMAP_PIPELINE_BYTES = 4 * 1024 * 1024;
diff --git a/src/lib/import/jobs.ts b/src/lib/import/jobs.ts
index 2f6a443..e50d87e 100644
--- a/src/lib/import/jobs.ts
+++ b/src/lib/import/jobs.ts
@@ -4,7 +4,7 @@ import { importJobs, mailboxes } from "@/db/schema";
import { newId } from "@/lib/ids";
import { getImportMessageUserId, parseImportDestination } from "./destination";
import { fetchImapMessages } from "./imap";
-import { getImapChunkSize, growImapChunkSize, isPermanentImapError } from "./imap-utils";
+import { getImapChunkSize, growImapChunkSize, isMissingImapFolderError, isPermanentImapError } from "./imap-utils";
import { deleteStagedImport, importBatchKey, stageImportMessages, unpackImportMessages } from "./staging";
import { importMessagesToMailbox } from "./service";
import type {
@@ -176,10 +176,57 @@ export async function cancelOrDismissImportJob(env: CloudflareEnv, userId: strin
await cleanupJob(env, row);
} else {
await db.delete(importJobs).where(eq(importJobs.id, jobId));
+ await cleanupJob(env, row);
}
return true;
}
+export type ResumeImportResult =
+ | { ok: true; job: ImportJobSummary }
+ | { ok: false; status: 400 | 404 | 409; error: string };
+
+/**
+ * Continues a failed job from its saved cursor. IMAP passwords are never stored, so the caller supplies
+ * it again; file jobs continue from their staged batches when those still exist.
+ */
+export async function resumeImportJob(
+ env: CloudflareEnv,
+ userId: string,
+ jobId: string,
+ password?: string,
+): Promise<ResumeImportResult> {
+ const db = getDb(env);
+ const [row] = await db
+ .select()
+ .from(importJobs)
+ .where(and(eq(importJobs.id, jobId), eq(importJobs.userId, userId)))
+ .limit(1);
+ if (!row) return { ok: false, status: 404, error: "Import not found" };
+ if (row.status !== "failed") return { ok: false, status: 409, error: "Only a failed import can be resumed" };
+
+ let cursor = row.cursor;
+ if (row.kind === "imap") {
+ if (!password) return { ok: false, status: 400, error: "Enter the IMAP password to resume" };
+ const parsed = parseJson<ImapImportCursor>(row.cursor, { source: 0, done: 0, counted: false });
+ cursor = JSON.stringify({ ...parsed, failures: undefined });
+ } else {
+ const parsed = parseJson<FileImportCursor>(row.cursor, { batch: 0 });
+ if (!(await env.BUCKET.head(importBatchKey(row.id, parsed.batch)))) {
+ return { ok: false, status: 409, error: "Uploaded files are no longer available; upload them again" };
+ }
+ }
+
+ const now = new Date();
+ const [claimed] = await db
+ .update(importJobs)
+ .set({ status: "queued", error: null, finishedAt: null, cursor, updatedAt: now })
+ .where(and(eq(importJobs.id, row.id), eq(importJobs.status, "failed")))
+ .returning();
+ if (!claimed) return { ok: false, status: 409, error: "Only a failed import can be resumed" };
+ await enqueue(env, { kind: "import.run", jobId: row.id, ...(row.kind === "imap" ? { imapPassword: password } : {}) });
+ return { ok: true, job: toImportJobSummary(claimed) };
+}
+
/** Runs one chunk of a job and queues the next. Never throws for import failures; they land on the job. */
export async function processImportJob(env: CloudflareEnv, delivered: ImportJobMessage): Promise<void> {
const { attempt, ...message } = delivered;
@@ -210,7 +257,7 @@ export async function processImportJob(env: CloudflareEnv, delivered: ImportJobM
updatedAt: now,
})
.where(and(eq(importJobs.id, job.id), inArray(importJobs.status, [...ACTIVE])));
- await cleanupJob(env, job);
+ // Staged uploads stay so the job can be resumed; dismissing or pruning it deletes them.
return;
}
@@ -308,19 +355,32 @@ async function runImapChunk(
const base = imapBaseChunkSize(spec, cursor);
const batchSize = base > 0 ? getImapChunkSize(base, cursor.size, attempt) : 0;
- const fetched =
- batchSize > 0
- ? await fetchImapMessages({
- host: spec.host,
- port: spec.port,
- secure: spec.secure,
- username: spec.username,
- password,
- folder: source.folder,
- limit: batchSize,
- offset: cursor.done,
- })
- : { messages: [], total: 0, nextOffset: null };
+ let fetched: Awaited<ReturnType<typeof fetchImapMessages>> = { messages: [], total: 0, nextOffset: null };
+ if (batchSize > 0) {
+ try {
+ fetched = await fetchImapMessages({
+ host: spec.host,
+ port: spec.port,
+ secure: spec.secure,
+ username: spec.username,
+ password,
+ folder: source.folder,
+ limit: batchSize,
+ offset: cursor.done,
+ });
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : "";
+ if (!isMissingImapFolderError(reason)) throw error;
+ // A folder that cannot be opened (Gmail's `[Gmail]` parent) must not sink the whole import.
+ const nextSource = cursor.source + 1;
+ return {
+ result: { imported: 0, skipped: 0, errors: [`${source.folder}: ${reason}`] },
+ cursor: { source: nextSource, done: 0, counted: false },
+ totalDelta: 0,
+ done: nextSource >= spec.sources.length,
+ };
+ }
+ }
const result = fetched.messages.length
? await importMessagesToMailbox(env, {
@@ -359,14 +419,12 @@ export async function sweepImportJobs(env: CloudflareEnv, now = new Date()): Pro
.update(importJobs)
.set({ status: "failed", error: "Import was interrupted", finishedAt: now, updatedAt: now })
.where(and(eq(importJobs.id, row.id), inArray(importJobs.status, [...ACTIVE])));
- await cleanupJob(env, row);
}
- await db
- .delete(importJobs)
- .where(
- and(
- inArray(importJobs.status, ["completed", "failed", "cancelled"]),
- lt(importJobs.finishedAt, new Date(now.getTime() - KEEP_FINISHED_MS)),
- ),
- );
+ const expired = and(
+ inArray(importJobs.status, ["completed", "failed", "cancelled"]),
+ lt(importJobs.finishedAt, new Date(now.getTime() - KEEP_FINISHED_MS)),
+ );
+ const old = await db.select().from(importJobs).where(expired);
+ for (const row of old) await cleanupJob(env, row);
+ await db.delete(importJobs).where(expired);
}
diff --git a/tests/imap-chunk-size.test.mjs b/tests/imap-chunk-size.test.mjs
index a6cc6a8..5a91f79 100644
--- a/tests/imap-chunk-size.test.mjs
+++ b/tests/imap-chunk-size.test.mjs
@@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import test from "node:test";
-import { getImapChunkSize, planImapFetchGroups, growImapChunkSize, isPermanentImapError } from "../src/lib/import/imap-utils.ts";
+import { getImapChunkSize, planImapFetchGroups, growImapChunkSize, isPermanentImapError, isMissingImapFolderError } from "../src/lib/import/imap-utils.ts";
test("chunk size defaults to the base and never exceeds it", () => {
assert.equal(getImapChunkSize(50, undefined), 50);
@@ -35,3 +35,8 @@ test("fetch groups pipeline small mail, split big mail and drop the tail past th
assert.deepEqual(planImapFetchGroups(uids, big).flat(), ["5", "4", "3", "2"]);
assert.deepEqual(planImapFetchGroups(["2", "1"], new Map([["1", 5]])), [["2"]]);
});
+
+test("a folder the server cannot open is detected so the import can skip it", () => {
+ assert.equal(isMissingImapFolderError("IMAP command failed: A0002 NO [NONEXISTENT] Unknown Mailbox: [Gmail] (Failure)"), true);
+ assert.equal(isMissingImapFolderError("IMAP connection closed unexpectedly"), false);
+});