Skip to content

Email for professionals and teams

improve backup / export

Hieu Nguyen (Jack)47ff67c81f56
diff --git a/src/app/api/backups/restore/route.ts b/src/app/api/backups/restore/route.ts
new file mode 100644
index 0000000..bc29e30
--- /dev/null
+++ b/src/app/api/backups/restore/route.ts
@@ -0,0 +1,21 @@
+import { NextResponse } from "next/server";
+import { assertAdmin } from "@/lib/auth/admin";
+import { requireUser } from "@/lib/auth/cookies";
+import { restoreDatabaseRecords } from "@/lib/backups/export";
+import { getEnv } from "@/lib/cloudflare";
+
+export async function POST(request: Request) {
+ const env = getEnv();
+ try {
+ const user = await requireUser(env, request);
+ assertAdmin(user);
+ const form = await request.formData();
+ const file = form.get("backup");
+ if (!(file instanceof File)) return NextResponse.json({ error: "Choose a backup file" }, { status: 400 });
+ await restoreDatabaseRecords(env.DB, await file.arrayBuffer());
+ return NextResponse.json({ ok: true });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : "Failed to restore backup";
+ return NextResponse.json({ error: message }, { status: 400 });
+ }
+}
diff --git a/src/app/api/backups/route.ts b/src/app/api/backups/route.ts
index f86f982..8d46ffd 100644
--- a/src/app/api/backups/route.ts
+++ b/src/app/api/backups/route.ts
@@ -4,10 +4,7 @@ import { getDb } from "@/db";
import { backups } from "@/db/schema";
import { assertAdmin } from "@/lib/auth/admin";
import { requireUser } from "@/lib/auth/cookies";
-import {
- getD1ExportConfiguration,
- getD1ExportConfigurationStatus,
-} from "@/lib/backups/export";
+import { getD1ExportConfigurationStatus } from "@/lib/backups/export";
import {
createBackupRecord,
getBackupSettings,
@@ -61,7 +58,6 @@ export async function POST(request: Request) {
try {
const { env, user } = await requireAdmin(request);
const workflow = getBackupWorkflowBinding(env);
- getD1ExportConfiguration(env);
const backupId = await createBackupRecord(env, "manual", user.id);
try {
await workflow.create({
diff --git a/src/lib/backups/export.ts b/src/lib/backups/export.ts
index 6b9387f..30c4f6d 100644
--- a/src/lib/backups/export.ts
+++ b/src/lib/backups/export.ts
@@ -1,47 +1,63 @@
-import type { D1ExportResponse } from "./types";
-import { getExportError } from "./utils";
-
-export function getD1ExportConfigurationStatus(env: CloudflareEnv) {
- const accountId = env.CF_AID?.trim();
- const databaseId = env.D1_DATABASE_ID?.trim();
- const token = env.D1_BACKUP_TOKEN?.trim() || env.CF_TOKEN?.trim();
- const missing = [
- !accountId ? "CF_AID" : null,
- !databaseId ? "D1_DATABASE_ID" : null,
- !token ? "D1_BACKUP_TOKEN or CF_TOKEN" : null,
- ].filter((value): value is string => Boolean(value));
-
- return { configured: missing.length === 0, missing };
+import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";
+
+const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_bodies", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];
+const INSERT_BATCH_SIZE = 50;
+
+export function getD1ExportConfigurationStatus(_env?: CloudflareEnv) {
+ return { configured: true, missing: [] };
}
-export function getD1ExportConfiguration(env: CloudflareEnv) {
- const status = getD1ExportConfigurationStatus(env);
+export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array> {
+ const tables = {} as Record<DatabaseBackupTable, DatabaseRecord[]>;
+ for (const table of BACKUP_TABLES) {
+ const result = await db.prepare(`SELECT * FROM ${table}`).all<DatabaseRecord>();
+ tables[table] = result.results;
+ }
+ const document: DatabaseBackupDocument = { format: "mailflare-database-backup", version: 1, createdAt: new Date().toISOString(), tables };
+ return new TextEncoder().encode(JSON.stringify(document));
+}
- if (!status.configured) {
- throw new Error(
- `Database backups require ${status.missing.join(", ")}. Configure ${status.missing.length === 1 ? "it" : "them"} in the Worker's Variables and Secrets settings.`,
- );
+export async function restoreDatabaseRecords(db: D1Database, content: ArrayBuffer): Promise<void> {
+ const document = parseDatabaseBackup(content);
+ validateDatabaseBackup(document);
+ for (const table of [...BACKUP_TABLES].reverse()) await db.prepare(`DELETE FROM ${table}`).run();
+ for (const table of BACKUP_TABLES) {
+ const rows = document.tables[table];
+ for (let index = 0; index < rows.length; index += INSERT_BATCH_SIZE) {
+ const statements = rows.slice(index, index + INSERT_BATCH_SIZE).map((row) => createInsertStatement(db, table, row));
+ if (statements.length > 0) await db.batch(statements);
+ }
}
+}
- return {
- token: (env.D1_BACKUP_TOKEN?.trim() || env.CF_TOKEN?.trim())!,
- url: `https://api.cloudflare.com/client/v4/accounts/${env.CF_AID!.trim()}/d1/database/${env.D1_DATABASE_ID!.trim()}/export`,
- };
+function parseDatabaseBackup(content: ArrayBuffer): DatabaseBackupDocument {
+ let value: unknown;
+ try { value = JSON.parse(new TextDecoder().decode(content)); } catch { throw new Error("The selected file is not a valid Mailflare backup"); }
+ if (!isDatabaseBackupDocument(value)) throw new Error("The selected file is not a valid Mailflare backup");
+ return value;
}
-export async function requestD1Export(
- configuration: ReturnType<typeof getD1ExportConfiguration>,
- payload: object,
-): Promise<D1ExportResponse> {
- const response = await fetch(configuration.url, {
- method: "POST",
- headers: {
- Authorization: `Bearer ${configuration.token}`,
- "Content-Type": "application/json",
- },
- body: JSON.stringify(payload),
- });
- const result = (await response.json()) as D1ExportResponse;
- if (!response.ok || !result.success) throw new Error(getExportError(result));
- return result;
+function isDatabaseBackupDocument(value: unknown): value is DatabaseBackupDocument {
+ if (!value || typeof value !== "object") return false;
+ const document = value as Partial<DatabaseBackupDocument>;
+ return document.format === "mailflare-database-backup" && document.version === 1 && !!document.tables && BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]));
+}
+
+function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row: DatabaseRecord) {
+ const columns = Object.keys(row);
+ if (columns.length === 0) throw new Error(`Backup contains an invalid ${table} record`);
+ const placeholders = columns.map(() => "?").join(", ");
+ return db.prepare(`INSERT INTO ${table} (${columns.join(", ")}) VALUES (${placeholders})`).bind(...columns.map((column) => row[column]));
+}
+
+function validateDatabaseBackup(document: DatabaseBackupDocument): void {
+ for (const table of BACKUP_TABLES) {
+ for (const row of document.tables[table]) {
+ for (const [column, value] of Object.entries(row)) {
+ if (!/^[a-z_]+$/.test(column) || (value !== null && typeof value !== "string" && typeof value !== "number")) {
+ throw new Error(`Backup contains an invalid ${table} record`);
+ }
+ }
+ }
+ }
}
diff --git a/src/lib/backups/types.d.ts b/src/lib/backups/types.d.ts
index affba30..84f867e 100644
--- a/src/lib/backups/types.d.ts
+++ b/src/lib/backups/types.d.ts
@@ -12,20 +12,6 @@ export type BackupWorkflowBinding = {
}): Promise<unknown>;
};
-export type D1ExportResult = {
- at_bookmark?: string;
- status?: string;
- error?: string;
- signed_url?: string;
- filename?: string;
- result?: {
- signed_url?: string;
- filename?: string;
- };
-};
-
-export type D1ExportResponse = {
- success: boolean;
- errors?: Array<{ message?: string }>;
- result?: D1ExportResult;
-};
+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_bodies" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings";
+export type DatabaseRecord = Record<string, string | number | null>;
+export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };
diff --git a/src/lib/backups/utils.ts b/src/lib/backups/utils.ts
index 00f3e53..1a52da0 100644
--- a/src/lib/backups/utils.ts
+++ b/src/lib/backups/utils.ts
@@ -1,8 +1,4 @@
-import type {
- BackupScheduleType,
- BackupWorkflowBinding,
- D1ExportResponse,
-} from "./types";
+import type { BackupScheduleType, BackupWorkflowBinding } from "./types";
export const BACKUP_SETTINGS_ID = "default";
export const BACKUP_PREFIX = "backups/database";
@@ -39,11 +35,6 @@ export function getUtcDayBounds(now: Date): { start: number; end: number } {
return { start, end: start + 86_400_000 };
}
-export function getExportError(response: D1ExportResponse): string {
- return response.errors?.map((error) => error.message).filter(Boolean).join(", ")
- || "Cloudflare D1 export failed";
-}
-
export function createBackupFilename(now: Date): string {
- return `mailflare-${now.toISOString().replace(/[:.]/g, "-")}.sql`;
+ return `mailflare-${now.toISOString().replace(/[:.]/g, "-")}.json`;
}
diff --git a/src/lib/backups/workflow.ts b/src/lib/backups/workflow.ts
index 99f0a81..7de0bb9 100644
--- a/src/lib/backups/workflow.ts
+++ b/src/lib/backups/workflow.ts
@@ -2,7 +2,7 @@ import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloud
import { and, eq, inArray, lt } from "drizzle-orm";
import { getDb } from "@/db";
import { backups } from "@/db/schema";
-import { getD1ExportConfiguration, requestD1Export } from "./export";
+import { exportDatabaseRecords } from "./export";
import { createScheduledBackupIfDue, getBackupSettings } from "./service";
import type { BackupWorkflowParams } from "./types";
import {
@@ -10,8 +10,6 @@ import {
createBackupFilename,
} from "./utils";
-const EXPORT_RETRIES = 20;
-
export class DatabaseBackupWorkflow extends WorkflowEntrypoint<CloudflareEnv, BackupWorkflowParams> {
async run(event: Readonly<WorkflowEvent<BackupWorkflowParams>>, step: WorkflowStep) {
let backupId = event.payload?.backupId;
@@ -33,38 +31,12 @@ export class DatabaseBackupWorkflow extends WorkflowEntrypoint<CloudflareEnv, Ba
.where(eq(backups.id, backupId));
});
- const exportConfiguration = getD1ExportConfiguration(this.env);
- const bookmark = await step.do("Start D1 export", async () => {
- const response = await requestD1Export(exportConfiguration, { output_format: "polling" });
- if (!response.result?.at_bookmark) throw new Error("D1 export did not return a bookmark");
- return response.result.at_bookmark;
- });
-
- let signedUrl = "";
- let exportFilename = "";
- for (let attempt = 1; attempt <= EXPORT_RETRIES; attempt += 1) {
- const result = await step.do(`Poll D1 export ${attempt}`, async () =>
- requestD1Export(exportConfiguration, { output_format: "polling", current_bookmark: bookmark }),
- );
- // The polling response nests the finished export one level deeper
- // (result.result.result); fall back to the flat shape defensively.
- const finished = result.result?.result ?? result.result;
- if (finished?.signed_url) {
- signedUrl = finished.signed_url;
- exportFilename = finished.filename ?? "";
- break;
- }
- if (attempt < EXPORT_RETRIES) await step.sleep(`Wait for D1 export ${attempt}`, "15 seconds");
- }
- if (!signedUrl) throw new Error("D1 export did not finish before the polling limit");
-
const stored = await step.do("Store backup in R2", async () => {
- const response = await fetch(signedUrl);
- if (!response.ok || !response.body) throw new Error("Failed to download the D1 export");
- const filename = exportFilename || createBackupFilename(new Date());
+ const content = await exportDatabaseRecords(this.env.DB);
+ const filename = createBackupFilename(new Date());
const r2Key = `${BACKUP_PREFIX}/${backupId}/${filename}`;
- const object = await this.env.BUCKET.put(r2Key, response.body, {
- httpMetadata: { contentType: "application/sql" },
+ const object = await this.env.BUCKET.put(r2Key, content, {
+ httpMetadata: { contentType: "application/json" },
customMetadata: { backupId },
});
return { filename, r2Key, size: object.size };
diff --git a/src/lib/setup/migration.ts b/src/lib/setup/migration.ts
index b2d0ebf..f39c3e2 100644
--- a/src/lib/setup/migration.ts
+++ b/src/lib/setup/migration.ts
@@ -14,6 +14,9 @@ const MIGRATION_NAMES = [
"0012_add_app_settings.sql",
"0014_add_account_permissions.sql",
"0015_add_forwarding_email.sql",
+ "0016_add_message_snooze.sql",
+ "0017_add_message_star.sql",
+ "0018_add_mailbox_domain_aliases.sql",
];
const INITIAL_SCHEMA_SQL = `
@@ -22,7 +25,7 @@ CREATE INDEX IF NOT EXISTS users_created_by_idx ON users(created_by_user_id);
CREATE TABLE IF NOT EXISTS domains (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, hostname text NOT NULL, zone_id text NOT NULL, status text DEFAULT 'pending' NOT NULL, routing_status text, sending_subdomain_tag text, sending_enabled integer DEFAULT false NOT NULL, routing_enabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);
CREATE UNIQUE INDEX IF NOT EXISTS domains_hostname_idx ON domains(hostname);
CREATE INDEX IF NOT EXISTS domains_user_idx ON domains(user_id);
-CREATE TABLE IF NOT EXISTS mailboxes (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, domain_id text NOT NULL REFERENCES domains(id) ON DELETE cascade, local_part text NOT NULL, display_name text, avatar_key text, type text DEFAULT 'personal' NOT NULL, disabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);
+CREATE TABLE IF NOT EXISTS mailboxes (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, domain_id text NOT NULL REFERENCES domains(id) ON DELETE cascade, local_part text NOT NULL, display_name text, avatar_key text, type text DEFAULT 'personal' NOT NULL, use_all_domains integer DEFAULT true NOT NULL, disabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);
CREATE UNIQUE INDEX IF NOT EXISTS mailboxes_address_idx ON mailboxes(domain_id, local_part);
CREATE TABLE IF NOT EXISTS mailbox_access (id text PRIMARY KEY NOT NULL, mailbox_id text NOT NULL REFERENCES mailboxes(id) ON DELETE cascade, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, permission text DEFAULT 'read_only' NOT NULL, created_by_user_id text REFERENCES users(id) ON DELETE set null, created_at integer NOT NULL);
CREATE UNIQUE INDEX IF NOT EXISTS mailbox_access_mailbox_user_idx ON mailbox_access(mailbox_id, user_id);
@@ -36,7 +39,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS folders_mailbox_name_idx ON folders(mailbox_id
CREATE INDEX IF NOT EXISTS folders_user_idx ON folders(user_id);
CREATE INDEX IF NOT EXISTS folders_mailbox_idx ON folders(mailbox_id);
CREATE TABLE IF NOT EXISTS api_keys (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, name text NOT NULL, prefix text NOT NULL, key_hash text NOT NULL, scopes text NOT NULL, created_at integer NOT NULL, last_used_at integer);
-CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, mailbox_id text REFERENCES mailboxes(id) ON DELETE set null, direction text NOT NULL, provider_message_id text, folder_id text REFERENCES folders(id) ON DELETE set null, from_addr text NOT NULL, to_addr text NOT NULL, subject text, snippet text, status text DEFAULT 'received' NOT NULL, read integer DEFAULT false NOT NULL, thread_id text, created_at integer NOT NULL);
+CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, mailbox_id text REFERENCES mailboxes(id) ON DELETE set null, direction text NOT NULL, provider_message_id text, folder_id text REFERENCES folders(id) ON DELETE set null, from_addr text NOT NULL, to_addr text NOT NULL, subject text, snippet text, status text DEFAULT 'received' NOT NULL, read integer DEFAULT false NOT NULL, starred integer DEFAULT false NOT NULL, snoozed_until integer, thread_id text, created_at integer NOT NULL);
CREATE INDEX IF NOT EXISTS messages_user_created_idx ON messages(user_id, created_at);
CREATE INDEX IF NOT EXISTS messages_mailbox_idx ON messages(mailbox_id);
CREATE INDEX IF NOT EXISTS messages_folder_idx ON messages(folder_id);