Email for professionals and teams
improve backup / export
Hieu Nguyen (Jack)47ff67c81f56
diff --git a/src/app/api/backups/restore/route.ts b/src/app/api/backups/restore/route.tsnew file mode 100644index 0000000..bc29e30--- /dev/null+++ b/src/app/api/backups/restore/route.ts@@ -0,0 +1,21 @@+import { NextResponse } from "next/server";+import { assertAdmin } from "@/lib/auth/admin";+import { requireUser } from "@/lib/auth/cookies";+import { restoreDatabaseRecords } from "@/lib/backups/export";+import { getEnv } from "@/lib/cloudflare";++export async function POST(request: Request) {+ const env = getEnv();+ try {+ const user = await requireUser(env, request);+ assertAdmin(user);+ const form = await request.formData();+ const file = form.get("backup");+ if (!(file instanceof File)) return NextResponse.json({ error: "Choose a backup file" }, { status: 400 });+ await restoreDatabaseRecords(env.DB, await file.arrayBuffer());+ return NextResponse.json({ ok: true });+ } catch (error) {+ const message = error instanceof Error ? error.message : "Failed to restore backup";+ return NextResponse.json({ error: message }, { status: 400 });+ }+}diff --git a/src/app/api/backups/route.ts b/src/app/api/backups/route.tsindex f86f982..8d46ffd 100644--- a/src/app/api/backups/route.ts+++ b/src/app/api/backups/route.ts@@ -4,10 +4,7 @@ import { getDb } from "@/db";import { backups } from "@/db/schema";import { assertAdmin } from "@/lib/auth/admin";import { requireUser } from "@/lib/auth/cookies";-import {- getD1ExportConfiguration,- getD1ExportConfigurationStatus,-} from "@/lib/backups/export";+import { getD1ExportConfigurationStatus } from "@/lib/backups/export";import {createBackupRecord,getBackupSettings,@@ -61,7 +58,6 @@ export async function POST(request: Request) {try {const { env, user } = await requireAdmin(request);const workflow = getBackupWorkflowBinding(env);- getD1ExportConfiguration(env);const backupId = await createBackupRecord(env, "manual", user.id);try {await workflow.create({diff --git a/src/lib/backups/export.ts b/src/lib/backups/export.tsindex 6b9387f..30c4f6d 100644--- a/src/lib/backups/export.ts+++ b/src/lib/backups/export.ts@@ -1,47 +1,63 @@-import type { D1ExportResponse } from "./types";-import { getExportError } from "./utils";--export function getD1ExportConfigurationStatus(env: CloudflareEnv) {- const accountId = env.CF_AID?.trim();- const databaseId = env.D1_DATABASE_ID?.trim();- const token = env.D1_BACKUP_TOKEN?.trim() || env.CF_TOKEN?.trim();- const missing = [- !accountId ? "CF_AID" : null,- !databaseId ? "D1_DATABASE_ID" : null,- !token ? "D1_BACKUP_TOKEN or CF_TOKEN" : null,- ].filter((value): value is string => Boolean(value));-- return { configured: missing.length === 0, missing };+import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";++const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_bodies", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];+const INSERT_BATCH_SIZE = 50;++export function getD1ExportConfigurationStatus(_env?: CloudflareEnv) {+ return { configured: true, missing: [] };}-export function getD1ExportConfiguration(env: CloudflareEnv) {- const status = getD1ExportConfigurationStatus(env);+export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array> {+ const tables = {} as Record<DatabaseBackupTable, DatabaseRecord[]>;+ for (const table of BACKUP_TABLES) {+ const result = await db.prepare(`SELECT * FROM ${table}`).all<DatabaseRecord>();+ tables[table] = result.results;+ }+ const document: DatabaseBackupDocument = { format: "mailflare-database-backup", version: 1, createdAt: new Date().toISOString(), tables };+ return new TextEncoder().encode(JSON.stringify(document));+}- if (!status.configured) {- throw new Error(- `Database backups require ${status.missing.join(", ")}. Configure ${status.missing.length === 1 ? "it" : "them"} in the Worker's Variables and Secrets settings.`,- );+export async function restoreDatabaseRecords(db: D1Database, content: ArrayBuffer): Promise<void> {+ const document = parseDatabaseBackup(content);+ validateDatabaseBackup(document);+ for (const table of [...BACKUP_TABLES].reverse()) await db.prepare(`DELETE FROM ${table}`).run();+ for (const table of BACKUP_TABLES) {+ const rows = document.tables[table];+ for (let index = 0; index < rows.length; index += INSERT_BATCH_SIZE) {+ const statements = rows.slice(index, index + INSERT_BATCH_SIZE).map((row) => createInsertStatement(db, table, row));+ if (statements.length > 0) await db.batch(statements);+ }}+}- return {- token: (env.D1_BACKUP_TOKEN?.trim() || env.CF_TOKEN?.trim())!,- url: `https://api.cloudflare.com/client/v4/accounts/${env.CF_AID!.trim()}/d1/database/${env.D1_DATABASE_ID!.trim()}/export`,- };+function parseDatabaseBackup(content: ArrayBuffer): DatabaseBackupDocument {+ let value: unknown;+ try { value = JSON.parse(new TextDecoder().decode(content)); } catch { throw new Error("The selected file is not a valid Mailflare backup"); }+ if (!isDatabaseBackupDocument(value)) throw new Error("The selected file is not a valid Mailflare backup");+ return value;}-export async function requestD1Export(- configuration: ReturnType<typeof getD1ExportConfiguration>,- payload: object,-): Promise<D1ExportResponse> {- const response = await fetch(configuration.url, {- method: "POST",- headers: {- Authorization: `Bearer ${configuration.token}`,- "Content-Type": "application/json",- },- body: JSON.stringify(payload),- });- const result = (await response.json()) as D1ExportResponse;- if (!response.ok || !result.success) throw new Error(getExportError(result));- return result;+function isDatabaseBackupDocument(value: unknown): value is DatabaseBackupDocument {+ if (!value || typeof value !== "object") return false;+ const document = value as Partial<DatabaseBackupDocument>;+ return document.format === "mailflare-database-backup" && document.version === 1 && !!document.tables && BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]));+}++function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row: DatabaseRecord) {+ const columns = Object.keys(row);+ if (columns.length === 0) throw new Error(`Backup contains an invalid ${table} record`);+ const placeholders = columns.map(() => "?").join(", ");+ return db.prepare(`INSERT INTO ${table} (${columns.join(", ")}) VALUES (${placeholders})`).bind(...columns.map((column) => row[column]));+}++function validateDatabaseBackup(document: DatabaseBackupDocument): void {+ for (const table of BACKUP_TABLES) {+ for (const row of document.tables[table]) {+ for (const [column, value] of Object.entries(row)) {+ if (!/^[a-z_]+$/.test(column) || (value !== null && typeof value !== "string" && typeof value !== "number")) {+ throw new Error(`Backup contains an invalid ${table} record`);+ }+ }+ }+ }}diff --git a/src/lib/backups/types.d.ts b/src/lib/backups/types.d.tsindex affba30..84f867e 100644--- a/src/lib/backups/types.d.ts+++ b/src/lib/backups/types.d.ts@@ -12,20 +12,6 @@ export type BackupWorkflowBinding = {}): Promise<unknown>;};-export type D1ExportResult = {- at_bookmark?: string;- status?: string;- error?: string;- signed_url?: string;- filename?: string;- result?: {- signed_url?: string;- filename?: string;- };-};--export type D1ExportResponse = {- success: boolean;- errors?: Array<{ message?: string }>;- result?: D1ExportResult;-};+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_bodies" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings";+export type DatabaseRecord = Record<string, string | number | null>;+export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };diff --git a/src/lib/backups/utils.ts b/src/lib/backups/utils.tsindex 00f3e53..1a52da0 100644--- a/src/lib/backups/utils.ts+++ b/src/lib/backups/utils.ts@@ -1,8 +1,4 @@-import type {- BackupScheduleType,- BackupWorkflowBinding,- D1ExportResponse,-} from "./types";+import type { BackupScheduleType, BackupWorkflowBinding } from "./types";export const BACKUP_SETTINGS_ID = "default";export const BACKUP_PREFIX = "backups/database";@@ -39,11 +35,6 @@ export function getUtcDayBounds(now: Date): { start: number; end: number } {return { start, end: start + 86_400_000 };}-export function getExportError(response: D1ExportResponse): string {- return response.errors?.map((error) => error.message).filter(Boolean).join(", ")- || "Cloudflare D1 export failed";-}-export function createBackupFilename(now: Date): string {- return `mailflare-${now.toISOString().replace(/[:.]/g, "-")}.sql`;+ return `mailflare-${now.toISOString().replace(/[:.]/g, "-")}.json`;}diff --git a/src/lib/backups/workflow.ts b/src/lib/backups/workflow.tsindex 99f0a81..7de0bb9 100644--- a/src/lib/backups/workflow.ts+++ b/src/lib/backups/workflow.ts@@ -2,7 +2,7 @@ import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudimport { and, eq, inArray, lt } from "drizzle-orm";import { getDb } from "@/db";import { backups } from "@/db/schema";-import { getD1ExportConfiguration, requestD1Export } from "./export";+import { exportDatabaseRecords } from "./export";import { createScheduledBackupIfDue, getBackupSettings } from "./service";import type { BackupWorkflowParams } from "./types";import {@@ -10,8 +10,6 @@ import {createBackupFilename,} from "./utils";-const EXPORT_RETRIES = 20;-export class DatabaseBackupWorkflow extends WorkflowEntrypoint<CloudflareEnv, BackupWorkflowParams> {async run(event: Readonly<WorkflowEvent<BackupWorkflowParams>>, step: WorkflowStep) {let backupId = event.payload?.backupId;@@ -33,38 +31,12 @@ export class DatabaseBackupWorkflow extends WorkflowEntrypoint<CloudflareEnv, Ba.where(eq(backups.id, backupId));});- const exportConfiguration = getD1ExportConfiguration(this.env);- const bookmark = await step.do("Start D1 export", async () => {- const response = await requestD1Export(exportConfiguration, { output_format: "polling" });- if (!response.result?.at_bookmark) throw new Error("D1 export did not return a bookmark");- return response.result.at_bookmark;- });-- let signedUrl = "";- let exportFilename = "";- for (let attempt = 1; attempt <= EXPORT_RETRIES; attempt += 1) {- const result = await step.do(`Poll D1 export ${attempt}`, async () =>- requestD1Export(exportConfiguration, { output_format: "polling", current_bookmark: bookmark }),- );- // The polling response nests the finished export one level deeper- // (result.result.result); fall back to the flat shape defensively.- const finished = result.result?.result ?? result.result;- if (finished?.signed_url) {- signedUrl = finished.signed_url;- exportFilename = finished.filename ?? "";- break;- }- if (attempt < EXPORT_RETRIES) await step.sleep(`Wait for D1 export ${attempt}`, "15 seconds");- }- if (!signedUrl) throw new Error("D1 export did not finish before the polling limit");-const stored = await step.do("Store backup in R2", async () => {- const response = await fetch(signedUrl);- if (!response.ok || !response.body) throw new Error("Failed to download the D1 export");- const filename = exportFilename || createBackupFilename(new Date());+ const content = await exportDatabaseRecords(this.env.DB);+ const filename = createBackupFilename(new Date());const r2Key = `${BACKUP_PREFIX}/${backupId}/${filename}`;- const object = await this.env.BUCKET.put(r2Key, response.body, {- httpMetadata: { contentType: "application/sql" },+ const object = await this.env.BUCKET.put(r2Key, content, {+ httpMetadata: { contentType: "application/json" },customMetadata: { backupId },});return { filename, r2Key, size: object.size };diff --git a/src/lib/setup/migration.ts b/src/lib/setup/migration.tsindex b2d0ebf..f39c3e2 100644--- a/src/lib/setup/migration.ts+++ b/src/lib/setup/migration.ts@@ -14,6 +14,9 @@ const MIGRATION_NAMES = ["0012_add_app_settings.sql","0014_add_account_permissions.sql","0015_add_forwarding_email.sql",+ "0016_add_message_snooze.sql",+ "0017_add_message_star.sql",+ "0018_add_mailbox_domain_aliases.sql",];const INITIAL_SCHEMA_SQL = `@@ -22,7 +25,7 @@ CREATE INDEX IF NOT EXISTS users_created_by_idx ON users(created_by_user_id);CREATE TABLE IF NOT EXISTS domains (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, hostname text NOT NULL, zone_id text NOT NULL, status text DEFAULT 'pending' NOT NULL, routing_status text, sending_subdomain_tag text, sending_enabled integer DEFAULT false NOT NULL, routing_enabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);CREATE UNIQUE INDEX IF NOT EXISTS domains_hostname_idx ON domains(hostname);CREATE INDEX IF NOT EXISTS domains_user_idx ON domains(user_id);-CREATE TABLE IF NOT EXISTS mailboxes (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, domain_id text NOT NULL REFERENCES domains(id) ON DELETE cascade, local_part text NOT NULL, display_name text, avatar_key text, type text DEFAULT 'personal' NOT NULL, disabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);+CREATE TABLE IF NOT EXISTS mailboxes (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, domain_id text NOT NULL REFERENCES domains(id) ON DELETE cascade, local_part text NOT NULL, display_name text, avatar_key text, type text DEFAULT 'personal' NOT NULL, use_all_domains integer DEFAULT true NOT NULL, disabled integer DEFAULT false NOT NULL, created_at integer NOT NULL);CREATE UNIQUE INDEX IF NOT EXISTS mailboxes_address_idx ON mailboxes(domain_id, local_part);CREATE TABLE IF NOT EXISTS mailbox_access (id text PRIMARY KEY NOT NULL, mailbox_id text NOT NULL REFERENCES mailboxes(id) ON DELETE cascade, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, permission text DEFAULT 'read_only' NOT NULL, created_by_user_id text REFERENCES users(id) ON DELETE set null, created_at integer NOT NULL);CREATE UNIQUE INDEX IF NOT EXISTS mailbox_access_mailbox_user_idx ON mailbox_access(mailbox_id, user_id);@@ -36,7 +39,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS folders_mailbox_name_idx ON folders(mailbox_idCREATE INDEX IF NOT EXISTS folders_user_idx ON folders(user_id);CREATE INDEX IF NOT EXISTS folders_mailbox_idx ON folders(mailbox_id);CREATE TABLE IF NOT EXISTS api_keys (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, name text NOT NULL, prefix text NOT NULL, key_hash text NOT NULL, scopes text NOT NULL, created_at integer NOT NULL, last_used_at integer);-CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, mailbox_id text REFERENCES mailboxes(id) ON DELETE set null, direction text NOT NULL, provider_message_id text, folder_id text REFERENCES folders(id) ON DELETE set null, from_addr text NOT NULL, to_addr text NOT NULL, subject text, snippet text, status text DEFAULT 'received' NOT NULL, read integer DEFAULT false NOT NULL, thread_id text, created_at integer NOT NULL);+CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id text NOT NULL REFERENCES users(id) ON DELETE cascade, mailbox_id text REFERENCES mailboxes(id) ON DELETE set null, direction text NOT NULL, provider_message_id text, folder_id text REFERENCES folders(id) ON DELETE set null, from_addr text NOT NULL, to_addr text NOT NULL, subject text, snippet text, status text DEFAULT 'received' NOT NULL, read integer DEFAULT false NOT NULL, starred integer DEFAULT false NOT NULL, snoozed_until integer, thread_id text, created_at integer NOT NULL);CREATE INDEX IF NOT EXISTS messages_user_created_idx ON messages(user_id, created_at);CREATE INDEX IF NOT EXISTS messages_mailbox_idx ON messages(mailbox_id);CREATE INDEX IF NOT EXISTS messages_folder_idx ON messages(folder_id);