Skip to content

Email for professionals and teams

fix(backups): back up the tables added since the format shipped

Bruce McLachlana6c326d77c85
diff --git a/src/lib/backups/export.ts b/src/lib/backups/export.ts
index 5998a26..9444ba1 100644
--- a/src/lib/backups/export.ts
+++ b/src/lib/backups/export.ts
@@ -1,14 +1,48 @@
import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";
import { mergeLegacyMessageBodies } from "./utils";
-const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];
+const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries"];
+/**
+ * Tables every backup document must contain. Tables added to BACKUP_TABLES
+ * after the format shipped are absent from older documents, so they stay
+ * optional here and are filled in as empty on restore.
+ */
+const REQUIRED_BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];
const INSERT_BATCH_SIZE = 50;
export function getD1ExportConfigurationStatus(_env?: CloudflareEnv) {
return { configured: true, missing: [] };
}
+/**
+ * Tables D1 manages itself, which are intentionally absent from BACKUP_TABLES.
+ */
+const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%"];
+const INTERNAL_TABLES = ["d1_migrations"];
+
+/**
+ * Fails the backup when the database contains a table BACKUP_TABLES does not
+ * list. Without this, a migration that adds a table silently produces backups
+ * that omit it, and the omission only surfaces during a restore.
+ */
+export async function assertBackupTablesCoverDatabase(db: D1Database): Promise<void> {
+ const conditions = [
+ ...INTERNAL_TABLE_PATTERNS.map((pattern) => `name NOT LIKE '${pattern}'`),
+ ...INTERNAL_TABLES.map((name) => `name <> '${name}'`),
+ ].join(" AND ");
+ const result = await db
+ .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND ${conditions}`)
+ .all<{ name: string }>();
+ const covered = new Set<string>(BACKUP_TABLES);
+ const unlisted = result.results.map((row) => row.name).filter((name) => !covered.has(name));
+ if (unlisted.length === 0) return;
+ throw new Error(
+ `Backup aborted: ${unlisted.join(", ")} not listed in BACKUP_TABLES, so this backup would omit them. Add them to BACKUP_TABLES in src/lib/backups/export.ts.`,
+ );
+}
+
export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array> {
+ await assertBackupTablesCoverDatabase(db);
const tables = {} as Record<DatabaseBackupTable, DatabaseRecord[]>;
for (const table of BACKUP_TABLES) {
const result = await db.prepare(`SELECT * FROM ${table}`).all<DatabaseRecord>();
@@ -21,6 +55,7 @@ export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array>
export async function restoreDatabaseRecords(db: D1Database, content: ArrayBuffer): Promise<void> {
const document = parseDatabaseBackup(content);
mergeLegacyMessageBodies(document);
+ fillMissingBackupTables(document);
validateDatabaseBackup(document);
for (const table of [...BACKUP_TABLES].reverse()) await db.prepare(`DELETE FROM ${table}`).run();
for (const table of BACKUP_TABLES) {
@@ -42,7 +77,12 @@ function parseDatabaseBackup(content: ArrayBuffer): DatabaseBackupDocument {
function isDatabaseBackupDocument(value: unknown): value is DatabaseBackupDocument {
if (!value || typeof value !== "object") return false;
const document = value as Partial<DatabaseBackupDocument>;
- return document.format === "mailflare-database-backup" && document.version === 1 && !!document.tables && BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]));
+ if (document.format !== "mailflare-database-backup" || document.version !== 1 || !document.tables) return false;
+ if (!REQUIRED_BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]))) return false;
+ return BACKUP_TABLES.every((table) => {
+ const rows = document.tables?.[table];
+ return rows === undefined || Array.isArray(rows);
+ });
}
function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row: DatabaseRecord) {
@@ -53,6 +93,13 @@ function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row:
return db.prepare(`INSERT INTO ${table} (${identifiers}) VALUES (${placeholders})`).bind(...columns.map((column) => row[column]));
}
+/** Backups written before a table joined BACKUP_TABLES simply omit it. */
+function fillMissingBackupTables(document: DatabaseBackupDocument): void {
+ for (const table of BACKUP_TABLES) {
+ if (!document.tables[table]) document.tables[table] = [];
+ }
+}
+
function validateDatabaseBackup(document: DatabaseBackupDocument): void {
for (const table of BACKUP_TABLES) {
for (const row of document.tables[table]) {
diff --git a/src/lib/backups/types.d.ts b/src/lib/backups/types.d.ts
index eae5c9e..40d081d 100644
--- a/src/lib/backups/types.d.ts
+++ b/src/lib/backups/types.d.ts
@@ -12,6 +12,6 @@ export type BackupWorkflowBinding = {
}): Promise<unknown>;
};
-export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings";
+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries";
export type DatabaseRecord = Record<string, string | number | null>;
export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };