Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

fix: avoid full recounts during backup pagination

tianma-ife6c8c03da299
diff --git a/apps/api/src/backup-consistency.test.mjs b/apps/api/src/backup-consistency.test.mjs
index 60343a04..7964ea7e 100644
--- a/apps/api/src/backup-consistency.test.mjs
+++ b/apps/api/src/backup-consistency.test.mjs
@@ -75,6 +75,62 @@ describe("consistent backup reads across a real schema upgrade", () => {
expect((await db.request("/api/v1/backups/json")).status).toBe(200);
});
+ test("counts the source only for snapshots, never for each backup page", async () => {
+ const db = await installation();
+ const queries = [];
+ const prepare = db.storage.db.prepare.bind(db.storage.db);
+ db.storage.db.prepare = (sql) => { queries.push(sql); return prepare(sql); };
+ const state = await (await db.request("/api/v1/backups/state")).json();
+ const isFullCount = (sql) => /COUNT\s*\(/i.test(sql) && /backup_revision/.test(sql);
+ expect(queries.filter(isFullCount)).toHaveLength(1);
+ queries.length = 0;
+ for (const kind of ["memos", "notebooks", "prompts"]) {
+ let after = "";
+ let count = 0;
+ do {
+ const response = await db.request(pageUrl(state, kind, after));
+ expect(response.status).toBe(200);
+ const page = await response.json();
+ count += page[kind].length;
+ after = page.nextCursor;
+ } while (after !== null);
+ expect(count).toBe(state.counts[kind]);
+ }
+ expect(queries.filter(isFullCount)).toHaveLength(0);
+ const final = await (await db.request("/api/v1/backups/state")).json();
+ expect(final).toEqual(state);
+ expect(queries.filter(isFullCount)).toHaveLength(1);
+ });
+
+ for (const change of ["revision", "identity"]) test(`rejects a ${change} change while assembling a page`, async () => {
+ const db = await installation();
+ const state = await (await db.request("/api/v1/backups/state")).json();
+ const prepare = db.storage.db.prepare.bind(db.storage.db);
+ let checks = 0;
+ db.storage.db.prepare = (sql) => {
+ const statement = prepare(sql);
+ if (!/FROM workspaces/.test(sql)) return statement;
+ const bind = statement.bind.bind(statement);
+ statement.bind = (...values) => {
+ const bound = bind(...values);
+ const first = bound.first.bind(bound);
+ bound.first = async (...args) => {
+ if (++checks === 2) {
+ if (change === "revision") db.sqlite.query("UPDATE memo_contents SET content_markdown = 'Changed during read' WHERE memo_id = ?").run(db.note.id);
+ else db.sqlite.exec("UPDATE workspaces SET created_at = 'replacement-instance' WHERE id = 'ws_default'");
+ }
+ return first(...args);
+ };
+ return bound;
+ };
+ return statement;
+ };
+ const response = await db.request(pageUrl(state));
+ expect(checks).toBe(2);
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({ error: { code: "backup_source_changed" } });
+ });
+
test("covers content, revisions, resources, prompts and deletions in their transaction", async () => {
const db = await installation();
const changes = [
diff --git a/apps/api/src/backup-routes.ts b/apps/api/src/backup-routes.ts
index e5a67c6e..20af7e4d 100644
--- a/apps/api/src/backup-routes.ts
+++ b/apps/api/src/backup-routes.ts
@@ -147,6 +147,13 @@ type BackupStateRow = {
prompts: number;
};
+// Page checks need only the transactional revision. Recounting every table here
+// makes a full backup quadratic in source size and can exhaust D1's daily reads.
+const readBackupVersion = (database: DatabaseAdapter, workspaceId: string) => database.prepare(
+ `SELECT created_at AS identity, CAST(backup_revision AS TEXT) AS backup_revision
+ FROM workspaces WHERE id = ?`,
+).bind(workspaceId).first<Pick<BackupStateRow, "identity" | "backup_revision">>();
+
const readBackupState = (database: DatabaseAdapter, workspaceId: string) => database.prepare(
`SELECT w.created_at AS identity, CAST(w.backup_revision AS TEXT) AS backup_revision,
(SELECT COUNT(*) FROM notebooks WHERE workspace_id = w.id AND is_deleted = 0) AS notebooks,
@@ -275,7 +282,7 @@ export const registerBackupRoutes = (
const workspaceId = getWorkspaceId(context);
const database = context.env.storage.db;
const unchanged = async () => {
- const state = await readBackupState(database, workspaceId);
+ const state = await readBackupVersion(database, workspaceId);
return state?.identity === identity && state.backup_revision === revision;
};
const changed = () => conflict(context, "backup_source_changed", "The source changed during backup. Try again when it is idle.");
diff --git a/apps/desktop/src/backup/worker-integration.test.mjs b/apps/desktop/src/backup/worker-integration.test.mjs
index b3c267cf..230c3067 100644
--- a/apps/desktop/src/backup/worker-integration.test.mjs
+++ b/apps/desktop/src/backup/worker-integration.test.mjs
@@ -5,6 +5,7 @@ import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
+import { createServer } from "node:http";
import { fetchEdgeEverApp } from "../../../api/src/index.ts";
import { createSelfHostedStorageAdapter } from "../../../api/src/self-hosted-storage-adapter.ts";
import { BackupManager } from "../main/backup-manager.mjs";
@@ -70,6 +71,21 @@ test("fast upload initialization failures reach the restore handler without term
expect(JSON.parse(stdout)).toEqual({ status: 503, code: "object_storage_unavailable", message: "Target storage unavailable" });
});
+test("real socket failures retain safe request and underlying transport diagnostics", async () => {
+ const server = createServer((request) => request.socket.destroy());
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ cleanups.push(() => new Promise((resolve) => server.close(resolve)));
+ const root = await directory();
+ const job = await launch({ operation: "export", context: { baseUrl: `http://127.0.0.1:${server.address().port}`, accountId: null, token: "private-request-token" }, path: join(root, "failed.zip.part"), version: { edgeeverVersion: "2.16.0", buildId: "socket-failure" } });
+ await expect(job.next("done")).rejects.toMatchObject({ code: "backup_failed", message: "fetch failed" });
+ const diagnostic = job.messages.find((message) => message.type === "request-error")?.result;
+ expect(diagnostic).toMatchObject({ method: "GET", path: "/api/v1/backups/state", code: "backup_network_failed", message: "Backup network request failed." });
+ expect(diagnostic.causeCodes.some((code) => ["UND_ERR_SOCKET", "ECONNRESET"].includes(code))).toBe(true);
+ expect(diagnostic.elapsedMs).toBeGreaterThanOrEqual(0);
+ expect(JSON.stringify(diagnostic)).not.toContain("private-request-token");
+ expect(await readdir(root)).toEqual([]);
+}, 10000);
+
test("failed workers cancel unfinished sibling requests and exit", async () => {
let release;
const promptsStarted = new Promise((resolve) => { release = resolve; });
diff --git a/apps/desktop/src/backup/worker.ts b/apps/desktop/src/backup/worker.ts
index 76c8289a..e01e5008 100644
--- a/apps/desktop/src/backup/worker.ts
+++ b/apps/desktop/src/backup/worker.ts
@@ -25,8 +25,38 @@ parentPort?.on("message", (message) => {
});
const send = (type: string, result: unknown) => parentPort?.postMessage({ type, result });
const fail = (code: string, message: string, status?: number): never => { throw Object.assign(new Error(message), { code, ...(status === undefined ? {} : { status }) }); };
+const networkCauseCodes = (error: unknown): string[] => {
+ const codes = new Set<string>();
+ const pending = [error];
+ const seen = new Set<unknown>();
+ while (pending.length && seen.size < 16) {
+ const value = pending.shift();
+ if (!value || typeof value !== "object" || seen.has(value)) continue;
+ seen.add(value);
+ const detail = value as { code?: unknown; cause?: unknown; errors?: unknown };
+ // Do not log arbitrary messages, addresses, stacks or request credentials.
+ if (typeof detail.code === "string" && /^[A-Z][A-Z0-9_]{0,63}$/.test(detail.code)) codes.add(detail.code);
+ if (detail.cause) pending.push(detail.cause);
+ if (Array.isArray(detail.errors)) pending.push(...detail.errors.slice(0, 8));
+ }
+ return [...codes].slice(0, 8);
+};
const fetchWithDiagnostics: typeof fetch = async (url, init) => {
- const response = await fetch(url, init);
+ const startedAt = Date.now();
+ let response: Response;
+ try {
+ response = await fetch(url, init);
+ } catch (error) {
+ if (!signal.aborted) send("request-error", {
+ method: init?.method ?? "GET",
+ path: new URL(typeof url === "string" ? url : url instanceof URL ? url.href : url.url).pathname,
+ code: "backup_network_failed",
+ message: "Backup network request failed.",
+ causeCodes: networkCauseCodes(error),
+ elapsedMs: Date.now() - startedAt,
+ });
+ throw error;
+ }
if (!response.ok) {
const body = await response.clone().json().catch(() => null) as { error?: { code?: unknown; message?: unknown } } | null;
send("request-error", {
diff --git a/apps/desktop/src/main/backup-manager.mjs b/apps/desktop/src/main/backup-manager.mjs
index e15bf4bb..824299f6 100644
--- a/apps/desktop/src/main/backup-manager.mjs
+++ b/apps/desktop/src/main/backup-manager.mjs
@@ -317,6 +317,8 @@ export class BackupManager {
status: detail.status,
code: sanitizeDesktopDiagnosticText(detail.code),
message: sanitizeDesktopDiagnosticText(detail.message),
+ ...(Array.isArray(detail.causeCodes) ? { causeCodes: detail.causeCodes.filter((code) => typeof code === "string" && /^[A-Z][A-Z0-9_]{0,63}$/.test(code)).slice(0, 8) } : {}),
+ ...(Number.isFinite(detail.elapsedMs) && detail.elapsedMs >= 0 ? { elapsedMs: detail.elapsedMs } : {}),
})).catch(() => {});
} catch { /* Diagnostic output must not change the task result. */ }
}
diff --git a/apps/desktop/src/main/backup-manager.test.mjs b/apps/desktop/src/main/backup-manager.test.mjs
index 9d8c7a74..88b5d55b 100644
--- a/apps/desktop/src/main/backup-manager.test.mjs
+++ b/apps/desktop/src/main/backup-manager.test.mjs
@@ -475,6 +475,31 @@ test("preserves HTTP status and emits only redacted request diagnostics without
} finally { await f.cleanup(); }
});
+test("records bounded transport cause codes without changing the backup failure", async () => {
+ const f = await fixture();
+ try {
+ await f.manager.run();
+ const worker = f.workers.at(-1);
+ const events = [];
+ f.manager.onDiagnostic = (event, details) => events.push({ event, ...details });
+ worker.emit("message", { type: "request-error", result: {
+ method: "GET", path: "/api/v1/resources/res_fixture/blob?token=private-query",
+ code: "backup_network_failed", message: "Backup network request failed.",
+ causeCodes: ["UND_ERR_SOCKET", "ECONNRESET", "token=private-token", null, "https://private.example.com"], elapsedMs: 123,
+ headers: { Authorization: "private-header" }, stack: "private-stack",
+ } });
+ expect(events).toEqual([{
+ event: "backup.request-failed", kind: "export", phase: "exporting", method: "GET",
+ path: "/api/v1/resources/res_fixture/blob", code: "backup_network_failed", message: "Backup network request failed.",
+ causeCodes: ["UND_ERR_SOCKET", "ECONNRESET"], elapsedMs: 123,
+ }]);
+ worker.emit("message", { type: "error", error: { code: "backup_failed", message: "fetch failed" } });
+ await f.manager.waitForIdle();
+ expect(f.manager.status().error).toEqual({ code: "backup_failed", message: "fetch failed" });
+ expect(f.manager.status().history).toEqual([]);
+ } finally { await f.cleanup(); }
+});
+
const sourceState = { identity: 'instance-a', workspaceId: 'workspace-a', backupRevision: '8' };
const finishSnapshot = async (f, result = {}) => {
const worker = f.workers.at(-1);
diff --git a/docs/changes/2026-10-31-backup-read-budget.md b/docs/changes/2026-10-31-backup-read-budget.md
new file mode 100644
index 00000000..dd106e52
--- /dev/null
+++ b/docs/changes/2026-10-31-backup-read-budget.md
@@ -0,0 +1,24 @@
+# Backup pagination read budget and network diagnostics
+
+## Problem and scope
+
+Backup pagination called the full snapshot-count query before and after every page. Its counts scan notes, revisions, resources, notebooks and prompts. A library with 4,199 notes needs 168 default pages; measured at approximately 16,420 rows per count query, those page checks alone require approximately 5.52 million row reads. This can exhaust D1's free daily read allowance and interrupt other database requests.
+
+The API now reads only the workspace identity and transactional backup revision by primary key for each page check. Full counts remain in the snapshot endpoint used at export start and completion. Both page checks still run, including the check after page assembly. Protocol version 1, response fields, archive formats and migrations are unchanged, so installed clients benefit when the API is deployed. Cloudflare and self-hosted installations use the same route code.
+
+Desktop request diagnostics now retain the request method, path without query parameters, elapsed time and up to eight underlying network error codes. Cause traversal is bounded and records only code-shaped strings, without arbitrary error messages, addresses, headers or stacks. This adds diagnostics without adding retries or changing restore writes.
+
+## Risk and rollback
+
+Backup consistency is the primary risk: an incorrect lightweight check could accept data changed during export. The existing transactionally updated revision and workspace identity remain authoritative; stale pages and changes during page assembly still return `backup_source_changed`. Reverting the code restores the previous behavior without a database migration.
+
+## Validation
+
+- The regression initially failed with 26 full count queries during pagination. It now confirms zero full counts while traversing all three data kinds, with counts retained in the initial and final snapshots. The real schema-upgrade fixture retains existing notes and legacy export callers.
+- Concurrent revision and workspace-identity changes during page assembly both return HTTP 409. Existing workspace isolation, permissions, archive completeness, digest, cancellation and recovery tests pass.
+- Backup API/manager/IPC/worker suites: 76 tests passed. Restore, JSON/ZIP64 and encrypted-container suites: 31 tests passed.
+- Extracted the actual released v2.16.0 backup worker from the installed app, SHA-256 `2b7c074846f2cb37a30fd245cc460aca1af406266a6ac5ac306f249ee3c882a1`. On isolated databases, old-export/new-restore and new-export/old-restore both passed for plain and encrypted archives, including exact attachment-byte checks after shutting down the source server. Repeated these checks with the installed Electron 44.7 / Node 24.21 runtime.
+- A real local HTTP socket reset now produces a request diagnostic containing its underlying transport code while preserving the original backup failure. This passed on both the normal Node test runtime and the installed Electron Node runtime. Manager tests verify diagnostic redaction.
+- `bun run typecheck`, `bun run typecheck:mobile` and `bun run build:web` passed.
+
+No production database scans were used to validate this fix. Live D1 row-read measurements after deployment and Windows/Linux native execution remain unverified. The fix does not replenish an already exhausted daily allowance, and the precise transport cause of the earlier historical failure remains unknown.
diff --git a/docs/changes/2026-10-31-backup-read-budget.zh-CN.md b/docs/changes/2026-10-31-backup-read-budget.zh-CN.md
new file mode 100644
index 00000000..05f7f404
--- /dev/null
+++ b/docs/changes/2026-10-31-backup-read-budget.zh-CN.md
@@ -0,0 +1,24 @@
+# 备份分页读取预算与网络诊断
+
+## 问题与范围
+
+备份分页原先在每页读取前后各执行一次完整快照计数,扫描笔记、修订、资源、笔记本及提示词。4,199 条笔记需要 168 个默认分页;按实测每次计数约读取 16,420 行计算,仅分页校验就需要约 552 万行读取,可能耗尽 D1 免费每日读取额度,并使其他数据库请求中断。
+
+API 现在对每页校验仅按工作区主键读取实例身份与事务性备份版本号。完整计数保留在导出开始和完成时使用的快照接口中。分页前后两次检查均保留,包括组装分页结果后的检查。协议版本 1、响应字段、归档格式与 Migration 保持原样,API 部署后已安装客户端即可使用修复。Cloudflare 与自托管共用同一路由代码。
+
+桌面请求诊断新增请求方法、不含查询参数的路径、耗时,以及最多八个底层网络错误码。异常链遍历有数量限制,仅记录符合错误码格式的字符串,不记录任意错误消息、地址、请求头或堆栈。此变更只增加诊断,不增加重试,也不改变恢复写入。
+
+## 风险与回滚
+
+主要风险是备份一致性:轻量检查若实现错误,可能接受导出期间变更的数据。现有事务内递增版本号和工作区身份仍作为判断依据;过期分页及分页组装期间发生变更,仍返回 `backup_source_changed`。撤销代码即可恢复原行为,不需要数据库迁移。
+
+## 验证
+
+- 回归测试先复现分页期间执行 26 次全量计数,修复后遍历全部三类数据时为零次;开始、结束快照仍保留计数。真实 Schema 升级夹具保留已有笔记及旧导出调用方。
+- 分页组装期间修改修订号或工作区身份,均返回 HTTP 409;现有工作区隔离、权限、归档完整性、摘要、取消与恢复测试通过。
+- 备份 API、管理器、IPC、线程相关测试 76 项通过;恢复、JSON/ZIP64 与加密容器测试 31 项通过。
+- 从已安装应用提取真实已发布 v2.16.0 备份线程,SHA-256 为 `2b7c074846f2cb37a30fd245cc460aca1af406266a6ac5ac306f249ee3c882a1`。使用隔离数据库,普通及加密归档的“旧导出/新恢复”“新导出/旧恢复”均通过,包括关闭源服务后逐字节检查附件;使用已安装 Electron 44.7 / Node 24.21 运行时再次通过上述验证。
+- 真实本地 HTTP 连接断开后,请求诊断会保留底层传输错误码,并保持原备份失败结果;普通 Node 测试运行时与已安装 Electron Node 运行时均通过。管理器测试验证诊断脱敏。
+- `bun run typecheck`、`bun run typecheck:mobile`、`bun run build:web` 均通过。
+
+修复验证未扫描生产数据库。部署后的真实 D1 读取量、Windows/Linux 原生运行仍未验证。修复不会补回已耗尽的当日额度,先前历史故障的具体底层传输原因仍未知。