AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
perf(api): index backup and bootstrap memo pages (#657)
sivdead52d30294e90c
diff --git a/apps/api/src/backup-consistency.test.mjs b/apps/api/src/backup-consistency.test.mjsindex 9102dafb..21ca3498 100644--- a/apps/api/src/backup-consistency.test.mjs+++ b/apps/api/src/backup-consistency.test.mjs@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { Hono } from "hono";import { registerBackupRoutes } from "./backup-routes.ts";+import { registerSyncRoutes } from "./sync-routes.ts";import { createSelfHostedStorageAdapter } from "./self-hosted-storage-adapter.ts";import { createMemoRecord, getMemoDetail, mapMemoDetail } from "./memo-service.ts";import { restoreJsonMemos, restoreJsonNotebooks } from "./backup-service.ts";@@ -14,11 +15,13 @@ const cleanups = [];afterEach(() => cleanups.splice(0).forEach((cleanup) => cleanup()));const auth = { kind: "user", actorType: "user", actorId: "backup-test-user", username: "owner", displayName: "Owner", scopes: [], workspaceId: "ws_default", role: "owner" };-const installation = async ({ includeExportIndex = true } = {}) => {+const installation = async ({ includeExportIndex = true, includeMemoIdIndex = true } = {}) => {const sqlite = new Database(":memory:");const migrations = globSync("migrations/*.sql").sort();// Start with the existing schema and data, then upgrade it in place.- for (const file of migrations.filter((file) => !file.endsWith("0080_backup_revision.sql") && (includeExportIndex || !file.endsWith("0085_export_cursor_index.sql")))) sqlite.exec(readFileSync(file, "utf8"));+ for (const file of migrations.filter((file) => !file.endsWith("0080_backup_revision.sql")+ && (includeExportIndex || !file.endsWith("0085_export_cursor_index.sql"))+ && (includeMemoIdIndex || !file.endsWith("0086_memo_id_paging_index.sql")))) sqlite.exec(readFileSync(file, "utf8"));const directory = mkdtempSync(join(tmpdir(), "edgeever-backup-protocol-"));cleanups.push(() => { sqlite.close(); rmSync(directory, { recursive: true, force: true }); });const storage = createSelfHostedStorageAdapter(sqlite, directory);@@ -29,6 +32,7 @@ const installation = async ({ includeExportIndex = true } = {}) => {const app = new Hono();app.use("/api/v1/*", async (c, next) => { c.set("auth", activeAuth); await next(); });registerBackupRoutes(app, { clampNumber: (v, min, max) => Math.min(Math.max(Number.isFinite(v) ? v : min, min), max), getMemoDetail, mapMemoDetail, restoreJsonMemos, restoreJsonNotebooks, sha256Bytes, initiateResourceRestoreUpload: async () => { throw new Error("unused"); } });+ registerSyncRoutes(app, { clampNumber: (v, min, max) => Math.min(Math.max(v, min), max), mapMemoDetail });return {sqlite, storage, note, inbox,revision: (workspace = "ws_default") => String(sqlite.query("SELECT backup_revision FROM workspaces WHERE id = ?").get(workspace).backup_revision),@@ -40,8 +44,92 @@ const installation = async ({ includeExportIndex = true } = {}) => {const pageUrl = (state, kind = "memos", after = "") => `/api/v1/backups/data?${new URLSearchParams({ kind, identity: state.identity, revision: state.backupRevision, after, limit: "1" })}`;describe("consistent backup reads across a real schema upgrade", () => {+ test("upgrades populated backup and bootstrap pages to workspace-scoped ID seeks without changing their data", async () => {+ const db = await installation({ includeMemoIdIndex: false });+ db.sqlite.exec("INSERT INTO workspaces(id,name) VALUES('ws_other','Other'); INSERT INTO notebooks(id,workspace_id,name) VALUES('nb_other','ws_other','Other');");+ const addMemo = db.sqlite.query(`INSERT INTO memos+ (id,workspace_id,notebook_id,title,tags_json,is_deleted,deleted_at,created_at,updated_at)+ VALUES(?,?,?,?,?,?,?,?,?)`);+ const addContent = db.sqlite.query(`INSERT INTO memo_contents+ (memo_id,content_json,content_markdown,content_text,content_hash,revision) VALUES(?,?,?,?,?,2)`);+ db.sqlite.transaction(() => {+ for (let index = 0; index < 64; index++) for (const workspaceId of ["ws_default", "ws_other"]) {+ const id = `memo_paging_${String(index).padStart(3, "0")}_${workspaceId}`;+ const deleted = index % 4 === 0;+ const body = `Body ${id}`;+ addMemo.run(id, workspaceId, workspaceId === "ws_default" ? db.inbox : "nb_other", `Title ${index}`, '["paging"]',+ Number(deleted), deleted ? "2021-02-03T00:00:00.000Z" : null, "2020-01-01T00:00:00.000Z", "2021-02-03T00:00:00.000Z");+ addContent.run(id, JSON.stringify({ type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text: body }] }] }), body, body, `hash-${id}`);+ db.sqlite.query("INSERT INTO resources(id,memo_id,object_key,kind,byte_size,filename) VALUES(?,?,?,'attachment',4,'sample.txt')").run(`resource_${id}`, id, `object_${id}`);+ db.sqlite.query("INSERT INTO memo_revisions(id,memo_id,revision,content_json,content_markdown,content_text,content_hash,created_by) VALUES(?,?,1,'{\"type\":\"doc\",\"content\":[]}','Previous','Previous','previous','user')").run(`revision_${id}`, id);+ }+ })();+ const originalMemos = db.sqlite.query("SELECT * FROM memos ORDER BY id").all();+ const dataSnapshot = () => Object.fromEntries([+ ["memos", "id"], ["memo_contents", "memo_id"], ["resources", "id"], ["memo_revisions", "id"], ["notebooks", "id"], ["workspaces", "id"],+ ].map(([table, key]) => [table, db.sqlite.query(`SELECT * FROM ${table} ORDER BY ${key}`).all()]));+ const beforeData = dataSnapshot();+ const pageQueries = [];+ const prepare = db.storage.db.prepare.bind(db.storage.db);+ db.storage.db.prepare = (sql) => {+ const statement = prepare(sql), bind = statement.bind.bind(statement);+ statement.bind = (...values) => {+ if (sql.includes("INNER JOIN memo_contents") && sql.includes("ORDER BY m.id ASC")) pageQueries.push({ sql, values });+ return bind(...values);+ };+ return statement;+ };+ const readAll = async () => {+ const values = {};+ for (const workspaceId of ["ws_default", "ws_other"]) {+ db.setAuth({ ...auth, workspaceId });+ const state = await (await db.request("/api/v1/backups/state")).json();+ const backup = [];+ let after = "";+ do {+ const response = await db.request(`/api/v1/backups/data?${new URLSearchParams({ kind: "memos", identity: state.identity, revision: state.backupRevision, after, limit: "7" })}`);+ expect(response.status).toBe(200);+ const page = await response.json();+ backup.push(page); after = page.nextCursor;+ } while (after !== null);+ const bootstrap = [];+ after = "";+ do {+ // No opt-in flags: preserve the metadata and defaults used by old clients.+ const response = await db.request(`/api/v1/sync/bootstrap?${new URLSearchParams({ afterId: after, limit: "7" })}`);+ expect(response.status).toBe(200);+ const page = await response.json();+ bootstrap.push(page); after = page.nextAfterId;+ } while (after !== null);+ const expected = originalMemos.filter(memo => memo.workspace_id === workspaceId);+ expect(backup.flatMap(page => page.memos.map(memo => memo.id))).toEqual(expected.filter(memo => !memo.is_deleted).map(memo => memo.id));+ expect(bootstrap.flatMap(page => page.memos.map(memo => memo.id))).toEqual(expected.map(memo => memo.id));+ expect(backup.flatMap(page => page.resources)).toHaveLength(48);+ expect(backup.flatMap(page => page.revisions)).toHaveLength(48);+ values[workspaceId] = { state, backup, bootstrap };+ }+ return values;+ };+ // Keep the planner's ordinary pre-migration statistics: no ANALYZE warm-up.+ const before = await readAll();+ const explain = ({ sql, values }) => db.sqlite.query(`EXPLAIN QUERY PLAN ${sql}`).all(...values);+ expect(pageQueries.some(query => explain(query).some(row => row.parent === 0 && row.detail.includes("TEMP B-TREE FOR ORDER BY")))).toBe(true);+ pageQueries.length = 0;+ db.sqlite.exec(readFileSync("migrations/0086_memo_id_paging_index.sql", "utf8"));+ expect(dataSnapshot()).toEqual(beforeData);+ expect(await readAll()).toEqual(before);+ expect(pageQueries.length).toBeGreaterThan(2);+ for (const query of pageQueries) {+ const plan = explain(query);+ expect(plan.some(row => row.detail.includes("idx_memos_workspace_id (workspace_id=? AND id>?)"))).toBe(true);+ // Bootstrap's orphan-notebook fallback may sort; only the main page must not.+ expect(plan.some(row => row.parent === 0 && row.detail.includes("TEMP B-TREE FOR ORDER BY"))).toBe(false);+ }+ expect(db.sqlite.query("SELECT name FROM sqlite_master WHERE name='idx_memos_workspace_created_export'").get()).not.toBeNull();+ });+test("upgrades existing data to indexed, stable browser-export continuation pages", async () => {- const db = await installation({ includeExportIndex: false });+ const db = await installation({ includeExportIndex: false, includeMemoIdIndex: false });for (let index = 0; index < 60; index++) await createMemoRecord(db.storage.db, "ws_default", {notebookId: db.inbox, title: `Cursor ${index}`, contentMarkdown: `Body ${index}`, tags: ["budget"],createdAt: index % 2 ? "2020-01-02T00:00:00.000Z" : "2020-01-01T00:00:00.000Z",diff --git a/docs/changes/2026-10-27-third-party-import.md b/docs/changes/2026-10-27-third-party-import.mdindex ab0bd209..d40eb298 100644--- a/docs/changes/2026-10-27-third-party-import.md+++ b/docs/changes/2026-10-27-third-party-import.md@@ -26,7 +26,7 @@ Extend the former Evernote command-line guide entry with online, additive Web/de## Unchanged boundaries-- Preserve MCP imports, ordinary editing, offline queues, Rust structures, sync, backup and encrypted restore contracts. Apart from R28's export-index migration, add no database structures, global required settings or authorization scopes.+- Preserve MCP imports, ordinary editing, offline queues, Rust structures, sync, backup and encrypted restore contracts. Apart from R28/R39's paging-index migrations, add no database structures, global required settings or authorization scopes.- No native mobile import entry, cross-run merging, resume after exiting the app, Obsidian plugin/inter-note conversion, remote-image capture or whole-library transaction rollback.- Keep installers, automatic updates, versions and release/deployment mechanisms unchanged. Existing Cloudflare upgrades retain the Worker, D1, R2, administrator and authentication Secret without resetting passwords or replacing the instance. Tests use authored sources and isolated directories, never personal libraries.@@ -77,6 +77,7 @@ Per-note creation previously called `getNotebook` with repeated `COUNT/MAX`, pro| --- | --- | --- || R27 | Use a lightweight ID/workspace/non-deleted lookup for boolean notebook existence checks in import/create, duplication, move/merge, child creation and parent changes. Preserve `getNotebook` statistics, permissions and existing missing-target errors. | Reproduce cumulative aggregates through real imports/folder creation, then verify large note/folder operations do not recount notes. Missing, foreign-workspace and deleted targets retain their existing rejection contract. || R28 | Add optional opaque `cursor` / `nextCursor` to browser JSON/ZIP and Markdown exports, retaining creation-time/ID ordering. A cursor carries the continuation position and traversed offset. Cursor requests omit repeated totals by default; explicit `includeTotal=1` still returns totals. Only malformed new cursors return `invalid_export_cursor` / HTTP 400. Preserve legacy OFFSET requests, response fields, defaults and ordering. Updated consumers retain OFFSET traversal when a server provides no cursor, without adding older-server prerequisites. Add an incremental migration for a `(workspace_id, is_deleted, created_at, id)` memo index. | Reproduce full page scans first. Verify complete ZIP/Markdown ordering, resources and revisions without gaps/duplicates, tied timestamps and selected-note filters. Upgrade a real prior schema/data set, preserving fields and old calls. Local whole-traversal work grows approximately linearly with indexed continuation queries. |+| R39 | Add `0086_memo_id_paging_index.sql` with only a `(workspace_id, id)` index on `memos`, allowing consistent-backup memo pages and initial sync to seek by workspace and ID range. Retain `0085`; do not change business fields, triggers, queries, filters, cursors, page sizes, APIs or client defaults. | Write a failing prior-schema regression before applying the migration. Retain existing data, complete traversal, workspace isolation and deleted-note behavior. Compare 1,000/4,000 notes without preceding `ANALYZE` for approximately linear reads, measuring actual D1 `rows_read` / `rows_written`, construction and insert costs. Existing creation-time exports must retain their `0085` index. |`0085_export_cursor_index.sql` adds one-time index-build reads and ongoing index write/storage costs without changing business fields. The main risk is skipped or duplicate page boundaries. Application rollback can restore old requests while leaving the index; removal requires another migration, never editing an executed file. Multipart upload retains `P + 2` R2 Class A operations per resource and at most three attempts per part. Add no import caps, required settings, automatic retries or remote reads. Measure representative local data rather than load-testing Cloudflare.@@ -104,6 +105,12 @@ Per-note creation previously called `getNotebook` with repeated `COUNT/MAX`, proKeep screenshots and operational records outside Git and attach images to the PR. The repository retains bilingual conclusions, maintainable tests and the authored source fixtures they consume. Preserve an independent recovery copy before cleaning unpublished history; do not rewrite other branches.+## D1 paging reads++R39 adds a `(workspace_id, id)` index, its migration, paging regressions and read/write-cost validation. Retain APIs, clients, the backup protocol and existing migration `0085`. See [plan section 9](../third-party-import-plan.md#9-d1-quota-optimization) for the choice, index costs, prior-schema upgrade acceptance and rollback. Unchanged-backup probing and search-index deduplication are outside this iteration.++R39's migration, prior-schema upgrade and contract regressions pass locally. Actual D1 row counts confirm approximately linear reads and one additional index-row write per inserted memo. APIs, clients and runtime business code are unchanged. Production migration and complete-flow usage remain deployment acceptance work.+## Risks and rollbackImport is high-risk data writing: partial new objects, duplicates, incorrect timestamps and resource references are the worst outcomes. Append only, retain sources, item outcomes and commit baselines; cancellation does not undo completed writes and code rollback does not delete imported notes. Removing the entry/routes leaves imported data usable as ordinary notes. Cleanup requires confirmed actual IDs, never bulk deletion by name.@@ -112,6 +119,6 @@ Shared codec repairs affect editing round-trips for images, lists and attachment## Completion reconciliation-I01–I11, R01–R38 and C01/C02 are implemented; every behavior in the task diff maps to these entries, with no additional input restrictions, permissions or sync/backup protocol changes. Focused checks cover source formats, hierarchy/metadata/original bytes, failure retries, concurrent-edit protection and resource cleanup. See [acceptance conclusions](evidence/2026-10-27-third-party-import/README.md) for the conditions and measurements of large-library reads, cursor exports, real-browser preflight and large-body result retention. Both typechecks, the Web build and targeted lint pass.+I01–I11, R01–R39 and C01/C02 are implemented; every behavior in the task diff maps to these entries, with no additional input restrictions, permissions or sync/backup protocol changes. Focused checks cover source formats, hierarchy/metadata/original bytes, failure retries, concurrent-edit protection and resource cleanup. See [acceptance conclusions](evidence/2026-10-27-third-party-import/README.md) for the conditions and measurements of large-library reads, cursor exports, real-browser preflight and large-body result retention. Both typechecks, the Web build and targeted lint pass.-Cloudflare has R27–R30 and `0085`; the acceptance document records the real prior-instance upgrade and readback. R31–R38 are not deployed. Earlier macOS/Linux native results do not cover the latest repairs. Windows native, USB mobile devices, older-client editing, AppImage installation/cross-version updates and Docker container deployment remain unverified; builds and driver-level checks do not replace those flows.+Cloudflare has R27–R30 and `0085`; the acceptance document records the real prior-instance upgrade and readback. R31–R39 and `0086` are not deployed. Earlier macOS/Linux native results do not cover the latest repairs. Windows native, USB mobile devices, older-client editing, AppImage installation/cross-version updates and Docker container deployment remain unverified; builds and driver-level checks do not replace those flows.diff --git a/docs/changes/2026-10-27-third-party-import.zh-CN.md b/docs/changes/2026-10-27-third-party-import.zh-CN.mdindex 00645e68..abbbf36d 100644--- a/docs/changes/2026-10-27-third-party-import.zh-CN.md+++ b/docs/changes/2026-10-27-third-party-import.zh-CN.md@@ -26,7 +26,7 @@## 保持不变的边界-- 保留 MCP 导入、普通编辑、离线队列、Rust 数据结构、同步协议、备份和加密恢复契约。除 R28 的导出索引 migration 外,不新增数据库结构、全局必填配置或权限。+- 保留 MCP 导入、普通编辑、离线队列、Rust 数据结构、同步协议、备份和加密恢复契约。除 R28/R39 的分页索引 migration 外,不新增数据库结构、全局必填配置或权限。- 不新增移动端导入入口、跨次自动合并、退出应用后续传、Obsidian 插件或跨笔记语法转换、远程图片抓取及全库事务回滚。- 不改变安装器、自动升级、版本号和发布/部署机制。既有 Cloudflare 升级复用原 Worker、D1、R2、管理员和认证 Secret;不重置密码或替换实例。测试使用自造来源和隔离目录,不修改个人知识库。@@ -77,6 +77,7 @@ Markdown 位置识别和序列化复用成熟解析器,不继续维护正则| --- | --- | --- || R27 | 笔记本存在性校验使用按 ID、工作区及未删除状态的轻量查询,覆盖导入/普通创建、复制、移动/合并及创建子目录、更换父级的布尔校验。不改变原 `getNotebook` 的统计字段、权限或不存在时的错误。 | 先用真实导入与目录创建复现累计聚合,再验证大量笔记/目录不重复计数;缺失、跨工作区和已删除目标仍按原契约拒绝。 || R28 | Web JSON/ZIP 和 Markdown 导出增加可选的不透明 `cursor` 与 `nextCursor`,按原创建时间、ID 顺序续页。游标包含续页位置和已遍历偏移;携带游标时默认不重复统计,显式 `includeTotal=1` 仍可取总数。仅新游标格式错误返回 `invalid_export_cursor` / HTTP 400。旧 OFFSET 请求、字段、默认值和排序保持;新版消费者在服务端未提供游标时继续原 OFFSET 路径,不增加旧服务端前置条件。新增递增 migration,为笔记添加 `(workspace_id, is_deleted, created_at, id)` 索引。 | 先复现分页全扫,再验证完整 ZIP/Markdown 的顺序、资源及修订无遗漏/重复,覆盖相同时间和选中笔记过滤;真实旧 Schema/数据迁至新索引后原字段与旧调用仍可用;本地完整续页累计工作量近似线性,相关查询走索引。 |+| R39 | 新增 `0086_memo_id_paging_index.sql`,仅为 `memos` 添加 `(workspace_id, id)` 索引,使一致性备份笔记页及首次同步按工作区和 ID 范围读取。保留 `0085`,不修改业务字段、触发器、查询、过滤条件、游标、分页大小、API 或客户端默认行为。 | 先补旧结构失败回归,再应用 migration;存量数据、完整分页、跨工作区和已删除笔记保持原契约。无预先 `ANALYZE` 的 1,000/4,000 篇规模对照应近似线性;按实际 D1 `rows_read` / `rows_written` 核对读量、索引构建和新增写入成本,旧创建时间导出继续使用 `0085` 索引。 |新增 `0085_export_cursor_index.sql` 有一次性构建读取及持续的索引写入/空间成本,不修改业务字段。主要风险为续页边界漏项或重复。应用回退可恢复旧请求方式,索引可保留;需要删除时另增 migration,不能修改已执行文件。分片上传继续沿用每资源 `P + 2` 次 R2 Class A 操作及每片最多三次尝试;不增加导入上限、必填配置、自动重试或远程读取。规模测量使用本地代表数据,不在 Cloudflare 压测。@@ -104,6 +105,12 @@ Markdown 位置识别和序列化复用成熟解析器,不继续维护正则截图和运行记录留在仓库外,PR 使用附件;仓库保留双语结论、可维护的测试及其实际消费的自造来源样本。提交历史整理前保留独立恢复副本,不改写其他分支。+## D1 分页读取优化++R39 的实施范围为新增 `(workspace_id, id)` 索引及其迁移、分页回归和读写成本验证;API、客户端、备份协议和现有 `0085` 保持不变。选择理由、索引成本、旧结构升级验收及回退见[方案第 9 节](../third-party-import-plan.zh-CN.md#9-d1-额度优化)。无变更备份探测与搜索索引去重不并入本轮。++R39 已完成正式 migration、本地旧结构升级和契约回归,实际 D1 读写行数验证了近似线性读取与每篇新增一行索引写入。未改动 API、客户端或运行时业务代码。线上迁移及完整流程用量仍归部署验收。+## 风险与回退导入是高风险数据写入:最坏结果为部分新对象、重复项、错误时间或资源引用。只追加对象,保留来源、逐项结果及提交基线;取消不撤销完成的写入,代码回退不删除已导入笔记。移除入口和新增路由后,已导入数据仍可按普通笔记使用。清理须依据实际 ID 确认,不能按名称批量删除。@@ -112,6 +119,6 @@ Markdown 位置识别和序列化复用成熟解析器,不继续维护正则## 完成核对-I01–I11、R01–R38、C01/C02 已实现,实际 diff 的行为变化均对应上述条目;没有额外输入限制、权限或同步/备份协议变更。来源格式、目录/元数据/原字节、失败重试、并发编辑保护及资源清理通过针对性验证。大库读量、导出续页、真实浏览器预检及大正文结果留存的条件与测量见[验收结论](evidence/2026-10-27-third-party-import/README.zh-CN.md)。两项类型检查、Web 构建和目标 lint 通过。+I01–I11、R01–R39、C01/C02 已实现,实际 diff 的行为变化均对应上述条目;没有额外输入限制、权限或同步/备份协议变更。来源格式、目录/元数据/原字节、失败重试、并发编辑保护及资源清理通过针对性验证。大库读量、导出续页、真实浏览器预检及大正文结果留存的条件与测量见[验收结论](evidence/2026-10-27-third-party-import/README.zh-CN.md)。两项类型检查、Web 构建和目标 lint 通过。-Cloudflare 已部署 R27–R30 与 `0085`,真实旧实例升级及数据读回结论见验收文档;R31–R38 尚未部署。此前 macOS/Linux 原生结果未覆盖最新修复。Windows 原生、USB 移动真机、旧客户端编辑、AppImage 安装/跨版本升级和 Docker 容器部署仍未验证;不得将构建或驱动级检查作为这些验收的替代。+Cloudflare 已部署 R27–R30 与 `0085`,真实旧实例升级及数据读回结论见验收文档;R31–R39 和 `0086` 尚未部署。此前 macOS/Linux 原生结果未覆盖最新修复。Windows 原生、USB 移动真机、旧客户端编辑、AppImage 安装/跨版本升级和 Docker 容器部署仍未验证;不得将构建或驱动级检查作为这些验收的替代。diff --git a/docs/changes/evidence/2026-10-27-third-party-import/README.md b/docs/changes/evidence/2026-10-27-third-party-import/README.mdindex 3c77df8f..0d16150d 100644--- a/docs/changes/evidence/2026-10-27-third-party-import/README.md+++ b/docs/changes/evidence/2026-10-27-third-party-import/README.md@@ -26,12 +26,34 @@ See the [real-source fixtures](../../../../tests/fixtures/note-import/real-apps/- Cloudflare: an earlier candidate merged with upstream 2.16.1 passed an existing-instance upgrade and actual three-source imports. The `0072` → `0084` upgrade retained four existing notes and one resource, while upstream `0078` cleared factory colors on five default notebooks. The current R27–R30 API/Web and `0085` are now deployed; all pre-upgrade business fields in 18 notes, 19 notebooks, 23 resources and the workspace remain identical. The existing authenticated session, health and OpenAPI probes pass. Deployed Settings imports one additional ENEX note with original timestamps/tags and five matching attachment payloads; PNG/SVG load and the PDF is readable.- Upgrading the original API against the same database retained existing bodies, tags, timestamps and resource bytes. Driver-level checks do not establish Docker container deployment acceptance.-Earlier native candidates do not cover the latest image/list and R31–R38 repairs. R31–R38 are not deployed to Cloudflare. Windows native, physical mobile devices, older-client editing, AppImage installation and cross-version automatic updates remain unverified. Current Cloudflare checks did not repeat password login or complete saving a Web ZIP to the local filesystem; the user can confirm these through the login and export entries.+Earlier native candidates do not cover the latest image/list and R31–R39 repairs. R31–R39 and `0086` are not deployed to Cloudflare. Windows native, physical mobile devices, older-client editing, AppImage installation and cross-version automatic updates remain unverified. Current Cloudflare checks did not repeat password login or complete saving a Web ZIP to the local filesystem; the user can confirm these through the login and export entries.## Large libraries and quotasThese are local measurements, not production D1 billing or Worker CPU data.+### Native D1 row-count comparison++Real API with local workerd D1, the complete schema through `0085`, and no preceding `ANALYZE`. Authored data includes 4,047 notes, nine notebooks, 42 revisions, 38 resources and six prompts. Add only a candidate `(workspace_id, id)` index to an isolated database copy and compare complete traversal using `meta.rows_read`; both sides exclude session authentication and resource-payload downloads:++| Flow | Current-schema rows read | With candidate index | Reduction |+| --- | ---: | ---: | ---: |+| Complete backup, 164 pages | 1,356,449 | 50,073 | 96.31% |+| Initial sync, 21 pages | 195,818 | 24,412 | 87.53% |++Complete returned data and ordering match. Backup originally reads the creation-time index's workspace prefix and sorts; bootstrap reads the archive index's workspace prefix and sorts. The candidate lets the memo paging query seek directly by workspace and ID range. Index construction reads 8,203 rows, writes 4,048 and increases database size by 224 KiB.++Migration `0086_memo_id_paging_index.sql` is implemented. A separate scale check runs the real backup/sync routes with local D1 and 1,000/4,000 fixed-body notes in each of the target and foreign workspaces. This dataset has no resources or revisions and likewise excludes authentication and resource-payload downloads:++| Flow | 1,000 notes before migration | 1,000 notes after migration | 4,000 notes after migration |+| --- | ---: | ---: | ---: |+| Full-backup rows read | 91,226 | 12,265 | 48,985 |+| Initial-sync rows read | 15,030 | 6,034 | 24,109 |++Four times the notes consumes 3.99/4.00 times the reads. Complete pre/post-migration responses match and exclude foreign-workspace data. A create-service comparison changes from 25 to 26 written rows; ordinary body updates remain at 26. The index adds only the expected new-memo maintenance cost. Schema-upgrade regression also covers interleaved workspace IDs, deleted notes, resources/revisions and legacy default responses. All 32 backup-consistency, creation-time export and sync-contract checks pass.++`0086` has not run in production; deployed construction costs and complete-flow usage remain unverified. See the [optimization scope and rollback](../../../third-party-import-plan.md#9-d1-quota-optimization). Aggregate-row contributions and SQLite VM instructions below retain their own definitions rather than substituting for D1 row counts.+| Notes imported into one notebook | Pre-repair memo-row aggregate contributions | After R27 || --- | ---: | ---: || 100 | 9,900 | 0 |diff --git a/docs/changes/evidence/2026-10-27-third-party-import/README.zh-CN.md b/docs/changes/evidence/2026-10-27-third-party-import/README.zh-CN.mdindex 13717ebb..48790788 100644--- a/docs/changes/evidence/2026-10-27-third-party-import/README.zh-CN.md+++ b/docs/changes/evidence/2026-10-27-third-party-import/README.zh-CN.md@@ -26,12 +26,34 @@ Web 设置入口及此前隔离的原生运行时已验证以下自造来源样- Cloudflare:此前合并上游 2.16.1 的候选完成既有实例升级与三来源实际导入。`0072` → `0084` 保持 4 篇既有笔记及 1 个资源,5 个默认笔记本配色按上游 `0078` 清除。当前 R27–R30 的 API/Web 及 `0085` 已部署,升级前的 18 篇笔记、19 个笔记本、23 个资源和工作区业务字段保持一致;既有登录会话、健康检查和 OpenAPI 探针通过。线上设置入口再导入 1 篇 ENEX,原始时间、标签和 5 个附件字节一致,PNG/SVG 加载且 PDF 可阅读。- 同一数据库从原 API 基线升级后,已有正文、标签、时间和资源字节保持一致。Docker 驱动的接口检查不等同于 Docker 容器部署验收。-旧原生候选不覆盖最新图片/列表及 R31–R38 修复;R31–R38 尚未部署到 Cloudflare。Windows 原生、移动端真机、旧客户端编辑、AppImage 安装及跨版本自动升级仍未验证。当前 Cloudflare 验证未重新输入密码登录,Web ZIP 的本机保存也未完成;这两项需用户通过登录页和导出入口确认。+旧原生候选不覆盖最新图片/列表及 R31–R39 修复;R31–R39 和 `0086` 尚未部署到 Cloudflare。Windows 原生、移动端真机、旧客户端编辑、AppImage 安装及跨版本自动升级仍未验证。当前 Cloudflare 验证未重新输入密码登录,Web ZIP 的本机保存也未完成;这两项需用户通过登录页和导出入口确认。## 大库与额度以下均为本地测量,不是线上 D1 账单或 Worker CPU 数据。+### D1 原生行数对照++真实 API 与本地 workerd D1,完整 Schema 至 `0085`,未预先执行 `ANALYZE`。自造数据包含 4,047 篇笔记、9 个笔记本、42 个修订、38 个资源和 6 个提示词。仅在隔离数据库副本中增加 `(workspace_id, id)` 候选索引,用 `meta.rows_read` 对比完整遍历;两边均不含会话认证和资源字节下载开销:++| 流程 | 现有 Schema 读取行数 | 候选索引后 | 降幅 |+| --- | ---: | ---: | ---: |+| 完整备份,164 页 | 1,356,449 | 50,073 | 96.31% |+| 首次同步,21 页 | 195,818 | 24,412 | 87.53% |++两边完整返回数据和顺序一致。备份原计划按创建时间索引的工作区前缀读取后排序,首次同步按归档索引的工作区前缀读取后排序;候选索引使笔记主分页直接按工作区和 ID 范围定位。索引构建读取 8,203 行、写入 4,048 行,数据库大小增加 224 KiB。++正式 migration `0086_memo_id_paging_index.sql` 已实现。另以目标和外部工作区各 1,000/4,000 篇固定正文笔记,通过实际备份/同步路由与本地 D1 核对规模增长;这组笔记没有资源或修订,认证和资源字节下载同样不计入:++| 流程 | 1,000 篇迁移前 | 1,000 篇迁移后 | 4,000 篇迁移后 |+| --- | ---: | ---: | ---: |+| 完整备份读取行数 | 91,226 | 12,265 | 48,985 |+| 首次同步读取行数 | 15,030 | 6,034 | 24,109 |++四倍笔记的读取分别增至 3.99/4.00 倍;迁移前后完整返回数据一致,外部工作区数据不混入。创建服务写入对照由 25 行变为 26 行;普通正文更新前后均为 26 行。索引只增加预期的新增笔记维护成本。Schema 升级回归还覆盖交错工作区 ID、已删除笔记、资源/修订和旧客户端默认响应;备份一致性、旧创建时间导出及同步契约的 32 项检查通过。++`0086` 尚未在线上执行,部署后的构建成本和完整流程用量仍待验证;实施边界与回退见[优化方案](../../../third-party-import-plan.zh-CN.md#9-d1-额度优化)。下方聚合行参与次数和 SQLite VM 指令保留各自测量口径,不能代替 D1 行数。+| 单目录导入笔记数 | 修前累计笔记聚合行参与次数 | R27 修后 || --- | ---: | ---: || 100 | 9,900 | 0 |diff --git a/docs/third-party-import-plan.md b/docs/third-party-import-plan.mdindex e78cec59..18c14c4e 100644--- a/docs/third-party-import-plan.md+++ b/docs/third-party-import-plan.md@@ -2,7 +2,7 @@[简体中文](third-party-import-plan.zh-CN.md)-Related: [Issue #657](https://github.com/tianma-if/edgeever/issues/657). Implemented contract; [acceptance scope and results](changes/evidence/2026-10-27-third-party-import/README.md).+Related: [Issue #657](https://github.com/tianma-if/edgeever/issues/657). Sections 1–8 describe the implemented contract; section 9 describes the implemented D1 optimization and acceptance. [Acceptance scope and results](changes/evidence/2026-10-27-third-party-import/README.md).## 1. Approach and first-release boundaries@@ -140,4 +140,50 @@ Feature rollback can remove the new entry and REST routes without changing old eThe [acceptance conclusions](changes/evidence/2026-10-27-third-party-import/README.md) distinguish the current Web/API candidate, earlier macOS/Linux native flows, existing-instance Cloudflare upgrades and large-library measurements. Real Evernote 11.40.2 ENEX, Obsidian 1.14.5 Vault and Joplin 3.7.21 RAW/JEX/both Markdown exports preserve declared bodies, hierarchy, source timestamps and original resource bytes.-R27–R30 and the `0085` index are deployed to the existing Cloudflare instance. R31–R38 are verified locally and are not deployed; earlier native acceptance does not cover these latest repairs. Windows native, physical mobile devices, older-client editing, AppImage installation/cross-version updates and Docker container deployment remain unverified. Fresh password login and saving a Web ZIP to the local filesystem were not covered by the latest Cloudflare acceptance; they require checks through those entries when that deployment is accepted.+R27–R30 and the `0085` index are deployed to the existing Cloudflare instance. R31–R39 and `0086` are verified locally and are not deployed; earlier native acceptance does not cover these latest repairs. Windows native, physical mobile devices, older-client editing, AppImage installation/cross-version updates and Docker container deployment remain unverified. Fresh password login and saving a Web ZIP to the local filesystem were not covered by the latest Cloudflare acceptance; they require checks through those entries when that deployment is accepted.++## 9. D1 quota optimization++### Add the paging index first++Limit this iteration to a `(workspace_id, id)` memo index supporting consistent-backup memo pages and initial sync. It offers the largest measured benefit without API, client or backup-protocol changes. Treat unchanged-backup probing and search-index deduplication as separate follow-ups rather than changing these established paths together.++Both endpoints continue by `id` without an index matching the workspace and ID range. Local D1 chooses the creation-time index's workspace prefix for backup and the archive index's workspace prefix for bootstrap, then repeatedly scans and sorts. Earlier SQLite VM measurements do not establish D1 row consumption in this environment; the [native row-count comparison](changes/evidence/2026-10-27-third-party-import/README.md#native-d1-row-count-comparison) supplements them.++For 4,047 notes, adding only the candidate index reduces full-backup reads from 1,356,449 to 50,073 and initial-sync reads from 195,818 to 24,412. Complete returned data and ordering match. These are local workerd D1 `meta.rows_read` values; equivalent production-scale validation remains outstanding.++### Implementation scope and choices++Add incrementally numbered migration `0086_memo_id_paging_index.sql`. Its only statement is:++```sql+CREATE INDEX idx_memos_workspace_id ON memos(workspace_id, id);+```++- Retain executed migration `0085` and its creation-time index: Web JSON/Markdown exports still order by creation time and ID.+- Choose `(workspace_id, id)`, not `(workspace_id, is_deleted, id)`: bootstrap includes deleted notes and does not constrain `is_deleted`, so the latter cannot serve both endpoints with a contiguous ID-range lookup. Backup retains its existing deleted-note filter.+- Retain queries, cursors, page sizes, response fields and ordering. Cloudflare D1 and self-hosted SQLite use the same migration; existing clients need no new parameters.+- Do not force `INDEXED BY` or substitute periodic `ANALYZE` for the missing index: neither an incompatible index order nor changing statistics should be required for this access path.+- Add no import limits, quota configuration, retries, scheduling delays or fallback reads. Retain idempotency records, audit events and concurrency protection.++### Read/write costs and risks++Building the candidate index on this dataset reads 8,203 rows, writes 4,048 and increases database size by 224 KiB. With the actual migration, create-service writes increase from 25 to 26 rows while ordinary body updates remain at 26. Deletion and workspace/ID changes also maintain the index. Import write budgets must still account for existing tags, resources, audits and the additional index maintenance.++This is a high-risk storage migration. Risks include database queuing during construction, insufficient space or remaining daily reads/writes, and other queries choosing a worse plan with the new index. Record database size and remaining usage before migration; stop subsequent deployment steps if migration fails. The migration adds only an index, without rewriting business fields or triggers.++### Acceptance and rollback++1. Reuse the complete-schema fixture in `backup-consistency.test.mjs` to exercise backup and sync routes in the same workspace context. Reproduce repeated scanning/sorting in the old schema before applying the migration, and retain existing `sync-routes.test.mjs` contract checks. Do not run `ANALYZE` beforehand and mask the defect.+2. Upgrade existing data and compare complete backup, bootstrap and legacy default calls: notes, notebooks, bodies, tags, timestamps, resource metadata, revisions and order must match. Cover multiple workspaces with interleaved IDs, deleted notes and page boundaries. Retain authorization, consistency-version checks and rejection when the source changes.+3. Reuse the valid 4,047-note isolated comparison and check only affected premises against the final migration. Add a scale regression with fixed-size content at 1,000/4,000 notes: cumulative memo-page reads should be approximately linear, with four times the notes consuming at most five times the reads. Plans should seek by workspace and ID range instead of rescanning on every page. Check the paging query itself, not legitimate sorting inside existing fallback subqueries. Measure actual `rows_read` / `rows_written`, including new writes and creation-time exports to detect regressions.+4. Deployment acceptance distinguishes prior-instance migration, API data readback and D1 usage observation. Local prior-schema upgrade, API contracts and the actual migration’s scale/write-cost checks pass; deployed construction costs, plans and complete-flow row counts remain to be recorded. Reuse applicable native archive-compatibility evidence for unchanged behavior.++Application rollback does not remove the index. If the index causes a regression, add a later migration dropping it while retaining business data. Do not edit executed migrations or remove the existing creation-time index. Retain the previous Worker version, database backup and independent recovery entry before deployment.++### Follow-up candidates outside this iteration++| Scenario | Candidate and evidence | Reason to defer |+| --- | --- | --- |+| Unchanged scheduled backup | Compare workspace identity and backup revision cheaply before reading full counts when changed. The current unchanged check still reads 12,313 rows at 4,047 notes. Retain old endpoint responses and pre/post-page consistency checks. | Saves one due check rather than repeated page scans. Backup API/native Worker compatibility needs its own design and acceptance. |+| Updates with unchanged searchable content | Compare title, plain text and tags before updating an existing search document, while still inserting missing documents. An isolated SQL guard reduces four written rows to one, not zero because of autoincrement-sequence maintenance. | Smaller benefit than the paging index; the shared helper also serves ordinary editing, restoration and tag operations. Validate those consumers separately rather than changing global behavior for a local import gain. |diff --git a/docs/third-party-import-plan.zh-CN.md b/docs/third-party-import-plan.zh-CN.mdindex a113a84a..6c9ee29f 100644--- a/docs/third-party-import-plan.zh-CN.md+++ b/docs/third-party-import-plan.zh-CN.md@@ -2,7 +2,7 @@[English](third-party-import-plan.md)-关联:[Issue #657](https://github.com/tianma-if/edgeever/issues/657)。本文为已实现契约;[验收范围与结论](changes/evidence/2026-10-27-third-party-import/README.zh-CN.md)。+关联:[Issue #657](https://github.com/tianma-if/edgeever/issues/657)。第 1–8 节为已实现契约;第 9 节为已实施的 D1 优化范围与验收。[验收范围与结论](changes/evidence/2026-10-27-third-party-import/README.zh-CN.md)。## 1. 实现方向与首版边界@@ -140,4 +140,50 @@ ENEX 第一遍只扫描元数据和资源大小,确认后第二遍按文件顺[验收结论](changes/evidence/2026-10-27-third-party-import/README.zh-CN.md)区分当前 Web/API 候选、此前 macOS/Linux 原生流程、Cloudflare 旧实例升级和大库测量。Evernote 11.40.2 原始 ENEX、Obsidian 1.14.5 Vault 及 Joplin 3.7.21 RAW/JEX/两种 Markdown 导出均按声明保留正文、层级、原始时间及资源字节。-R27–R30 和 `0085` 索引已部署到既有 Cloudflare 实例。R31–R38 已在本地验证、尚未部署;此前原生验收未覆盖最新修复。Windows 原生、移动端真机、旧客户端编辑、AppImage 安装/跨版本升级及 Docker 容器部署仍未验证。最近一次 Cloudflare 验收未覆盖密码重新登录和 Web ZIP 本机保存,接受该次部署时需通过对应入口检查。+R27–R30 和 `0085` 索引已部署到既有 Cloudflare 实例。R31–R39 和 `0086` 已在本地验证、尚未部署;此前原生验收未覆盖最新修复。Windows 原生、移动端真机、旧客户端编辑、AppImage 安装/跨版本升级及 Docker 容器部署仍未验证。最近一次 Cloudflare 验收未覆盖密码重新登录和 Web ZIP 本机保存,接受该次部署时需通过对应入口检查。++## 9. D1 额度优化++### 先补分页索引++本轮只为笔记增加 `(workspace_id, id)` 索引,覆盖一致性备份的笔记分页和首次同步。收益最大,且不需要改 API、客户端或备份协议。无变更备份探测和搜索索引去重作为后续独立优化,不一起改动成熟链路。++当前两个入口均按 `id` 续页,但缺少匹配工作区和 ID 范围的索引。本地 D1 的备份查询选择创建时间索引的工作区前缀,首次同步选择归档索引的工作区前缀,然后重复扫描、排序。此前 SQLite VM 测量不能代替这一环境的 D1 行数,相关结论以[原生行数对照](changes/evidence/2026-10-27-third-party-import/README.zh-CN.md#d1-原生行数对照)补充。++4,047 篇笔记的隔离对照中,仅添加候选索引,完整备份读取从 1,356,449 行降至 50,073 行,首次同步从 195,818 行降至 24,412 行;完整返回数据和顺序一致。这些是本地 workerd D1 的 `meta.rows_read`,尚未完成线上同规模验证。++### 实现范围与选择++新增递增 migration `0086_memo_id_paging_index.sql`,只包含:++```sql+CREATE INDEX idx_memos_workspace_id ON memos(workspace_id, id);+```++- 保留已执行的 `0085` 及其创建时间索引:Web JSON/Markdown 导出仍按创建时间、ID 排序。+- 选择 `(workspace_id, id)` 而非 `(workspace_id, is_deleted, id)`:首次同步需要包含已删除笔记,不约束 `is_deleted`;后者不能让两个入口共用连续的 ID 范围查找。备份继续按原条件过滤已删除笔记。+- 保持现有查询、游标、分页大小、响应字段与顺序;Cloudflare D1 和自托管 SQLite 共用同一 migration。旧客户端无需新增参数。+- 不强制 `INDEXED BY`,也不以定期 `ANALYZE` 代替缺失索引;避免依赖另一种不匹配顺序或变化中的统计信息。+- 不增加导入上限、额度配置、自动重试、调度延迟或降级读取,不删幂等记录、审计或并发保护。++### 读写成本与风险++上述数据的候选索引构建读取 8,203 行、写入 4,048 行,数据库大小增加 224 KiB。正式迁移的创建服务对照由 25 行写入增至 26 行,普通正文更新前后同为 26 行;删除或改变工作区/ID 时也需要维护索引。导入写预算仍需计入原有标签、资源、审计及新增索引维护。++这是高风险存储迁移。主要风险是构建期间的数据库排队、空间或当日读写余量不足,以及其他查询选择新索引后退化。实施前记录数据库大小与读写余量;迁移失败时停止后续部署步骤。迁移只增加索引,不重写业务字段或触发器。++### 验收与回退++1. 复用 `backup-consistency.test.mjs` 的完整 Schema 夹具,在同一工作区上下文验证备份与同步路由;旧结构复现分页主查询重复扫描/排序,再应用新 migration。既有 `sync-routes.test.mjs` 继续检查同步合同。不要预先执行 `ANALYZE` 掩盖问题。+2. 验证旧结构和存量数据升级后,完整备份、首次同步及旧客户端默认调用的笔记、目录、正文、标签、时间、资源元数据、修订与顺序一致。覆盖交错 ID 的多个工作区、已删除笔记和跨页边界;权限、一致性版本检查及来源变更时的拒绝行为保持原样。+3. 复用当前有效的 4,047 篇隔离对照;对最终 migration 的受影响部分补验。新增规模回归比较固定内容的 1,000/4,000 篇:笔记分页累计读量应近似线性,四倍笔记不超过五倍读取;查询计划应直接按工作区和 ID 范围定位,而非每页重扫。只检查分页主查询,不能把既有回退子查询的排序误判为分页退化。按实际 `rows_read` / `rows_written` 计量,同时确认新写入和创建时间导出未退化。+4. 实际部署验收区分旧实例迁移、API 数据读回与 D1 用量观察。当前本地旧结构升级、接口契约和正式 migration 的规模/写入成本检查通过;部署后的索引构建成本、计划和完整流程读量仍需记录。未变化的原生归档兼容证据继续复用。++应用回退不会自动移除索引。若索引引起退化,新增后续 migration 删除该索引,保留业务数据;不修改已执行 migration,不删除已有创建时间索引。部署前保留旧 Worker 版本、数据库备份及独立恢复入口。++### 后续候选,不纳入本轮推荐范围++| 场景 | 候选方向与证据 | 暂后原因 |+| --- | --- | --- |+| 无变更自动备份 | 先轻量比较工作区身份和备份版本,变更后再取完整计数;当前 4,047 篇的无变更检查仍读取 12,313 行。旧接口响应和分页前后的一致性检查须保留。 | 节省一次到期检查,而非每页扫描;涉及备份 API 与原生 Worker 的新旧版本协作,应独立设计和验收。 |+| 可搜索内容未变的更新 | 搜索文档 upsert 比较标题、纯文本和标签,仅实际变化时更新;缺失文档仍应插入。隔离 SQL 对照由 4 行写入降到 1 行,因自增序列维护并非零写入。 | 收益小于分页索引,且 helper 被普通编辑、恢复和标签操作共用;需另行验证这些消费方,不为导入局部收益直接改全局行为。 |diff --git a/migrations/0086_memo_id_paging_index.sql b/migrations/0086_memo_id_paging_index.sqlnew file mode 100644index 00000000..251e6dd0--- /dev/null+++ b/migrations/0086_memo_id_paging_index.sql@@ -0,0 +1,3 @@+-- Seek backup and bootstrap pages by workspace and memo ID, including deleted notes for sync.+CREATE INDEX idx_memos_workspace_id+ ON memos(workspace_id, id);