AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
fix: avoid redundant full counts across pagination and task polling
tianma-if90ae22689450
diff --git a/apps/api/src/backup-consistency.test.mjs b/apps/api/src/backup-consistency.test.mjsindex 7964ea7e..ab3ce5ef 100644--- a/apps/api/src/backup-consistency.test.mjs+++ b/apps/api/src/backup-consistency.test.mjs@@ -102,6 +102,37 @@ describe("consistent backup reads across a real schema upgrade", () => {expect(queries.filter(isFullCount)).toHaveLength(1);});+ for (const endpoint of ["backups/json", "exports/markdown"]) test(`${endpoint} can traverse continuation pages without recounting or leaking lookahead resources`, async () => {+ const db = await installation();+ const memos = db.sqlite.query("SELECT id FROM memos WHERE workspace_id = 'ws_default' AND is_deleted = 0 ORDER BY created_at, id").all();+ for (const [index, memo] of memos.entries()) {+ db.sqlite.query("INSERT INTO resources (id,memo_id,object_key,kind,byte_size) VALUES (?,?,?,'attachment',4)").run(`resource-${index}`, memo.id, `object-${index}`);+ }+ const queries = [];+ const prepare = db.storage.db.prepare.bind(db.storage.db);+ db.storage.db.prepare = (sql) => { queries.push(sql); return prepare(sql); };+ let offset = 0;+ const ids = [];+ const resourceIds = [];+ do {+ const response = await db.request(`/api/v1/${endpoint}?limit=1&offset=${offset}&includeTotal=0`);+ expect(response.status).toBe(200);+ const page = await response.json();+ if (offset === 0) expect(page.totalCount).toBe(memos.length);+ else expect(page.totalCount).toBeUndefined();+ expect(page.memos).toHaveLength(1);+ expect(page.resources.every((resource) => resource.memoId === page.memos[0].id)).toBe(true);+ ids.push(...page.memos.map((memo) => memo.id));+ resourceIds.push(...page.resources.map((resource) => resource.id));+ offset = page.nextOffset;+ } while (offset !== null);+ expect(ids).toEqual(memos.map((memo) => memo.id));+ expect(new Set(resourceIds).size).toBe(memos.length);+ expect(queries.filter((sql) => /COUNT\s*\(/i.test(sql))).toHaveLength(1);+ const legacy = await (await db.request(`/api/v1/${endpoint}?limit=1&offset=1`)).json();+ expect(legacy.totalCount).toBe(memos.length);+ });+for (const change of ["revision", "identity"]) test(`rejects a ${change} change while assembling a page`, async () => {const db = await installation();const state = await (await db.request("/api/v1/backups/state")).json();diff --git a/apps/api/src/backup-routes.ts b/apps/api/src/backup-routes.tsindex 20af7e4d..7aada387 100644--- a/apps/api/src/backup-routes.ts+++ b/apps/api/src/backup-routes.ts@@ -188,16 +188,17 @@ const listBackupPage = async (workspaceId: string,limit: number,offset: number,+ includeTotal = true,) => Promise.all([database.prepare(`${MEMO_EXPORT_SELECT_SQL}WHERE m.workspace_id = ? AND m.is_deleted = 0ORDER BY m.created_at ASC, m.id ASCLIMIT ? OFFSET ?`,- ).bind(workspaceId, limit, offset).all<BackupMemoDetailRow>(),- database.prepare(+ ).bind(workspaceId, includeTotal ? limit : limit + 1, offset).all<BackupMemoDetailRow>(),+ includeTotal ? database.prepare(`SELECT COUNT(*) AS count FROM memos WHERE workspace_id = ? AND is_deleted = 0`,- ).bind(workspaceId).first<{ count: number }>(),+ ).bind(workspaceId).first<{ count: number }>() : Promise.resolve(null),]);const emptyMarkdownExportPage = async () => [@@ -211,13 +212,14 @@ const listMarkdownExportPage = async (limit: number,offset: number,memoIds: string[] | null,+ includeTotal = true,) => {if (memoIds && memoIds.length === 0) {return emptyMarkdownExportPage();}if (!memoIds) {- return listBackupPage(database, workspaceId, limit, offset);+ return listBackupPage(database, workspaceId, limit, offset, includeTotal);}// json_each keeps the D1 bound-parameter count fixed (workspace + JSON + limit/offset).@@ -229,12 +231,12 @@ const listMarkdownExportPage = async (AND m.id IN (SELECT value FROM json_each(?))ORDER BY m.created_at ASC, m.id ASCLIMIT ? OFFSET ?`,- ).bind(workspaceId, memoIdsJson, limit, offset).all<BackupMemoDetailRow>(),- database.prepare(+ ).bind(workspaceId, memoIdsJson, includeTotal ? limit : limit + 1, offset).all<BackupMemoDetailRow>(),+ includeTotal ? database.prepare(`SELECT COUNT(*) AS count FROM memosWHERE workspace_id = ? AND is_deleted = 0AND id IN (SELECT value FROM json_each(?))`,- ).bind(workspaceId, memoIdsJson).first<{ count: number }>(),+ ).bind(workspaceId, memoIdsJson).first<{ count: number }>() : Promise.resolve(null),]);};@@ -322,6 +324,7 @@ export const registerBackupRoutes = (const limit = dependencies.clampNumber(Number(context.req.query("limit") ?? 50), 1, 100);const offset = dependencies.clampNumber(Number(context.req.query("offset") ?? 0), 0, 1_000_000);const parsedIds = parseMarkdownExportMemoIds(context.req.query("ids"));+ const includeTotal = offset === 0 || context.req.query("includeTotal") !== "0";if (parsedIds.error === "too_many") {return badRequest(context, `Export at most ${MAX_MARKDOWN_EXPORT_MEMO_IDS} notes at a time.`);}@@ -332,8 +335,10 @@ export const registerBackupRoutes = (limit,offset,parsedIds.ids,+ includeTotal,);- const memoIds = memoRows.results.map((row) => row.id);+ const page = memoRows.results.slice(0, limit);+ const memoIds = page.map((row) => row.id);let resources: Resource[] = [];if (memoIds.length > 0) {@@ -348,13 +353,13 @@ export const registerBackupRoutes = (resources = resourceRows.results.map(mapResource);}- const totalCount = totalRow?.count ?? memoRows.results.length;+ const totalCount = totalRow?.count ?? page.length;return context.json({- memos: memoRows.results.map(dependencies.mapMemoDetail),+ memos: page.map(dependencies.mapMemoDetail),resources,- totalCount,- nextOffset: offset + memoRows.results.length < totalCount- ? offset + memoRows.results.length+ ...(includeTotal ? { totalCount } : {}),+ nextOffset: (includeTotal ? offset + page.length < totalCount : memoRows.results.length > limit)+ ? offset + page.length: null,});});@@ -366,8 +371,10 @@ export const registerBackupRoutes = (const limit = dependencies.clampNumber(Number(context.req.query("limit") ?? 25), 1, 50);const offset = dependencies.clampNumber(Number(context.req.query("offset") ?? 0), 0, 1_000_000);const workspaceId = getWorkspaceId(context);- const [memoRows, totalRow] = await listBackupPage(context.env.storage.db, workspaceId, limit, offset);- const memoIds = memoRows.results.map((row) => row.id);+ const includeTotal = offset === 0 || context.req.query("includeTotal") !== "0";+ const [memoRows, totalRow] = await listBackupPage(context.env.storage.db, workspaceId, limit, offset, includeTotal);+ const page = memoRows.results.slice(0, limit);+ const memoIds = page.map((row) => row.id);let resources: Resource[] = [];let revisions: JsonBackupRevision[] = [];@@ -393,14 +400,14 @@ export const registerBackupRoutes = (revisions = revisionRows.results.map(mapJsonBackupRevision);}- const totalCount = totalRow?.count ?? memoRows.results.length;+ const totalCount = totalRow?.count ?? page.length;return context.json({- memos: memoRows.results.map(dependencies.mapMemoDetail),+ memos: page.map(dependencies.mapMemoDetail),resources,revisions,- totalCount,- nextOffset: offset + memoRows.results.length < totalCount- ? offset + memoRows.results.length+ ...(includeTotal ? { totalCount } : {}),+ nextOffset: (includeTotal ? offset + page.length < totalCount : memoRows.results.length > limit)+ ? offset + page.length: null,});});diff --git a/apps/api/src/memo-list-service.test.mjs b/apps/api/src/memo-list-service.test.mjsindex c9576e29..5e8b1bdf 100644--- a/apps/api/src/memo-list-service.test.mjs+++ b/apps/api/src/memo-list-service.test.mjs@@ -38,6 +38,19 @@ const createDatabase = ({ rows = [], totalCount = rows.length } = {}) => {};describe("memo list service", () => {+ for (const query of [undefined, "hello", "hello world", "%"]) test(`skips exact counts for opted-in ${query ?? "unfiltered"} result pages, while retaining lookahead cursors`, async () => {+ const database = createDatabase({ rows: [memoRow("memo_1"), memoRow("memo_2"), memoRow("memo_3")] });+ const result = await listMemos(database, { workspaceId: "ws_1", query, limit: 2, includeTotal: false });+ expect(result.memos.map((memo) => memo.id)).toEqual(["memo_1", "memo_2"]);+ expect(result.totalCount).toBeUndefined();+ expect(result.nextCursor).toBeString();+ expect(database.calls).toHaveLength(1);+ expect(database.calls[0].sql).not.toMatch(/COUNT\s*\(/i);+ const end = await listMemos(createDatabase({ rows: [memoRow("memo_3")] }), { workspaceId: "ws_1", query, limit: 2, includeTotal: false, cursor: result.nextCursor });+ expect(end.memos.map((memo) => memo.id)).toEqual(["memo_3"]);+ expect(end.nextCursor).toBeNull();+ });+test("identifies diagram notes without exposing their source as the note type", () => {const content_markdown = serializeDiagramDocument(createDefaultDiagramDocument("architecture"));diff --git a/apps/api/src/memo-list-service.ts b/apps/api/src/memo-list-service.tsindex 929339b4..0a85e1a8 100644--- a/apps/api/src/memo-list-service.ts+++ b/apps/api/src/memo-list-service.ts@@ -45,6 +45,7 @@ type MemoListCursor = {export type ListMemosInput = {workspaceId: string;+ includeTotal?: boolean;notebookId?: string;includeNotebookDescendants?: boolean;query?: string;@@ -58,7 +59,7 @@ export type ListMemosInput = {export type ListMemosResult = {memos: MemoSummary[];- totalCount: number;+ totalCount?: number;nextCursor: string | null;};@@ -244,7 +245,7 @@ export const listMemos = async (const page = rows.slice(0, limit);return {memos: page.map(mapMemoSummary),- totalCount: totalCount ?? page.length,+ ...(input.includeTotal === false ? {} : { totalCount: totalCount ?? page.length }),nextCursor: rows.length > limit? encodeMemoListCursor(page[page.length - 1], sort, includeTrash): null,@@ -287,7 +288,7 @@ export const listMemos = async (ORDER BY ${getOrderBy()}LIMIT ?`,).bind(...searchPrefix, ...cursorBinds, pageLimit).all<MemoSummaryRow>(),- database.prepare(+ input.includeTotal === false ? Promise.resolve(null) : database.prepare(`WITH raw_matches(memo_id) AS (SELECT memo_idFROM memos_fts@@ -332,7 +333,7 @@ export const listMemos = async (ORDER BY ${getOrderBy()}LIMIT ?`,).bind(...searchCursorBinds, pageLimit).all<MemoSummaryRow>(),- database.prepare(+ input.includeTotal === false ? Promise.resolve(null) : database.prepare(`SELECT COUNT(*) AS countFROM memos mINNER JOIN memo_contents mc ON mc.memo_id = m.id@@ -353,7 +354,7 @@ export const listMemos = async (ORDER BY ${getOrderBy()}LIMIT ?`,).bind(...cursorBinds, pageLimit).all<MemoSummaryRow>(),- database.prepare(+ input.includeTotal === false ? Promise.resolve(null) : database.prepare(`SELECT COUNT(*) AS countFROM memos mWHERE ${baseConditions.join(" AND ")}`,diff --git a/apps/api/src/memo-routes.test.mjs b/apps/api/src/memo-routes.test.mjsindex fabac15e..6d22acf4 100644--- a/apps/api/src/memo-routes.test.mjs+++ b/apps/api/src/memo-routes.test.mjs@@ -121,6 +121,7 @@ describe("memo route contracts", () => {expect(response.status).toBe(200);expect(received[1]).toEqual({workspaceId: "ws_1",+ includeTotal: true,notebookId: "nb_1",includeNotebookDescendants: true,query: "hello",@@ -133,6 +134,15 @@ describe("memo route contracts", () => {});});+ test("forwards an explicit count opt-out for bounded result consumers", async () => {+ let input;+ const dependencies = createDependencies({ listMemos: async (_db, value) => { input = value; return { memos: [], nextCursor: null }; } });+ const response = await createApp(dependencies).request("/api/v1/memos?q=hello&includeTotal=0", {}, environment);+ expect(response.status).toBe(200);+ expect(input.includeTotal).toBe(false);+ expect((await response.json()).totalCount).toBeUndefined();+ });+test("validates and delegates memo creation", async () => {let received;const dependencies = createDependencies({diff --git a/apps/api/src/memo-routes.ts b/apps/api/src/memo-routes.tsindex 7b264923..27d2744b 100644--- a/apps/api/src/memo-routes.ts+++ b/apps/api/src/memo-routes.ts@@ -141,6 +141,7 @@ export const registerMemoRoutes = (return context.json(await dependencies.listMemos(context.env.storage.db, {workspaceId: getWorkspaceId(context),+ includeTotal: context.req.query("includeTotal") !== "0",notebookId: context.req.query("notebookId"),includeNotebookDescendants: context.req.query("includeDescendants") === "1",query: context.req.query("q"),diff --git a/apps/api/src/notebook-service.ts b/apps/api/src/notebook-service.tsindex 2f87e66e..5fce3d29 100644--- a/apps/api/src/notebook-service.ts+++ b/apps/api/src/notebook-service.ts@@ -291,16 +291,15 @@ const listActiveNotebookTree = async (db: DatabaseAdapter, workspaceId: string,return rows.results;};-const countActiveMemosInNotebooks = async (db: DatabaseAdapter, workspaceId: string, notebookIds: string[]) => {- let total = 0;+const hasActiveMemosInNotebooks = async (db: DatabaseAdapter, workspaceId: string, notebookIds: string[]) => {for (const chunk of chunkValues(notebookIds, NOTEBOOK_DELETE_ID_CHUNK)) {const placeholders = chunk.map(() => "?").join(", ");const row = await db.prepare(- `SELECT COUNT(*) AS count FROM memos WHERE workspace_id = ? AND is_deleted = 0 AND notebook_id IN (${placeholders})`- ).bind(workspaceId, ...chunk).first<{ count: number }>();- total += Number(row?.count ?? 0);+ `SELECT 1 AS found FROM memos WHERE workspace_id = ? AND is_deleted = 0 AND notebook_id IN (${placeholders}) LIMIT 1`+ ).bind(workspaceId, ...chunk).first<{ found: number }>();+ if (row) return true;}- return total;+ return false;};export const deleteNotebookTree = async (@@ -332,7 +331,7 @@ export const deleteNotebookTree = async (const notebookIds = tree.map((notebook) => notebook.id);// Empty child notebooks are removed with the parent. Notes anywhere in the// subtree block the whole delete so a parent click cannot orphan them.- if (await countActiveMemosInNotebooks(db, workspaceId, notebookIds) > 0) {+ if (await hasActiveMemosInNotebooks(db, workspaceId, notebookIds)) {throw new AppError("notebook_not_empty","Move or delete notes in this notebook and its child notebooks before deleting it.",diff --git a/apps/api/src/scheduled-run-counts.test.mjs b/apps/api/src/scheduled-run-counts.test.mjsnew file mode 100644index 00000000..c371c96a--- /dev/null+++ b/apps/api/src/scheduled-run-counts.test.mjs@@ -0,0 +1,76 @@+import { afterEach, expect, test } from "bun:test";+import { Database } from "bun:sqlite";+import { globSync, readFileSync } from "node:fs";+import { Hono } from "hono";+import { createSelfHostedStorageAdapter } from "./self-hosted-storage-adapter.ts";+import { registerScheduledTaskRoutes } from "./scheduled-task-routes.ts";++const databases = [];+afterEach(() => databases.splice(0).forEach((db) => db.close()));+const upgradeFixture = () => {+ const db = new Database(":memory:"); databases.push(db);+ for (const file of globSync("migrations/*.sql").sort().filter((file) => file < "migrations/0084_scheduled_run_counts.sql")) db.exec(readFileSync(file, "utf8"));+ db.exec("INSERT INTO workspaces (id,name) VALUES ('ws_a','A'),('ws_b','B');");+ const now = new Date().toISOString();+ for (const [id, workspace] of [["task_a", "ws_a"], ["task_a2", "ws_a"], ["task_b", "ws_b"]]) db.query(+ `INSERT INTO scheduled_tasks (id,workspace_id,name,task_type,task_payload_json,cron_expression,timezone,executor_device_id,created_at,updated_at)+ VALUES (?,?,?,'plugin-command','{"pluginId":"fixture","commandId":"run"}','* * * * *','UTC','desktop-device-1234567890',?,?)`,+ ).run(id, workspace, id, now, now);+ let sequence = 0;+ const insert = (id, task = "task_a", workspace = "ws_a", started = now) => db.query(+ "INSERT INTO scheduled_task_runs (id,task_id,workspace_id,scheduled_for,executor_device_id,status,started_at) VALUES (?,?,?,?,?,'running',?)",+ ).run(id, task, workspace, new Date(Date.now() + ++sequence * 1000).toISOString(), "desktop-device-1234567890", started);+ insert("run_a"); insert("run_a2"); insert("run_a3", "task_a2"); insert("run_b", "task_b", "ws_b");+ const before = db.query("SELECT * FROM scheduled_task_runs ORDER BY id").all();+ db.exec(readFileSync("migrations/0084_scheduled_run_counts.sql", "utf8"));+ const assertCounts = () => {+ for (const row of db.query("SELECT id,scheduled_run_count FROM workspaces").all()) expect(row.scheduled_run_count).toBe(db.query("SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE workspace_id = ?").get(row.id).n);+ for (const row of db.query("SELECT id,workspace_id,run_count FROM scheduled_tasks").all()) expect(row.run_count).toBe(db.query("SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE task_id = ? AND workspace_id = ?").get(row.id, row.workspace_id).n);+ };+ return { db, insert, before, assertCounts };+};++test("upgrades existing task history without changing records and maintains exact transactional counts", () => {+ const { db, insert, before, assertCounts } = upgradeFixture();+ expect(db.query("SELECT * FROM scheduled_task_runs ORDER BY id").all()).toEqual(before);+ assertCounts();+ insert("run_new"); assertCounts();+ db.exec("UPDATE scheduled_task_runs SET status = 'succeeded' WHERE id = 'run_new'"); assertCounts();+ db.exec("UPDATE scheduled_task_runs SET task_id = 'task_a2' WHERE id = 'run_new'"); assertCounts();+ db.exec("UPDATE scheduled_task_runs SET workspace_id = 'ws_b', task_id = 'task_b' WHERE id = 'run_new'"); assertCounts();+ expect(() => db.transaction(() => { insert("rolled_back"); throw new Error("rollback"); })()).toThrow("rollback"); assertCounts();+ db.exec("DELETE FROM scheduled_task_runs WHERE id = 'run_a'"); assertCounts();+ db.exec("DELETE FROM scheduled_tasks WHERE id = 'task_a2'"); assertCounts();+ db.exec("UPDATE scheduled_tasks SET id = 'renamed_task' WHERE id = 'task_a'"); assertCounts();+ db.exec("UPDATE workspaces SET id = 'renamed_workspace' WHERE id = 'ws_a'"); assertCounts();+ db.exec("DELETE FROM workspaces WHERE id = 'ws_b'"); assertCounts();+});++test("ignored claims and mismatched historical scopes do not corrupt task totals", () => {+ const { db, insert, assertCounts } = upgradeFixture();+ db.exec("INSERT OR IGNORE INTO scheduled_task_runs SELECT * FROM scheduled_task_runs WHERE id = 'run_a'"); assertCounts();+ insert("scope_mismatch", "task_a", "ws_b"); assertCounts();+ db.exec("UPDATE scheduled_tasks SET workspace_id = 'ws_b' WHERE id = 'task_a'"); assertCounts();+ db.exec("UPDATE scheduled_tasks SET workspace_id = 'ws_a' WHERE id = 'task_a'"); assertCounts();+ db.exec("DELETE FROM scheduled_task_runs WHERE id = 'scope_mismatch'"); assertCounts();+});++test("history polling uses primary-key totals and indexed retention cleanup after a real schema upgrade", async () => {+ const { db, insert, assertCounts } = upgradeFixture();+ insert("expired", "task_a", "ws_a", new Date(Date.now() - 31 * 86400000).toISOString());+ const storage = createSelfHostedStorageAdapter(db, "/private/tmp/edgeever-count-unused");+ const queries = []; const prepare = storage.db.prepare.bind(storage.db);+ storage.db.prepare = (sql) => { queries.push(sql); return prepare(sql); };+ const app = new Hono();+ app.use("*", async (c, next) => { c.set("auth", { kind: "user", actorId: "owner", workspaceId: "ws_a", scopes: [], role: "owner" }); await next(); });+ registerScheduledTaskRoutes(app);+ const history = await app.request("/api/v1/scheduled-task-runs?limit=1", {}, { storage });+ expect(history.status).toBe(200);+ expect(await history.json()).toMatchObject({ totalCount: 3, nextOffset: 1 });+ const task = await app.request("/api/v1/scheduled-tasks/task_a/runs?limit=1", {}, { storage });+ expect(task.status).toBe(200);+ expect(await task.json()).toMatchObject({ totalCount: 2, nextOffset: 1 });+ expect(queries.some((sql) => /COUNT\s*\(/i.test(sql))).toBe(false);+ expect(db.query("EXPLAIN QUERY PLAN DELETE FROM scheduled_task_runs WHERE workspace_id = ? AND started_at < ?").all("ws_a", new Date().toISOString()).some((row) => row.detail.includes("idx_scheduled_task_runs_workspace_started"))).toBe(true);+ assertCounts();+});diff --git a/apps/api/src/scheduled-task-routes.ts b/apps/api/src/scheduled-task-routes.tsindex 8e38a38f..82aa52ca 100644--- a/apps/api/src/scheduled-task-routes.ts+++ b/apps/api/src/scheduled-task-routes.ts@@ -205,7 +205,7 @@ export const registerScheduledTaskRoutes = (app: Hono<AppEnv>) => {LIMIT ? OFFSET ?`,).bind(workspaceId, limit, offset).all<ScheduledTaskRunHistoryRow>(),c.env.storage.db.prepare(- "SELECT COUNT(*) AS total_count FROM scheduled_task_runs WHERE workspace_id = ?",+ "SELECT scheduled_run_count AS total_count FROM workspaces WHERE id = ?",).bind(workspaceId).first<{ total_count: number }>(),]);const totalCount = Number(countRow?.total_count ?? 0);@@ -416,7 +416,7 @@ export const registerScheduledTaskRoutes = (app: Hono<AppEnv>) => {LIMIT ? OFFSET ?`,).bind(id, workspaceId, limit, offset).all<ScheduledTaskRunRow>(),c.env.storage.db.prepare(- "SELECT COUNT(*) AS total_count FROM scheduled_task_runs WHERE task_id = ? AND workspace_id = ?",+ "SELECT run_count AS total_count FROM scheduled_tasks WHERE id = ? AND workspace_id = ?",).bind(id, workspaceId).first<{ total_count: number }>(),]);const totalCount = Number(countRow?.total_count ?? 0);diff --git a/apps/api/src/sync-routes.test.mjs b/apps/api/src/sync-routes.test.mjsindex ebc00315..9f1b9b88 100644--- a/apps/api/src/sync-routes.test.mjs+++ b/apps/api/src/sync-routes.test.mjs@@ -89,6 +89,31 @@ const createSyncIntegrationFixture = () => {};describe("sync route contracts", () => {+ test("opted-in continuation pages skip global counts and notebook aggregation, retaining the cursor and identity", async () => {+ const fixture = createSyncIntegrationFixture();+ try {+ const queries = [];+ const prepare = fixture.storage.db.prepare.bind(fixture.storage.db);+ fixture.storage.db.prepare = (sql) => { queries.push(sql); return prepare(sql); };+ const request = (query) => fixture.app.request(`/api/v1/sync/bootstrap?${query}`, {}, { storage: fixture.storage });+ const first = await (await request("limit=1&includeMetadata=0")).json();+ expect(first.totalCount).toBe(1);+ expect(first.notebooks.length).toBeGreaterThan(0);+ queries.length = 0;+ const page = await (await request("afterId=memo_0&limit=1&includeMetadata=0")).json();+ expect(page.memos.map((memo) => memo.id)).toEqual(["memo_trashed"]);+ expect(page.totalCount).toBeUndefined();+ expect(page.notebooks).toEqual([]);+ expect(page.syncIdentity).toBe(first.syncIdentity);+ expect(page.snapshotCursor).toBe(first.snapshotCursor);+ expect(page.nextAfterId).toBeNull();+ expect(queries.some((sql) => /COUNT\s*\(|GROUP BY/i.test(sql))).toBe(false);+ const legacy = await (await request("afterId=memo_0&limit=1")).json();+ expect(legacy.totalCount).toBe(1);+ expect(legacy.notebooks).toEqual(first.notebooks);+ } finally { fixture.sqlite.close(); }+ });+test("keeps bootstrap references inside the active notebook snapshot", async () => {const fixture = createSyncIntegrationFixture();try {diff --git a/apps/api/src/sync-routes.ts b/apps/api/src/sync-routes.tsindex b6a4c6d6..ee0af9f4 100644--- a/apps/api/src/sync-routes.ts+++ b/apps/api/src/sync-routes.ts@@ -103,8 +103,11 @@ export const registerSyncRoutes = (const workspaceId = getWorkspaceId(context);const limit = dependencies.clampNumber(Number(context.req.query("limit") ?? 100), 1, 200);const afterId = context.req.query("afterId")?.trim() ?? "";+ // Initial metadata is needed once; later pages can use the first snapshot.+ // Keep the historical response for clients that do not opt out.+ const includeMetadata = !afterId || context.req.query("includeMetadata") !== "0";const [notebookRows, memoRows, totalRow, cursorRow] = await Promise.all([- context.env.storage.db.prepare(+ includeMetadata ? context.env.storage.db.prepare(`SELECT n.id, ${syncNotebookParentIdSql} AS parent_id, n.name, n.slug, n.icon, n.color, n.text_color, n.color_target, n.is_bold, n.icon_inverted, n.row_inverted, n.sort_order,n.created_at, n.updated_at, COUNT(m.id) AS memo_count, MAX(m.updated_at) AS last_memo_updated_atFROM notebooks n@@ -112,7 +115,7 @@ export const registerSyncRoutes = (WHERE n.workspace_id = ? AND n.is_deleted = 0GROUP BY n.id, n.parent_id, n.name, n.slug, n.icon, n.color, n.text_color, n.color_target, n.is_bold, n.icon_inverted, n.row_inverted, n.sort_order, n.created_at, n.updated_atORDER BY n.sort_order ASC, n.name ASC`,- ).bind(workspaceId).all<NotebookRow>(),+ ).bind(workspaceId).all<NotebookRow>() : Promise.resolve({ results: [] as NotebookRow[] }),context.env.storage.db.prepare(`SELECT m.id, ${syncMemoNotebookIdSql} AS notebook_id, m.title, m.excerpt, m.tags_json, m.is_pinned,m.is_archived, m.is_deleted, m.created_at, m.updated_at, m.deleted_at, mc.revision,@@ -124,9 +127,9 @@ export const registerSyncRoutes = (ORDER BY m.id ASCLIMIT ?`,).bind(workspaceId, afterId, limit + 1).all<SyncMemoDetailRow>(),- context.env.storage.db.prepare(+ includeMetadata ? context.env.storage.db.prepare(`SELECT COUNT(*) AS count FROM memos WHERE workspace_id = ?`,- ).bind(workspaceId).first<{ count: number }>(),+ ).bind(workspaceId).first<{ count: number }>() : Promise.resolve(null),context.env.storage.db.prepare(`SELECT w.created_at AS sync_identity,COALESCE((@@ -144,7 +147,7 @@ export const registerSyncRoutes = (memos: page.map(dependencies.mapMemoDetail),snapshotCursor: cursorRow?.cursor ?? 0,syncIdentity: cursorRow?.sync_identity,- totalCount: totalRow?.count ?? page.length,+ ...(includeMetadata ? { totalCount: totalRow?.count ?? page.length } : {}),nextAfterId: memoRows.results.length > limit ? page.at(-1)?.id ?? null : null,});});diff --git a/apps/ios/EdgeEver/Data/Models/Models.swift b/apps/ios/EdgeEver/Data/Models/Models.swiftindex fb09b2e6..d4906090 100644--- a/apps/ios/EdgeEver/Data/Models/Models.swift+++ b/apps/ios/EdgeEver/Data/Models/Models.swift@@ -352,7 +352,7 @@ struct MobileSyncBootstrapPage: Codable, Sendable {var memos: [MemoDetail]var snapshotCursor: Intvar syncIdentity: String?- var totalCount: Int+ var totalCount: Int?var nextAfterId: String?}diff --git a/apps/ios/EdgeEver/Data/Network/APIClient.swift b/apps/ios/EdgeEver/Data/Network/APIClient.swiftindex 4e7934cf..be7e8ad4 100644--- a/apps/ios/EdgeEver/Data/Network/APIClient.swift+++ b/apps/ios/EdgeEver/Data/Network/APIClient.swift@@ -85,7 +85,10 @@ actor APIClient {func getMobileSyncBootstrapPage(afterId: String?, limit: Int = 200) async throws -> MobileSyncBootstrapPage {var items: [URLQueryItem] = [.init(name: "limit", value: String(limit))]- if let afterId { items.append(.init(name: "afterId", value: afterId)) }+ if let afterId, !afterId.isEmpty {+ items.append(.init(name: "afterId", value: afterId))+ items.append(.init(name: "includeMetadata", value: "0"))+ }return try await request(path: "/api/v1/sync/bootstrap", query: items)}diff --git a/apps/ios/EdgeEver/Data/Sync/SyncEngine.swift b/apps/ios/EdgeEver/Data/Sync/SyncEngine.swiftindex e51766b4..a043c315 100644--- a/apps/ios/EdgeEver/Data/Sync/SyncEngine.swift+++ b/apps/ios/EdgeEver/Data/Sync/SyncEngine.swift@@ -29,6 +29,7 @@ actor SyncEngine {var afterId: String?var loaded = 0var snapshotCursor = 0+ var snapshotTotalCount = 0var snapshotIdentity = "legacy"var firstPage = true@@ -39,6 +40,7 @@ actor SyncEngine {)if firstPage {snapshotCursor = page.snapshotCursor+ snapshotTotalCount = page.totalCount ?? 0snapshotIdentity = page.syncIdentity ?? "legacy"firstPage = false}@@ -53,7 +55,7 @@ actor SyncEngine {memos: batch)loaded += batch.count- onBootstrapProgress?(BootstrapProgress(loadedCount: loaded, totalCount: page.totalCount))+ onBootstrapProgress?(BootstrapProgress(loadedCount: loaded, totalCount: page.totalCount ?? snapshotTotalCount))}guard let next = page.nextAfterId else { break }afterId = nextdiff --git a/apps/ios/Tests/EdgeEverTests/SyncProtocolTests.swift b/apps/ios/Tests/EdgeEverTests/SyncProtocolTests.swiftindex 4a78375d..72b17cbe 100644--- a/apps/ios/Tests/EdgeEverTests/SyncProtocolTests.swift+++ b/apps/ios/Tests/EdgeEverTests/SyncProtocolTests.swift@@ -2,6 +2,19 @@ import XCTest@testable import EdgeEverfinal class SyncProtocolTests: XCTestCase {+ func testBootstrapDecodesLegacyAndMetadataFreeContinuation() throws {+ let legacy = try EdgeEverJSON.decoder.decode(MobileSyncBootstrapPage.self, from: Data("""+ {"notebooks":[],"memos":[],"snapshotCursor":42,"syncIdentity":"workspace","totalCount":200,"nextAfterId":"memo_1"}+ """.utf8))+ XCTAssertEqual(legacy.totalCount, 200)+ let continuation = try EdgeEverJSON.decoder.decode(MobileSyncBootstrapPage.self, from: Data("""+ {"notebooks":[],"memos":[],"snapshotCursor":42,"syncIdentity":"workspace","nextAfterId":null}+ """.utf8))+ XCTAssertNil(continuation.totalCount)+ XCTAssertEqual(continuation.snapshotCursor, legacy.snapshotCursor)+ XCTAssertEqual(continuation.syncIdentity, legacy.syncIdentity)+ }+func testCursorRewound() {XCTAssertTrue(SyncProtocol.hasCursorRewound(localCursor: 10, serverCursor: 3))XCTAssertFalse(SyncProtocol.hasCursorRewound(localCursor: 10, serverCursor: 10))diff --git a/apps/mobile/src/lib/local-mirror.ts b/apps/mobile/src/lib/local-mirror.tsindex 01f65914..5bbbe250 100644--- a/apps/mobile/src/lib/local-mirror.ts+++ b/apps/mobile/src/lib/local-mirror.ts@@ -289,6 +289,7 @@ const performMobileLocalMirrorSync = async (let afterId: string | null = null;let loadedCount = 0;let snapshotCursor = 0;+ let snapshotTotalCount = 0;let snapshotIdentity = "legacy";await db.withExclusiveTransactionAsync(async (tx) => {await tx.runAsync(`DELETE FROM mobile_notebooks WHERE scope = ?`, scope);@@ -301,6 +302,7 @@ const performMobileLocalMirrorSync = async (const page = await client.getMobileSyncBootstrapPage(afterId, BOOTSTRAP_PAGE_SIZE);if (afterId === null) {snapshotCursor = page.snapshotCursor;+ snapshotTotalCount = page.totalCount ?? 0;snapshotIdentity = page.syncIdentity || "legacy";}const writeBatches = splitSyncBootstrapWriteBatches(page.memos, BOOTSTRAP_WRITE_BATCH_SIZE);@@ -316,7 +318,7 @@ const performMobileLocalMirrorSync = async (}});loadedCount += memos.length;- await options.onBootstrapProgress?.({ loadedCount, totalCount: page.totalCount });+ await options.onBootstrapProgress?.({ loadedCount, totalCount: page.totalCount ?? snapshotTotalCount });}if (page.nextAfterId === null) {break;diff --git a/apps/mobile/src/lib/mobile-memo-list-cache.ts b/apps/mobile/src/lib/mobile-memo-list-cache.tsindex 944a716a..55aec4aa 100644--- a/apps/mobile/src/lib/mobile-memo-list-cache.ts+++ b/apps/mobile/src/lib/mobile-memo-list-cache.ts@@ -18,7 +18,7 @@ export const removeMobileMemosFromListCache = (const pages = current.pages.map((page) => ({...page,memos: page.memos.filter((memo) => !removedIds.has(memo.id)),- totalCount: Math.max(0, page.totalCount - removedIds.size),+ totalCount: page.totalCount === undefined ? undefined : Math.max(0, page.totalCount - removedIds.size),}));return { ...current, pages };diff --git a/apps/mobile/src/screens/workspace-memo-cache.ts b/apps/mobile/src/screens/workspace-memo-cache.tsindex b546a32b..40990ad5 100644--- a/apps/mobile/src/screens/workspace-memo-cache.ts+++ b/apps/mobile/src/screens/workspace-memo-cache.ts@@ -102,7 +102,8 @@ export const applyOptimisticMemoToCache = (queryClient: QueryClient, previousMemconst flattened = data.pages.flatMap((page) => page.memos);const withoutMemo = flattened.filter((memo) => memo.id !== nextMemo.id && memo.id !== previousMemo.id);const nextMemos = nextMatches ? sortMemoSummaries([nextMemo, ...withoutMemo], query.queryKey[5]) : withoutMemo;- const totalCount = Math.max(0, data.pages[0].totalCount + (nextMatches ? 1 : 0) - (previouslyMatched ? 1 : 0));+ const previousTotal = data.pages[0].totalCount;+ const totalCount = previousTotal === undefined ? undefined : Math.max(0, previousTotal + (nextMatches ? 1 : 0) - (previouslyMatched ? 1 : 0));let cursor = 0;const pages = data.pages.map((page, index) => {const pageSize = index === data.pages.length - 1 ? Math.min(page.memos.length, Math.max(0, nextMemos.length - cursor)) : page.memos.length;diff --git a/apps/web/src/components/CompanionChat.tsx b/apps/web/src/components/CompanionChat.tsxindex 715b9fe2..7a4b7498 100644--- a/apps/web/src/components/CompanionChat.tsx+++ b/apps/web/src/components/CompanionChat.tsx@@ -113,7 +113,7 @@ export function CompanionChat({let cancelled = false;const timer = window.setTimeout(() => {void Promise.all([- api.listMemos({ q: query, trash: false, sort: "updated-desc", limit: 8 }),+ api.listMemos({ q: query, trash: false, sort: "updated-desc", limit: 8, includeTotal: false }),api.listNotebooks(),api.listTags(),]).then(([memos, notebooks, tags]) => {diff --git a/apps/web/src/components/EditorPane.tsx b/apps/web/src/components/EditorPane.tsxindex 5f239752..d68f0d45 100644--- a/apps/web/src/components/EditorPane.tsx+++ b/apps/web/src/components/EditorPane.tsx@@ -611,7 +611,7 @@ const RichEditorPane = ({}, []);const noteLinkResultsQuery = useQuery({queryKey: ["memo-link-search", noteLinkQuery],- queryFn: () => repository.listMemos({ q: noteLinkQuery, limit: 20 }),+ queryFn: () => repository.listMemos({ q: noteLinkQuery, limit: 20, includeTotal: false }),enabled: noteLinkPickerOpen,});const [editorScrollContainer, setEditorScrollContainer] = useState<HTMLDivElement | null>(null);@@ -824,7 +824,7 @@ const RichEditorPane = ({getCurrentMemoId: () => memoRef.current?.id ?? null,getLabels: () => noteLinkSuggestionLabelsRef.current,searchMemos: async (query) => {- const result = await repository.listMemos({ q: query, limit: 20 });+ const result = await repository.listMemos({ q: query, limit: 20, includeTotal: false });return result.memos;},});diff --git a/apps/web/src/components/WorkspaceApp.tsx b/apps/web/src/components/WorkspaceApp.tsxindex 86f9ee5f..a7bc9e71 100644--- a/apps/web/src/components/WorkspaceApp.tsx+++ b/apps/web/src/components/WorkspaceApp.tsx@@ -1111,6 +1111,8 @@ export const WorkspaceApp = ({networkMode: "offlineFirst",queryKey: ["memos", memoView, selectedNotebookId, search, memoFilterMode, memoSortMode, selectedNotebookScopeIds, selectedTag],queryFn: ({ pageParam }) => repository.listMemos({+ includeTotal: !pageParam,+ cursor: pageParam,notebookId: memoView === "notebook" && !selectedTag ? selectedNotebookId : null,notebookIds: memoView === "notebook" && !selectedTag ? selectedNotebookScopeIds : undefined,q: search,diff --git a/apps/web/src/lib/local-mirror.ts b/apps/web/src/lib/local-mirror.tsindex a41dde27..be7bd04c 100644--- a/apps/web/src/lib/local-mirror.ts+++ b/apps/web/src/lib/local-mirror.ts@@ -34,6 +34,9 @@ import { createClientUuid } from "@/lib/client-id";export type LocalMemoListParams = {notebookId?: string | null;+ includeTotal?: boolean;+ /** Remote continuation token; local mirrors use offset instead. */+ cursor?: string | null;notebookIds?: string[];q?: string;tag?: string;@@ -46,7 +49,7 @@ export type LocalMemoListParams = {export type LocalMemoListResponse = {memos: MemoSummary[];- totalCount: number;+ totalCount?: number;nextCursor: string | null;};@@ -277,7 +280,7 @@ export const listLocalMemos = async (scope: string, params: LocalMemoListParams)const page = memos.slice(offset, offset + (params.limit ?? 50));return {memos: page.map(toSummary),- totalCount,+ ...(params.includeTotal === false ? {} : { totalCount }),nextCursor: offset + page.length < totalCount ? String(offset + page.length) : null,};};diff --git a/apps/web/src/lib/markdown-export.ts b/apps/web/src/lib/markdown-export.tsindex 50cf9b04..3cc641b8 100644--- a/apps/web/src/lib/markdown-export.ts+++ b/apps/web/src/lib/markdown-export.ts@@ -11,7 +11,7 @@ export type MarkdownExportProgress = {export type MarkdownExportPage = {memos: MemoDetail[];resources: Resource[];- totalCount: number;+ totalCount?: number;nextOffset: number | null;};@@ -72,10 +72,12 @@ export const createMarkdownExport = async (try {let offset = 0;let completed = 0;+ let total = 0;while (true) {const page = await source.getPage(offset, EXPORT_PAGE_SIZE);- onProgress?.({ completed, total: page.totalCount });+ total = page.totalCount ?? Math.max(total, completed + page.memos.length);+ onProgress?.({ completed, total });const resourcesByMemo = new Map<string, Resource[]>();for (const resource of page.resources) {@@ -114,7 +116,7 @@ export const createMarkdownExport = async (zip.add(markdownFile);markdownFile.push(strToU8(buildNoteMarkdown(markdown, memo, notebookPath)), true);completed += 1;- onProgress?.({ completed, total: page.totalCount });+ onProgress?.({ completed, total });}if (page.nextOffset === null) {diff --git a/apps/web/src/lib/plugins/plugin-host.ts b/apps/web/src/lib/plugins/plugin-host.tsindex 0a7a49ed..7e789b3e 100644--- a/apps/web/src/lib/plugins/plugin-host.ts+++ b/apps/web/src/lib/plugins/plugin-host.ts@@ -1154,6 +1154,7 @@ export class EdgeEverPluginHost {assertPermission(manifest, "notes:read");const result = await this.repository.listMemos({notebookId: input.notebookId,+ includeTotal: true,q: input.text,sort: input.sort,limit: Math.min(Math.max(input.limit ?? 50, 1), 200),@@ -1163,12 +1164,14 @@ export class EdgeEverPluginHost {.filter((note) => !input.tags?.length || input.tags.every((tag) => note.tags.includes(tag))).map(toPluginNoteSummary);const offset = Math.max(input.offset ?? 0, 0);+ if (result.totalCount === undefined) throw new Error("note_query_total_missing");return { notes, totalCount: result.totalCount, nextOffset: result.nextCursor ? offset + result.memos.length : null };},queryContent: async (input = {}) => {assertPermission(manifest, "notes:read");const result = await this.repository.listMemos({notebookId: input.notebookId,+ includeTotal: true,q: input.text,sort: input.sort,limit: Math.min(Math.max(input.limit ?? 50, 1), 200),@@ -1180,6 +1183,7 @@ export class EdgeEverPluginHost {const notes = await Promise.all(summaries.map(async (note) =>toPluginNote((await this.repository.getMemo(note.id)).memo)));const offset = Math.max(input.offset ?? 0, 0);+ if (result.totalCount === undefined) throw new Error("note_query_total_missing");return { notes, totalCount: result.totalCount, nextOffset: result.nextCursor ? offset + result.memos.length : null };},get: async (noteId) => {diff --git a/apps/web/src/lib/repository.test.mjs b/apps/web/src/lib/repository.test.mjsindex 8e6cf052..70f51e31 100644--- a/apps/web/src/lib/repository.test.mjs+++ b/apps/web/src/lib/repository.test.mjs@@ -36,6 +36,19 @@ afterEach(async () => {});describe("web repository offline boundaries", () => {+ test("keeps remote continuation tokens opaque when skipping later-page counts", async () => {+ const scope = "https://demo.edgeever.org|user-1";+ const original = api.listMemos;+ let received;+ api.listMemos = async (params) => { received = params; return { memos: [], nextCursor: null }; };+ try {+ const result = await createWebRepository(scope).listMemos({ cursor: "opaque-continuation", offset: Number.NaN, includeTotal: false });+ expect(received.cursor).toBe("opaque-continuation");+ expect(received.includeTotal).toBe(false);+ expect(result.totalCount).toBeUndefined();+ } finally { api.listMemos = original; }+ });+test("empty trash records the removed memo ids for conflict cleanup after sync", async () => {const restoreWindow = installTestWindow();const scope = "https://demo.edgeever.org|user-1";diff --git a/apps/web/src/lib/repository.ts b/apps/web/src/lib/repository.tsindex a6c209f2..8bb37e34 100644--- a/apps/web/src/lib/repository.ts+++ b/apps/web/src/lib/repository.ts@@ -471,6 +471,7 @@ export const createWebRepository = (scope: string): EdgeEverRepository => {const remote = await api.listMemos({notebookId: params.notebookId,+ includeTotal: params.includeTotal,// The scope reaches sub-notebooks only when it lists more than the notebook itself.includeDescendants: Boolean(params.notebookIds?.some((notebookId) => notebookId !== params.notebookId)),q: params.q,@@ -478,7 +479,7 @@ export const createWebRepository = (scope: string): EdgeEverRepository => {trash: params.trash,filter: params.filter,sort: params.sort,- cursor: params.offset ? String(params.offset) : null,+ cursor: params.cursor ?? (params.offset ? String(params.offset) : null),limit: params.limit,});return remote;diff --git a/apps/web/src/lib/selected-markdown-export.ts b/apps/web/src/lib/selected-markdown-export.tsindex 4561202f..9fcefe32 100644--- a/apps/web/src/lib/selected-markdown-export.ts+++ b/apps/web/src/lib/selected-markdown-export.ts@@ -57,7 +57,7 @@ export const exportSelectedMemosAsMarkdownZip = async ({listNotebooks,getPage: async (offset, limit) => {const page = await getPage(offset, limit, exportable);- exported = page.totalCount;+ exported += page.memos.length;return page;},getResourceBlob,diff --git a/docs/changes/2026-10-31-count-query-audit.md b/docs/changes/2026-10-31-count-query-audit.mdnew file mode 100644index 00000000..6588dc1e--- /dev/null+++ b/docs/changes/2026-10-31-count-query-audit.md@@ -0,0 +1,38 @@+# Count-query audit++## Scope and changes++Reviewed API aggregate queries and their Web, Android, iOS, and desktop consumers after the backup read-budget fix. The target is repeated full-set counting on pagination or polling, and counting solely to test existence. Exact statistics and concurrency guards remain explicit.++| Path | Previous behavior | Change |+| --- | --- | --- |+| Sync bootstrap | Recounted all notes and aggregated notebooks on every page | Updated clients request metadata only on the first page; continuation pages preserve the snapshot cursor and identity. |+| Legacy JSON / Markdown export | Recounted the export set on every page | First-page count retained for progress; continuation uses one lookahead row, without loading its resources or revisions. |+| Memo feeds and search | Counted all matches for subsequent pages and bounded suggestions | Optional count; feed counts on its first page, note-link and companion suggestions skip it. Remote opaque pagination cursors are preserved. |+| Notebook deletion preflight | Counted all active notes just to reject nonempty deletion | Indexed existence query with `LIMIT 1`; atomic tree-deletion guards remain. |+| Task history polling | Counted all retained runs every 15 seconds; retention cleanup lacked a workspace/time index | Migration `0084` backfills workspace/task counters and maintains them with transactional triggers; polling reads exact totals by primary key. Added an index for retention and history ordering. |++Default API responses still include totals and bootstrap metadata for older clients. New clients tolerate old servers returning metadata on every page. Plugin note queries continue to require exact totals. Optional totals do not overwrite the initial progress estimate or produce invalid cache counts.++## Aggregates retained++- Explicit workspace, storage, user, tag, and notebook statistics, plus initial/final backup snapshot counts: consumers need these exact results.+- Bounded capacity and quota guards for companion memories/actions/turns, extensions, public-form submissions, and poster rendering: counts enforce limits and some participate in atomic writes.+- Empty-trash result counts and recursive notebook-tree counts: these report affected records or protect concurrent deletion.+- Local SQLite counts: these do not consume D1 reads, and their consumers still require totals. This change does not replace all aggregates with persisted counters.++## Risk and rollback++Pagination changes can lose records or progress if continuation contracts drift. Regression tests cover page termination, lookahead exclusion, resources, selection exports, search, and opaque cursors.++Task counters change stored schema. Migration `0084` builds an index and performs a one-time backfill, which consumes database reads/writes. Trigger tests compare persisted totals with actual counts after insertion, ignored duplicate claims, updates, task/workspace moves, cascades, deletion, and transaction rollback. Normal history reads do not prepare a `COUNT` query. Rare parent-identity changes recompute affected counters.++Code can be rolled back to the previous count queries while leaving the additive columns, index, and triggers in place. Do not edit or remove an already-applied migration. No production migration, deployment, or production load test was performed for this audit.++## Verification++- Required TypeScript checks for Web/API and Android, and Web production build.+- Full project tests, Rust unit tests, freshly built desktop sidecar tests, and plugin sync integration.+- Actual v2.16.0 bootstrap route versus the current route on isolated migrated SQLite; actual old/current Swift DTOs decode the corresponding responses.+- Actual v2.16.0 task writer on the old schema, followed by migration `0084` and writes/reads through both old and current routes: exact totals preserved.+- iOS simulator-target build-for-testing and native Swift decoding checks. No iOS device/UI execution or Android USB-device validation; no production performance measurements.diff --git a/docs/changes/2026-10-31-count-query-audit.zh-CN.md b/docs/changes/2026-10-31-count-query-audit.zh-CN.mdnew file mode 100644index 00000000..5e092850--- /dev/null+++ b/docs/changes/2026-10-31-count-query-audit.zh-CN.md@@ -0,0 +1,38 @@+# 全量计数查询审计++## 范围与修复++在备份读取预算修复之后,检查 API 聚合查询及其 Web、Android、iOS 和桌面端调用方。重点处理分页或轮询时重复扫描全集计数,以及只为判断存在性而计数的路径;需要精确结果的统计和并发保护仍然保留。++| 路径 | 原行为 | 修复 |+| --- | --- | --- |+| 同步首次拉取 | 每页重新统计全部笔记并聚合笔记本 | 新客户端仅首批请求元数据;后续页保留快照游标和身份信息。 |+| 旧版 JSON / Markdown 导出 | 每页重算导出集合总数 | 首批总数供进度使用;后续页多取一行判断是否结束,不加载预读行的附件和历史版本。 |+| 笔记列表与搜索 | 后续页和有限条数的候选建议也统计全部匹配项 | 支持省略总数;列表仅首批计数,笔记链接及陪伴建议跳过计数。同时保留远程分页的不透明游标。 |+| 删除笔记本前置检查 | 为拒绝非空删除而统计全部有效笔记 | 改用带索引的 `LIMIT 1` 存在性查询;保留原子删除中的目录树并发保护。 |+| 任务历史轮询 | 每 15 秒重算全部保留记录;过期清理缺少工作区与时间索引 | 迁移 `0084` 回填工作区和任务计数,并通过事务内触发器维护;轮询按主键读取精确总数。补充过期清理和历史排序索引。 |++旧客户端的默认 API 响应仍包含总数和同步元数据。新客户端也兼容旧服务端每页返回元数据。插件笔记查询继续要求精确总数。省略后续页总数不会覆盖首批进度估计,也不会产生无效缓存计数。++## 保留的聚合++- 显式请求的工作区、存储、用户、标签和笔记本统计,以及备份开始和结束时的快照计数:调用方需要精确结果。+- 陪伴记忆、操作及轮次、扩展、公开表单待处理提交、海报渲染等有界容量和配额检查:计数用于限制数量,部分参与原子写入。+- 清空废纸篓的结果计数及笔记本递归树计数:用于报告影响数量或保护并发删除。+- 本地 SQLite 计数:不消耗 D1 读取额度,调用方仍需要总数。本次没有把所有聚合都改成持久化计数器。++## 风险与回滚++分页协议处理错误可能漏数据或破坏进度。回归测试覆盖分页终止、预读行隔离、附件、选择导出、搜索和不透明游标。++任务计数涉及持久化结构。迁移 `0084` 建立索引并一次性回填,会消耗数据库读写。触发器测试在插入、忽略重复领取、更新、任务和工作区移动、级联修改、删除及事务回滚后,将持久化计数与真实统计逐项比较。正常历史查询不准备 `COUNT` SQL;罕见的父记录身份变化会重新统计受影响计数。++代码可回滚到原计数查询,同时保留新增字段、索引和触发器。不得修改或删除已经执行的迁移。本次审计未执行生产迁移、部署或生产负载测试。++## 验证++- Web/API 与 Android 必需的 TypeScript 检查、Web 生产构建。+- 项目完整测试、Rust 单元测试、重新构建后的桌面侧车测试及插件同步集成。+- 在隔离的已迁移 SQLite 上对比实际 v2.16.0 与当前同步接口;实际新旧 Swift 数据模型分别解码对应响应。+- 在旧结构上使用实际 v2.16.0 任务写入接口,再执行 `0084`,并交替通过新旧接口写入和读取:总数保持准确。+- iOS 模拟器目标的 build-for-testing 构建与本机 Swift 解码验证。未运行 iOS 真机/UI 或 Android USB 真机验证,未测量生产性能。diff --git a/migrations/0084_scheduled_run_counts.sql b/migrations/0084_scheduled_run_counts.sqlnew file mode 100644index 00000000..6f535ab1--- /dev/null+++ b/migrations/0084_scheduled_run_counts.sql@@ -0,0 +1,15 @@+-- Keep exact history totals without rescanning runs on every poll/page.+ALTER TABLE workspaces ADD COLUMN scheduled_run_count INTEGER NOT NULL DEFAULT 0 CHECK (scheduled_run_count >= 0);+ALTER TABLE scheduled_tasks ADD COLUMN run_count INTEGER NOT NULL DEFAULT 0 CHECK (run_count >= 0);++CREATE INDEX idx_scheduled_task_runs_workspace_started ON scheduled_task_runs(workspace_id, started_at DESC, id DESC);++UPDATE workspaces SET scheduled_run_count = (SELECT COUNT(*) FROM scheduled_task_runs r WHERE r.workspace_id = workspaces.id);+UPDATE scheduled_tasks SET run_count = (SELECT COUNT(*) FROM scheduled_task_runs r WHERE r.task_id = scheduled_tasks.id AND r.workspace_id = scheduled_tasks.workspace_id);++-- Each trigger is one line for the existing D1 migration splitter.+CREATE TRIGGER scheduled_run_count_insert AFTER INSERT ON scheduled_task_runs BEGIN UPDATE workspaces SET scheduled_run_count = scheduled_run_count + 1 WHERE id = NEW.workspace_id; UPDATE scheduled_tasks SET run_count = run_count + 1 WHERE id = NEW.task_id AND workspace_id = NEW.workspace_id; END;+CREATE TRIGGER scheduled_run_count_delete AFTER DELETE ON scheduled_task_runs BEGIN UPDATE workspaces SET scheduled_run_count = scheduled_run_count - 1 WHERE id = OLD.workspace_id; UPDATE scheduled_tasks SET run_count = run_count - 1 WHERE id = OLD.task_id AND workspace_id = OLD.workspace_id; END;+CREATE TRIGGER scheduled_run_count_move AFTER UPDATE OF workspace_id, task_id ON scheduled_task_runs WHEN OLD.workspace_id IS NOT NEW.workspace_id OR OLD.task_id IS NOT NEW.task_id BEGIN UPDATE workspaces SET scheduled_run_count = scheduled_run_count - 1 WHERE id = OLD.workspace_id AND OLD.workspace_id IS NOT NEW.workspace_id; UPDATE workspaces SET scheduled_run_count = scheduled_run_count + 1 WHERE id = NEW.workspace_id AND OLD.workspace_id IS NOT NEW.workspace_id; UPDATE scheduled_tasks SET run_count = run_count - 1 WHERE id = OLD.task_id AND workspace_id = OLD.workspace_id; UPDATE scheduled_tasks SET run_count = run_count + 1 WHERE id = NEW.task_id AND workspace_id = NEW.workspace_id; END;+CREATE TRIGGER scheduled_run_count_task_identity AFTER UPDATE OF id, workspace_id ON scheduled_tasks WHEN OLD.id IS NOT NEW.id OR OLD.workspace_id IS NOT NEW.workspace_id BEGIN UPDATE scheduled_tasks SET run_count = (SELECT COUNT(*) FROM scheduled_task_runs r WHERE r.task_id = NEW.id AND r.workspace_id = NEW.workspace_id) WHERE id = NEW.id; END;+CREATE TRIGGER scheduled_run_count_workspace_identity AFTER UPDATE OF id ON workspaces WHEN OLD.id IS NOT NEW.id BEGIN UPDATE workspaces SET scheduled_run_count = (SELECT COUNT(*) FROM scheduled_task_runs r WHERE r.workspace_id = NEW.id) WHERE id = NEW.id; END;diff --git a/packages/client/src/index.test.mjs b/packages/client/src/index.test.mjsindex a270bd77..e53c6927 100644--- a/packages/client/src/index.test.mjs+++ b/packages/client/src/index.test.mjs@@ -8,6 +8,24 @@ const jsonResponse = (body, init = {}) => new Response(JSON.stringify(body), {});describe("EdgeEver client HTTP contract", () => {+ test("requests full metadata initially and opts out only for later sync and export pages", async () => {+ const calls = [];+ const client = createEdgeEverClient({ baseUrl: "https://notes.example", fetch: async (input) => { calls.push(new URL(input)); return jsonResponse({}); } });+ await client.syncBootstrap({ limit: 1 });+ await client.syncBootstrap({ afterId: "memo_1", limit: 1 });+ await client.getMobileSyncBootstrapPage(null, 1);+ await client.getMobileSyncBootstrapPage("memo_1", 1);+ await client.getJsonBackupPage(0, 1);+ await client.getJsonBackupPage(1, 1);+ await client.getMarkdownExportPage(0, 1, ["memo_1", "memo_2"]);+ await client.getMarkdownExportPage(1, 1, ["memo_1", "memo_2"]);+ expect(calls.map((url) => url.searchParams.get("includeMetadata"))).toEqual([null, "0", null, "0", null, null, null, null]);+ expect(calls.map((url) => url.searchParams.get("includeTotal"))).toEqual([null, null, null, null, null, "0", null, "0"]);+ expect(calls[7].searchParams.get("ids")).toBe("memo_1,memo_2");+ await client.listMemos({ q: "hello", includeTotal: false });+ expect(calls.at(-1).searchParams.get("includeTotal")).toBe("0");+ });+test("companion prepare errors authenticate and never retry on a server generation endpoint", async()=>{const paths=[];const client=createEdgeEverClient({baseUrl:"https://notes.example",token:"test-token",fetch:async(url,init)=>{paths.push(url);expect(new Headers(init.headers).get("Authorization")).toBe("Bearer test-token");return Response.json({error:{code:"ai_not_configured",message:"Choose a model"}},{status:409});}});await expect(client.streamCompanion({id:"request",message:"hi"},{onEvent(){}})).rejects.toMatchObject({code:"ai_not_configured"});expect(paths).toEqual(["https://notes.example/api/v1/companion/turns/prepare"]);@@ -50,7 +68,7 @@ describe("EdgeEver client HTTP contract", () => {});await client.getJsonBackupPage(25, 10);- expect(calls[0].input).toBe("https://notes.example/api/v1/backups/json?offset=25&limit=10");+ expect(calls[0].input).toBe("https://notes.example/api/v1/backups/json?offset=25&limit=10&includeTotal=0");expect(new Headers(calls[0].init.headers).get("Authorization")).toBe("Bearer secret");expect(calls[0].init.credentials).toBe("include");});diff --git a/packages/client/src/index.ts b/packages/client/src/index.tsindex d95bc9f0..bd53393f 100644--- a/packages/client/src/index.ts+++ b/packages/client/src/index.ts@@ -146,7 +146,7 @@ export type ListNotebooksResponse = {export type ListMemosResponse = {memos: MemoSummary[];- totalCount: number;+ totalCount?: number;nextCursor: string | null;};@@ -276,7 +276,7 @@ export type ResourceUploadResponse = {export type MarkdownExportPage = {memos: MemoDetail[];resources: Resource[];- totalCount: number;+ totalCount?: number;nextOffset: number | null;};@@ -1143,7 +1143,10 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {syncBootstrap: (params?: { afterId?: string | null; limit?: number }) => {const search = new URLSearchParams();- if (params?.afterId) search.set("afterId", params.afterId);+ if (params?.afterId) {+ search.set("afterId", params.afterId);+ search.set("includeMetadata", "0");+ }if (params?.limit) search.set("limit", String(params.limit));const suffix = search.toString() ? `?${search.toString()}` : "";return request<SyncBootstrapResponse>(`/api/v1/sync/bootstrap${suffix}`);@@ -1159,6 +1162,7 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {const search = new URLSearchParams({ limit: String(limit) });if (afterId) {search.set("afterId", afterId);+ search.set("includeMetadata", "0");}return request<MobileSyncBootstrapPage>(`/api/v1/sync/bootstrap?${search.toString()}`);},@@ -1216,6 +1220,7 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {listMemos: (params: {notebookId?: string | null;+ includeTotal?: boolean;includeDescendants?: boolean;q?: string;tag?: string;@@ -1263,6 +1268,7 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {search.set("limit", String(params.limit));}+ if (params.includeTotal === false) search.set("includeTotal", "0");return request<ListMemosResponse>(`/api/v1/memos?${search.toString()}`);},@@ -1549,11 +1555,12 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {if (memoIds && memoIds.length > 0) {search.set("ids", memoIds.join(","));}+ if (offset > 0) search.set("includeTotal", "0");return request<MarkdownExportPage>(`/api/v1/exports/markdown?${search.toString()}`);},getJsonBackupPage: (offset = 0, limit = 25) =>- request<JsonBackupPage>(`/api/v1/backups/json?offset=${offset}&limit=${limit}`),+ request<JsonBackupPage>(`/api/v1/backups/json?offset=${offset}&limit=${limit}${offset > 0 ? "&includeTotal=0" : ""}`),restoreJsonNotebooks: (notebooks: JsonBackupNotebook[]) =>request<{ ok: true }>("/api/v1/restores/json/notebooks", {diff --git a/packages/shared/src/backup-archive.ts b/packages/shared/src/backup-archive.tsindex 9e0bf631..c2fb7c4f 100644--- a/packages/shared/src/backup-archive.ts+++ b/packages/shared/src/backup-archive.ts@@ -37,7 +37,7 @@ type JsonBackupPage = {memos: MemoDetail[];resources: Resource[];revisions: JsonBackupRevision[];- totalCount: number;+ totalCount?: number;nextOffset: number | null;};@@ -134,7 +134,7 @@ export const createEdgeEverZipStream = async (while (true) {abort.signal.throwIfAborted();const page = await source.getPage(offset, 25);- total = page.totalCount;+ total = page.totalCount ?? Math.max(total, completed + page.memos.length);onProgress?.({ completed, total });for (const memo of page.memos) {const notebookPath = notebookPaths.get(memo.notebookId) ?? "Unfiled";diff --git a/packages/shared/src/sync.ts b/packages/shared/src/sync.tsindex cbccf40c..d6168c2a 100644--- a/packages/shared/src/sync.ts+++ b/packages/shared/src/sync.ts@@ -18,7 +18,8 @@ export type SyncBootstrapResponse = {memos: MemoDetail[];snapshotCursor: number;syncIdentity?: string;- totalCount: number;+ /** Omitted on continuation pages requested with includeMetadata=0. */+ totalCount?: number;nextAfterId: string | null;};diff --git a/tests/json-backup.test.ts b/tests/json-backup.test.tsindex 241742cf..b24bd5a4 100644--- a/tests/json-backup.test.ts+++ b/tests/json-backup.test.ts@@ -117,6 +117,24 @@ describe("EdgeEver ZIP", () => {})]))).rejects.toEqual(expect.objectContaining({ code: "unsupportedVersion" }));});+ test("keeps archive counts and progress when later pages omit totals", async () => {+ const progress: { completed: number; total: number }[] = [];+ const blob = await createEdgeEverZip({+ listNotebooks: async () => ({ notebooks: [notebook("nb_child", "Child")] }),+ listPrompts: async () => ({ prompts: [] }),+ getPage: async (offset) => ({+ memos: [{ ...memo, id: `memo_page_${offset}`, contentMarkdown: "body" }],+ resources: [], revisions: [],+ ...(offset === 0 ? { totalCount: 2 } : {}), nextOffset: offset === 0 ? 1 : null,+ }),+ getResourceResponse: async () => new Response(),+ }, { edgeeverVersion: "2.16.0", buildId: "pagination" }, (value) => progress.push(value));+ const parsed = await parseEdgeEverZip(blob);+ expect(parsed.manifest.counts.memos).toBe(2);+ expect(progress.every((value) => value.total === 2)).toBe(true);+ expect(progress.at(-1)?.completed).toBe(2);+ });+test("combines readable Markdown with versioned recovery data", async () => {const resourceChunks: number[] = [];const blob = await createEdgeEverZip(diff --git a/tests/markdown-export.test.ts b/tests/markdown-export.test.tsindex a8544f0e..296bc99f 100644--- a/tests/markdown-export.test.ts+++ b/tests/markdown-export.test.ts@@ -104,4 +104,19 @@ describe("Markdown export", () => {"");});++ test("retains the initial total when continuation pages omit counts", async () => {+ const progress: { completed: number; total: number }[] = [];+ const blob = await createMarkdownExport({+ listNotebooks: async () => ({ notebooks: [notebook({ id: "nb_child", name: "Notes" })] }),+ getPage: async (offset) => ({+ memos: [memo({ id: `memo_${offset}`, title: `Note ${offset}`, contentMarkdown: "body" })],+ resources: [], ...(offset === 0 ? { totalCount: 2 } : {}), nextOffset: offset === 0 ? 1 : null,+ }),+ getResourceBlob: async () => new Blob(),+ }, (value) => progress.push(value));+ expect(Object.keys(unzipSync(new Uint8Array(await blob.arrayBuffer())))).toHaveLength(2);+ expect(progress.every((value) => value.total === 2)).toBe(true);+ expect(progress.at(-1)?.completed).toBe(2);+ });});diff --git a/tests/selected-markdown-export.test.ts b/tests/selected-markdown-export.test.tsindex dc46fec4..9a893cb9 100644--- a/tests/selected-markdown-export.test.ts+++ b/tests/selected-markdown-export.test.ts@@ -60,6 +60,17 @@ const resource: Resource = {};describe("selected Markdown ZIP export", () => {+ test("reports the actual selected export count when continuation totals are omitted", async () => {+ const notes = [memo({ id: "memo_first", title: "First" }), memo({ id: "memo_second", title: "Second" })];+ const result = await exportSelectedMemosAsMarkdownZip({+ memoIds: notes.map((note) => note.id),+ listNotebooks: async () => ({ notebooks: [notebook({ id: "nb_root", name: "Root" })] }),+ getPage: async (offset) => ({ memos: [notes[offset]], resources: [], ...(offset === 0 ? { totalCount: 2 } : {}), nextOffset: offset === 0 ? 1 : null }),+ getResourceBlob: async () => new Blob(), save: () => {},+ });+ expect(result).toEqual({ status: "ok", exported: 2, skippedLocal: 0 });+ });+test("drops unsynced local ids before talking to the export API", () => {expect(partitionExportableMemoIds(["memo_1", "local_draft", "memo_1", "local_other"])).toEqual({exportable: ["memo_1"],diff --git a/tests/wrangler-runner.test.ts b/tests/wrangler-runner.test.tsindex 27e8d813..7f1a14f8 100644--- a/tests/wrangler-runner.test.ts+++ b/tests/wrangler-runner.test.ts@@ -193,7 +193,7 @@ describe("cross-platform Wrangler runner", () => {.split("\n").filter((line) => line.startsWith("CREATE TRIGGER "));- expect(triggerLines).toHaveLength(30);+ expect(triggerLines).toHaveLength(35);for (const triggerLine of triggerLines) {expect(triggerLine).toEndWith(" END;");}