Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

perf(history): bound history pages without truncating records

Chris6333df42ba95
diff --git a/apps/desktop/src/main/localDb/__tests__/remoteHistoryPagination.test.ts b/apps/desktop/src/main/localDb/__tests__/remoteHistoryPagination.test.ts
index 1e9e41b615..8f57d83df6 100644
--- a/apps/desktop/src/main/localDb/__tests__/remoteHistoryPagination.test.ts
+++ b/apps/desktop/src/main/localDb/__tests__/remoteHistoryPagination.test.ts
@@ -2,7 +2,8 @@ import path from 'node:path';
import Database from 'better-sqlite3';
import { drizzle } from 'drizzle-orm/better-sqlite3';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { listSessionsForHistory } from '../chatHistoryReader';
+import { getMessagesForHistory, listSessionsForHistory } from '../chatHistoryReader';
+import { XdtHelperToolRegistry, registerGetChatHistoryTool, historyPayload } from '@cindy/mcps';
import { searchChatHistoryHybrid } from '../chatHistorySearch';
import { readRemoteBotSessionAccessBatch } from '../ipc/botRemoteSessionAccess';
import { setChatEmbeddingEnabled } from '../../embedders/chat-history-embedder';
@@ -159,6 +160,79 @@ describe.each(modes)('%s remote history visibility', (mode) => {
return projectRemoteSessionResult(channel, result);
};
+ // Real SQLite + public tool cursor: byte budgeting must not skip same-ms
+ // records or truncate a large individual message. Unit, no artificial waits.
+ it.each(['asc', 'desc'] as const)('pages complete UTF-8 messages without losing rowid ties (%s)', async (order) => {
+ const ids = ['budget-z', 'budget-a', 'budget-m', 'budget-b'];
+ const texts = ['中'.repeat(18000), '文'.repeat(18000), '大'.repeat(30000), 'last'];
+ for (let i = 0; i < ids.length; i++) {
+ db.prepare("INSERT INTO messages (id, session_id, role, content, created_at) VALUES (?, 'ordinary', 'user', ?, 1000)")
+ .run(ids[i], JSON.stringify(texts[i]));
+ }
+ const registry = new XdtHelperToolRegistry();
+ registerGetChatHistoryTool(registry, { history: {
+ listWorkdirs: vi.fn(), listSessions: vi.fn(), searchChatHistory: vi.fn(),
+ getMessages: async args => ({ ok: true, page: await getMessagesForHistory(args) }),
+ } });
+ const collected: Array<{ id: string; content: unknown }> = [];
+ let cursor: string | undefined;
+ for (let i = 0; i < 5; i++) {
+ const page = historyPayload(await registry.call('get_chat_history', {
+ session_ids: ['ordinary'], from: '1970-01-01T00:00:01.000Z', limit: 100, order, ...(cursor ? { cursor } : {}),
+ })) as any;
+ expect(page.messages.length).toBeGreaterThan(0);
+ if (Buffer.byteLength(JSON.stringify(page.messages), 'utf8') > 64 * 1024) {
+ expect(page.messages).toHaveLength(1);
+ }
+ collected.push(...page.messages);
+ if (!page.hasMore) { expect(page.nextCursor).toBeNull(); break; }
+ expect(page.nextCursor).not.toBe(cursor);
+ cursor = page.nextCursor;
+ }
+ const expected = ids.map((id, i) => ({ id, content: texts[i] }));
+ if (order === 'desc') expected.reverse();
+ expect(collected.map(({ id, content }) => ({ id, content }))).toEqual(expected);
+ });
+
+ it('advances the activity cursor at the last byte-bounded session through the remote tool entry', async () => {
+ db.prepare("UPDATE sessions SET title=?, updated_at=1000 WHERE id IN ('ordinary', 'visible', 'zero')")
+ .run('标题'.repeat(6000));
+ const ids: string[] = [];
+ let cursor: string | undefined;
+ for (let i = 0; i < 3; i++) {
+ const page = await invokeHistory('list_sessions', { limit: 100, time_field: 'activity', order: 'asc',
+ ...(cursor ? { cursor } : {}) }) as any;
+ expect(page.sessions).toHaveLength(1);
+ ids.push(page.sessions[0].id);
+ expect(page.sessions[0].title).toBe('标题'.repeat(6000));
+ cursor = page.nextCursor;
+ expect(page.hasMore).toBe(i < 2);
+ }
+ expect(ids).toEqual(['ordinary', 'visible', 'zero']);
+ });
+
+ it('continues search at the consumed candidate rather than the requested page limit', async () => {
+ setChatEmbeddingEnabled(false);
+ const text = 'budgetneedle ' + '文'.repeat(18000);
+ const ids = ['budget-z', 'budget-a', 'budget-m'];
+ for (const id of ids) {
+ db.prepare("INSERT INTO messages (id, session_id, role, content, created_at) VALUES (?, 'ordinary', 'user', ?, 1000)")
+ .run(id, JSON.stringify(text));
+ db.prepare('INSERT INTO messages_fts VALUES (?, ?)').run(id, 'budgetneedle');
+ }
+ const found: string[] = [];
+ let offset = 0;
+ for (let i = 0; i < 3; i++) {
+ const page = await searchChatHistoryHybrid({ ...searchArgs, query: 'budgetneedle', limit: 10, offset });
+ expect(page.hits).toHaveLength(1);
+ expect(page.hits[0].context[0].content).toBe(text);
+ found.push(page.hits[0].messageId);
+ expect(page.hasMore).toBe(i < 2);
+ if (page.nextOffset !== null) { expect(page.nextOffset).toBe(offset + 1); offset = page.nextOffset; }
+ }
+ expect(found.sort()).toEqual(ids.sort());
+ });
+
// Regression: old-created, recently active chats were lost by workdir/creation
// filters; known ownership must include archived channels/topics without claiming
// unrelated legacy Telegram records. Unit: real readers + MCP/remote handler + SQLite.
diff --git a/apps/desktop/src/main/localDb/chatHistoryReader.ts b/apps/desktop/src/main/localDb/chatHistoryReader.ts
index 4569255388..dd6b0a8f1a 100644
--- a/apps/desktop/src/main/localDb/chatHistoryReader.ts
+++ b/apps/desktop/src/main/localDb/chatHistoryReader.ts
@@ -18,6 +18,7 @@ import { and, asc, desc, eq, gt, gte, lt, inArray, isNotNull, isNull, ne, or, sq
import type { HistoryCompanion } from '@cindy/mcps';
import { historyCompanionJson, parseHistoryCompanion, teammateHistoryFilter } from './historyCompanion';
import { getDbClient } from './client/current';
+import { historyPageWithinBudget } from './historyPageBudget';
import { sessions, messages } from './schema';
import { messageToCamel } from './mapper';
import { parseOrcaRemoteLead, type OrcaRemoteLead } from '../../shared/orcaRemoteWorker.js';
@@ -282,8 +283,7 @@ export async function listSessionsForHistory(
.orderBy(orderFn(time), orderFn(sessions.id))
.limit(fetchLimit);
- const hasMore = sessionRows.length > params.limit;
- const sliced = hasMore ? sessionRows.slice(0, params.limit) : sessionRows;
+ const sliced = sessionRows.slice(0, params.limit);
// 批量查 messageCount (已过滤 rewindAt 软删)
const countMap = new Map<string, number>();
@@ -300,7 +300,7 @@ export async function listSessionsForHistory(
for (const r of countRows) countMap.set(r.sessionId, Number(r.cnt));
}
- const items: SessionSummary[] = sliced.map((r) => ({
+ const items: SessionSummary[] = historyPageWithinBudget(sliced.map((r) => ({
lastActivityAt: r.lastActivityAt,
companion: parseHistoryCompanion(r.companionJson),
id: r.id,
@@ -318,7 +318,8 @@ export async function listSessionsForHistory(
updatedAt: r.updatedAt,
userSendAt: r.userSendAt,
messageCount: countMap.get(r.id) ?? 0,
- }));
+ })));
+ const hasMore = sessionRows.length > items.length;
const last = items[items.length - 1];
const nextCursor: HistoryCursor | null = hasMore && last
? { createdAt: params.timeField === 'activity' ? last.lastActivityAt : last.createdAt, id: last.id }
@@ -485,9 +486,8 @@ export async function getMessagesForHistory(
: messages.id))
.limit(fetchLimit);
- const hasMore = rows.length > params.limit;
- const sliced = hasMore ? rows.slice(0, params.limit) : rows;
- const items: HistoryMessage[] = sliced.map((r) => {
+ const sliced = rows.slice(0, params.limit);
+ const items: HistoryMessage[] = historyPageWithinBudget(sliced.map((r) => {
// 复用 messageToCamel 拿 content/agentMeta JSON 解析 + 错误兜底
const camel = messageToCamel(r.m);
return {
@@ -504,14 +504,15 @@ export async function getMessagesForHistory(
createdAt: r.m.createdAt,
rewindAt: r.m.rewindAt,
};
- });
+ }));
+ const hasMore = rows.length > items.length;
const last = items[items.length - 1];
const nextCursor: HistoryCursor | null = hasMore && last
? {
createdAt: last.createdAt,
id: last.id,
- ...(typeof sliced[sliced.length - 1]?.rowid === 'number'
- ? { rowid: sliced[sliced.length - 1].rowid }
+ ...(typeof sliced[items.length - 1]?.rowid === 'number'
+ ? { rowid: sliced[items.length - 1].rowid }
: {}),
}
: null;
diff --git a/apps/desktop/src/main/localDb/chatHistorySearch.ts b/apps/desktop/src/main/localDb/chatHistorySearch.ts
index bf09d45c9b..0ce7ebc7b6 100644
--- a/apps/desktop/src/main/localDb/chatHistorySearch.ts
+++ b/apps/desktop/src/main/localDb/chatHistorySearch.ts
@@ -39,6 +39,7 @@ import type {
import { historyCompanionJson, parseHistoryCompanion } from './historyCompanion';
import { getDbClient } from './client/current';
+import { historyPageWithinBudget } from './historyPageBudget';
import { messages as messagesTable, sessions as sessionsTable } from './schema';
import { messageToCamel } from './mapper';
import { fuseRRF, buildMessagesFtsMatch, extractMessagesFtsTokens } from './chatHistorySearch.pure';
@@ -194,7 +195,7 @@ export async function searchChatHistoryHybrid(
}
const start = Math.max(0, Math.min(args.offset, pool.length));
const pageEntries = pool.slice(start, start + args.limit);
- const nextOffset = start + args.limit < pool.length ? start + args.limit : null;
+ let nextOffset = start + args.limit < pool.length ? start + args.limit : null;
// radius=0 is the conversation-search hot path. Hydrate the whole page in
// one worker RPC; callers that need neighbours keep the original window query.
@@ -234,10 +235,15 @@ export async function searchChatHistoryHybrid(
}
// 6) session 元数据(顶层 map, 避免逐条命中重复)
- const sessions = await fetchSessionsMeta(hits.map((h) => h.sessionId));
+ const boundedHits = historyPageWithinBudget(hits);
+ if (boundedHits.length < hits.length) {
+ const last = boundedHits[boundedHits.length - 1];
+ nextOffset = start + pageEntries.findIndex(entry => entry.messageId === last.messageId) + 1;
+ }
+ const sessions = await fetchSessionsMeta(boundedHits.map((h) => h.sessionId));
return {
- hits,
+ hits: boundedHits,
sessions,
vectorUsed: vec.skipReason === null,
vectorSkipReason: vec.skipReason,
diff --git a/apps/desktop/src/main/localDb/historyPageBudget.ts b/apps/desktop/src/main/localDb/historyPageBudget.ts
new file mode 100644
index 0000000000..02c80ecbc2
--- /dev/null
+++ b/apps/desktop/src/main/localDb/historyPageBudget.ts
@@ -0,0 +1,15 @@
+/** Soft budget for complete history records, before the tool's small envelope.
+ * A single oversized record always makes progress and remains readable in full.
+ * Never slice content: callers advance their native cursor at the last kept row.
+ */
+export function historyPageWithinBudget<T>(items: T[]): T[] {
+ let bytes = 2; // array brackets
+ let count = 0;
+ for (const item of items) {
+ const size = Buffer.byteLength(JSON.stringify(item), 'utf8') + (count ? 1 : 0);
+ if (count > 0 && bytes + size > 64 * 1024) break;
+ bytes += size;
+ count++;
+ }
+ return count === items.length ? items : items.slice(0, count);
+}
diff --git a/docs/product-rules/agent-user-parity.md b/docs/product-rules/agent-user-parity.md
index 8830ae16be..a8e9d223e7 100644
--- a/docs/product-rules/agent-user-parity.md
+++ b/docs/product-rules/agent-user-parity.md
@@ -164,6 +164,14 @@ Host 不因当前输入缺少主人标记再追加行动同意判定。只认运
执行任务的模型可用 `cindy_helper` 的 cindy 类 `report_task_delivery` 报告本轮是否新交付最终结果;只限当前任务执行者,子 Agent 不可代报。三个 Harness 共用此入口;它不结束任务或授予操作权限。跟进通知门控见 `cindy-bots-runtime.md` §6.3。
+## 历史查询的大结果分页
+
+历史工具 `list_sessions`、`get_chat_history`、`search_chat_history` 的 `limit` 是条数上限。
+源设备另按完整记录约 64 KiB 的软预算提前分页,消息、搜索命中及其上下文不被截断;
+单条记录超过预算时仍完整返回,保证继续读取可以前进。调用方以 `hasMore` / `nextCursor`
+续读,不以返回条数少于 limit 判断结束。原始历史不改写、不另存摘要,所有 Harness 共用;
+跨电脑沿源设备同一查询与游标,旧设备保留其原有分页行为。
+
## 社区插件贡献
普通个人作者的独立入口为 `ghost_contribution_prepare`、`ghost_contribution_submit`、
diff --git a/packages/lizi-mcps/src/xdt-helper/get_chat_history.ts b/packages/lizi-mcps/src/xdt-helper/get_chat_history.ts
index d208d99bd0..fefea6a0d8 100644
--- a/packages/lizi-mcps/src/xdt-helper/get_chat_history.ts
+++ b/packages/lizi-mcps/src/xdt-helper/get_chat_history.ts
@@ -84,6 +84,7 @@ const DESCRIPTION = [
' 要拿某条 message 的 workdir / agentKind / title, 用 sessions[message.sessionId] 查。',
'',
'【分页】游标分页, 默认 200/次, 最大 1000。串联 nextCursor 多次调用拿全量, 信息不丢。',
+ '源设备可按完整记录的 64 KiB 软预算提前分页;单条超大记录仍完整返回。limit 是条数上限,继续读 nextCursor,不要按返回条数判断结束。',
'默认按 createdAt desc + id desc, 同毫秒多消息靠 id 兜底排序稳定。',
].join('\n');
diff --git a/packages/lizi-mcps/src/xdt-helper/list_sessions.ts b/packages/lizi-mcps/src/xdt-helper/list_sessions.ts
index 0d53a95a54..5652ebb36f 100644
--- a/packages/lizi-mcps/src/xdt-helper/list_sessions.ts
+++ b/packages/lizi-mcps/src/xdt-helper/list_sessions.ts
@@ -45,6 +45,7 @@ const DESCRIPTION = [
'orcaRole / parentSessionId / userSendAt 仅在非 null 时出现(默认场景几乎全为 null, 已 omit)。',
'',
'【分页】游标分页, 默认 100 条/次, 最大 1000。串联 nextCursor 拿全量。',
+ '源设备可按完整记录的 64 KiB 软预算提前分页;单条超大记录仍完整返回。limit 是条数上限,结束以 hasMore 为准。',
].join('\n');
export interface ListSessionsToolDeps {
diff --git a/packages/lizi-mcps/src/xdt-helper/search_chat_history.ts b/packages/lizi-mcps/src/xdt-helper/search_chat_history.ts
index 657cc47cb5..f207d2627e 100644
--- a/packages/lizi-mcps/src/xdt-helper/search_chat_history.ts
+++ b/packages/lizi-mcps/src/xdt-helper/search_chat_history.ts
@@ -72,6 +72,7 @@ const DESCRIPTION = [
' - vector_used / vector_skip_reason / pool_size / pool_capped 诊断字段。',
'',
'【分页】游标分页, 默认 limit=10(最大 50)。hasMore=true 时用 nextCursor 再调拿下一页。',
+ '源设备可按完整命中及其上下文的 64 KiB 软预算提前分页;不截断单个命中,单条超大命中仍完整返回。limit 是上限,结束以 hasMore 为准。',
'pool_capped=true 表示候选池已达上限, 更靠后的相关结果可能未进池(可缩小过滤范围再搜)。',
].join('\n');