Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(companions): batch import receipts by serialized bytes

Chrise9364544663c
diff --git a/apps/desktop/src/main/bot-import/README.md b/apps/desktop/src/main/bot-import/README.md
index a8e4a9e4de..c6802dace7 100644
--- a/apps/desktop/src/main/bot-import/README.md
+++ b/apps/desktop/src/main/bot-import/README.md
@@ -535,9 +535,15 @@ it cannot accept a command as a simple local reminder. Native `heartbeat` payloa
are preserved disabled with their schedule and original definition; native context
semantics remain explicitly pending, rather than being silently reduced to a reminder.
-Ordinary content/routine progress uses indexed checks and at most about 100 full
-receipt snapshots per pass, sized from the selected input. An interrupted batch
-replays idempotent writes and `createOnce` under the same IDs. Handover intent,
+Ordinary content/routine progress uses indexed checks and batches by serialized
+UTF-8 bytes. Each changed record is charged against the last complete receipt's
+size (including the display-name index), with a 64 KiB floor. Growing receipts
+therefore save progressively less often instead of rewriting every name/check
+roughly 100 times; measuring a change never scans the accumulated receipt. The
+host returns the byte count from its single serialization. Phase boundaries and
+terminal results still save complete, backward-compatible atomic receipts.
+An interrupted batch replays idempotent writes and `createOnce` under the same
+IDs; a large name index can mean replaying a larger unsaved batch. Handover intent,
source pause, target enable and rollback milestones still persist immediately.
New hosts also advertise `selectionChunks` on previews. The shared remote client
diff --git a/apps/desktop/src/main/bot-import/__tests__/host.test.ts b/apps/desktop/src/main/bot-import/__tests__/host.test.ts
index 55395fbf63..100db2a404 100644
--- a/apps/desktop/src/main/bot-import/__tests__/host.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/host.test.ts
@@ -109,6 +109,37 @@ beforeEach(async () => {
});
afterEach(async () => { vi.restoreAllMocks(); await fs.rm(h.root, { recursive: true, force: true }); });
+it('bounds actual receipt writes with long entry names while retaining retryable failures', async () => {
+ const count = 1_000;
+ h.snapshot.items = Array.from({ length: count }, (_, index) => ({
+ view: { id: `memory-${index}`, name: `${'长名称'.repeat(60)}-${index}`, category: 'memory', selected: true }, text: `Original ${index}`,
+ }));
+ h.importDocument.mockImplementation(async (_bot, id) => { if (id === 'memory-999') throw Object.assign(new Error('full'), { code: 'ENOSPC' }); });
+ const [source] = await listCompanionImportSources('fixture');
+ const preview = await previewCompanionImport(source!.id, 'fixture');
+ const selection = { requestId: 'receipt-byte-budget-fixture', previewId: preview.id, name: 'Ada', entryIds: h.snapshot.items.map(item => item.view.id), takeover: false, deferSetup: true };
+ const receiptPath = path.join(h.root, 'companion-imports', `${selection.requestId}.json`);
+ let writtenBytes = 0;
+ const write = fsSync.writeFileSync;
+ vi.spyOn(fsSync, 'writeFileSync').mockImplementation((file, data, options) => {
+ if (String(file).startsWith(`${receiptPath}.`) && typeof data === 'string') writtenBytes += Buffer.byteLength(data);
+ return write(file, data, options);
+ });
+ const accepted = await startCompanionImport(selection, 'fixture');
+ const result = await withBotProfileLocks([accepted.botId], () => getCompanionImportResult(selection.requestId));
+ const saved = JSON.parse(await fs.readFile(receiptPath, 'utf8'));
+ expect(writtenBytes).toBeGreaterThan(0);
+ expect(writtenBytes).toBeLessThan((await fs.stat(receiptPath)).size * 12);
+ expect(Object.keys(saved.entryNames)).toHaveLength(count);
+ expect(result?.savedEntryIds).toHaveLength(count - 1);
+ expect(result?.checks.find(check => check.entryId === 'memory-999')).toMatchObject({ status: 'needs-attention', message: 'IMPORT_DISK_FULL' });
+ expect((await h.store.read(h.root, accepted.botId, () => {}))?.pendingImport).toBeDefined();
+ h.importDocument.mockResolvedValue(undefined);
+ await startCompanionImport(selection, 'after-restart');
+ const retried = await withBotProfileLocks([accepted.botId], () => getCompanionImportResult(selection.requestId));
+ expect(retried?.status).toBe('complete'); expect(retried?.savedEntryIds).toHaveLength(count);
+}, 10_000);
+
it('rejects a simultaneous normalized-name conflict, removes only the loser checkpoint, and accepts a renamed request', async () => {
h.snapshot.items = [{ view: { id: 'env', name: 'Key', category: 'connections', selected: true }, env: { API_KEY: 'fixture-private-key' } }];
const profiles = new Map<string, string>();
diff --git a/apps/desktop/src/main/bot-import/__tests__/transfer.test.ts b/apps/desktop/src/main/bot-import/__tests__/transfer.test.ts
index 578d043c6a..d19b876653 100644
--- a/apps/desktop/src/main/bot-import/__tests__/transfer.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/transfer.test.ts
@@ -138,10 +138,10 @@ it.each(['created', 'source-paused'] as const)('resumes a pre-upgrade %s receipt
describe('companion takeover transaction', () => {
it('copies exactly the selection and verifies before pausing source, idempotently', async () => {
- const { deps } = harness(); const order: string[] = [];
+ const { deps, receipt } = harness(); const order: string[] = [];
vi.mocked(deps.verifyAutomation).mockImplementation(async () => { order.push('verify'); return { verified: true }; });
- vi.mocked(deps.pauseSource).mockImplementation(async () => { order.push('pause'); });
- vi.mocked(deps.enableRoutine).mockImplementation(async () => { order.push('enable'); });
+ vi.mocked(deps.pauseSource).mockImplementation(async () => { expect(receipt()?.routines.task?.phase).toBe('pausing-source'); order.push('pause'); });
+ vi.mocked(deps.enableRoutine).mockImplementation(async () => { expect(receipt()?.routines.task?.phase).toBe('source-paused'); order.push('enable'); });
const result = await transferCompanion(snapshot, selection, deps);
expect(result.status).toBe('complete'); expect(order).toEqual(['verify', 'pause', 'enable']);
expect(deps.importItem).toHaveBeenCalledTimes(1);
@@ -438,36 +438,63 @@ it('validates 100,000 sparse selections in linear work and rejects oversized ran
.toThrow('SELECTION_CHANGED');
});
-it('saves 100,000 content entries with bounded receipt rewrites and indexed progress', async () => {
+it.each([{ count: 100_000, name: 'Note' }, { count: 5_000, name: '长名称'.repeat(60) }])('bounds serialized receipt bytes for $count entries including the name index', async ({ count, name }) => {
const { deps } = harness();
- const count = 100_000;
- const items = Array.from({ length: count }, (_, index) => ({ view: { id: `memory-${index}`, name: 'Note', category: 'memory' as const, selected: true }, text: 'Original' }));
- let writes = 0, serializedChecks = 0;
- deps.saveReceipt = async receipt => { writes++; serializedChecks += receipt.result.checks.length; };
+ const items = Array.from({ length: count }, (_, index) => ({ view: { id: `memory-${index}`, name, category: 'memory' as const, selected: true }, text: 'Original' }));
+ const entryNames = Object.fromEntries(items.map(item => [item.view.id, item.view.name]));
+ let serializedBytes = 0, finalBytes = 0;
+ deps.saveReceipt = async receipt => {
+ receipt.entryNames = entryNames;
+ finalBytes = Buffer.byteLength(JSON.stringify(receipt)); serializedBytes += finalBytes;
+ return finalBytes;
+ };
const result = await transferCompanion({ ...snapshot, items }, { ...selection, entryIds: items.map(item => item.view.id), takeover: false, deferSetup: true }, deps);
expect(result.savedEntryIds).toHaveLength(count);
expect(result.checks).toHaveLength(count);
- expect(writes).toBeLessThan(110);
- expect(serializedChecks).toBeLessThan(count * 60);
+ expect(serializedBytes).toBeLessThan(finalBytes * 10);
expect(deps.importItem).toHaveBeenCalledTimes(count);
+}, 30_000);
+
+it('batches disabled and deferred routine records without losing their durable states', async () => {
+ const { deps } = harness();
+ const count = 10_000;
+ const template = snapshot.items.find(item => item.automation)!;
+ const items = Array.from({ length: count }, (_, index) => ({ ...template,
+ view: { ...template.view, id: `task-${index}`, enabled: index % 2 === 0, dependsOn: [] },
+ }));
+ let bytes = 0, finalText = '';
+ deps.saveReceipt = async value => { finalText = JSON.stringify(value); const length = Buffer.byteLength(finalText); bytes += length; return length; };
+ const result = await transferCompanion({ ...snapshot, items }, { ...selection, entryIds: items.map(item => item.view.id), deferSetup: true }, deps);
+ expect(bytes).toBeLessThan(Buffer.byteLength(finalText) * 10);
+ const saved = JSON.parse(finalText) as ImportReceipt;
+ expect(Object.keys(saved.routines)).toHaveLength(count);
+ expect(saved.routines['task-0']?.phase).toBe('created');
+ expect(saved.routines['task-1']?.phase).toBe('complete');
+ expect(saved.result).toEqual(result);
+ expect(result.savedEntryIds).toHaveLength(count);
+ expect(deps.pauseSource).not.toHaveBeenCalled(); expect(deps.enableRoutine).not.toHaveBeenCalled();
}, 10_000);
it('resumes an interrupted progress batch using stable item IDs without replaying saved batches', async () => {
const { deps, receipt } = harness();
- const items = Array.from({ length: 400 }, (_, index) => ({ view: { id: `memory-${index}`, name: 'Note', category: 'memory' as const, selected: true }, text: 'Original' }));
+ const items = Array.from({ length: 3_000 }, (_, index) => ({ view: { id: `memory-${index}`, name: 'Note', category: 'memory' as const, selected: true }, text: 'Original' }));
const input = { ...selection, entryIds: items.map(item => item.view.id), takeover: false, deferSetup: true };
- let interrupted = false;
+ let interrupted = false, durableCount = 0;
const calls: string[] = [];
+ const save = deps.saveReceipt;
+ deps.saveReceipt = async value => { await save(value); durableCount = value.copied.length; };
deps.assertOwner = () => { if (interrupted) throw new Error('OWNER_CHANGED'); };
- deps.importItem = async (_bot, item) => { calls.push(item.view.id); if (calls.length === 6) interrupted = true; };
+ deps.importItem = async (_bot, item) => { calls.push(item.view.id); if (durableCount && calls.length === durableCount + 2) interrupted = true; };
await expect(transferCompanion({ ...snapshot, items }, input, deps)).rejects.toThrow('OWNER_CHANGED');
- expect(receipt()?.copied).toEqual(items.slice(0, 4).map(item => item.view.id));
+ const savedCount = durableCount;
+ expect(savedCount).toBeGreaterThan(0); expect(savedCount).toBeLessThan(items.length - 2);
+ expect(receipt()?.copied).toEqual(items.slice(0, savedCount).map(item => item.view.id));
interrupted = false;
const result = await transferCompanion({ ...snapshot, items }, input, deps);
- expect(result.savedEntryIds).toHaveLength(400);
+ expect(result.savedEntryIds).toHaveLength(items.length);
expect(calls.filter(id => id === 'memory-0')).toHaveLength(1);
- expect(calls.filter(id => id === 'memory-4')).toHaveLength(2);
- expect(calls.filter(id => id === 'memory-5')).toHaveLength(2);
+ expect(calls.filter(id => id === `memory-${savedCount}`)).toHaveLength(2);
+ expect(calls.filter(id => id === `memory-${savedCount + 1}`)).toHaveLength(2);
});
it('does not reuse a legacy boolean link exception to read an external credential on retry', async ctx => {
diff --git a/apps/desktop/src/main/bot-import/host.ts b/apps/desktop/src/main/bot-import/host.ts
index 1bd241573e..7e68dde039 100644
--- a/apps/desktop/src/main/bot-import/host.ts
+++ b/apps/desktop/src/main/bot-import/host.ts
@@ -247,7 +247,9 @@ async function readReceipt(root: string, requestId: string): Promise<ImportRecei
async function saveReceipt(root: string, receipt: ImportReceipt) {
const file = receiptFile(root, receipt.result.requestId);
await fs.mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
- atomicWriteFileSync(file, JSON.stringify(receipt));
+ const contents = JSON.stringify(receipt);
+ atomicWriteFileSync(file, contents);
+ return Buffer.byteLength(contents);
}
/** A definite DB rejection must not strand credentials without a deletable profile. */
diff --git a/apps/desktop/src/main/bot-import/transfer.ts b/apps/desktop/src/main/bot-import/transfer.ts
index 1c3efad9fc..31cd0affa5 100644
--- a/apps/desktop/src/main/bot-import/transfer.ts
+++ b/apps/desktop/src/main/bot-import/transfer.ts
@@ -36,7 +36,8 @@ export interface TransferDeps {
readReceipt(requestId: string): Promise<ImportReceipt | undefined>;
readLegacyDeferSetup?(botId: string): Promise<boolean | undefined>;
readSkillConfig?(source: ImportSource, budget: ImportReadBudget): Promise<Record<string, unknown>>;
- saveReceipt(receipt: ImportReceipt): Promise<void>;
+ /** Return serialized UTF-8 bytes when available, avoiding a second full serialization. */
+ saveReceipt(receipt: ImportReceipt): Promise<number | void>;
createCompanion(botId: string, selection: CompanionImportSelection): Promise<void>;
validateItems?(items: ImportItem[]): void;
importItem(botId: string, item: ImportItem, snapshot: ImportSnapshot): Promise<void>;
@@ -132,13 +133,23 @@ export async function transferCompanion(snapshot: ImportSnapshot, selection: Com
const copied = new Set(receipt.copied);
const checkIndexes = new Map(receipt.result.checks.map((item, index) => [item.entryId, index]));
const selected = new Set(selectedIds);
- // Cap full progress snapshots per pass, not imported items. A lost batch is
- // replayed through idempotent content writes/createOnce; source handover
- // milestones still call save immediately before/after each external mutation.
- const progressBatchSize = Math.max(1, Math.ceil(items.length / 100));
- let pendingProgress = 0;
- const save = async () => { deps.assertOwner(); await deps.saveReceipt(receipt); deps.assertOwner(); pendingProgress = 0; };
- const saveProgress = async () => { if (++pendingProgress >= progressBatchSize) await save(); };
+ // Charge changed records against the last serialized receipt (including the
+ // immutable name index). As the receipt grows, ordinary snapshots get farther
+ // apart instead of rewriting the full catalog ~100 times. Lost batches replay
+ // idempotently; source handover milestones still save before/after mutations.
+ let progressBatchBytes = 64 * 1024, pendingProgressBytes = 0;
+ const save = async () => {
+ deps.assertOwner();
+ const bytes = await deps.saveReceipt(receipt);
+ deps.assertOwner();
+ progressBatchBytes = Math.max(64 * 1024, bytes ?? Buffer.byteLength(JSON.stringify(receipt)));
+ pendingProgressBytes = 0;
+ };
+ const saveProgress = async (entryId: string) => {
+ const index = checkIndexes.get(entryId);
+ pendingProgressBytes += Buffer.byteLength(JSON.stringify([entryId, index === undefined ? null : receipt.result.checks[index], receipt.routines[entryId]]));
+ if (pendingProgressBytes >= progressBatchBytes) await save();
+ };
const check = (entryId: string, status: CompanionImportResult['checks'][number]['status'], message?: string, progress?: { saved: number; total: number }) => {
const index = checkIndexes.get(entryId) ?? receipt.result.checks.length;
checkIndexes.set(entryId, index);
@@ -252,7 +263,7 @@ export async function transferCompanion(snapshot: ImportSnapshot, selection: Com
deps.assertOwner();
check(item.view.id, 'needs-attention', importFailureCode(error), (error as { importProgress?: { saved: number; total: number } })?.importProgress);
}
- await saveProgress();
+ await saveProgress(item.view.id);
}
await save();
if (!receipt.environmentSaved) {
@@ -300,21 +311,21 @@ export async function transferCompanion(snapshot: ImportSnapshot, selection: Com
} catch (error) {
deps.assertOwner();
check(item.view.id, 'needs-attention', importFailureCode(error));
- await saveProgress(); continue;
+ await saveProgress(item.view.id); continue;
}
}
if (record.phase === 'complete') { check(item.view.id, item.view.enabled && selection.takeover ? 'taken-over' : 'paused'); continue; }
if (item.view.issues?.length) {
- check(item.view.id, 'needs-attention', item.view.issues[0]); await saveProgress(); continue;
+ check(item.view.id, 'needs-attention', item.view.issues[0]); await saveProgress(item.view.id); continue;
}
if (!item.view.enabled || !selection.takeover) {
- record.phase = 'complete'; check(item.view.id, 'paused'); await saveProgress(); continue;
+ record.phase = 'complete'; check(item.view.id, 'paused'); await saveProgress(item.view.id); continue;
}
if (selection.deferSetup && !resumeSetup && record.phase === 'created') {
- check(item.view.id, 'needs-attention', 'IMPORT_SETUP_DEFERRED'); await saveProgress(); continue;
+ check(item.view.id, 'needs-attention', 'IMPORT_SETUP_DEFERRED'); await saveProgress(item.view.id); continue;
}
if (item.view.dependsOn?.some(id => missingDependency(id))) {
- check(item.view.id, 'needs-attention', 'AUTOMATION_DEPENDENCY_NOT_SELECTED'); await saveProgress(); continue;
+ check(item.view.id, 'needs-attention', 'AUTOMATION_DEPENDENCY_NOT_SELECTED'); await saveProgress(item.view.id); continue;
}
if (record.phase === 'created') {
try {