Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(companions): bound remote reads and recover failed import subtrees

zqchris48ad8639cfa3
diff --git a/apps/desktop/src/main/bot-import/__tests__/files.test.ts b/apps/desktop/src/main/bot-import/__tests__/files.test.ts
index d6b13d1300..10cfc2d70d 100644
--- a/apps/desktop/src/main/bot-import/__tests__/files.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/files.test.ts
@@ -10,7 +10,7 @@ it('rejects the next file before allocating its buffer when the cumulative budge
try {
const file = path.join(root, 'resource');
await fs.writeFile(file, '1234');
- const budget = createImportBudget(7);
+ const budget = createImportBudget(270);
expect((await readImportFile(root, file, budget)).bytes.toString()).toBe('1234');
const allocate = vi.spyOn(Buffer, 'alloc');
try {
@@ -52,3 +52,36 @@ it('reads every resource in a skill with more than 4096 small files', async () =
expect(files.find(file => file.name === 'resource-4099.txt')?.bytes.toString()).toBe('Resource 4099');
} finally { await fs.rm(root, { recursive: true, force: true }); }
});
+
+it('charges metadata for empty files and stops streaming instead of retaining unlimited errors', async () => {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-import-empty-files-'));
+ try {
+ for (let index = 0; index < 20; index++) await fs.writeFile(path.join(root, `${index}.txt`), '');
+ const onError = vi.fn();
+ await expect(readImportTree(root, undefined, createImportBudget(1000), onError)).rejects.toThrow('SOURCE_SNAPSHOT_TOO_LARGE');
+ expect(onError).not.toHaveBeenCalled();
+ } finally { await fs.rm(root, { recursive: true, force: true }); }
+});
+
+it('retries a selected subtree inside its original root and still rejects escapes and cycles', async () => {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-import-subtree-'));
+ try {
+ const memory = path.join(root, 'memory'); const outside = path.join(root, 'outside');
+ await fs.mkdir(memory); await fs.mkdir(outside);
+ await fs.writeFile(path.join(outside, 'note.md'), 'Outside');
+ const link = path.join(memory, 'broken');
+ const linkType = process.platform === 'win32' ? 'junction' : 'dir';
+ await fs.symlink(outside, link, linkType);
+ const failures: unknown[] = [];
+ expect(await readImportTree(memory, undefined, createImportBudget(), (name, error, kind) => failures.push([name, (error as Error).message, kind]))).toEqual([]);
+ expect(failures).toEqual([['broken', 'SOURCE_LINK_OUTSIDE_FOLDER', 'directory']]);
+ await expect(readImportTree(memory, undefined, createImportBudget(), undefined, link)).rejects.toThrow('SOURCE_LINK_OUTSIDE_FOLDER');
+ await fs.rm(link, { recursive: true });
+ await fs.symlink(memory, link, linkType);
+ await expect(readImportTree(memory, undefined, createImportBudget(), undefined, link)).rejects.toThrow('SOURCE_LINK_CYCLE');
+ await fs.rm(link, { recursive: true });
+ await fs.mkdir(link); await fs.writeFile(path.join(link, 'note.md'), 'Repaired');
+ const files = await readImportTree(memory, undefined, createImportBudget(), undefined, link);
+ expect(files.map(file => [file.name, file.bytes.toString()])).toEqual([['broken/note.md', 'Repaired']]);
+ } finally { await fs.rm(root, { recursive: true, force: true }); }
+});
diff --git a/apps/desktop/src/main/bot-import/__tests__/host.test.ts b/apps/desktop/src/main/bot-import/__tests__/host.test.ts
index ecc1c11ff2..729e10fab2 100644
--- a/apps/desktop/src/main/bot-import/__tests__/host.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/host.test.ts
@@ -1,3 +1,4 @@
+import { remoteCompanionImportApi, compactCompanionImportSelection } from '@cindy/maker-shared/companion-import';
import fsSync, { promises as fs } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
@@ -62,7 +63,7 @@ vi.mock('../../routines/service.js', () => ({
h.routines = h.routines.map(row => row.id === id ? { ...row, ...input, revision: row.revision + 1 } : row);
} }),
}));
-import { listCompanionImportSources, previewCompanionImport, startCompanionImport, getCompanionImportResult, recoverCompanionImports, prepareCompanionImportDeletion, cancelCompanionImportsForDeletion, ensureImportedAutomationReady } from '../host.js';
+import { readRemoteCompanionImport, listCompanionImportSources, previewCompanionImport, startCompanionImport, getCompanionImportResult, recoverCompanionImports, prepareCompanionImportDeletion, cancelCompanionImportsForDeletion, ensureImportedAutomationReady } from '../host.js';
import { withBotProfileLocks } from '../../maker-ipc/botProfileLock.js';
import { assertImportedAutomationReady, prepareImportedAutomation } from '../automationRuntime.js';
import { decodeBotAvatarImage } from '../../localDb/ipc/botAvatarSelection.js';
@@ -879,3 +880,65 @@ it('refreshes masks and encrypted originals when a previously failed credential-
expect(Buffer.from(original!.bytes, 'base64').toString()).toContain(secret);
expect(Object.values(environment.contentRedactions ?? {})).toContain(secret);
});
+
+it('transfers a 10,000-entry preview in bounded, immutable chunks and preserves selection indexes', async () => {
+ h.snapshot.items = Array.from({ length: 10_000 }, (_, index) => ({ view: { id: `skill-${index}`, name: `技能 ${index} ` + 'long name '.repeat(20), description: '説明😀'.repeat(60), category: 'skills' as const, selected: true } }));
+ let reads = 0;
+ let continuation = '';
+ const api = remoteCompanionImportApi(async id => {
+ const data = await readRemoteCompanionImport(id, 'phone', true);
+ const response = { blocks: [{ primitive: 'companion-import', data }] };
+ expect(Buffer.byteLength(JSON.stringify(response))).toBeLessThan(2 * 1024 * 1024 - 4096);
+ reads++;
+ if (id.startsWith('chunk:')) continuation = id;
+ return response;
+ }, async () => {});
+ const [source] = await api.sources();
+ const preview = await api.preview(source!.id);
+ expect(preview.entries).toEqual(h.snapshot.items.map(item => item.view));
+ expect(reads).toBeGreaterThan(2);
+ expect(inspectImportSource).toHaveBeenCalledOnce();
+ expect(compactCompanionImportSelection(preview, preview.entries.filter((_, index) => index !== 500).map(entry => entry.id)))
+ .toEqual({ entryIds: [], entryRanges: [[0, 499], [501, 9999]] });
+ const again = await readRemoteCompanionImport(continuation, 'phone', true);
+ expect(await readRemoteCompanionImport(continuation, 'phone', true)).toEqual(again);
+ await expect(readRemoteCompanionImport(continuation, 'other-phone', true)).rejects.toThrow('PREVIEW_EXPIRED');
+ h.boundary = true;
+ await expect(readRemoteCompanionImport(continuation, 'phone', true)).rejects.toThrow('OWNER_CHANGED');
+ h.boundary = false;
+ const [legacySource] = await listCompanionImportSources('old-phone');
+ const legacy = await readRemoteCompanionImport(`preview:${legacySource!.id}`, 'old-phone', false);
+ expect(legacy).not.toHaveProperty('chunk');
+ expect((legacy.preview as { entries: unknown[] }).entries).toHaveLength(10_000);
+});
+
+it.each(['directory', undefined] as const)('retries a repaired memory directory with its original request, including legacy kind=%s', async kind => {
+ const directory = path.join(h.root, 'memory', 'broken');
+ const memoryRoot = path.join(h.root, 'memory');
+ await fs.mkdir(memoryRoot);
+ h.snapshot.items = [
+ { view: { id: 'healthy', name: 'healthy.md', category: 'memory', selected: true }, text: 'Healthy' },
+ { view: { id: 'broken', name: 'broken', category: 'memory', selected: true }, captureIssue: 'IMPORT_ITEM_FAILED', sourceFile: { root: memoryRoot, file: directory, ...(kind ? { kind } : {}) } },
+ ];
+ const [source] = await listCompanionImportSources('phone');
+ const preview = await previewCompanionImport(source!.id, 'phone');
+ const selection = { previewId: preview.id, requestId: 'retry-memory-directory', name: 'Ada', entryIds: ['healthy', 'broken'], takeover: false, deferSetup: true };
+ const first = await startCompanionImport(selection, 'phone');
+ expect(first.status).toBe('needs-attention');
+ expect(h.importDocument).toHaveBeenCalledTimes(1);
+ await fs.mkdir(path.join(directory, 'nested'), { recursive: true });
+ await fs.writeFile(path.join(directory, 'nested', 'note.md'), 'Recovered note');
+ await fs.writeFile(path.join(directory, 'USER.md'), 'User preferences');
+ await fs.writeFile(path.join(directory, 'ignored.txt'), 'Not a memory');
+ await startCompanionImport(selection, 'reconnected-phone');
+ await vi.waitFor(async () => expect((await getCompanionImportResult(selection.requestId))?.status).toBe('complete'));
+ const result = (await getCompanionImportResult(selection.requestId))!;
+ expect(result).toMatchObject({ status: 'complete', botId: first.botId, savedEntryIds: ['healthy', 'broken'] });
+ expect(h.importDocument).toHaveBeenCalledTimes(3);
+ expect(h.importDocument).toHaveBeenCalledWith(first.botId, expect.stringMatching(/^broken-/), 'broken/nested/note.md', 'Recovered note', 'reference');
+ expect(h.importDocument).toHaveBeenCalledWith(first.botId, expect.stringMatching(/^broken-/), 'broken/USER.md', 'User preferences', 'user');
+ const environment = await h.store.read(h.root, first.botId, () => {});
+ expect(Object.values(environment!.documents!)).toEqual(expect.arrayContaining(['Healthy', 'Recovered note', 'User preferences']));
+ await startCompanionImport(selection, 'phone');
+ expect(h.importDocument).toHaveBeenCalledTimes(3);
+});
diff --git a/apps/desktop/src/main/bot-import/__tests__/openclawCronWorker.test.ts b/apps/desktop/src/main/bot-import/__tests__/openclawCronWorker.test.ts
index ed65495aa5..0343ece945 100644
--- a/apps/desktop/src/main/bot-import/__tests__/openclawCronWorker.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/openclawCronWorker.test.ts
@@ -30,3 +30,39 @@ it.each(['esm', 'cjs'] as const)('reads committed WAL jobs for exactly the selec
expect(db.prepare('SELECT COUNT(*) AS count FROM cron_jobs').get()).toEqual({ count: 1103 });
} finally { await worker.terminate(); db.close(); }
});
+
+it.each(['tiny', 'utf8'] as const)('stops the worker iterator before retaining an unbounded %s row set', async scenario => {
+ root = await fs.mkdtemp(path.join(os.tmpdir(), 'cindy-import-sqlite-limit-'));
+ const database = path.join(root, 'iterator-count.json');
+ const fakeDatabase = path.join(root, 'database.cjs');
+ await fs.writeFile(fakeDatabase, `
+ const fs = require('node:fs');
+ module.exports = class {
+ constructor(file) { this.file = file; this.read = 0; this.closed = false; }
+ prepare() {
+ const db = this;
+ return { *iterate() {
+ try { for (let i = 0; i < 2000000; i++) {
+ db.read++;
+ yield { job_json: ${scenario === 'tiny' ? "'{}'" : "JSON.stringify({ note: '漢'.repeat(500000) })"}, state_json: '{}' };
+ } } finally { db.closed = true; }
+ } };
+ }
+ close() { fs.writeFileSync(this.file, JSON.stringify({ read: this.read, closed: this.closed })); }
+ };
+ `);
+ const source = await fs.readFile(new URL('../openclawCronWorker.ts', import.meta.url), 'utf8');
+ const module = path.join(root, 'worker.cjs');
+ await fs.writeFile(module, transformSync(source, { loader: 'ts', format: 'cjs', platform: 'node', target: 'node22', logLevel: 'silent' }).code);
+ const worker = new Worker(module, { workerData: { database, storeKey: 'store', agentId: 'main', defaultAgent: true, modulePath: fakeDatabase } });
+ try {
+ const exit = new Promise<void>((resolve, reject) => { worker.once('exit', code => code ? reject(new Error(`worker exit ${code}`)) : resolve()); worker.once('error', reject); });
+ const reply = await new Promise<unknown>((resolve, reject) => { worker.once('message', resolve); worker.once('error', reject); });
+ await exit;
+ expect(reply).toEqual({ ok: false });
+ const counts = JSON.parse(await fs.readFile(database, 'utf8'));
+ expect(counts.closed).toBe(true);
+ expect(counts.read).toBeGreaterThan(1);
+ expect(counts.read).toBeLessThan(scenario === 'tiny' ? 65_000 : 13);
+ } finally { await worker.terminate(); }
+});
diff --git a/apps/desktop/src/main/bot-import/__tests__/registration.test.ts b/apps/desktop/src/main/bot-import/__tests__/registration.test.ts
index 2a361a5618..ff3c622c8e 100644
--- a/apps/desktop/src/main/bot-import/__tests__/registration.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/registration.test.ts
@@ -1,9 +1,9 @@
import { beforeAll, expect, it, vi } from 'vitest';
-const h = vi.hoisted(() => ({ start: vi.fn(), sources: vi.fn(), preview: vi.fn(), status: vi.fn(), handlers: new Map<string, (event: unknown, operation: unknown, input?: unknown) => Promise<unknown>>() }));
+const h = vi.hoisted(() => ({ channel: 'maker:remote-resources:invoke', remote: vi.fn(), start: vi.fn(), sources: vi.fn(), preview: vi.fn(), status: vi.fn(), handlers: new Map<string, (event: unknown, operation: unknown, input?: unknown) => Promise<unknown>>() }));
vi.mock('electron', () => ({ ipcMain: { handle: (name: string, fn: (event: unknown, operation: unknown, input?: unknown) => Promise<unknown>) => h.handlers.set(name, fn) } }));
-vi.mock('../host.js', () => ({ startCompanionImport: h.start, listCompanionImportSources: h.sources, previewCompanionImport: h.preview, getCompanionImportResult: h.status }));
+vi.mock('../host.js', () => ({ startCompanionImport: h.start, listCompanionImportSources: h.sources, previewCompanionImport: h.preview, getCompanionImportResult: h.status, readRemoteCompanionImport: h.remote }));
vi.mock('../../security/trustedAppRenderer.js', () => ({ assertTrustedAppRendererEvent: vi.fn() }));
-vi.mock('../../device-link/invoke-context.js', () => ({ getDeviceLinkInvokeContext: () => ({ controllerDeviceId: 'phone', channel: 'maker:remote-resources:invoke' }) }));
+vi.mock('../../device-link/invoke-context.js', () => ({ getDeviceLinkInvokeContext: () => ({ controllerDeviceId: 'phone', channel: h.channel }) }));
import { registerCompanionImport } from '../registration.js';
import { registerRemoteResourcesIpc } from '../../device-link/remoteResourcesIpc.js';
import { CompanionImportError } from '../types.js';
@@ -43,3 +43,10 @@ it('continues to hide unexpected provider failures from mobile', async () => {
h.start.mockRejectedValueOnce(new Error('fixture private filesystem path'));
await expect(invoke()).rejects.toThrow('[INTERNAL] remote resource provider failed');
});
+
+it.each([false, true])('negotiates chunk reads without changing the legacy resource (%s)', async supported => {
+ h.channel = 'maker:remote-resources:get';
+ h.remote.mockResolvedValueOnce({ preview: { id: 'preview', entries: [] } });
+ await h.handlers.get('maker:remote-resources:get')!({}, { ref: { collectionId: 'companion-import', kind: 'import', id: 'preview:source' }, client: { protocolVersion: 1, primitives: ['companion-import', ...(supported ? ['companion-import-chunks-v1'] : [])] } });
+ expect(h.remote).toHaveBeenLastCalledWith('preview:source', 'phone', supported);
+});
diff --git a/apps/desktop/src/main/bot-import/__tests__/sources.test.ts b/apps/desktop/src/main/bot-import/__tests__/sources.test.ts
index 465ce185f8..5563dda44c 100644
--- a/apps/desktop/src/main/bot-import/__tests__/sources.test.ts
+++ b/apps/desktop/src/main/bot-import/__tests__/sources.test.ts
@@ -54,11 +54,11 @@ it('retains an over-budget skill for retry without dropping healthy siblings', a
const jobs = (skills: string[]) => JSON.stringify([{ id: 'read', skills, prompt: 'Read resources', schedule: { kind: 'interval', minutes: 5 } }]);
await write('.hermes/cron/jobs.json', jobs(['first']));
const reader = deps(); const [source] = await discoverImportSources(reader);
- const snapshot = await inspectImportSource(source!, reader, createImportBudget(3000));
+ const snapshot = await inspectImportSource(source!, reader, createImportBudget(4500));
expect(snapshot.items.find(item => item.view.name === 'first')?.files?.find(file => file.name === 'data.txt')?.bytes.length).toBe(2048);
expect(snapshot.items.find(item => item.view.name === 'second')?.files).toHaveLength(1);
await write('.hermes/cron/jobs.json', jobs(['first', 'second']));
- const partial = await inspectImportSource(source!, reader, createImportBudget(3000));
+ const partial = await inspectImportSource(source!, reader, createImportBudget(4500));
expect(partial.items.find(item => item.view.name === 'second')?.captureIssue).toBe('SOURCE_SNAPSHOT_TOO_LARGE');
expect(partial.items.find(item => item.view.name === 'first')?.files).toHaveLength(2);
});
@@ -68,7 +68,7 @@ it('counts configuration includes and memory against the same source budget', as
await write('.openclaw/extra.json', JSON.stringify({ name: 'x'.repeat(1000) }));
await write('.openclaw/workspace/MEMORY.md', 'x'.repeat(1000));
const reader = deps(); const [source] = await discoverImportSources(reader);
- const partial = await inspectImportSource(source!, reader, createImportBudget(1500));
+ const partial = await inspectImportSource(source!, reader, createImportBudget(2100));
expect(partial.items.find(item => item.view.name === 'MEMORY.md')?.captureIssue).toBe('SOURCE_SNAPSHOT_TOO_LARGE');
});
@@ -347,7 +347,7 @@ it('shares discovery config/include reads with name masking and charges all Herm
await write(`.hermes/profiles/${id}/.env`, `API_KEY=${'z'.repeat(80)}\n`);
}
const reader = deps();
- const metadata = createImportSourceReader(reader, createImportBudget(400));
+ const metadata = createImportSourceReader(reader, createImportBudget(2350));
const opened = vi.spyOn(fs, 'open');
const sources = await discoverImportSources(reader, metadata);
expect(sources).toHaveLength(3);
@@ -435,3 +435,26 @@ it('retains unmapped skill authentication privately without deselecting the skil
expect(skill.credential).toEqual({ format: 'source-skill-auth', value: { apiKey: secret } });
expect(JSON.stringify(skill.view)).not.toContain(secret);
});
+
+it.each(['hermes', 'openclaw'] as const)('records a failed %s memory subtree separately from a failed document', async kind => {
+ await write(`.${kind}/${kind === 'hermes' ? 'config.yaml' : 'openclaw.json'}`, '{}');
+ const folder = kind === 'hermes' ? '.hermes/memories' : '.openclaw/workspace/memory';
+ await write(`${folder}/healthy.md`, 'Keep healthy');
+ await write(`${folder}/broken/note.md`, 'Repair directory');
+ await write(`${folder}/bad.md`, 'Repair file');
+ const open = fs.opendir.bind(fs);
+ vi.spyOn(fs, 'opendir').mockImplementation(((...args: Parameters<typeof fs.opendir>) => {
+ if (String(args[0]).endsWith(`${path.sep}broken`)) return Promise.reject(Object.assign(new Error('denied'), { code: 'EACCES' }));
+ return open(...args);
+ }) as typeof fs.opendir);
+ const read = fs.open.bind(fs);
+ vi.spyOn(fs, 'open').mockImplementation(((...args: Parameters<typeof fs.open>) => {
+ if (String(args[0]).endsWith(`${path.sep}bad.md`)) return Promise.reject(Object.assign(new Error('denied'), { code: 'EACCES' }));
+ return read(...args);
+ }) as typeof fs.open);
+ const reader = deps(); const [source] = await discoverImportSources(reader);
+ const snapshot = await inspectImportSource(source!, reader);
+ expect(snapshot.items.find(item => item.view.name === 'broken')?.sourceFile).toMatchObject({ root: path.join(home, folder), file: path.join(home, folder, 'broken'), kind: 'directory' });
+ expect(snapshot.items.find(item => item.view.name === 'bad.md')?.sourceFile?.kind).toBe('file');
+ expect(snapshot.items.find(item => item.view.name === 'healthy.md')?.text).toBe('Keep healthy');
+});
diff --git a/apps/desktop/src/main/bot-import/files.ts b/apps/desktop/src/main/bot-import/files.ts
index 1909e65c45..44a85f9c3d 100644
--- a/apps/desktop/src/main/bot-import/files.ts
+++ b/apps/desktop/src/main/bot-import/files.ts
@@ -16,7 +16,8 @@ export function createImportBudget(limit = MAX_SNAPSHOT_BYTES) {
remaining -= size;
};
return { reserve, reserveFile(size: number) {
- reserve(size);
+ // Even empty files retain a Buffer, metadata and checkpoint representation.
+ reserve(size + 256);
} };
}
export type ImportReadBudget = ReturnType<typeof createImportBudget>;
@@ -111,6 +112,7 @@ export async function readImportFile(root: string, file: string, budget?: Import
if (!stat.isFile()) throw new CompanionImportError('SOURCE_NOT_REGULAR_FILE');
if (stat.size > MAX_FILE_BYTES) throw new CompanionImportError('SOURCE_FILE_TOO_LARGE');
budget?.reserveFile(stat.size);
+ budget?.reserve(Buffer.byteLength(path.relative(root, file)));
// A bounded read also handles files growing after fstat.
const bytes = Buffer.alloc(Math.min(stat.size + 1, MAX_FILE_BYTES + 1));
let size = 0;
@@ -133,7 +135,7 @@ export async function optionalText(root: string, file: string, budget?: ImportRe
}
/** Only traverses the selected skill/document subtree; never copies a whole Agent home. */
-export async function readImportTree(root: string, include: (name: string) => boolean = () => true, budget?: ImportReadBudget, onError?: (name: string, error: unknown) => void): Promise<ImportFile[]> {
+export async function readImportTree(root: string, include: (name: string) => boolean = () => true, budget?: ImportReadBudget, onError?: (name: string, error: unknown, kind: 'file' | 'directory' | 'unknown') => void, directory = root): Promise<ImportFile[]> {
const result: ImportFile[] = [];
const visited = new Set<string>();
let size = 0;
@@ -143,26 +145,35 @@ export async function readImportTree(root: string, include: (name: string) => bo
if (!inside(realRoot, real)) throw new CompanionImportError('SOURCE_LINK_OUTSIDE_FOLDER');
if (visited.has(real)) throw new CompanionImportError('SOURCE_LINK_CYCLE');
visited.add(real);
- for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
- if (entry.name === '.git' || entry.name === '__pycache__' || entry.name === '.DS_Store') continue;
- const file = path.join(dir, entry.name);
- try {
- const target = entry.isSymbolicLink() ? await fs.stat(file) : entry;
- if (target.isDirectory()) await visit(file);
- else if (include(path.relative(root, file).split(path.sep).join('/'))) {
- const item = await readImportFile(root, file, budget);
- size += item.bytes.length;
- if (size > MAX_ITEM_BYTES) throw new CompanionImportError('SOURCE_ITEM_TOO_LARGE');
- result.push(item);
+ try {
+ budget?.reserve(128 + Buffer.byteLength(path.relative(root, dir)));
+ // Stream directory entries too: empty files and rejected links must not
+ // allocate an unbounded readdir array or an unbounded list of errors.
+ for await (const entry of await fs.opendir(dir)) {
+ if (entry.name === '.git' || entry.name === '__pycache__' || entry.name === '.DS_Store') continue;
+ const file = path.join(dir, entry.name);
+ const name = path.relative(root, file).split(path.sep).join('/');
+ budget?.reserve(128 + Buffer.byteLength(name));
+ let kind: 'file' | 'directory' | 'unknown' = entry.isDirectory() ? 'directory' : entry.isSymbolicLink() ? 'unknown' : 'file';
+ try {
+ const target = entry.isSymbolicLink() ? await fs.stat(file) : entry;
+ kind = target.isDirectory() ? 'directory' : 'file';
+ if (target.isDirectory()) await visit(file);
+ else if (include(name)) {
+ const item = await readImportFile(root, file, budget);
+ size += item.bytes.length;
+ if (size > MAX_ITEM_BYTES) throw new CompanionImportError('SOURCE_ITEM_TOO_LARGE');
+ result.push(item);
+ }
+ } catch (error) {
+ if (!onError || error instanceof CompanionImportError && error.code === 'SOURCE_SNAPSHOT_TOO_LARGE') throw error;
+ onError(name, error, kind);
}
- } catch (error) {
- if (!onError) throw error;
- onError(path.relative(root, file).split(path.sep).join('/'), error);
}
- }
- visited.delete(real);
+ } finally { visited.delete(real); }
}
- await visit(root);
+ if (!inside(path.resolve(root), path.resolve(directory))) throw new CompanionImportError('SOURCE_LINK_OUTSIDE_FOLDER');
+ await visit(directory);
return result.sort((a, b) => a.name.localeCompare(b.name));
}
diff --git a/apps/desktop/src/main/bot-import/host.ts b/apps/desktop/src/main/bot-import/host.ts
index 7aa92da301..3d4f62cc3f 100644
--- a/apps/desktop/src/main/bot-import/host.ts
+++ b/apps/desktop/src/main/bot-import/host.ts
@@ -10,6 +10,7 @@ import { parseRoutineInput, type RoutineInput } from '@cindy/maker-scheduler';
import { normalizeBotName } from '../../shared/botCreation.js';
import { listBotRemoteResourceSources } from '../localDb/ipc/bots.js';
import { validateBotAvatarBuffer, decodeBotAvatarImage } from '../localDb/ipc/botAvatarSelection.js';
+import { COMPANION_IMPORT_CHUNK_LENGTH, COMPANION_IMPORT_READ_MAX_LENGTH } from '@cindy/maker-shared/companion-import';
import type { CompanionImportPreview, CompanionImportResult, CompanionImportSelection, CompanionImportSource } from '@cindy/maker-shared/companion-import';
import { activeOwnerScopeKey, getActiveAppSession, isAppSessionBoundaryPending, ownerScopedUserDataPath } from '../appSessionState.js';
import { createBotCanonicalSession, createBotProfile, getBotMemoryService, getBotRemoteResourceSource, reconcileBotProfileFolder } from '../localDb/ipc/bots.js';
@@ -24,7 +25,7 @@ import { readOpenClawCronDatabase } from './openclawCron.js';
import { companionEnvironmentStore, recoverCompanionEnvironmentRemovals } from './runtime.js';
import { deserializeImportSnapshotAsync, fingerprint, serializeImportSnapshotAsync, reserveSnapshotItems, MAX_SNAPSHOT_BYTES } from './files.js';
import { transferCompanion, validateImportSelection, type ImportReceipt, type TransferDeps } from './transfer.js';
-import { CompanionImportError, type ImportSnapshot, type ImportSource } from './types.js';
+import { CompanionImportError, type ImportItem, type ImportSnapshot, type ImportSource } from './types.js';
import { changeSourceAutomationState } from './takeover.js';
import { verifyImportedAutomation } from './verification.js';
import { projectImportedSkill } from './skillResources.js';
@@ -33,6 +34,7 @@ import { assertImportedAutomationReady } from './automationRuntime.js';
interface Owned<T> { owner: string; controller: string; value: T; createdAt: number }
const sources = new Map<string, Owned<ImportSource>>();
const previews = new Map<string, Owned<ImportSnapshot> & { bytes: number }>();
+const remoteReads = new Map<string, Owned<string>>();
const jobs = new Map<string, Promise<CompanionImportResult>>();
const TTL = 30 * 60_000;
function owner() {
@@ -119,6 +121,47 @@ export async function previewCompanionImport(sourceId: string, controller: strin
...(view.description === undefined ? {} : { description: redact(view.description) }) })) };
}
+/** Freeze each public read while transferring it; never re-inspect a source between chunks.
+ * UTF-16 slices are at most 1.5 MiB after JSON escaping, well below a 2 MiB frame. */
+export async function readRemoteCompanionImport(id: string, controller: string, chunked: boolean): Promise<Record<string, unknown>> {
+ const scope = owner();
+ const chunk = (token: string, text: string, offset: number) => ({
+ chunk: { id: token, offset, total: text.length, text: text.slice(offset, offset + COMPANION_IMPORT_CHUNK_LENGTH) },
+ });
+ if (id.startsWith('chunk:')) {
+ const match = /^chunk:([a-zA-Z0-9-]{1,64}):(0|[1-9][0-9]*)$/.exec(id);
+ if (!chunked || !match) throw new CompanionImportError('INVALID_REQUEST');
+ const text = owned(remoteReads, match[1]!, controller);
+ const offset = Number(match[2]);
+ if (!Number.isSafeInteger(offset) || offset >= text.length || offset % COMPANION_IMPORT_CHUNK_LENGTH) throw new CompanionImportError('INVALID_REQUEST');
+ return chunk(match[1]!, text, offset);
+ }
+ const data = id === 'sources' ? { sources: await listCompanionImportSources(controller) }
+ : id.startsWith('preview:') ? { preview: await previewCompanionImport(id.slice(8), controller) }
+ : id.startsWith('result:') ? { result: await getCompanionImportResult(id.slice(7)) ?? null } : undefined;
+ scope.assert();
+ if (!data) throw new CompanionImportError('INVALID_REQUEST');
+ if (!chunked) return data;
+ const text = JSON.stringify(data);
+ if (text.length <= COMPANION_IMPORT_CHUNK_LENGTH) return data;
+ if (text.length > COMPANION_IMPORT_READ_MAX_LENGTH || text.length * 2 > MAX_SNAPSHOT_BYTES) throw new CompanionImportError('SOURCE_SNAPSHOT_TOO_LARGE');
+ prune(remoteReads);
+ for (const [key, entry] of remoteReads) if (entry.controller === controller) remoteReads.delete(key);
+ let bytes = [...remoteReads.values()].reduce((sum, entry) => sum + entry.value.length * 2, 0);
+ for (const [key, entry] of remoteReads) {
+ if (remoteReads.size < 4 && bytes + text.length * 2 <= MAX_SNAPSHOT_BYTES) break;
+ remoteReads.delete(key); bytes -= entry.value.length * 2;
+ }
+ const token = randomUUID();
+ remoteReads.set(token, { owner: scope.scope, controller, value: text, createdAt: Date.now() });
+ return chunk(token, text, 0);
+}
+
+function documentEntries(item: ImportItem): Array<[string, string]> {
+ return [...(item.text === undefined ? [] : [[item.view.id, item.text] as [string, string]]),
+ ...(item.documents ?? []).map(document => [document.id, document.text] as [string, string])];
+}
+
function receiptFile(root: string, requestId: string) {
if (!/^[A-Za-z0-9_-]{16,100}$/.test(requestId)) throw new CompanionImportError('INVALID_REQUEST');
return path.join(root, 'companion-imports', `${requestId}.json`);
@@ -395,11 +438,15 @@ export async function startCompanionImport(selection: CompanionImportSelection,
const slug = skillSlug(item);
const projected = projectImportedSkill(item.files ?? [], slug, contentSecrets);
await importBotSkillFiles(scope.root, botId, slug, projected.files, scope.assert, item.view.enabled !== false);
- } else if (item.text && (item.view.category === 'memory' || item.view.category === 'personality')) {
- await getBotMemoryService().importDocument(botId, item.view.id, redactText(item.view.name), redactText(item.text), item.role === 'user' ? 'user' : 'reference');
+ } else if (item.view.category === 'memory' || item.view.category === 'personality') {
+ const documents = [...(item.text ? [{ id: item.view.id, name: item.view.name, text: item.text, role: item.role }] : []), ...item.documents ?? []];
+ for (const document of documents) {
+ scope.assert();
+ await getBotMemoryService().importDocument(botId, document.id, redactText(document.name), redactText(document.text), document.role === 'user' ? 'user' : 'reference');
+ }
}
if (prior?.environmentSaved) await companionEnvironmentStore.update(scope.root, botId, scope.assert, environment => {
- if (item.text !== undefined) { environment.documents ??= {}; environment.documents[item.view.id] = item.text; }
+ for (const [id, text] of documentEntries(item)) { environment.documents ??= {}; environment.documents[id] = text; }
if (item.view.category === 'skills') {
const slug = skillSlug(item);
const originals = projectImportedSkill(item.files ?? [], slug, contentSecrets).originals;
@@ -438,14 +485,14 @@ export async function startCompanionImport(selection: CompanionImportSelection,
credentials: items.flatMap(item => item.credential && !item.view.dependsOn?.some(id => !chosen.has(id)) && (!item.view.issues?.length || item.credential.format !== 'telegram') ? [{ id: item.view.id, ...item.credential, ...(item.credential.format === 'telegram' ? { value: resolveReferences(item.credential.value) } : {}) }] : []),
files: Object.fromEntries(items.flatMap(item => item.asset ? [[item.asset.name, item.asset.bytes.toString('base64')]] : [])),
skillFiles,
- documents: Object.fromEntries(items.flatMap(item => item.text === undefined ? [] : [[item.view.id, item.text]])),
+ documents: Object.fromEntries(items.flatMap(documentEntries)),
contentRedactions: publicationRedactions(items),
sourceAutomations: items.flatMap(item => item.automation ? [{ entryId: item.view.id, kind: snapshot.source.kind, original: item.automation.original }] : []),
pendingImport: previous?.pendingImport ?? { selection, snapshotJson: await serializeImportSnapshotAsync({ ...snapshot, avatarImageBase64: selection.avatarImageBase64, items, publicationRedactions: publicationRedactions(items) }, scope.assert) },
automations: previous?.automations ?? {},
}, scope.assert);
const roleText = (role: 'identity' | 'user' | 'instructions') => {
- const text = redactText(items.filter(item => item.role === role).map(item => item.text).join('\n\n'));
+ const text = redactText(items.flatMap(item => [...(item.role === role ? [item.text] : []), ...(item.documents ?? []).filter(document => document.role === role).map(document => document.text)]).join('\n\n'));
return Buffer.byteLength(text, 'utf8') <= BOT_PROFILE_TEXT_MAX_BYTES ? text : `The imported ${role} documents are preserved in your personal memory. Search and read those original documents when applying the imported profile.`;
};
const folder = await readBotProfileFolder(scope.root, botId); scope.assert();
@@ -574,7 +621,8 @@ async function accepted(task: Promise<CompanionImportResult>, root: string, requ
const receipt = (async () => {
while (!finished) {
const value = await readReceipt(root, requestId);
- if (value && (value.companionCreated || value.environmentSaved)) return value.result;
+ // A retry must not acknowledge the previous attempt's terminal receipt.
+ if (value?.result.status === 'running' && (value.companionCreated || value.environmentSaved)) return value.result;
await new Promise(resolve => setTimeout(resolve, 40));
}
return task;
diff --git a/apps/desktop/src/main/bot-import/openclawCronWorker.ts b/apps/desktop/src/main/bot-import/openclawCronWorker.ts
index 2170709d7f..9d1fdb1c6d 100644
--- a/apps/desktop/src/main/bot-import/openclawCronWorker.ts
+++ b/apps/desktop/src/main/bot-import/openclawCronWorker.ts
@@ -15,9 +15,17 @@ try {
const rows = db.prepare(`SELECT job_json, state_json FROM cron_jobs
WHERE store_key = ? AND (COALESCE(agent_id, owner_agent_id) = ?
OR (? = 1 AND COALESCE(agent_id, owner_agent_id) IS NULL))
- ORDER BY sort_order`).all(storeKey, agentId, defaultAgent ? 1 : 0) as Array<{ job_json: string; state_json: string }>;
- if (rows.reduce((n, row) => n + row.job_json.length + row.state_json.length, 0) > 16 * 1024 * 1024) throw new Error('limit');
- parentPort?.postMessage({ ok: true, jobs: rows.map(row => ({ ...JSON.parse(row.job_json), state: JSON.parse(row.state_json) })) });
+ ORDER BY sort_order`).iterate(storeKey, agentId, defaultAgent ? 1 : 0) as Iterable<{ job_json: string; state_json: string }>;
+ const jobs: Array<Record<string, unknown>> = [];
+ let bytes = 0;
+ for (const row of rows) {
+ // Account for retained row/object overhead too, including empty JSON rows.
+ // Check before parsing or retaining this row, never materialize the result set.
+ bytes += Buffer.byteLength(row.job_json) + Buffer.byteLength(row.state_json) + 256;
+ if (bytes > 16 * 1024 * 1024) throw new Error('limit');
+ jobs.push({ ...JSON.parse(row.job_json), state: JSON.parse(row.state_json) });
+ }
+ parentPort?.postMessage({ ok: true, jobs });
} catch {
// SQL errors may include source paths/content; only a stable code crosses back.
parentPort?.postMessage({ ok: false });
diff --git a/apps/desktop/src/main/bot-import/registration.ts b/apps/desktop/src/main/bot-import/registration.ts
index b832b79fbe..0497c8d027 100644
--- a/apps/desktop/src/main/bot-import/registration.ts
+++ b/apps/desktop/src/main/bot-import/registration.ts
@@ -1,8 +1,8 @@
import { ipcMain } from 'electron';
-import type { CompanionImportSelection } from '@cindy/maker-shared/companion-import';
+import { COMPANION_IMPORT_CHUNK_PRIMITIVE, type CompanionImportSelection } from '@cindy/maker-shared/companion-import';
import { remoteResourceRegistry, RemoteResourceRegistryError } from '../device-link/remoteResourceRegistry.js';
import { assertTrustedAppRendererEvent } from '../security/trustedAppRenderer.js';
-import { listCompanionImportSources, previewCompanionImport, startCompanionImport, getCompanionImportResult } from './host.js';
+import { listCompanionImportSources, previewCompanionImport, startCompanionImport, getCompanionImportResult, readRemoteCompanionImport } from './host.js';
import { CompanionImportError } from './types.js';
import { isIpcErrorCode } from '../../shared/ipc-errors.js';
import { throwIpcError } from '../utils/ipcValidate.js';
@@ -29,10 +29,8 @@ export function registerCompanionImport(): void {
async get(context, request) {
try {
const id = request.ref.id;
- const data = id === 'sources' ? { sources: await listCompanionImportSources(context.controllerDeviceId) }
- : id.startsWith('preview:') ? { preview: await previewCompanionImport(id.slice(8), context.controllerDeviceId) }
- : id.startsWith('result:') ? { result: await getCompanionImportResult(id.slice(7)) ?? null } : undefined;
- if (!data) throw new CompanionImportError('INVALID_REQUEST');
+ const data = await readRemoteCompanionImport(id, context.controllerDeviceId,
+ request.client.primitives.includes(COMPANION_IMPORT_CHUNK_PRIMITIVE));
return { ref: request.ref, revision: '1', display: { title: 'Import' }, links: [],
blocks: [{ id: 'import', primitive: 'companion-import', fallbackMarkdown: 'Hermes / OpenClaw', data }],
actions: id.startsWith('preview:') ? [{ id: 'import', label: 'Import' }] : [] };
diff --git a/apps/desktop/src/main/bot-import/sources.ts b/apps/desktop/src/main/bot-import/sources.ts
index cbd844c21e..bc0705a8ac 100644
--- a/apps/desktop/src/main/bot-import/sources.ts
+++ b/apps/desktop/src/main/bot-import/sources.ts
@@ -124,11 +124,11 @@ async function document(items: ImportItem[], root: string, name: string, role: I
async function memoryDocuments(items: ImportItem[], root: string, prefix: string, budget: ImportReadBudget) {
if (!(await directories(root)).includes(prefix)) return;
const directory = path.join(root, prefix);
- const failed = (name: string, error: unknown) => items.push({ view: { id: entryId('memory', `${prefix}/${name}`), category: 'memory', name, selected: true },
- sourceFile: { root: directory, file: path.join(directory, name) }, captureIssue: error instanceof CompanionImportError ? error.code : 'IMPORT_ITEM_FAILED' });
+ const failed = (name: string, error: unknown, kind: 'file' | 'directory' | 'unknown') => items.push({ view: { id: entryId('memory', `${prefix}/${name}`), category: 'memory', name: name || prefix, selected: true },
+ sourceFile: { root: directory, file: path.join(directory, name), kind }, captureIssue: error instanceof CompanionImportError ? error.code : 'IMPORT_ITEM_FAILED' });
let files;
try { files = await readImportTree(directory, name => /\.md$/i.test(name), budget, failed); }
- catch (error) { failed(prefix, error); return; }
+ catch (error) { failed('', error, 'directory'); return; }
for (const file of files.filter(file => /\.md$/i.test(file.name))) {
if (!file.bytes.toString('utf8').trim()) continue;
items.push({ view: { id: entryId('memory', `${prefix}/${file.name}`), category: 'memory', name: file.name, selected: true },
diff --git a/apps/desktop/src/main/bot-import/transfer.ts b/apps/desktop/src/main/bot-import/transfer.ts
index 7e3e2988a7..b52923305a 100644
--- a/apps/desktop/src/main/bot-import/transfer.ts
+++ b/apps/desktop/src/main/bot-import/transfer.ts
@@ -1,3 +1,4 @@
+import { promises as fs } from 'node:fs';
import type { CompanionImportResult, CompanionImportSelection } from '@cindy/maker-shared/companion-import';
import type { RoutineInput } from '@cindy/maker-scheduler';
import { createImportBudget, fingerprint, readImportFile, readImportTree, reserveSnapshotItems } from './files.js';
@@ -94,7 +95,17 @@ export async function transferCompanion(snapshot: ImportSnapshot, selection: Com
}
if (item.captureIssue && item.sourceFile) {
try {
- item.text = (await readImportFile(item.sourceFile.root, item.sourceFile.file, budget)).bytes.toString('utf8');
+ const source = item.sourceFile;
+ // Missing kind is a legacy checkpoint. Re-evaluate repaired paths, but
+ // keep containment anchored to the original memory root in either case.
+ const directory = source.kind === 'directory' || source.kind !== 'file' && (await fs.stat(source.file)).isDirectory();
+ if (directory) {
+ const files = await readImportTree(source.root, name => /\.md$/i.test(name), budget, undefined, source.file);
+ item.documents = files.filter(file => file.bytes.toString('utf8').trim()).map(file => ({
+ id: `${item.view.id}-${fingerprint(file.name).slice(0, 20)}`, name: file.name, text: file.bytes.toString('utf8'),
+ ...(/(^|\/)USER\.md$/i.test(file.name) ? { role: 'user' as const } : {}),
+ }));
+ } else item.text = (await readImportFile(source.root, source.file, budget)).bytes.toString('utf8');
delete item.captureIssue;
} catch (error) { item.captureIssue = error instanceof CompanionImportError ? error.code : 'IMPORT_ITEM_FAILED'; }
deps.assertOwner();
diff --git a/apps/desktop/src/main/bot-import/types.ts b/apps/desktop/src/main/bot-import/types.ts
index 74ce03061b..44ea67e5cf 100644
--- a/apps/desktop/src/main/bot-import/types.ts
+++ b/apps/desktop/src/main/bot-import/types.ts
@@ -43,8 +43,10 @@ export interface ImportItem {
files?: ImportFile[];
sourceDirectory?: string;
sourceAlias?: string;
- /** Failed individual document read; retained for an explicit retry. */
- sourceFile?: { root: string; file: string };
+ /** Failed document or subtree read; retained for an explicit retry. */
+ sourceFile?: { root: string; file: string; kind?: 'file' | 'directory' | 'unknown' };
+ /** Recovered subtree documents stay under the original selected entry/receipt. */
+ documents?: Array<{ id: string; name: string; text: string; role?: 'user' }>;
/** Selected skill resources have been captured for copying, verification and restart. */
filesComplete?: boolean;
/** Item-level capture failure retained for a later retry, without discarding healthy siblings. */
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/botSkillStore.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/botSkillStore.test.ts
index b62218efb3..6c7f79c61d 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/botSkillStore.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/botSkillStore.test.ts
@@ -316,3 +316,21 @@ it('retains a disabled imported skill without mounting or re-enabling it when ed
expect(await deleteBotSkill(userDataDir, 'bot-1', 'disabled')).toBe(true);
expect(await listBotSkills(userDataDir, 'bot-1')).toEqual([]);
});
+
+it.each([false, true])('preserves the other skill when an import collides across enabled stores (%s)', async enabled => {
+ const files = [{ name: 'SKILL.md', bytes: Buffer.from('# Existing'), executable: false }];
+ await importBotSkillFiles(userDataDir, 'bot-1', 'report', files, () => {}, enabled);
+ await expect(importBotSkillFiles(userDataDir, 'bot-1', 'report', [{ ...files[0]!, bytes: Buffer.from('# Different') }], () => {}, !enabled)).rejects.toThrow('conflicts with an existing skill');
+ expect(await listBotSkills(userDataDir, 'bot-1')).toHaveLength(1);
+ expect((await readBotSkill(userDataDir, 'bot-1', 'report'))?.body).toBe('# Existing');
+});
+
+it('deletes only the read/edit target if older data contains duplicate enabled and disabled slugs', async () => {
+ await saveBotSkill(userDataDir, 'bot-1', { ...SAMPLE, slug: 'report' });
+ const disabled = path.join(botSkillRootDir(userDataDir, 'bot-1'), 'disabled-skills', 'report');
+ await fs.mkdir(disabled, { recursive: true });
+ await fs.writeFile(path.join(disabled, 'SKILL.md'), '# Retain disabled copy');
+ expect(await deleteBotSkill(userDataDir, 'bot-1', 'report')).toBe(true);
+ expect(await fs.readFile(path.join(disabled, 'SKILL.md'), 'utf8')).toBe('# Retain disabled copy');
+ expect(await readBotSkill(userDataDir, 'bot-1', 'report')).toMatchObject({ enabled: false, body: '# Retain disabled copy' });
+});
diff --git a/apps/desktop/src/main/maker-ipc/botSkillStore.ts b/apps/desktop/src/main/maker-ipc/botSkillStore.ts
index b3706b940f..5579dfcbf6 100644
--- a/apps/desktop/src/main/maker-ipc/botSkillStore.ts
+++ b/apps/desktop/src/main/maker-ipc/botSkillStore.ts
@@ -498,6 +498,14 @@ export async function importBotSkillFiles(userDataDir: string, botId: string, sl
validateBotSkillFiles(slug, files);
await ensureLayout(userDataDir, botId);
assertOwner();
+ // Slugs identify one skill across both stores. A source must not create a
+ // second, indistinguishable entry or overwrite a skill learned during retry.
+ try {
+ await fs.lstat(resolveSkillDir(userDataDir, botId, slug, !enabled));
+ throw new BotSkillStoreError('INVALID_ARGS', 'Imported skill conflicts with an existing skill');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+ }
const target = resolveSkillDir(userDataDir, botId, slug, enabled);
await fs.mkdir(path.dirname(target), { recursive: true });
const temporary = path.join(botSkillRootDir(userDataDir, botId), `.import-skill-${randomUUID()}`);
@@ -535,13 +543,14 @@ export async function deleteBotSkill(
botId: string,
slug: string,
): Promise<boolean> {
- let deleted = false;
+ // Match read/edit precedence, including any pre-existing duplicate directories.
+ // Deleting one visible skill must never remove another directory implicitly.
for (const enabled of [true, false]) {
const skillDir = resolveSkillDir(userDataDir, botId, slug, enabled);
try { if (!(await fs.stat(skillDir)).isDirectory()) continue; }
catch { continue; }
await fs.rm(skillDir, { recursive: true, force: true });
- deleted = true;
+ return true;
}
- return deleted;
+ return false;
}
diff --git a/apps/mobile/src/session/CompanionImportSheet.tsx b/apps/mobile/src/session/CompanionImportSheet.tsx
index 996c3c45c1..55cb49c995 100644
--- a/apps/mobile/src/session/CompanionImportSheet.tsx
+++ b/apps/mobile/src/session/CompanionImportSheet.tsx
@@ -3,7 +3,7 @@ import { Pressable, ScrollView, StyleSheet, Switch, View } from 'react-native';
import { randomUUID } from 'expo-crypto';
import { useTranslation } from 'react-i18next';
import { parseRemoteActionInvokeRequest, REMOTE_RESOURCE_GET_CHANNEL, REMOTE_RESOURCE_PROTOCOL_VERSION, type RemoteResourceRef } from '@cindy/device-link';
-import { compactCompanionImportSelection, areCompanionImportEntriesSelected, toggleCompanionImportEntries, companionImportCategories, remoteCompanionImportApi, type CompanionImportPreview, type CompanionImportResult, type CompanionImportSelection, type CompanionImportSource } from '@cindy/maker-shared/companion-import';
+import { COMPANION_IMPORT_CHUNK_PRIMITIVE, compactCompanionImportSelection, areCompanionImportEntriesSelected, toggleCompanionImportEntries, companionImportCategories, remoteCompanionImportApi, type CompanionImportPreview, type CompanionImportResult, type CompanionImportSelection, type CompanionImportSource } from '@cindy/maker-shared/companion-import';
import { Text, TextInput } from '@/components/AppText';
import { MainWindowActionButton } from '@/components/MobilePrimitives';
import { useDeviceLink } from '@/device-link/DeviceLinkContext';
@@ -20,7 +20,7 @@ export function CompanionImportSheet({ visible, onClose, onClosed, deviceId, dev
const styles = useThemedStyles(makeStyles);
const { invoke, openLink } = useDeviceLink();
const api = useMemo(() => remoteCompanionImportApi(
- async id => { await openLink(deviceId); return invoke(deviceId, REMOTE_RESOURCE_GET_CHANNEL, [{ ref: { collectionId: 'companion-import', kind: 'import', id }, client: { protocolVersion: REMOTE_RESOURCE_PROTOCOL_VERSION, primitives: ['companion-import'], locale: i18n.language } }]); },
+ async id => { await openLink(deviceId); return invoke(deviceId, REMOTE_RESOURCE_GET_CHANNEL, [{ ref: { collectionId: 'companion-import', kind: 'import', id }, client: { protocolVersion: REMOTE_RESOURCE_PROTOCOL_VERSION, primitives: ['companion-import', COMPANION_IMPORT_CHUNK_PRIMITIVE], locale: i18n.language } }]); },
(sourceId, selection) => invokeRemoteResourceAction(invoke, { deviceId, deviceName }, { collectionId: 'companion-import', resourceRef: { collectionId: 'companion-import', kind: 'import', id: `preview:${sourceId}` }, actionId: 'import', input: { ...selection } }, i18n.language),
), [deviceId, deviceName, invoke, openLink, i18n.language]);
const [sources, setSources] = useState<CompanionImportSource[]>();
diff --git a/packages/maker-shared/src/__tests__/companionImport.test.ts b/packages/maker-shared/src/__tests__/companionImport.test.ts
index 9ec969d1c5..5805e4b848 100644
--- a/packages/maker-shared/src/__tests__/companionImport.test.ts
+++ b/packages/maker-shared/src/__tests__/companionImport.test.ts
@@ -1,5 +1,5 @@
import { expect, it, vi } from 'vitest';
-import { compactCompanionImportSelection, remoteCompanionImportApi, areCompanionImportEntriesSelected, toggleCompanionImportEntries, type CompanionImportEntry, type CompanionImportPreview } from '../companionImport.js';
+import { COMPANION_IMPORT_CHUNK_LENGTH, compactCompanionImportSelection, remoteCompanionImportApi, areCompanionImportEntriesSelected, toggleCompanionImportEntries, type CompanionImportEntry, type CompanionImportPreview } from '../companionImport.js';
const entries: CompanionImportEntry[] = [
{ id: 'work', name: 'Work', category: 'connections', selected: false, exclusiveWith: ['personal'] },
{ id: 'personal', name: 'Personal', category: 'connections', selected: false, exclusiveWith: ['work'] },
@@ -31,3 +31,30 @@ it.each([false, true])('negotiates compact selections only when the host adverti
expect(invoke).toHaveBeenCalledWith('source', expect.objectContaining(supported ? { entryIds: [], entryRanges: [[0, 99], [101, 2099]] } : { entryIds: chosen }));
if (!supported) expect(invoke.mock.calls[0]?.[1]).not.toHaveProperty('entryRanges');
});
+
+it('reassembles source and result reads without dropping saved IDs or checks', async () => {
+ const sources = Array.from({ length: 3000 }, (_, index) => ({ id: `source-${index}`, kind: 'hermes', name: 'Name'.repeat(20) }));
+ const result = { requestId: 'request-1234567890', botId: 'bot', status: 'complete', checks: sources.map(source => ({ entryId: source.id, status: 'copied' })), savedEntryIds: sources.map(source => source.id) };
+ let text = '';
+ const api = remoteCompanionImportApi(async id => {
+ if (!id.startsWith('chunk:')) text = JSON.stringify(id === 'sources' ? { sources } : { result });
+ const offset = id.startsWith('chunk:') ? Number(id.split(':')[2]) : 0;
+ return { blocks: [{ primitive: 'companion-import', data: { chunk: { id: 'token', offset, total: text.length, text: text.slice(offset, offset + COMPANION_IMPORT_CHUNK_LENGTH) } } }] };
+ }, async () => {});
+ expect(await api.sources()).toEqual(sources);
+ expect(await api.status(result.requestId)).toEqual(result);
+});
+
+it.each(['token', 'offset', 'total', 'missing', 'disconnect'])('rejects %s drift during chunk reads without publishing a partial preview', async mode => {
+ const first = { id: 'stable', offset: 0, total: COMPANION_IMPORT_CHUNK_LENGTH + 10, text: 'x'.repeat(COMPANION_IMPORT_CHUNK_LENGTH) };
+ const read = vi.fn(async (id: string) => {
+ if (id.startsWith('chunk:') && mode === 'disconnect') throw new Error('DISCONNECTED');
+ const chunk = !id.startsWith('chunk:') ? first : { ...first, offset: COMPANION_IMPORT_CHUNK_LENGTH, text: 'x'.repeat(10),
+ ...(mode === 'token' ? { id: 'changed' } : {}), ...(mode === 'offset' ? { offset: 0 } : {}), ...(mode === 'total' ? { total: first.total + 1 } : {}),
+ };
+ return { blocks: [{ primitive: 'companion-import', data: mode === 'missing' && id.startsWith('chunk:') ? {} : { chunk } }] };
+ });
+ const api = remoteCompanionImportApi(read, async () => {});
+ await expect(api.preview('source')).rejects.toThrow(mode === 'disconnect' ? 'DISCONNECTED' : 'INVALID_IMPORT_RESPONSE');
+ expect(read).toHaveBeenCalledTimes(2);
+});
diff --git a/packages/maker-shared/src/companionImport.ts b/packages/maker-shared/src/companionImport.ts
index bb554e9f7a..1c0b5e511b 100644
--- a/packages/maker-shared/src/companionImport.ts
+++ b/packages/maker-shared/src/companionImport.ts
@@ -125,17 +125,47 @@ export function companionImportIssueKey(code?: string): string {
return 'itemAttention';
}
+/** Additive read capability; older hosts continue returning the original projection. */
+export const COMPANION_IMPORT_CHUNK_PRIMITIVE = 'companion-import-chunks-v1';
+export const COMPANION_IMPORT_CHUNK_LENGTH = 256 * 1024;
+export const COMPANION_IMPORT_READ_MAX_LENGTH = 128 * 1024 * 1024;
+
/** Transport-neutral client: Desktop and Mobile use their existing Remote Resource adapters. */
export function remoteCompanionImportApi(
read: (id: string) => Promise<unknown>,
invoke: (sourceId: string, selection: CompanionImportSelection) => Promise<unknown>,
): CompanionImportApi {
- const data = async (id: string): Promise<Record<string, unknown>> => {
+ const readData = async (id: string): Promise<Record<string, unknown>> => {
const raw = await read(id) as { blocks?: Array<{ primitive?: string; data?: Record<string, unknown> }> };
const block = raw?.blocks?.find(block => block.primitive === 'companion-import');
if (!block?.data) throw new Error('IMPORT_UNAVAILABLE');
return block.data;
};
+ const data = async (id: string): Promise<Record<string, unknown>> => {
+ let part = await readData(id);
+ if (!part.chunk) return part;
+ const pieces: string[] = [];
+ let offset = 0;
+ let token: string | undefined;
+ let total: number | undefined;
+ for (;;) {
+ const chunk = part.chunk as { id?: unknown; offset?: unknown; total?: unknown; text?: unknown } | undefined;
+ if (!chunk || typeof chunk.id !== 'string' || !/^[a-zA-Z0-9-]{1,64}$/.test(chunk.id)
+ || chunk.offset !== offset || !Number.isSafeInteger(chunk.total)
+ || typeof chunk.total !== 'number' || chunk.total <= 0 || chunk.total > COMPANION_IMPORT_READ_MAX_LENGTH
+ || typeof chunk.text !== 'string' || !chunk.text.length || chunk.text.length > COMPANION_IMPORT_CHUNK_LENGTH
+ || (token !== undefined && (token !== chunk.id || total !== chunk.total))
+ || offset + chunk.text.length > chunk.total) throw new Error('INVALID_IMPORT_RESPONSE');
+ token = chunk.id; total = chunk.total;
+ pieces.push(chunk.text); offset += chunk.text.length;
+ if (offset === total) break;
+ if (chunk.text.length !== COMPANION_IMPORT_CHUNK_LENGTH) throw new Error('INVALID_IMPORT_RESPONSE');
+ part = await readData(`chunk:${token}:${offset}`);
+ }
+ const result: unknown = JSON.parse(pieces.join(''));
+ if (!result || typeof result !== 'object' || Array.isArray(result)) throw new Error('INVALID_IMPORT_RESPONSE');
+ return result as Record<string, unknown>;
+ };
const status = async (requestId: string) => {
const result = (await data(`result:${requestId}`)).result as CompanionImportResult | null;
if (result && (result.requestId !== requestId || !Array.isArray(result.checks)