Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* GBRA-283 Wave C: settings.storage and file_verify, for an embedding host that keeps a brain's files in its own store.
* Contributed by @bradgessler (#8457); rewritten on master. Real PGLite engines and a real local storage backend.
*/
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';
import { createHash, randomUUID } from 'node:crypto';
import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { embeddedSettings, openEngine, type EngineSettings, type EngineStorageSettings } from '../src/core/engine-open.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { importFromContent } from '../src/core/import-file.ts';
import { operationsByName, type OperationContext } from '../src/core/operations.ts';
import { dispatchToolCall } from '../src/mcp/dispatch.ts';
import { isCallable } from '../src/core/ops/callable.ts';
import { __resetPrivateVisibilityCacheForTests } from '../src/core/search/private-visibility.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { withEnv } from './helpers/with-env.ts';
const root = mkdtempSync(join(tmpdir(), 'gbrain-embedded-file-store-'));
const storageRoot = join(root, 'store');
const storage: EngineStorageSettings = { backend: 'local', localPath: storageRoot };
const unreachable: EngineStorageSettings = { backend: 's3', bucket: 'test', endpoint: 'http://127.0.0.1:1', region: 'us-east-1', accessKeyId: 'test-key', secretAccessKey: 'test-secret' };
const settings = (over: Partial<EngineSettings> = {}): EngineSettings => ({ embedded_core: true, engine: 'pglite', database: {}, host: { id: randomUUID() }, embedding: { model: 'openai:text-embedding-3-small', dimensions: 1536 }, storage, ...over });
const ctxFor = (engine: BrainEngine, over: Partial<OperationContext> = {}): OperationContext => ({
engine, config: {}, logger: { info: () => {}, warn: () => {}, error: () => {} }, dryRun: false, remote: false, ...over,
} as unknown as OperationContext);
const run = (op: string, ctx: OperationContext, params: Record<string, unknown>) => operationsByName[op]!.handler(ctx, params);
const failure = async (work: () => Promise<unknown>) => {
try { await work(); return null; } catch (error) { return error as Error & { code?: string }; }
};
const sha = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex');
function store(path: string, bytes: Buffer): void {
mkdirSync(dirname(join(storageRoot, path)), { recursive: true });
writeFileSync(join(storageRoot, path), bytes);
}
const addPage = (engine: BrainEngine, slug: string) => importFromContent(engine, slug, ['---', 'type: note', `title: ${slug}`, '---', `# ${slug}`, '', 'Plan body.'].join('\n'));
const addRow = (engine: BrainEngine, path: string, bytes: Buffer, pageSlug: string | null = null) =>
engine.executeRaw('INSERT INTO files (page_slug, filename, storage_path, content_hash, size_bytes) VALUES ($1, $2, $3, $4, $5)', [pageSlug, path.split('/').pop()!, path, sha(bytes), bytes.length]);
const fileRows = async (engine: BrainEngine) => Number((await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM files'))[0]!.n);
const engines: BrainEngine[] = [];
async function embedded(over: Partial<EngineSettings> = {}): Promise<BrainEngine> {
const engine = await openEngine(settings(over));
await engine.initSchema();
engines.push(engine);
return engine;
}
afterEach(async () => {
await Promise.all(engines.splice(0).map(engine => engine.disconnect()));
rmSync(storageRoot, { recursive: true, force: true });
__resetPrivateVisibilityCacheForTests();
});
afterAll(() => rmSync(root, { recursive: true, force: true }));
describe('embedded file store', () => {
test('an engine opened without settings.storage refuses even when the process has a storage config', async () => {
const engine = await embedded({ storage: undefined });
const ctx = ctxFor(engine, { config: { storage } } as never);
expect((await failure(() => run('file_verify', ctx, {})))?.code).toBe('storage_error');
expect(await fileRows(engine)).toBe(0);
});
test('two brains sharing one store under different prefixes see only their own objects', async () => {
const a = await embedded({ storage: { ...storage, prefix: 'brain-a' } });
const b = await embedded({ storage: { ...storage, prefix: 'brain-b' } });
await Promise.all([addPage(a, 'notes/plan'), addPage(b, 'notes/plan')]);
const source = join(root, 'a.txt');
writeFileSync(source, 'brain a bytes');
await run('file_upload', ctxFor(a), { path: source, page_slug: 'notes/plan' });
await addRow(b, 'notes/plan/a.txt', Buffer.from('brain a bytes'), 'notes/plan');
expect(existsSync(join(storageRoot, 'brain-a/notes/plan/a.txt'))).toBe(true);
expect(await run('file_verify', ctxFor(b), { slug: 'notes/plan' })).toEqual([{ storage_path: 'notes/plan/a.txt', page_slug: 'notes/plan', present: false }]);
expect(await run('file_verify', ctxFor(a), { storage_path: 'notes/plan/a.txt' })).toEqual({ storage_path: 'notes/plan/a.txt', present: true, size_matches: true, hash_matches: true });
const [{ metadata }] = await a.executeRaw<{ metadata: unknown }>('SELECT metadata FROM files');
expect(metadata).toEqual({ storage: 'local', prefix: 'brain-a' });
});
test('file_upload and file_url on an engine opened from settings use its store and prefix, never the process store', async () => {
const engine = await embedded({ storage: { ...storage, prefix: 'brain-a' } });
await addPage(engine, 'notes/plan');
const source = join(root, 'upload.txt');
writeFileSync(source, 'uploaded bytes');
const processStore = { backend: 'local', bucket: 'test', localPath: join(root, 'process-store') };
const ctx = ctxFor(engine, { config: { storage: processStore } } as never);
expect(await run('file_upload', ctx, { path: source, page_slug: 'notes/plan' })).toMatchObject({ status: 'uploaded', storage_path: 'notes/plan/upload.txt' });
expect(readFileSync(join(storageRoot, 'brain-a/notes/plan/upload.txt'), 'utf8')).toBe('uploaded bytes');
expect(existsSync(join(root, 'process-store'))).toBe(false);
expect(await run('file_url', ctx, { storage_path: 'notes/plan/upload.txt' })).toEqual({ storage_path: 'notes/plan/upload.txt', url: `file://${join(storageRoot, 'brain-a/notes/plan/upload.txt')}` });
expect(await run('file_verify', ctx, { storage_path: 'notes/plan/upload.txt' })).toMatchObject({ present: true, hash_matches: true });
});
test('file_upload on an engine opened without settings.storage refuses and never writes the process store', async () => {
const engine = await embedded({ storage: undefined });
await addPage(engine, 'notes/plan');
const source = join(root, 'refused.txt');
writeFileSync(source, 'refused bytes');
const ctx = ctxFor(engine, { config: { storage: { backend: 'local', bucket: 'test', localPath: join(root, 'process-store-2') } } } as never);
expect((await failure(() => run('file_upload', ctx, { path: source, page_slug: 'notes/plan' })))?.code).toBe('storage_error');
expect([existsSync(join(root, 'process-store-2')), await fileRows(engine)]).toEqual([false, 0]);
});
test('a purge on an engine opened from settings deletes its blobs from that store, never the process store', async () => {
const engine = await embedded({ storage: { ...storage, prefix: 'brain-a' } });
const processDir = join(root, 'process-store-3');
store('brain-a/notes/plan/p.txt', Buffer.from('embedded'));
mkdirSync(join(processDir, 'notes/plan'), { recursive: true });
writeFileSync(join(processDir, 'notes/plan/p.txt'), 'another brain');
const { finishPagePurge } = await import('../src/core/persistence/page-purge.ts');
const purged = { status: 'purged', source_id: 'default', slug: 'notes/plan', request_id: randomUUID(), purge: { blobs: ['notes/plan/p.txt'], removed: {} } };
const result = await finishPagePurge({ engine, config: { storage: { backend: 'local', bucket: 'test', localPath: processDir } } } as never, purged) as { purge: { blobs_remaining: string[] } };
expect([result.purge.blobs_remaining, existsSync(join(storageRoot, 'brain-a/notes/plan/p.txt')), existsSync(join(processDir, 'notes/plan/p.txt'))]).toEqual([[], false, true]);
const bare = await embedded({ storage: undefined });
const kept = await finishPagePurge({ engine: bare, config: { storage: { backend: 'local', bucket: 'test', localPath: processDir } } } as never, { ...purged, request_id: randomUUID() }) as { purge: { blobs_remaining: string[] } };
expect([kept.purge.blobs_remaining, existsSync(join(processDir, 'notes/plan/p.txt'))]).toEqual([['notes/plan/p.txt'], true]);
});
test('a purge never counts a blob missing from settings.storage as deleted (an older row whose object is in the process store)', async () => {
const engine = await embedded({ storage: { ...storage, prefix: 'brain-a' } });
const processDir = join(root, 'process-store-4');
mkdirSync(join(processDir, 'notes/plan'), { recursive: true });
writeFileSync(join(processDir, 'notes/plan/old.txt'), 'uploaded before this release');
store('brain-a/notes/plan/new.txt', Buffer.from('embedded'));
const { finishPagePurge } = await import('../src/core/persistence/page-purge.ts');
const purged = { status: 'purged', source_id: 'default', slug: 'notes/plan', request_id: randomUUID(), purge: { blobs: ['notes/plan/old.txt', 'notes/plan/new.txt'], removed: {} } };
const result = await finishPagePurge({ engine, config: { storage: { backend: 'local', bucket: 'test', localPath: processDir } } } as never, purged) as { purge: { blobs_remaining: string[]; blobs_deleted: number } };
expect([result.purge.blobs_remaining, result.purge.blobs_deleted]).toEqual([['notes/plan/old.txt'], 1]);
expect([existsSync(join(processDir, 'notes/plan/old.txt')), existsSync(join(storageRoot, 'brain-a/notes/plan/new.txt'))]).toEqual([true, false]);
});
test('a store that cannot be reached is store_unavailable, never a file reported missing', async () => {
const engine = await embedded({ storage: unreachable });
await addRow(engine, 'unsorted/a.txt', Buffer.from('x'));
for (const params of [{}, { storage_path: 'unsorted/a.txt' }]) expect((await failure(() => run('file_verify', ctxFor(engine), params)))?.code).toBe('store_unavailable');
expect((await failure(() => run('file_url', ctxFor(engine), { storage_path: 'unsorted/a.txt' })))?.code).toBe('store_unavailable');
});
});
describe('settings.storage', () => {
test('openEngine refuses a store missing a value it needs, an unknown backend, field or prefix, and the view hides its secret', async () => {
const cases: Array<[unknown, string]> = [
[{ backend: 'nope' }, 'storage.backend'], [{ backend: 'local' }, 'storage.localPath'],
[{ backend: 'supabase', bucket: 'b' }, 'storage.backend'], [{ ...unreachable, endpoint: undefined }, 'storage.endpoint'], [{ ...unreachable, secretAccessKey: '' }, 'storage.secretAccessKey'],
[{ ...storage, region2: 'x' }, 'storage.region2'], [{ ...storage, prefix: '../a' }, 'storage.prefix'], [{ ...storage, prefix: 'a/b' }, 'storage.prefix'],
];
for (const [given, setting] of cases) {
const error = await failure(() => openEngine(settings({ storage: given as EngineStorageSettings })));
expect([setting, error?.code, error?.message]).toEqual([setting, 'engine_settings_invalid', `The ${setting} setting is invalid.`]);
}
const engine = await embedded({ storage: unreachable });
expect(embeddedSettings(engine)!.storage).toEqual({ backend: 's3', bucket: 'test', endpoint: 'http://127.0.0.1:1', region: 'us-east-1', accessKeyId: 'test-key', hasSecret: true });
expect(JSON.stringify(embeddedSettings(engine))).not.toContain('test-secret');
});
});
describe('prefixedStorage', () => {
test("refuses a key that would leave its prefix, so one brain never reaches another brain's objects", async () => {
const { LocalStorage } = await import('../src/core/storage/local.ts');
const { prefixedStorage } = await import('../src/core/storage.ts');
store('brain-b/notes/x.txt', Buffer.from('brain b bytes'));
const a = prefixedStorage(new LocalStorage(storageRoot), 'brain-a');
for (const path of ['../brain-b/notes/x.txt', 'notes/../../brain-b/notes/x.txt', './notes/x.txt', 'notes//x.txt', '/notes/x.txt', '', 'notes/x.txt/']) {
for (const call of [() => a.exists(path), () => a.download(path), () => a.delete(path), () => a.upload(path, Buffer.from('overwrite')), () => a.getUrl(path)]) {
expect([path, (await failure(call))?.message]).toEqual([path, `Path traversal blocked: ${path} resolves outside storage prefix brain-a`]);
}
}
expect((await failure(() => a.list('../brain-b')))?.message).toBe('Path traversal blocked: ../brain-b resolves outside storage prefix brain-a');
expect(readFileSync(join(storageRoot, 'brain-b/notes/x.txt'), 'utf8')).toBe('brain b bytes');
await a.upload('notes/x.txt', Buffer.from('brain a bytes'));
expect([await a.exists('notes/x.txt'), readFileSync(join(storageRoot, 'brain-a/notes/x.txt'), 'utf8')]).toEqual([true, 'brain a bytes']);
expect(await a.list('notes')).toEqual(['notes/x.txt']);
});
});
describe('hashing a stored object', () => {
test('S3 streams the object body into its size and SHA-256', async () => {
const { S3Storage } = await import('../src/core/storage/s3.ts');
const parts = [Buffer.from('first part, '), Buffer.from('second part')];
const client = { send: async () => ({ Body: (async function* () { yield* parts; })() }) };
const s3 = new S3Storage({ backend: 's3', bucket: 'test' }, client as never);
expect(await s3.hashObject('a/b.txt')).toEqual({ size: 23, sha256: sha(Buffer.concat(parts)) });
});
test('an s3 store with an endpoint builds file URLs from it', async () => {
const { S3Storage } = await import('../src/core/storage/s3.ts');
const s3 = new S3Storage({ backend: 's3', bucket: 'test', endpoint: 'http://127.0.0.1:9000', region: 'us-east-1', accessKeyId: 'k', secretAccessKey: 's' });
const url = new URL(await s3.getUrl('notes/plan/a.txt'));
expect(`${url.origin}${url.pathname}`).toBe('http://127.0.0.1:9000/test/notes/plan/a.txt');
expect(url.searchParams.get('X-Amz-Expires')).toBe('3600');
expect(url.searchParams.get('X-Amz-Signature')).toMatch(/^[0-9a-f]{64}$/);
});
test('a backend without hashObject is hashed from one download', async () => {
const { hashStoredObject } = await import('../src/core/storage.ts');
const bytes = Buffer.from('downloaded');
expect(await hashStoredObject({ download: async () => bytes } as never, 'a.txt')).toEqual({ size: bytes.length, sha256: sha(bytes) });
});
test('an s3 store sends every request to its settings endpoint, never one named by AWS_* variables', async () => {
const hits = { settings: 0, env: 0 };
const settingsServer = Bun.serve({ port: 0, fetch: () => { hits.settings++; return new Response(null, { status: 404 }); } });
const envServer = Bun.serve({ port: 0, fetch: () => { hits.env++; return new Response(null, { status: 200 }); } });
try {
await withEnv({ AWS_ENDPOINT_URL_S3: `http://127.0.0.1:${envServer.port}`, AWS_ENDPOINT_URL: `http://127.0.0.1:${envServer.port}` }, async () => {
const engine = await embedded({ storage: { ...unreachable, endpoint: `http://127.0.0.1:${settingsServer.port}` } });
await addRow(engine, 'unsorted/s.txt', Buffer.from('s3'));
const result = await run('file_verify', ctxFor(engine), { storage_path: 'unsorted/s.txt' }) as { present: boolean };
expect([result.present, hits.env, hits.settings > 0]).toEqual([false, 0, true]);
});
} finally {
settingsServer.stop(true);
envServer.stop(true);
}
});
});
describe('file_verify', () => {
test('lists presence per row, reads one object back, and notices bytes changed underneath', async () => {
const engine = await embedded();
await addPage(engine, 'notes/plan');
const a = Buffer.from('alpha'), b = Buffer.from('bravo');
store('notes/plan/a.txt', a);
store('notes/plan/b.txt', b);
for (const [name, bytes] of [['a.txt', a], ['b.txt', b]] as const) await addRow(engine, `notes/plan/${name}`, bytes, 'notes/plan');
rmSync(join(storageRoot, 'notes/plan/b.txt'));
expect(await run('file_verify', ctxFor(engine), { slug: 'notes/plan' })).toEqual([
{ storage_path: 'notes/plan/a.txt', page_slug: 'notes/plan', present: true },
{ storage_path: 'notes/plan/b.txt', page_slug: 'notes/plan', present: false },
]);
expect(await run('file_verify', ctxFor(engine), { storage_path: 'notes/plan/a.txt' })).toEqual({ storage_path: 'notes/plan/a.txt', present: true, size_matches: true, hash_matches: true });
store('notes/plan/a.txt', Buffer.from('ALPHA'));
expect(await run('file_verify', ctxFor(engine), { storage_path: 'notes/plan/a.txt' })).toEqual({ storage_path: 'notes/plan/a.txt', present: true, size_matches: true, hash_matches: false });
});
});
describe('who may call them', () => {
let cliEngine: PGLiteEngine;
beforeAll(async () => {
cliEngine = new PGLiteEngine();
await cliEngine.connect({});
await cliEngine.initSchema();
});
afterAll(() => cliEngine.disconnect());
test('an engine the CLI opens refuses file_verify with embedded_engine_required', async () => {
expect((await failure(() => run('file_verify', ctxFor(cliEngine), {})))?.code).toBe('embedded_engine_required');
});
test('file_verify is not listed or callable over MCP, stdio included', async () => {
const engine = await embedded();
for (const op of ['file_verify']) {
for (const transport of ['stdio', 'http'] as const) expect(isCallable(operationsByName[op]!, { transport, surface: 'full', scopes: ['admin'], publishGates: {} })).toBe(false);
const result = await dispatchToolCall(engine, op, {}, { remote: true, transport: 'stdio' } as never);
expect(result.isError).toBe(true);
expect(await fileRows(engine)).toBe(0);
}
});
test('a caller under the read policy cannot see or check a file on a page it cannot read; a local caller can', async () => {
const engine = await embedded();
await importFromContent(engine, 'notes/secret', ['---', 'type: note', 'title: Secret', 'visibility: private', '---', '# Secret', '', 'Private body.'].join('\n'));
const bytes = Buffer.from('secret attachment');
store('notes/secret/s.txt', bytes);
await addRow(engine, 'notes/secret/s.txt', bytes, 'notes/secret');
const remote = ctxFor(engine, { remote: true });
expect(await run('file_verify', remote, {})).toEqual([]);
expect((await failure(() => run('file_verify', remote, { storage_path: 'notes/secret/s.txt' })))?.code).toBe('storage_error');
expect(await run('file_verify', ctxFor(engine), {})).toEqual([{ storage_path: 'notes/secret/s.txt', page_slug: 'notes/secret', present: true }]);
});
test('a page naming its readers stays closed to an unlisted remote caller even with the private-pages hatch open', async () => {
const engine = await embedded();
await importFromContent(engine, 'notes/members', ['---', 'type: note', 'title: Members', 'readable_by: [alice-example]', '---', '# Members', '', 'Member body.'].join('\n'));
const bytes = Buffer.from('member attachment');
store('notes/members/m.txt', bytes);
await addRow(engine, 'notes/members/m.txt', bytes, 'notes/members');
await withEnv({ GBRAIN_REMOTE_PRIVATE_PAGES: '1' }, async () => {
__resetPrivateVisibilityCacheForTests();
const remote = ctxFor(engine, { remote: true });
expect(await run('file_verify', remote, { slug: 'notes/members' })).toEqual([]);
expect((await failure(() => run('file_verify', remote, { storage_path: 'notes/members/m.txt' })))?.code).toBe('storage_error');
});
});
});