Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { closeSync, openSync, readSync } from 'fs';
import type { FileHandle } from 'fs/promises';
import { createHash } from 'crypto';
import { embeddedSettingsRecord } from './embedded-engine.ts';
/**
* StorageBackend — pluggable interface for binary file storage.
*
* GBrain is agnostic about where files live. The setup skill picks
* the backend (Supabase Storage or S3/R2/MinIO), gbrain doesn't care.
*/
export interface StorageBackend {
upload(path: string, data: Buffer, mime?: string): Promise<void>;
/**
* Upload a file from disk with memory bounded independently of its size (#7427). With `sha256`, bytes that differ
* from it are refused before anything is published, so a stored object is never replaced by them.
*/
uploadFile(path: string, filePath: string, mime?: string, sha256?: string): Promise<void>;
download(path: string): Promise<Buffer>;
delete(path: string): Promise<void>;
exists(path: string): Promise<boolean>;
list(prefix: string): Promise<string[]>;
getUrl(path: string): Promise<string>;
/** Size and SHA-256 of a stored object, streamed; a backend without it is hashed from one download (hashStoredObject). */
hashObject?(path: string): Promise<{ size: number; sha256: string }>;
}
export interface StorageConfig {
backend: 's3' | 'supabase' | 'local';
bucket: string;
region?: string;
endpoint?: string;
// S3 credentials
accessKeyId?: string;
secretAccessKey?: string;
// Supabase credentials
projectUrl?: string;
serviceRoleKey?: string;
// Local (for testing)
localPath?: string;
}
/**
* Create a StorageBackend from config.
*/
export async function createStorage(config: StorageConfig): Promise<StorageBackend> {
switch (config.backend) {
case 's3': {
const { S3Storage } = await import('./storage/s3.ts');
return new S3Storage(config);
}
case 'supabase': {
const { SupabaseStorage } = await import('./storage/supabase.ts');
return new SupabaseStorage(config);
}
case 'local': {
const { LocalStorage } = await import('./storage/local.ts');
return new LocalStorage(config.localPath || '/tmp/gbrain-storage');
}
default:
throw new Error(`Unknown storage backend: ${config.backend}`);
}
}
/** SHA-256 of a file read in fixed-size chunks, so large originals are never held in memory. */
export function hashStorageFile(filePath: string): string {
const fd = openSync(filePath, 'r');
try {
const hash = createHash('sha256');
const chunk = Buffer.allocUnsafe(1024 * 1024);
let length: number;
while ((length = readSync(fd, chunk, 0, chunk.length, null)) > 0) {
hash.update(chunk.subarray(0, length));
}
return hash.digest('hex');
} finally {
closeSync(fd);
}
}
/** The store an engine opened from settings keeps its files in (settings.storage under its prefix); undefined for any other engine or with no settings.storage. */
export async function settingsStorage(engine: object): Promise<StorageBackend | undefined> {
const settings = embeddedSettingsRecord(engine)?.storage;
if (!settings) return undefined;
const { prefix, ...config } = settings;
return prefixedStorage(await createStorage({ bucket: '', ...config }), prefix);
}
/**
* `backend` with every object key under `<prefix>/`; list() returns keys without it. No prefix returns `backend` itself.
* A key with an empty, `.` or `..` segment (a leading `/` included) is refused, so no key reaches outside the prefix.
*/
export function prefixedStorage(backend: StorageBackend, prefix: string | undefined): StorageBackend {
if (!prefix) return backend;
const key = (path: string, listing = false) => {
const segments = listing && path === '' ? [] : path.split('/');
if (segments.some(segment => segment === '' || segment === '.' || segment === '..')) {
throw new Error(`Path traversal blocked: ${path} resolves outside storage prefix ${prefix}`);
}
return `${prefix}/${path}`;
};
return {
upload: async (path, data, mime) => backend.upload(key(path), data, mime),
uploadFile: async (path, filePath, mime, sha256) => backend.uploadFile(key(path), filePath, mime, sha256),
download: async path => backend.download(key(path)),
delete: async path => backend.delete(key(path)),
exists: async path => backend.exists(key(path)),
list: async listPrefix => (await backend.list(key(listPrefix, true))).map(path => path.slice(prefix.length + 1)),
getUrl: async path => backend.getUrl(key(path)),
...(backend.hashObject ? { hashObject: async (path: string) => backend.hashObject!(key(path)) } : {}),
};
}
export async function hashStoredObject(storage: StorageBackend, path: string): Promise<{ size: number; sha256: string }> {
if (storage.hashObject) return storage.hashObject(path);
const bytes = await storage.download(path);
return { size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
}
/** Upload a file under the hash the caller recorded; a file that changed since then is refused before it is published. */
export async function uploadUnchangedFile(storage: StorageBackend, path: string, filePath: string, sha256: string, mime?: string): Promise<void> {
await storage.uploadFile(path, filePath, mime, sha256);
}
export function fileChangedDuringUpload(filePath: string): Error {
return Object.assign(new Error(`${filePath} changed during the upload, so its bytes no longer match the hash taken before it; nothing was stored or recorded. Upload it again once it stops changing.`), { code: 'file_changed_during_upload' });
}
/**
* Hashes the bytes a backend reads from a file in order (re-reads of a retried range count once), so the backend can
* refuse before its publishing step when they differ from the hash the caller recorded.
*/
export class FileDigest {
private readonly hash = createHash('sha256');
private hashed = 0;
constructor(private readonly filePath: string, private readonly expected?: string) {}
async read(file: FileHandle, buffer: Buffer, length: number, position: number): Promise<Buffer> {
const bytes = await readFileRange(file, buffer, length, position);
if (position <= this.hashed && position + length > this.hashed) {
this.hash.update(bytes.subarray(this.hashed - position));
this.hashed = position + length;
}
return bytes;
}
assertMatches(size: number): void {
if (this.expected !== undefined && (this.hashed !== size || this.hash.copy().digest('hex') !== this.expected)) throw fileChangedDuringUpload(this.filePath);
}
}
/** Read exactly `length` bytes at `position` into the start of `buffer`. */
export async function readFileRange(file: FileHandle, buffer: Buffer, length: number, position: number): Promise<Buffer> {
for (let filled = 0; filled < length;) {
const { bytesRead } = await file.read(buffer, filled, length - filled, position + filled);
if (!bytesRead) throw new Error('File shrank during upload');
filled += bytesRead;
}
return buffer.subarray(0, length);
}