Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { open } from 'fs/promises';
import { FileDigest, type StorageBackend, type StorageConfig } from '../storage.ts';
/** Size thresholds for upload method selection */
const TUS_THRESHOLD = 100 * 1024 * 1024; // 100 MB — use TUS resumable above this
const TUS_CHUNK_SIZE = 6 * 1024 * 1024; // 6 MB chunks for TUS uploads
const SIGNED_URL_EXPIRY = 3600; // 1 hour
/** Injectable fetch seam (same shape as github-source.ts); tests stub the network. */
export type FetchImpl = (url: string, init: RequestInit) => Promise<Response>;
/**
* Supabase Storage — uses the Supabase Storage REST API.
* Auth via the service role key (not the anon key).
*
* Upload method auto-selected by file size:
* < 100 MB → standard POST (single request)
* >= 100 MB → TUS resumable upload (6 MB chunks with retry)
*/
export class SupabaseStorage implements StorageBackend {
private projectUrl: string;
private serviceRoleKey: string;
private bucket: string;
private fetchImpl: FetchImpl;
constructor(config: StorageConfig, fetchImpl: FetchImpl = fetch) {
this.projectUrl = config.projectUrl || '';
this.serviceRoleKey = config.serviceRoleKey || '';
this.bucket = config.bucket;
this.fetchImpl = fetchImpl;
if (!this.projectUrl || !this.serviceRoleKey) {
throw new Error('Supabase storage requires projectUrl and serviceRoleKey in config');
}
}
private url(path: string): string {
return `${this.projectUrl}/storage/v1/object/${this.bucket}/${path}`;
}
/**
* Publishable and secret keys (`sb_publishable_`, `sb_secret_`) are API
* keys, not JWTs: Supabase takes them on `apikey` only, and a JWT check on
* `Authorization: Bearer` fails on them (#7205). Legacy JWT keys keep both.
*/
private headers(): Record<string, string> {
if (/^sb_(secret|publishable)_/.test(this.serviceRoleKey)) return { 'apikey': this.serviceRoleKey };
return {
'Authorization': `Bearer ${this.serviceRoleKey}`,
'apikey': this.serviceRoleKey,
};
}
async upload(path: string, data: Buffer, mime?: string): Promise<void> {
if (data.length >= TUS_THRESHOLD) {
await this.uploadTus(path, data.length, async (offset, length) => data.subarray(offset, offset + length), mime);
} else {
await this.uploadStandard(path, data, mime);
}
}
/** Upload from disk: TUS reads each chunk from the file, so memory is bounded by the chunk size (#7427). */
async uploadFile(path: string, filePath: string, mime?: string, sha256?: string): Promise<void> {
const file = await open(filePath, 'r');
const digest = new FileDigest(filePath, sha256);
try {
const { size } = await file.stat();
if (size < TUS_THRESHOLD) {
const data = await digest.read(file, Buffer.allocUnsafe(size), size, 0);
digest.assertMatches(size);
await this.uploadStandard(path, data, mime);
return;
}
const chunk = Buffer.allocUnsafe(TUS_CHUNK_SIZE);
await this.uploadTus(path, size, async (offset, length) => {
const bytes = await digest.read(file, chunk, length, offset);
if (offset + length === size) digest.assertMatches(size);
return bytes;
}, mime);
} finally {
await file.close();
}
}
/** Standard single-request upload for files < 100 MB */
private async uploadStandard(path: string, data: Buffer, mime?: string): Promise<void> {
const res = await this.fetchImpl(this.url(path), {
method: 'POST',
headers: {
...this.headers(),
'Content-Type': mime || 'application/octet-stream',
'x-upsert': 'true',
},
body: new Uint8Array(data.buffer, data.byteOffset, data.byteLength) as BodyInit,
});
if (!res.ok) {
const body = await res.text();
throw new Error(`Supabase upload failed: ${res.status} ${body}`);
}
}
/**
* TUS resumable upload for files >= 100 MB.
* Sends in 6 MB chunks with retry + exponential backoff; `readChunk` supplies
* each chunk's bytes, from memory or from disk.
*/
private async uploadTus(
path: string,
size: number,
readChunk: (offset: number, length: number) => Promise<Buffer>,
mime?: string,
): Promise<void> {
const tusUrl = `${this.projectUrl}/storage/v1/upload/resumable`;
const objectName = `${this.bucket}/${path}`;
// Step 1: Create the upload session
const createRes = await this.fetchImpl(tusUrl, {
method: 'POST',
headers: {
...this.headers(),
'Tus-Resumable': '1.0.0',
'Upload-Length': String(size),
'Upload-Metadata': [
`bucketName ${btoa(this.bucket)}`,
`objectName ${btoa(path)}`,
`contentType ${btoa(mime || 'application/octet-stream')}`,
].join(','),
'x-upsert': 'true',
},
});
if (!createRes.ok) {
const body = await createRes.text();
throw new Error(`TUS create failed: ${createRes.status} ${body}`);
}
const uploadUrl = createRes.headers.get('Location');
if (!uploadUrl) throw new Error('TUS create did not return Location header');
// Step 2: Upload chunks
let offset = 0;
let held: { start: number; bytes: Buffer } | null = null;
while (offset < size) {
let attempt = 0;
const maxAttempts = 3;
while (attempt < maxAttempts) {
try {
// On retry, check server's actual offset (TUS spec requirement)
if (attempt > 0) {
const headRes = await this.fetchImpl(uploadUrl, {
method: 'HEAD',
headers: { ...this.headers(), 'Tus-Resumable': '1.0.0' },
});
if (headRes.ok) {
const serverOffset = headRes.headers.get('Upload-Offset');
if (serverOffset) offset = parseInt(serverOffset, 10);
}
}
const end = Math.min(offset + TUS_CHUNK_SIZE, size);
if (held && offset < held.start) throw new Error(`TUS server offset ${offset} is behind acknowledged offset ${held.start}`);
const heldEnd = held ? held.start + held.bytes.length : 0;
let chunk: Buffer;
if (held && offset < heldEnd) {
const tail = Buffer.from(held.bytes.subarray(offset - held.start, Math.min(heldEnd, end) - held.start));
chunk = end > heldEnd ? Buffer.concat([tail, await readChunk(heldEnd, end - heldEnd)]) : tail;
} else {
chunk = await readChunk(offset, end - offset);
}
held = { start: offset, bytes: chunk };
const patchRes = await this.fetchImpl(uploadUrl, {
method: 'PATCH',
headers: {
...this.headers(),
'Tus-Resumable': '1.0.0',
'Upload-Offset': String(offset),
'Content-Type': 'application/offset+octet-stream',
'Content-Length': String(chunk.length),
},
body: new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) as BodyInit,
});
if (!patchRes.ok) {
const body = await patchRes.text();
throw new Error(`TUS PATCH failed: ${patchRes.status} ${body}`);
}
const newOffset = patchRes.headers.get('Upload-Offset');
offset = newOffset ? parseInt(newOffset, 10) : end;
break; // Success, move to next chunk
} catch (err) {
attempt++;
if (attempt >= maxAttempts) throw err;
// Exponential backoff: 1s, 2s, 4s
await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
}
}
}
}
async download(path: string): Promise<Buffer> {
const res = await this.fetchImpl(this.url(path), {
headers: this.headers(),
});
if (!res.ok) throw new Error(`Supabase download failed: ${res.status}`);
return Buffer.from(await res.arrayBuffer());
}
async delete(path: string): Promise<void> {
const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/${this.bucket}`, {
method: 'DELETE',
headers: { ...this.headers(), 'Content-Type': 'application/json' },
body: JSON.stringify({ prefixes: [path] }),
});
if (!res.ok && res.status !== 404) throw new Error(`Supabase delete failed: ${res.status}`);
}
async exists(path: string): Promise<boolean> {
const res = await this.fetchImpl(this.url(path), {
method: 'HEAD',
headers: this.headers(),
});
return res.ok;
}
async list(prefix: string): Promise<string[]> {
const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/list/${this.bucket}`, {
method: 'POST',
headers: { ...this.headers(), 'Content-Type': 'application/json' },
body: JSON.stringify({ prefix, limit: 1000 }),
});
if (!res.ok) throw new Error(`Supabase list failed: ${res.status}`);
const items = await res.json() as { name: string }[];
return items.map(i => `${prefix}/${i.name}`);
}
/** Generate a signed URL with 1-hour expiry for private bucket access */
async getSignedUrl(path: string, expiresIn: number = SIGNED_URL_EXPIRY): Promise<string> {
const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/sign/${this.bucket}/${path}`, {
method: 'POST',
headers: { ...this.headers(), 'Content-Type': 'application/json' },
body: JSON.stringify({ expiresIn }),
});
if (!res.ok) {
const body = await res.text();
throw new Error(`Supabase signed URL failed: ${res.status} ${body}`);
}
const result = await res.json() as { signedURL: string };
// Supabase returns `signedURL` relative to the Storage API root, e.g.
// "/object/sign/<bucket>/<path>?token=...". Prepend projectUrl + "/storage/v1"
// (not just projectUrl) or the link 404s. Tolerate an already-absolute URL or a
// value that already carries the /storage/v1 prefix.
const signed = result.signedURL;
if (/^https?:\/\//.test(signed)) return signed;
if (signed.startsWith('/storage/v1')) return `${this.projectUrl}${signed}`;
return `${this.projectUrl}/storage/v1${signed.startsWith('/') ? '' : '/'}${signed}`;
}
async getUrl(path: string): Promise<string> {
// Try signed URL first (works for private buckets)
try {
return await this.getSignedUrl(path);
} catch {
// Fall back to public URL
return `${this.projectUrl}/storage/v1/object/public/${this.bucket}/${path}`;
}
}
}