Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.2 KB
Raw Blame
import { open } from 'fs/promises';
import { FileDigest, type StorageBackend, type StorageConfig } from '../storage.ts';

/** Size thresholds for upload method selection */
const TUS_THRESHOLD = 100 * 1024 * 1024;   // 100 MB — use TUS resumable above this
const TUS_CHUNK_SIZE = 6 * 1024 * 1024;     // 6 MB chunks for TUS uploads
const SIGNED_URL_EXPIRY = 3600;             // 1 hour

/** Injectable fetch seam (same shape as github-source.ts); tests stub the network. */
export type FetchImpl = (url: string, init: RequestInit) => Promise<Response>;

/**
 * Supabase Storage — uses the Supabase Storage REST API.
 * Auth via the service role key (not the anon key).
 *
 * Upload method auto-selected by file size:
 *   < 100 MB  → standard POST (single request)
 *   >= 100 MB → TUS resumable upload (6 MB chunks with retry)
 */
export class SupabaseStorage implements StorageBackend {
  private projectUrl: string;
  private serviceRoleKey: string;
  private bucket: string;
  private fetchImpl: FetchImpl;

  constructor(config: StorageConfig, fetchImpl: FetchImpl = fetch) {
    this.projectUrl = config.projectUrl || '';
    this.serviceRoleKey = config.serviceRoleKey || '';
    this.bucket = config.bucket;
    this.fetchImpl = fetchImpl;
    if (!this.projectUrl || !this.serviceRoleKey) {
      throw new Error('Supabase storage requires projectUrl and serviceRoleKey in config');
    }
  }

  private url(path: string): string {
    return `${this.projectUrl}/storage/v1/object/${this.bucket}/${path}`;
  }

  /**
   * Publishable and secret keys (`sb_publishable_`, `sb_secret_`) are API
   * keys, not JWTs: Supabase takes them on `apikey` only, and a JWT check on
   * `Authorization: Bearer` fails on them (#7205). Legacy JWT keys keep both.
   */
  private headers(): Record<string, string> {
    if (/^sb_(secret|publishable)_/.test(this.serviceRoleKey)) return { 'apikey': this.serviceRoleKey };
    return {
      'Authorization': `Bearer ${this.serviceRoleKey}`,
      'apikey': this.serviceRoleKey,
    };
  }

  async upload(path: string, data: Buffer, mime?: string): Promise<void> {
    if (data.length >= TUS_THRESHOLD) {
      await this.uploadTus(path, data.length, async (offset, length) => data.subarray(offset, offset + length), mime);
    } else {
      await this.uploadStandard(path, data, mime);
    }
  }

  /** Upload from disk: TUS reads each chunk from the file, so memory is bounded by the chunk size (#7427). */
  async uploadFile(path: string, filePath: string, mime?: string, sha256?: string): Promise<void> {
    const file = await open(filePath, 'r');
    const digest = new FileDigest(filePath, sha256);
    try {
      const { size } = await file.stat();
      if (size < TUS_THRESHOLD) {
        const data = await digest.read(file, Buffer.allocUnsafe(size), size, 0);
        digest.assertMatches(size);
        await this.uploadStandard(path, data, mime);
        return;
      }
      const chunk = Buffer.allocUnsafe(TUS_CHUNK_SIZE);
      await this.uploadTus(path, size, async (offset, length) => {
        const bytes = await digest.read(file, chunk, length, offset);
        if (offset + length === size) digest.assertMatches(size);
        return bytes;
      }, mime);
    } finally {
      await file.close();
    }
  }

  /** Standard single-request upload for files < 100 MB */
  private async uploadStandard(path: string, data: Buffer, mime?: string): Promise<void> {
    const res = await this.fetchImpl(this.url(path), {
      method: 'POST',
      headers: {
        ...this.headers(),
        'Content-Type': mime || 'application/octet-stream',
        'x-upsert': 'true',
      },
      body: new Uint8Array(data.buffer, data.byteOffset, data.byteLength) as BodyInit,
    });
    if (!res.ok) {
      const body = await res.text();
      throw new Error(`Supabase upload failed: ${res.status} ${body}`);
    }
  }

  /**
   * TUS resumable upload for files >= 100 MB.
   * Sends in 6 MB chunks with retry + exponential backoff; `readChunk` supplies
   * each chunk's bytes, from memory or from disk.
   */
  private async uploadTus(
    path: string,
    size: number,
    readChunk: (offset: number, length: number) => Promise<Buffer>,
    mime?: string,
  ): Promise<void> {
    const tusUrl = `${this.projectUrl}/storage/v1/upload/resumable`;
    const objectName = `${this.bucket}/${path}`;

    // Step 1: Create the upload session
    const createRes = await this.fetchImpl(tusUrl, {
      method: 'POST',
      headers: {
        ...this.headers(),
        'Tus-Resumable': '1.0.0',
        'Upload-Length': String(size),
        'Upload-Metadata': [
          `bucketName ${btoa(this.bucket)}`,
          `objectName ${btoa(path)}`,
          `contentType ${btoa(mime || 'application/octet-stream')}`,
        ].join(','),
        'x-upsert': 'true',
      },
    });

    if (!createRes.ok) {
      const body = await createRes.text();
      throw new Error(`TUS create failed: ${createRes.status} ${body}`);
    }

    const uploadUrl = createRes.headers.get('Location');
    if (!uploadUrl) throw new Error('TUS create did not return Location header');

    // Step 2: Upload chunks
    let offset = 0;
    let held: { start: number; bytes: Buffer } | null = null;
    while (offset < size) {
      let attempt = 0;
      const maxAttempts = 3;
      while (attempt < maxAttempts) {
        try {
          // On retry, check server's actual offset (TUS spec requirement)
          if (attempt > 0) {
            const headRes = await this.fetchImpl(uploadUrl, {
              method: 'HEAD',
              headers: { ...this.headers(), 'Tus-Resumable': '1.0.0' },
            });
            if (headRes.ok) {
              const serverOffset = headRes.headers.get('Upload-Offset');
              if (serverOffset) offset = parseInt(serverOffset, 10);
            }
          }

          const end = Math.min(offset + TUS_CHUNK_SIZE, size);
          if (held && offset < held.start) throw new Error(`TUS server offset ${offset} is behind acknowledged offset ${held.start}`);
          const heldEnd = held ? held.start + held.bytes.length : 0;
          let chunk: Buffer;
          if (held && offset < heldEnd) {
            const tail = Buffer.from(held.bytes.subarray(offset - held.start, Math.min(heldEnd, end) - held.start));
            chunk = end > heldEnd ? Buffer.concat([tail, await readChunk(heldEnd, end - heldEnd)]) : tail;
          } else {
            chunk = await readChunk(offset, end - offset);
          }
          held = { start: offset, bytes: chunk };

          const patchRes = await this.fetchImpl(uploadUrl, {
            method: 'PATCH',
            headers: {
              ...this.headers(),
              'Tus-Resumable': '1.0.0',
              'Upload-Offset': String(offset),
              'Content-Type': 'application/offset+octet-stream',
              'Content-Length': String(chunk.length),
            },
            body: new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) as BodyInit,
          });

          if (!patchRes.ok) {
            const body = await patchRes.text();
            throw new Error(`TUS PATCH failed: ${patchRes.status} ${body}`);
          }

          const newOffset = patchRes.headers.get('Upload-Offset');
          offset = newOffset ? parseInt(newOffset, 10) : end;
          break; // Success, move to next chunk
        } catch (err) {
          attempt++;
          if (attempt >= maxAttempts) throw err;
          // Exponential backoff: 1s, 2s, 4s
          await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
        }
      }
    }
  }

  async download(path: string): Promise<Buffer> {
    const res = await this.fetchImpl(this.url(path), {
      headers: this.headers(),
    });
    if (!res.ok) throw new Error(`Supabase download failed: ${res.status}`);
    return Buffer.from(await res.arrayBuffer());
  }

  async delete(path: string): Promise<void> {
    const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/${this.bucket}`, {
      method: 'DELETE',
      headers: { ...this.headers(), 'Content-Type': 'application/json' },
      body: JSON.stringify({ prefixes: [path] }),
    });
    if (!res.ok && res.status !== 404) throw new Error(`Supabase delete failed: ${res.status}`);
  }

  async exists(path: string): Promise<boolean> {
    const res = await this.fetchImpl(this.url(path), {
      method: 'HEAD',
      headers: this.headers(),
    });
    return res.ok;
  }

  async list(prefix: string): Promise<string[]> {
    const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/list/${this.bucket}`, {
      method: 'POST',
      headers: { ...this.headers(), 'Content-Type': 'application/json' },
      body: JSON.stringify({ prefix, limit: 1000 }),
    });
    if (!res.ok) throw new Error(`Supabase list failed: ${res.status}`);
    const items = await res.json() as { name: string }[];
    return items.map(i => `${prefix}/${i.name}`);
  }

  /** Generate a signed URL with 1-hour expiry for private bucket access */
  async getSignedUrl(path: string, expiresIn: number = SIGNED_URL_EXPIRY): Promise<string> {
    const res = await this.fetchImpl(`${this.projectUrl}/storage/v1/object/sign/${this.bucket}/${path}`, {
      method: 'POST',
      headers: { ...this.headers(), 'Content-Type': 'application/json' },
      body: JSON.stringify({ expiresIn }),
    });
    if (!res.ok) {
      const body = await res.text();
      throw new Error(`Supabase signed URL failed: ${res.status} ${body}`);
    }
    const result = await res.json() as { signedURL: string };
    // Supabase returns `signedURL` relative to the Storage API root, e.g.
    // "/object/sign/<bucket>/<path>?token=...". Prepend projectUrl + "/storage/v1"
    // (not just projectUrl) or the link 404s. Tolerate an already-absolute URL or a
    // value that already carries the /storage/v1 prefix.
    const signed = result.signedURL;
    if (/^https?:\/\//.test(signed)) return signed;
    if (signed.startsWith('/storage/v1')) return `${this.projectUrl}${signed}`;
    return `${this.projectUrl}/storage/v1${signed.startsWith('/') ? '' : '/'}${signed}`;
  }

  async getUrl(path: string): Promise<string> {
    // Try signed URL first (works for private buckets)
    try {
      return await this.getSignedUrl(path);
    } catch {
      // Fall back to public URL
      return `${this.projectUrl}/storage/v1/object/public/${this.bucket}/${path}`;
    }
  }
}