Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(storage): S3 and MinIO file URLs are presigned for one hour
garrytan35952ddcaa1d
diff --git a/bun.lock b/bun.lockindex 02bf070fb4..b89ebddfb6 100644--- a/bun.lock+++ b/bun.lock@@ -12,6 +12,7 @@"@ai-sdk/provider-utils": "4.0.56","@anthropic-ai/sdk": "^0.30.0","@aws-sdk/client-s3": "^3.1028.0",+ "@aws-sdk/s3-request-presigner": "3.1028.0","@dqbd/tiktoken": "^1.0.22","@electric-sql/pglite": "0.4.3","@jsquash/avif": "^2.1.1",@@ -54,6 +55,7 @@],"overrides": {"@ai-sdk/provider-utils": "4.0.56",+ "@aws-sdk/util-format-url": "3.972.9","@hono/node-server": "^2.0.5","body-parser": "^2.4.0","fast-uri": "^3.1.9",@@ -144,6 +146,8 @@"@aws-sdk/region-config-resolver": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/config-resolver": "^4.4.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-6Q8B1dcx6BBqUTY1Mc/eROKA0FImEEY5VPSd6AGPEUf0ErjExz4snVqa9kNJSoVDV1rKaNf3qrWojgcKW+SdDg=="],+ "@aws-sdk/s3-request-presigner": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/signature-v4-multi-region": "^3.996.16", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-format-url": "^3.972.9", "@smithy/middleware-endpoint": "^4.4.29", "@smithy/protocol-http": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-T46EyIIHUaF/I2zhjtSUO4/87QdhqobClCscJawrxe2h/8nZJoO3DQDVZ4tMDl5UV/B5SPz6+xwki8jGylTF4Q=="],+"@aws-sdk/signature-v4-multi-region": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/middleware-sdk-s3": "^3.972.28", "@aws-sdk/types": "^3.973.7", "@smithy/protocol-http": "^5.3.13", "@smithy/signature-v4": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-EMdXYB4r/k5RWq86fugjRhid5JA+Z6MpS7n4sij4u5/C+STrkvuf9aFu41rJA9MjUzxCLzv8U2XL8cH2GSRYpQ=="],"@aws-sdk/token-providers": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-Ieq/HiRrbEtrYP387Nes0XlR7H1pJiJOZKv+QyQzMYpvTiDs0VKy2ZB3E2Zf+aFovWmeE7lRE4lXyF7dYM6GgA=="],@@ -154,6 +158,8 @@"@aws-sdk/util-endpoints": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-endpoints": "^3.3.4", "tslib": "^2.6.2" } }, "sha512-2nUQ+2ih7CShuKHpGSIYvvAIOHy52dOZguYG36zptBukhw6iFwcvGfG0tes0oZFWQqEWvgZe9HLWaNlvXGdOrg=="],+ "@aws-sdk/util-format-url": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/querystring-builder": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-fNJXHrs0ZT7Wx0KGIqKv7zLxlDXt2vqjx9z6oKUQFmpE5o4xxnSryvVHfHpIifYHWKz94hFccIldJ0YSZjlCBw=="],+"@aws-sdk/util-locate-window": ["@aws-sdk/[email protected]", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ=="],"@aws-sdk/util-user-agent-browser": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-sn/LMzTbGjYqCCF24390WxPd6hkpoSptiUn5DzVp4cD71yqw+yGEGm1YCxyEoPXyc8qciM8UzLJcZBFslxo5Uw=="],diff --git a/package.json b/package.jsonindex 5b8e6c6363..35caca6703 100644--- a/package.json+++ b/package.json@@ -210,6 +210,7 @@"@ai-sdk/provider-utils": "4.0.56","@anthropic-ai/sdk": "^0.30.0","@aws-sdk/client-s3": "^3.1028.0",+ "@aws-sdk/s3-request-presigner": "3.1028.0","@dqbd/tiktoken": "^1.0.22","@electric-sql/pglite": "0.4.3","@jsquash/avif": "^2.1.1",@@ -254,6 +255,7 @@"license": "MIT","version": "0.60.620.0","overrides": {+ "@aws-sdk/util-format-url": "3.972.9","@ai-sdk/provider-utils": "4.0.56","@hono/node-server": "^2.0.5","fast-uri": "^3.1.9",diff --git a/src/core/storage/s3.ts b/src/core/storage/s3.tsindex 5a081b1c97..f8ac5cbad7 100644--- a/src/core/storage/s3.ts+++ b/src/core/storage/s3.ts@@ -12,11 +12,14 @@ import {HeadObjectCommand,ListObjectsV2Command,} from '@aws-sdk/client-s3';+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';import { FileDigest, type StorageBackend, type StorageConfig } from '../storage.ts';const MIN_PART_SIZE = 16 * 1024 * 1024;const MAX_PART_SIZE = 5 * 1024 * 1024 * 1024;const MAX_PARTS = 10_000;+/** Lifetime of a presigned GET (getUrl): one hour, as the Supabase backend's signed URL. */+const SIGNED_URL_EXPIRY_S = 3600;/** Multipart part size for `size` bytes within S3's limits: 5 MiB..5 GiB per part, at most 10,000 parts. */export function multipartPartSize(size: number): number {@@ -169,16 +172,9 @@ export class S3Storage implements StorageBackend {return (res.Contents || []).map(obj => obj.Key!).filter(Boolean);}+ /** A SigV4 presigned GET for `path` that expires after SIGNED_URL_EXPIRY_S, so a private bucket's object opens from anywhere. */async getUrl(path: string): Promise<string> {- // For custom endpoints (R2, MinIO), use the endpoint URL- const endpoint = (this.client.config as any).endpoint;- if (endpoint) {- const e = typeof endpoint === 'function' ? await endpoint() : endpoint;- const base = typeof e === 'string' ? e : e.url ? e.url.toString() : `${e.protocol}//${e.hostname}${e.port ? `:${e.port}` : ''}${e.path ?? ''}`;- return `${base.replace(/\/+$/, '')}/${this.bucket}/${path}`;- }- const region = await this.client.config.region();- return `https://${this.bucket}.s3.${region}.amazonaws.com/${path}`;+ return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: path }), { expiresIn: SIGNED_URL_EXPIRY_S });}async getContentHash(path: string): Promise<string | null> {diff --git a/test/storage-s3-signed-url.test.ts b/test/storage-s3-signed-url.test.tsnew file mode 100644index 0000000000..d26b4bb3f6--- /dev/null+++ b/test/storage-s3-signed-url.test.ts@@ -0,0 +1,42 @@+/**+ * #8793: S3/MinIO file URLs are presigned, as the help promises ("Generate signed URL (1-hour)").+ *+ * 1. Protects: file_url and gbrain files signed-url hand back a SigV4 presigned GET that expires after one hour, so a+ * remote client can open a file in a private bucket, the way the Supabase backend's signed URL works.+ * 2. Fails when: getUrl returns a plain <endpoint>/<bucket>/<key> URL with no signature (403 on a private bucket), the+ * expiry is unbounded, or the secret key appears in the URL.+ * 3. Existing coverage: storage-s3 pins exists/list against a stubbed client; nothing pinned getUrl.+ * 4. Real seams: a real S3Client built from config; presigning is offline, so no network or bucket is needed.+ */+import { describe, expect, test } from 'bun:test';+import { S3Storage } from '../src/core/storage/s3.ts';++const KEYS = { accessKeyId: 'test-access-key-a', secretAccessKey: 'test-secret-value-a' };++function signed(url: string) {+ const u = new URL(url);+ return { u, q: u.searchParams };+}++describe('#8793 S3 getUrl presigns a bounded GET', () => {+ test('a MinIO/R2 endpoint gets a path-style presigned URL that expires in one hour', async () => {+ const storage = new S3Storage({ backend: 's3', bucket: 'bucket-a', region: 'us-east-1', endpoint: 'http://127.0.0.1:9000', ...KEYS });+ const { u, q } = signed(await storage.getUrl('files/report a.pdf'));+ expect(u.origin).toBe('http://127.0.0.1:9000');+ expect(u.pathname).toBe('/bucket-a/files/report%20a.pdf');+ expect(q.get('X-Amz-Algorithm')).toBe('AWS4-HMAC-SHA256');+ expect(q.get('X-Amz-Expires')).toBe('3600');+ expect(q.get('X-Amz-Credential')?.startsWith('test-access-key-a/')).toBe(true);+ expect(q.get('X-Amz-Signature')).toMatch(/^[0-9a-f]{64}$/);+ expect(u.toString()).not.toContain(KEYS.secretAccessKey);+ });++ test('AWS S3 without an endpoint is presigned too', async () => {+ const storage = new S3Storage({ backend: 's3', bucket: 'bucket-a', region: 'us-west-2', ...KEYS });+ const { u, q } = signed(await storage.getUrl('files/a.pdf'));+ expect(u.hostname).toBe('bucket-a.s3.us-west-2.amazonaws.com');+ expect(u.pathname).toBe('/files/a.pdf');+ expect(q.get('X-Amz-Expires')).toBe('3600');+ expect(q.get('X-Amz-Signature')).toMatch(/^[0-9a-f]{64}$/);+ });+});diff --git a/test/storage-s3.test.ts b/test/storage-s3.test.tsindex eb5dfa63fd..b0e3b97ff1 100644--- a/test/storage-s3.test.ts+++ b/test/storage-s3.test.ts@@ -80,24 +80,22 @@ describe('S3Storage exists', () => {});});-describe('S3Storage getUrl', () => {+describe('S3Storage getUrl (presigned, #8793)', () => {+ const KEYS = { accessKeyId: 'test-access-key-b', secretAccessKey: 'test-secret-value-b' };+ const base = (url: string) => { const u = new URL(url); expect(u.searchParams.get('X-Amz-Signature')).toBeTruthy(); return u.origin + u.pathname; };+test('string endpoint (R2/MinIO): endpoint/bucket/path', async () => {- const { client } = stubClient({ endpoint: 'https://acct.r2.example.com' });- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))+ expect(base(await new S3Storage({ ...CONFIG, endpoint: 'https://acct.r2.example.com', ...KEYS }).getUrl('img/a.png'))).toBe('https://acct.r2.example.com/blobs/img/a.png');});- test('function endpoint: resolves the provider and uses its url', async () => {- const { client } = stubClient({- endpoint: async () => ({ url: new URL('https://minio.local:9000/s3') }),- });- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))+ test('endpoint with a path: keeps the path before bucket/key', async () => {+ expect(base(await new S3Storage({ ...CONFIG, endpoint: 'https://minio.local:9000/s3', ...KEYS }).getUrl('img/a.png'))).toBe('https://minio.local:9000/s3/blobs/img/a.png');});test('no endpoint: regional virtual-hosted AWS URL', async () => {- const { client } = stubClient({ region: 'eu-central-1' });- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))+ expect(base(await new S3Storage({ ...CONFIG, region: 'eu-central-1', ...KEYS }).getUrl('img/a.png'))).toBe('https://blobs.s3.eu-central-1.amazonaws.com/img/a.png');});});