Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(storage): S3 and MinIO file URLs are presigned for one hour

garrytan35952ddcaa1d
diff --git a/bun.lock b/bun.lock
index 02bf070fb4..b89ebddfb6 100644
--- a/bun.lock
+++ b/bun.lock
@@ -12,6 +12,7 @@
"@ai-sdk/provider-utils": "4.0.56",
"@anthropic-ai/sdk": "^0.30.0",
"@aws-sdk/client-s3": "^3.1028.0",
+ "@aws-sdk/s3-request-presigner": "3.1028.0",
"@dqbd/tiktoken": "^1.0.22",
"@electric-sql/pglite": "0.4.3",
"@jsquash/avif": "^2.1.1",
@@ -54,6 +55,7 @@
],
"overrides": {
"@ai-sdk/provider-utils": "4.0.56",
+ "@aws-sdk/util-format-url": "3.972.9",
"@hono/node-server": "^2.0.5",
"body-parser": "^2.4.0",
"fast-uri": "^3.1.9",
@@ -144,6 +146,8 @@
"@aws-sdk/region-config-resolver": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/config-resolver": "^4.4.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-6Q8B1dcx6BBqUTY1Mc/eROKA0FImEEY5VPSd6AGPEUf0ErjExz4snVqa9kNJSoVDV1rKaNf3qrWojgcKW+SdDg=="],
+ "@aws-sdk/s3-request-presigner": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/signature-v4-multi-region": "^3.996.16", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-format-url": "^3.972.9", "@smithy/middleware-endpoint": "^4.4.29", "@smithy/protocol-http": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-T46EyIIHUaF/I2zhjtSUO4/87QdhqobClCscJawrxe2h/8nZJoO3DQDVZ4tMDl5UV/B5SPz6+xwki8jGylTF4Q=="],
+
"@aws-sdk/signature-v4-multi-region": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/middleware-sdk-s3": "^3.972.28", "@aws-sdk/types": "^3.973.7", "@smithy/protocol-http": "^5.3.13", "@smithy/signature-v4": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-EMdXYB4r/k5RWq86fugjRhid5JA+Z6MpS7n4sij4u5/C+STrkvuf9aFu41rJA9MjUzxCLzv8U2XL8cH2GSRYpQ=="],
"@aws-sdk/token-providers": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-Ieq/HiRrbEtrYP387Nes0XlR7H1pJiJOZKv+QyQzMYpvTiDs0VKy2ZB3E2Zf+aFovWmeE7lRE4lXyF7dYM6GgA=="],
@@ -154,6 +158,8 @@
"@aws-sdk/util-endpoints": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-endpoints": "^3.3.4", "tslib": "^2.6.2" } }, "sha512-2nUQ+2ih7CShuKHpGSIYvvAIOHy52dOZguYG36zptBukhw6iFwcvGfG0tes0oZFWQqEWvgZe9HLWaNlvXGdOrg=="],
+ "@aws-sdk/util-format-url": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/querystring-builder": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-fNJXHrs0ZT7Wx0KGIqKv7zLxlDXt2vqjx9z6oKUQFmpE5o4xxnSryvVHfHpIifYHWKz94hFccIldJ0YSZjlCBw=="],
+
"@aws-sdk/util-locate-window": ["@aws-sdk/[email protected]", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ=="],
"@aws-sdk/util-user-agent-browser": ["@aws-sdk/[email protected]", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-sn/LMzTbGjYqCCF24390WxPd6hkpoSptiUn5DzVp4cD71yqw+yGEGm1YCxyEoPXyc8qciM8UzLJcZBFslxo5Uw=="],
diff --git a/package.json b/package.json
index 5b8e6c6363..35caca6703 100644
--- a/package.json
+++ b/package.json
@@ -210,6 +210,7 @@
"@ai-sdk/provider-utils": "4.0.56",
"@anthropic-ai/sdk": "^0.30.0",
"@aws-sdk/client-s3": "^3.1028.0",
+ "@aws-sdk/s3-request-presigner": "3.1028.0",
"@dqbd/tiktoken": "^1.0.22",
"@electric-sql/pglite": "0.4.3",
"@jsquash/avif": "^2.1.1",
@@ -254,6 +255,7 @@
"license": "MIT",
"version": "0.60.620.0",
"overrides": {
+ "@aws-sdk/util-format-url": "3.972.9",
"@ai-sdk/provider-utils": "4.0.56",
"@hono/node-server": "^2.0.5",
"fast-uri": "^3.1.9",
diff --git a/src/core/storage/s3.ts b/src/core/storage/s3.ts
index 5a081b1c97..f8ac5cbad7 100644
--- a/src/core/storage/s3.ts
+++ b/src/core/storage/s3.ts
@@ -12,11 +12,14 @@ import {
HeadObjectCommand,
ListObjectsV2Command,
} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { FileDigest, type StorageBackend, type StorageConfig } from '../storage.ts';
const MIN_PART_SIZE = 16 * 1024 * 1024;
const MAX_PART_SIZE = 5 * 1024 * 1024 * 1024;
const MAX_PARTS = 10_000;
+/** Lifetime of a presigned GET (getUrl): one hour, as the Supabase backend's signed URL. */
+const SIGNED_URL_EXPIRY_S = 3600;
/** Multipart part size for `size` bytes within S3's limits: 5 MiB..5 GiB per part, at most 10,000 parts. */
export function multipartPartSize(size: number): number {
@@ -169,16 +172,9 @@ export class S3Storage implements StorageBackend {
return (res.Contents || []).map(obj => obj.Key!).filter(Boolean);
}
+ /** A SigV4 presigned GET for `path` that expires after SIGNED_URL_EXPIRY_S, so a private bucket's object opens from anywhere. */
async getUrl(path: string): Promise<string> {
- // For custom endpoints (R2, MinIO), use the endpoint URL
- const endpoint = (this.client.config as any).endpoint;
- if (endpoint) {
- const e = typeof endpoint === 'function' ? await endpoint() : endpoint;
- const base = typeof e === 'string' ? e : e.url ? e.url.toString() : `${e.protocol}//${e.hostname}${e.port ? `:${e.port}` : ''}${e.path ?? ''}`;
- return `${base.replace(/\/+$/, '')}/${this.bucket}/${path}`;
- }
- const region = await this.client.config.region();
- return `https://${this.bucket}.s3.${region}.amazonaws.com/${path}`;
+ return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: path }), { expiresIn: SIGNED_URL_EXPIRY_S });
}
async getContentHash(path: string): Promise<string | null> {
diff --git a/test/storage-s3-signed-url.test.ts b/test/storage-s3-signed-url.test.ts
new file mode 100644
index 0000000000..d26b4bb3f6
--- /dev/null
+++ b/test/storage-s3-signed-url.test.ts
@@ -0,0 +1,42 @@
+/**
+ * #8793: S3/MinIO file URLs are presigned, as the help promises ("Generate signed URL (1-hour)").
+ *
+ * 1. Protects: file_url and gbrain files signed-url hand back a SigV4 presigned GET that expires after one hour, so a
+ * remote client can open a file in a private bucket, the way the Supabase backend's signed URL works.
+ * 2. Fails when: getUrl returns a plain <endpoint>/<bucket>/<key> URL with no signature (403 on a private bucket), the
+ * expiry is unbounded, or the secret key appears in the URL.
+ * 3. Existing coverage: storage-s3 pins exists/list against a stubbed client; nothing pinned getUrl.
+ * 4. Real seams: a real S3Client built from config; presigning is offline, so no network or bucket is needed.
+ */
+import { describe, expect, test } from 'bun:test';
+import { S3Storage } from '../src/core/storage/s3.ts';
+
+const KEYS = { accessKeyId: 'test-access-key-a', secretAccessKey: 'test-secret-value-a' };
+
+function signed(url: string) {
+ const u = new URL(url);
+ return { u, q: u.searchParams };
+}
+
+describe('#8793 S3 getUrl presigns a bounded GET', () => {
+ test('a MinIO/R2 endpoint gets a path-style presigned URL that expires in one hour', async () => {
+ const storage = new S3Storage({ backend: 's3', bucket: 'bucket-a', region: 'us-east-1', endpoint: 'http://127.0.0.1:9000', ...KEYS });
+ const { u, q } = signed(await storage.getUrl('files/report a.pdf'));
+ expect(u.origin).toBe('http://127.0.0.1:9000');
+ expect(u.pathname).toBe('/bucket-a/files/report%20a.pdf');
+ expect(q.get('X-Amz-Algorithm')).toBe('AWS4-HMAC-SHA256');
+ expect(q.get('X-Amz-Expires')).toBe('3600');
+ expect(q.get('X-Amz-Credential')?.startsWith('test-access-key-a/')).toBe(true);
+ expect(q.get('X-Amz-Signature')).toMatch(/^[0-9a-f]{64}$/);
+ expect(u.toString()).not.toContain(KEYS.secretAccessKey);
+ });
+
+ test('AWS S3 without an endpoint is presigned too', async () => {
+ const storage = new S3Storage({ backend: 's3', bucket: 'bucket-a', region: 'us-west-2', ...KEYS });
+ const { u, q } = signed(await storage.getUrl('files/a.pdf'));
+ expect(u.hostname).toBe('bucket-a.s3.us-west-2.amazonaws.com');
+ expect(u.pathname).toBe('/files/a.pdf');
+ expect(q.get('X-Amz-Expires')).toBe('3600');
+ expect(q.get('X-Amz-Signature')).toMatch(/^[0-9a-f]{64}$/);
+ });
+});
diff --git a/test/storage-s3.test.ts b/test/storage-s3.test.ts
index eb5dfa63fd..b0e3b97ff1 100644
--- a/test/storage-s3.test.ts
+++ b/test/storage-s3.test.ts
@@ -80,24 +80,22 @@ describe('S3Storage exists', () => {
});
});
-describe('S3Storage getUrl', () => {
+describe('S3Storage getUrl (presigned, #8793)', () => {
+ const KEYS = { accessKeyId: 'test-access-key-b', secretAccessKey: 'test-secret-value-b' };
+ const base = (url: string) => { const u = new URL(url); expect(u.searchParams.get('X-Amz-Signature')).toBeTruthy(); return u.origin + u.pathname; };
+
test('string endpoint (R2/MinIO): endpoint/bucket/path', async () => {
- const { client } = stubClient({ endpoint: 'https://acct.r2.example.com' });
- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))
+ expect(base(await new S3Storage({ ...CONFIG, endpoint: 'https://acct.r2.example.com', ...KEYS }).getUrl('img/a.png')))
.toBe('https://acct.r2.example.com/blobs/img/a.png');
});
- test('function endpoint: resolves the provider and uses its url', async () => {
- const { client } = stubClient({
- endpoint: async () => ({ url: new URL('https://minio.local:9000/s3') }),
- });
- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))
+ test('endpoint with a path: keeps the path before bucket/key', async () => {
+ expect(base(await new S3Storage({ ...CONFIG, endpoint: 'https://minio.local:9000/s3', ...KEYS }).getUrl('img/a.png')))
.toBe('https://minio.local:9000/s3/blobs/img/a.png');
});
test('no endpoint: regional virtual-hosted AWS URL', async () => {
- const { client } = stubClient({ region: 'eu-central-1' });
- expect(await new S3Storage(CONFIG, client).getUrl('img/a.png'))
+ expect(base(await new S3Storage({ ...CONFIG, region: 'eu-central-1', ...KEYS }).getUrl('img/a.png')))
.toBe('https://blobs.s3.eu-central-1.amazonaws.com/img/a.png');
});
});