Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: file_register records page and source and never moves a row; s3 file_url builds from the resolved endpoint

garrytan83ba306c923f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 29871c4b1d..35b7cfa421 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,8 @@ identifiers and attribution are available in the pre-removal Git revision
### Itemized changes
-- `src/core/ops/files.ts`: `file_register` accepts only the upload layout path (`<page_slug>/<filename>`, or `unsorted/<first 8 hash chars>-<filename>`), refuses `page_not_found` for a page that does not exist or that the caller's read policy hides (member access and privacy, the private-pages hatch included), and records nothing unless the stored object matches. Rows record their store's backend and prefix. Registering the same path again updates its row.
+- `src/core/ops/files.ts`: `file_register` accepts only the upload layout path (`<page_slug>/<filename>`, or `unsorted/<first 8 hash chars>-<filename>`), refuses `page_not_found` for a page that does not exist or that the caller's read policy hides (member access and privacy, the private-pages hatch included), and records nothing unless the stored object matches. Rows record their store's backend and prefix, and the page and source they belong to, so purging that page finds them. Registering the same path again updates its row, and a registration that would move a row to another page, filename or source is refused `invalid_params`.
+- `src/core/storage/s3.ts`: `file_url` on an s3 store with an endpoint builds the URL from the endpoint the SDK resolves, instead of throwing.
- A store that does not answer is the new `store_unavailable` code, never a file reported missing and never a row recorded.
- On an engine opened from settings, `delete_page` purge deletes a page's files from `settings.storage` too. A file it can't find there, or a store that doesn't answer, is reported in `blobs_remaining`, never as deleted; files an embedded engine's `file_upload` wrote before this release live in the process store, so delete those remaining paths by hand. Rows an embedded engine's `file_upload` wrote before this release point at the process store, so `file_url` and `file_verify` look for them in `settings.storage`.
- `openEngine` refuses `engine_settings_invalid` for a `storage` setting missing a value its backend needs (so no backend falls back to a default path or environment credentials), an unknown field, a backend other than local or s3, or a prefix that is not one segment; an s3 store must name its endpoint, so no AWS_* variable or ~/.aws file picks it; `embeddedSettings()` reads back `hasSecret` instead of the secret.
diff --git a/src/core/ops/files.ts b/src/core/ops/files.ts
index 936b429589..c495136093 100644
--- a/src/core/ops/files.ts
+++ b/src/core/ops/files.ts
@@ -247,10 +247,10 @@ function uploadLayoutPath(pageSlug: string | null, filename: string, hash: strin
return pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
}
-async function pageReadable(ctx: OperationContext, slug: string): Promise<boolean> {
- const rows = await ctx.engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM pages fp WHERE fp.slug = $1
- AND ${await resolveExcludePrivatePages(ctx.engine, ctx.remote) ? privatePagesFilterForCaller('fp') : 'TRUE'}`, [slug]);
- return rows[0]!.n > 0;
+async function readablePageId(ctx: OperationContext, slug: string, sourceId: string): Promise<number | undefined> {
+ const rows = await ctx.engine.executeRaw<{ id: number }>(`SELECT fp.id FROM pages fp WHERE fp.slug = $1 AND fp.source_id = $2
+ AND ${await resolveExcludePrivatePages(ctx.engine, ctx.remote) ? privatePagesFilterForCaller('fp') : 'TRUE'}`, [slug, sourceId]);
+ return rows[0]?.id;
}
const file_register: Operation = {
@@ -281,9 +281,15 @@ const file_register: Operation = {
const storagePath = uploadLayoutPath(pageSlug, filename, hash);
if (p.storage_path !== storagePath) throw opError('invalid_params', `storage_path must be ${storagePath} for this page, filename and hash.`,
`Store the object at ${storagePath} (the path file_upload uses) and register that path. Nothing was recorded.`);
- if (pageSlug && !await pageReadable(ctx, pageSlug)) throw opError('page_not_found', `No page '${pageSlug}' is visible to this caller.`, 'Attach the file to a page this caller can read. Nothing was recorded.');
+ const sourceId = ctx.sourceId ?? 'default';
+ const pageId = pageSlug ? await readablePageId(ctx, pageSlug, sourceId) : null;
+ if (pageId === undefined) throw opError('page_not_found', `No page '${pageSlug}' in source '${sourceId}' is visible to this caller.`, 'Attach the file to a page this caller can read. Nothing was recorded.');
const existing = await ctx.engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM files f WHERE f.storage_path = $1 AND NOT (${await fileReadableSql(ctx)})`, [storagePath]);
if (existing[0]!.n > 0) throw opError('page_not_found', `The file at ${storagePath} belongs to a page this caller cannot read.`, 'Nothing was recorded.');
+ const owned = await ctx.engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM files WHERE storage_path = $1
+ AND (page_slug IS DISTINCT FROM $2 OR filename <> $3 OR source_id IS DISTINCT FROM $4)`, [storagePath, pageSlug, filename, sourceId]);
+ if (owned[0]!.n > 0) throw opError('invalid_params', `${storagePath} is already registered to a different page, filename or source.`,
+ 'A registration never moves an existing row. Register under a path no other row uses. Nothing was recorded.');
if (ctx.dryRun) return { dry_run: true, action: 'file_register', storage_path: storagePath };
const { hashStoredObject } = await import('../storage.ts');
let stored: { size: number; sha256: string } | null;
@@ -293,16 +299,20 @@ const file_register: Operation = {
if (stored.size !== size || stored.sha256 !== hash) throw opError('storage_error', `The object at ${storagePath} does not match size_bytes and content_hash, so no row was recorded.`,
'Register the size and SHA-256 of the bytes actually stored at that path.');
const { executeRawJsonb, FILES_METADATA_MERGE_SQL } = await import('../sql-query.ts');
- await executeRawJsonb(ctx.engine,
- `INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
- VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
+ const written = await executeRawJsonb(ctx.engine,
+ `INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, source_id, page_id, metadata)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
ON CONFLICT (storage_path) DO UPDATE SET
content_hash = EXCLUDED.content_hash,
size_bytes = EXCLUDED.size_bytes,
mime_type = EXCLUDED.mime_type,
- ${FILES_METADATA_MERGE_SQL}`,
- [pageSlug, filename, storagePath, (p.mime_type as string | undefined) ?? null, size, hash],
+ ${FILES_METADATA_MERGE_SQL}
+ WHERE files.page_slug IS NOT DISTINCT FROM EXCLUDED.page_slug AND files.filename = EXCLUDED.filename
+ AND files.source_id IS NOT DISTINCT FROM EXCLUDED.source_id
+ RETURNING 1`,
+ [pageSlug, filename, storagePath, (p.mime_type as string | undefined) ?? null, size, hash, sourceId, pageId],
[metadata]);
+ if (!written.length) throw opError('invalid_params', `${storagePath} was registered to a different page, filename or source meanwhile.`, 'Nothing was recorded.');
return { status: 'registered', storage_path: storagePath, size_bytes: size };
},
};
diff --git a/src/core/storage/s3.ts b/src/core/storage/s3.ts
index a178ac03b5..5a081b1c97 100644
--- a/src/core/storage/s3.ts
+++ b/src/core/storage/s3.ts
@@ -173,8 +173,9 @@ export class S3Storage implements StorageBackend {
// For custom endpoints (R2, MinIO), use the endpoint URL
const endpoint = (this.client.config as any).endpoint;
if (endpoint) {
- const base = typeof endpoint === 'function' ? (await endpoint()).url.toString() : endpoint;
- return `${base}/${this.bucket}/${path}`;
+ const e = typeof endpoint === 'function' ? await endpoint() : endpoint;
+ const base = typeof e === 'string' ? e : e.url ? e.url.toString() : `${e.protocol}//${e.hostname}${e.port ? `:${e.port}` : ''}${e.path ?? ''}`;
+ return `${base.replace(/\/+$/, '')}/${this.bucket}/${path}`;
}
const region = await this.client.config.region();
return `https://${this.bucket}.s3.${region}.amazonaws.com/${path}`;
diff --git a/test/file-register.test.ts b/test/file-register.test.ts
index 7062b194a6..6bd858b700 100644
--- a/test/file-register.test.ts
+++ b/test/file-register.test.ts
@@ -69,6 +69,32 @@ describe('file_register', () => {
expect(await fileRows(engine)).toBe(1);
});
+ test('records the page and source a registration belongs to, so purging that page in a non-default source finds it', async () => {
+ const engine = await embedded();
+ await engine.executeRaw("INSERT INTO sources (id, name) VALUES ('source-a', 'source-a') ON CONFLICT DO NOTHING");
+ await importFromContent(engine, 'notes/plan', ['---', 'type: note', 'title: plan', '---', '# plan', '', 'Body.'].join('\n'), { sourceId: 'source-a' });
+ const bytes = Buffer.from('source-a bytes');
+ store('notes/plan/a.txt', bytes);
+ await run('file_register', ctxFor(engine, { sourceId: 'source-a' }), { storage_path: 'notes/plan/a.txt', filename: 'a.txt', page_slug: 'notes/plan', content_hash: sha(bytes), size_bytes: bytes.length });
+ const [page] = await engine.executeRaw<{ id: number }>("SELECT id FROM pages WHERE slug='notes/plan' AND source_id='source-a'");
+ expect(await engine.executeRaw('SELECT source_id, page_id FROM files')).toEqual([{ source_id: 'source-a', page_id: page!.id }]);
+ expect((await failure(() => run('file_register', ctxFor(engine, { sourceId: 'default' }), { storage_path: 'notes/plan/a.txt', filename: 'a.txt', page_slug: 'notes/plan', content_hash: sha(bytes), size_bytes: bytes.length })))?.code).toBe('page_not_found');
+ });
+
+ test('refuses a registration that would move an existing row to another page, keeping the row as it was', async () => {
+ const engine = await embedded();
+ await addPage(engine, 'unsorted');
+ const old = Buffer.from('loose file');
+ const path = `unsorted/${sha(old).slice(0, 8)}-x.txt`;
+ store(path, old);
+ await run('file_register', ctxFor(engine), { storage_path: path, filename: 'x.txt', content_hash: sha(old), size_bytes: old.length });
+ const next = Buffer.from('bytes for the page');
+ store(path, next);
+ const err = await failure(() => run('file_register', ctxFor(engine), { storage_path: path, filename: `${sha(old).slice(0, 8)}-x.txt`, page_slug: 'unsorted', content_hash: sha(next), size_bytes: next.length }));
+ expect(err?.code).toBe('invalid_params');
+ expect(await engine.executeRaw('SELECT page_slug, filename, content_hash FROM files')).toEqual([{ page_slug: null, filename: 'x.txt', content_hash: sha(old) }]);
+ });
+
test('refuses a page that does not exist and an object it cannot read back, recording nothing', async () => {
const engine = await embedded();
const bytes = Buffer.from('orphan');
@@ -218,6 +244,12 @@ describe('hashing a stored object', () => {
expect(await s3.hashObject('a/b.txt')).toEqual({ size: 23, sha256: sha(Buffer.concat(parts)) });
});
+ test('an s3 store with an endpoint builds file URLs from it', async () => {
+ const { S3Storage } = await import('../src/core/storage/s3.ts');
+ const s3 = new S3Storage({ backend: 's3', bucket: 'test', endpoint: 'http://127.0.0.1:9000', region: 'us-east-1', accessKeyId: 'k', secretAccessKey: 's' });
+ expect(await s3.getUrl('notes/plan/a.txt')).toBe('http://127.0.0.1:9000/test/notes/plan/a.txt');
+ });
+
test('a backend without hashObject is hashed from one download', async () => {
const { hashStoredObject } = await import('../src/core/storage.ts');
const bytes = Buffer.from('downloaded');