Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
feat(database): persist the cloud media catalog and upload handoff
Timothy Yin089964851225
diff --git a/packages/core/src/cloud-media.ts b/packages/core/src/cloud-media.tsnew file mode 100644index 00000000..6af53930--- /dev/null+++ b/packages/core/src/cloud-media.ts@@ -0,0 +1,58 @@+/** Internal durable cloud catalog contracts. Applications authorize the Site and+ * verify stored objects before completing uploads. No storage secrets or URLs.+ */+export interface UploadReservation {+ id: string+ ownerId: number+ photoId: string+ sourceVersion: string+ connectionId: string+ objectKey: string+ providerUploadId: string | null+ expectedBytes: number+ expiresAt: number+}+export interface MediaPhoto {+ id: string+ title: string+ source_version: string+}+export interface MediaUpload {+ id: string+ owner_id: number+ photo_id: string+ source_version: string+ connection_id: string+ object_key: string+ provider_upload_id: string | null+ expected_bytes: number+ expires_at: number+ state: 'pending' | 'completed' | 'cancelled'+}++export interface CloudMediaRepository {+ photo(id: string): Promise<MediaPhoto | null>+ album(id: string): Promise<{ id: string; title: string } | null>+ createAlbum(id: string, title: string): Promise<void>+ addToAlbum(albumId: string, photoId: string): Promise<void>+ /** Keyset page, at most 100 rows; pass the final ID to continue. */+ albumPhotos(albumId: string, after?: string): Promise<MediaPhoto[]>+ upload(id: string, ownerId: number): Promise<MediaUpload | null>+ reserveUpload(input: UploadReservation): Promise<void>+ completeUpload(input: {+ id: string+ ownerId: number+ jobId: string+ pipelineVersion: string+ title: string+ now: number+ }): Promise<string>+ cancelUpload(id: string, ownerId: number): Promise<void>+ job(id: string): Promise<{+ id: string+ photo_id: string+ source_version: string+ state: string+ } | null>+ deletePhoto(id: string): Promise<void>+}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 1396a38a..ebef0df5 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -149,3 +149,5 @@ export * from './site.ts'export * from './publication.ts'export * from './platform.ts'export * from './email-auth.ts'++export type * from './cloud-media.ts'diff --git a/packages/database/package.json b/packages/database/package.jsonindex 4ff80ac4..1f74ace1 100644--- a/packages/database/package.json+++ b/packages/database/package.json@@ -9,7 +9,8 @@"./repositories": "./src/repositories.ts","./dto": "./src/dto.ts","./album-membership": "./src/album-membership.ts",- "./album-photos-add": "./src/album-photos-add.ts"+ "./album-photos-add": "./src/album-photos-add.ts",+ "./cloud-media": "./src/cloud-media.ts"},"scripts": {"type-check": "tsc --noEmit"diff --git a/packages/database/src/cloud-media.ts b/packages/database/src/cloud-media.tsnew file mode 100644index 00000000..33d7e1d2--- /dev/null+++ b/packages/database/src/cloud-media.ts@@ -0,0 +1,230 @@+import {+ assertSiteContext,+ CoreError,+ type SiteContext,+} from '@chronoframe/core'++/** Minimal asynchronous SQL boundary shared by D1 and the SQLite behavior harness.+ * batch must execute all statements atomically, in order, or roll them all back.+ * This entry deliberately imports no native SQLite driver.+ */+export interface MediaStatement {+ bind(...values: (string | number | null)[]): MediaStatement+ all<T>(): Promise<{ results: T[] }>+}+export interface MediaDatabase {+ prepare(sql: string): MediaStatement+ batch(statements: MediaStatement[]): Promise<unknown>+}+import type {+ UploadReservation,+ MediaPhoto,+ MediaUpload,+ CloudMediaRepository,+} from '@chronoframe/core'+export type {+ UploadReservation,+ MediaPhoto,+ MediaUpload,+ CloudMediaRepository,+} from '@chronoframe/core'+function text(value: string) {+ if (typeof value !== 'string' || !value.trim() || value.length > 1024)+ throw new CoreError(400, 'Invalid catalog value')+ return value+}+function integer(value: number) {+ if (!Number.isSafeInteger(value) || value < 1)+ throw new CoreError(400, 'Invalid catalog number')+ return value+}++/** Internal repository: the application must authorize the trusted Site context.+ * No public DTOs, signed URLs, credentials, or provider callbacks are persisted.+ */+export function createCloudMediaRepository(+ db: MediaDatabase,+ site: SiteContext,+): CloudMediaRepository {+ assertSiteContext(site)+ const siteId = site.id+ const statement = (sql: string, ...values: (string | number | null)[]) =>+ db.prepare(sql).bind(siteId, ...values)+ async function first<T>(sql: string, ...values: (string | number | null)[]) {+ return (await statement(sql, ...values).all<T>()).results[0] ?? null+ }+ async function upload(id: string, ownerId: number) {+ return first<MediaUpload>(+ 'SELECT * FROM cloud_uploads WHERE site_id=? AND id=? AND owner_id=?',+ text(id),+ integer(ownerId),+ )+ }+ return {+ async photo(id: string) {+ return first<MediaPhoto>(+ 'SELECT id,title,source_version FROM cloud_photos WHERE site_id=? AND id=? AND deleted=0',+ text(id),+ )+ },+ async album(id: string) {+ return first<{ id: string; title: string }>(+ 'SELECT id,title FROM cloud_albums WHERE site_id=? AND id=?',+ text(id),+ )+ },+ async createAlbum(id: string, title: string) {+ await db.batch([+ statement(+ 'INSERT INTO cloud_albums(site_id,id,title) VALUES(?,?,?)',+ text(id),+ text(title),+ ),+ ])+ },+ async addToAlbum(albumId: string, photoId: string) {+ await db.batch([+ statement(+ `INSERT INTO cloud_album_photos(site_id,album_id,photo_id)+ SELECT ?,?,? WHERE EXISTS (SELECT 1 FROM cloud_photos WHERE site_id=? AND id=? AND deleted=0)+ ON CONFLICT DO NOTHING`,+ text(albumId),+ text(photoId),+ siteId,+ photoId,+ ),+ ])+ if (+ !(await first<{ photo_id: string }>(+ `SELECT a.photo_id FROM cloud_album_photos a JOIN cloud_photos p ON p.site_id=a.site_id AND p.id=a.photo_id WHERE a.site_id=? AND a.album_id=? AND a.photo_id=? AND p.deleted=0`,+ albumId,+ photoId,+ ))+ )+ throw new CoreError(404, 'Photo not found')+ },+ async albumPhotos(albumId: string, after = '') {+ return (+ await statement(+ `SELECT p.id,p.title,p.source_version FROM cloud_photos p+ JOIN cloud_album_photos a ON a.site_id=p.site_id AND a.photo_id=p.id+ WHERE p.site_id=? AND a.album_id=? AND p.deleted=0 AND p.id>? ORDER BY p.id LIMIT 100`,+ text(albumId),+ after,+ ).all<MediaPhoto>()+ ).results+ },+ upload,+ async reserveUpload(input: UploadReservation) {+ const values = [+ text(input.id),+ integer(input.ownerId),+ text(input.photoId),+ text(input.sourceVersion),+ text(input.connectionId),+ text(input.objectKey),+ input.providerUploadId === null ? null : text(input.providerUploadId),+ integer(input.expectedBytes),+ integer(input.expiresAt),+ ]+ await db.batch([+ statement(+ `INSERT INTO cloud_uploads(site_id,id,owner_id,photo_id,source_version,connection_id,object_key,provider_upload_id,expected_bytes,expires_at)+ VALUES(?,?,?,?,?,?,?,?,?,?)`,+ ...values,+ ),+ ])+ },+ /** Only call after provider-side object verification. Completion and durable+ * queued job are one batch; queue delivery is a separate later adapter.+ */+ async completeUpload(input: {+ id: string+ ownerId: number+ jobId: string+ pipelineVersion: string+ title: string+ now: number+ }) {+ const { id, ownerId, jobId, pipelineVersion, title, now } = input+ text(id)+ integer(ownerId)+ text(jobId)+ text(pipelineVersion)+ text(title)+ integer(now)+ const condition = `site_id=? AND id=? AND owner_id=? AND state='pending' AND expires_at>?`+ await db.batch([+ statement(+ `INSERT INTO cloud_photos(site_id,id,source_version,title)+ SELECT ?,photo_id,source_version,? FROM cloud_uploads WHERE ${condition}`,+ title,+ siteId,+ id,+ ownerId,+ now,+ ),+ statement(+ `INSERT INTO cloud_processing_jobs(site_id,id,upload_id,photo_id,source_version,pipeline_version,created_at)+ SELECT ?,?,id,photo_id,source_version,?,? FROM cloud_uploads WHERE ${condition}`,+ jobId,+ pipelineVersion,+ now,+ siteId,+ id,+ ownerId,+ now,+ ),+ statement(+ `UPDATE cloud_uploads SET state='completed' WHERE site_id=? AND id=? AND owner_id=? AND state='pending' AND expires_at>?`,+ id,+ ownerId,+ now,+ ),+ ])+ const result = await first<{ id: string; pipeline_version: string }>(+ `SELECT j.id,j.pipeline_version FROM cloud_processing_jobs j JOIN cloud_uploads u ON u.site_id=j.site_id AND u.id=j.upload_id+ WHERE j.site_id=? AND u.id=? AND u.owner_id=? AND u.state='completed'`,+ id,+ ownerId,+ )+ if (!result) throw new CoreError(409, 'Upload unavailable')+ if (result.id !== jobId || result.pipeline_version !== pipelineVersion)+ throw new CoreError(409, 'Upload already completed with another job')+ return result.id+ },+ async cancelUpload(id: string, ownerId: number) {+ await db.batch([+ statement(+ `UPDATE cloud_uploads SET state='cancelled' WHERE site_id=? AND id=? AND owner_id=? AND state='pending'`,+ text(id),+ integer(ownerId),+ ),+ ])+ },+ async job(id: string) {+ return first<{+ id: string+ photo_id: string+ source_version: string+ state: string+ }>(+ 'SELECT id,photo_id,source_version,state FROM cloud_processing_jobs WHERE site_id=? AND id=?',+ text(id),+ )+ },+ async deletePhoto(id: string) {+ text(id)+ await db.batch([+ statement(+ 'UPDATE cloud_photos SET deleted=1 WHERE site_id=? AND id=?',+ id,+ ),+ statement(+ "UPDATE cloud_processing_jobs SET state='cancelled' WHERE site_id=? AND photo_id=? AND state IN ('queued','running','retry_wait')",+ id,+ ),+ ])+ },+ }+}diff --git a/packages/platform/migrations/0005_media_catalog.sql b/packages/platform/migrations/0005_media_catalog.sqlnew file mode 100644index 00000000..35ed0e85--- /dev/null+++ b/packages/platform/migrations/0005_media_catalog.sql@@ -0,0 +1,54 @@+-- Additive cloud catalog; deliberately separate from Self-hosted tables.+CREATE TABLE cloud_photos (+ site_id INTEGER NOT NULL REFERENCES platform_sites(id),+ id TEXT NOT NULL,+ source_version TEXT NOT NULL,+ title TEXT NOT NULL,+ deleted INTEGER NOT NULL DEFAULT 0 CHECK (deleted IN (0,1)),+ PRIMARY KEY (site_id,id)+);+CREATE TABLE cloud_albums (+ site_id INTEGER NOT NULL REFERENCES platform_sites(id),+ id TEXT NOT NULL,+ title TEXT NOT NULL,+ PRIMARY KEY (site_id,id)+);+CREATE TABLE cloud_album_photos (+ site_id INTEGER NOT NULL,+ album_id TEXT NOT NULL,+ photo_id TEXT NOT NULL,+ PRIMARY KEY (site_id,album_id,photo_id),+ FOREIGN KEY (site_id,album_id) REFERENCES cloud_albums(site_id,id),+ FOREIGN KEY (site_id,photo_id) REFERENCES cloud_photos(site_id,id)+);+CREATE TABLE cloud_uploads (+ site_id INTEGER NOT NULL REFERENCES platform_sites(id),+ id TEXT NOT NULL,+ owner_id INTEGER NOT NULL REFERENCES platform_users(id),+ photo_id TEXT NOT NULL,+ source_version TEXT NOT NULL,+ connection_id TEXT NOT NULL,+ object_key TEXT NOT NULL,+ provider_upload_id TEXT,+ expected_bytes INTEGER NOT NULL CHECK (expected_bytes > 0),+ expires_at INTEGER NOT NULL,+ state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','completed','cancelled')),+ PRIMARY KEY (site_id,id),+ UNIQUE (site_id,connection_id,object_key)+);+CREATE TABLE cloud_processing_jobs (+ site_id INTEGER NOT NULL,+ id TEXT NOT NULL,+ upload_id TEXT NOT NULL,+ photo_id TEXT NOT NULL,+ source_version TEXT NOT NULL,+ pipeline_version TEXT NOT NULL,+ state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','running','retry_wait','succeeded','failed','cancelled')),+ created_at INTEGER NOT NULL,+ PRIMARY KEY (site_id,id),+ UNIQUE (site_id,upload_id),+ FOREIGN KEY (site_id,upload_id) REFERENCES cloud_uploads(site_id,id),+ FOREIGN KEY (site_id,photo_id) REFERENCES cloud_photos(site_id,id)+);+CREATE INDEX cloud_jobs_dispatch ON cloud_processing_jobs(site_id,state,created_at,id);+CREATE INDEX cloud_uploads_expiry ON cloud_uploads(site_id,state,expires_at);diff --git a/tests/cloud-media-repositories.test.ts b/tests/cloud-media-repositories.test.tsnew file mode 100644index 00000000..5f8d0b91--- /dev/null+++ b/tests/cloud-media-repositories.test.ts@@ -0,0 +1,294 @@+import assert from 'node:assert/strict'+import test from 'node:test'+import { createRequire } from 'node:module'+import { readFile, mkdtemp, rm } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import Database from 'better-sqlite3'+import {+ createCloudMediaRepository,+ type MediaDatabase,+ type MediaStatement,+} from '../packages/database/src/cloud-media.ts'+const require = createRequire(import.meta.url)+const { Miniflare, convertV4MiniflareOptions } = createRequire(+ require.resolve('wrangler/package.json'),+)('miniflare')++// Run identical behavioral assertions on synchronous SQLite transactions and+// actual workerd D1 batch transactions, rather than mocking SQL results.+for (const backend of ['SQLite', 'D1'] as const) {+ test(`${backend}: durable media catalog isolates tenants and atomically completes uploads`, async () => {+ const sqlite = backend === 'SQLite' ? new Database(':memory:') : null+ const mf =+ backend === 'D1'+ ? new Miniflare(+ convertV4MiniflareOptions({+ modules: true,+ compatibilityDate: '2026-10-22',+ script: 'export default {fetch(){return new Response("test")}}',+ d1Databases: ['DB'],+ }),+ )+ : null+ try {+ let db: MediaDatabase+ if (sqlite) {+ sqlite.pragma('foreign_keys = ON')+ class Statement implements MediaStatement {+ sql: string+ values: (string | number | null)[] = []+ constructor(sql: string) {+ this.sql = sql+ }+ bind(...values: (string | number | null)[]) {+ this.values = values+ return this+ }+ async all<T>() {+ return {+ results: sqlite!.prepare(this.sql).all(...this.values) as T[],+ }+ }+ }+ db = {+ prepare: (sql) => new Statement(sql),+ async batch(statements) {+ sqlite.transaction(() => {+ for (const statement of statements) {+ const s = statement as Statement+ sqlite.prepare(s.sql).run(...s.values)+ }+ })()+ },+ }+ } else db = await mf!.getD1Database('DB')+ async function run(sql: string) {+ await db.batch([db.prepare(sql)])+ }+ for (const file of [+ '0001_platform.sql',+ '0002_visitor_access.sql',+ '0003_account_security.sql',+ '0004_workspace_sites.sql',+ ]) {+ const sql = await readFile(+ new URL('../packages/platform/migrations/' + file, import.meta.url),+ 'utf8',+ )+ for (const part of sql+ .replace(/^--.*$/gm, '')+ .split(';')+ .map((s) => s.trim())+ .filter(Boolean))+ await run(part)+ }+ await run(+ "INSERT INTO platform_users(id,email,created_at) VALUES(1,'[email protected]',1),(2,'[email protected]',1)",+ )+ await run("INSERT INTO platform_workspaces(id,name) VALUES(1,'Original')")+ await run(+ "INSERT INTO platform_sites(id,workspace_id,name,published) VALUES(1,1,'One',0),(2,1,'Two',0)",+ )+ const migration = await readFile(+ new URL(+ '../packages/platform/migrations/0005_media_catalog.sql',+ import.meta.url,+ ),+ 'utf8',+ )+ for (const part of migration+ .replace(/^--.*$/gm, '')+ .split(';')+ .map((s) => s.trim())+ .filter(Boolean))+ await run(part)+ assert.equal(+ (+ await db+ .prepare('SELECT name FROM platform_workspaces WHERE id=1')+ .all<{ name: string }>()+ ).results[0]?.name,+ 'Original',+ )+ const a = createCloudMediaRepository(db, { id: 1 }),+ b = createCloudMediaRepository(db, { id: 2 })+ assert.throws(() => createCloudMediaRepository(db, { id: 0 }))+ const reserve = (id: string, photoId = id, expiresAt = 1000) => ({+ id,+ ownerId: 1,+ photoId,+ sourceVersion: 'v1',+ connectionId: 'storage-1',+ objectKey: `originals/${id}`,+ providerUploadId: 'multipart-1',+ expectedBytes: 42,+ expiresAt,+ })+ const complete = (id: string, jobId = id) => ({+ id,+ ownerId: 1,+ jobId,+ pipelineVersion: 'p1',+ title: 'Photo',+ now: 10,+ })+ await a.reserveUpload(reserve('same'))+ await b.reserveUpload(reserve('same'))+ assert.equal(await a.upload('same', 2), null)+ assert.equal(+ await a.upload('same', 1).then((u) => u?.provider_upload_id),+ 'multipart-1',+ )+ const parallel = await Promise.all([+ a.completeUpload(complete('same')),+ a.completeUpload(complete('same')),+ ])+ assert.deepEqual(parallel, ['same', 'same'])+ assert.equal(await a.completeUpload(complete('same')), 'same')+ await assert.rejects(a.completeUpload(complete('same', 'different')), {+ statusCode: 409,+ })+ assert.equal(await b.photo('same'), null)+ assert.equal(await b.job('same'), null)+ await b.completeUpload({ ...complete('same'), title: 'Other tenant' })+ assert.equal((await a.photo('same'))?.title, 'Photo')+ assert.equal((await b.photo('same'))?.title, 'Other tenant')+ await a.createAlbum('album', 'My album')+ await a.addToAlbum('album', 'same')+ await a.addToAlbum('album', 'same')+ assert.equal((await a.albumPhotos('album')).length, 1)+ assert.equal((await a.albumPhotos('album', 'same')).length, 0)+ assert.equal(await b.album('album'), null)+ await b.createAlbum('other-album', 'Other')+ await assert.rejects(b.addToAlbum('album', 'same'))+ await assert.rejects(a.addToAlbum('album', 'missing'))+ // Foreign key failure in the second operation must roll back the first.+ await assert.rejects(+ db.batch([+ db.prepare(+ "INSERT INTO cloud_albums VALUES(1,'rolled-back','Rollback')",+ ),+ db.prepare(+ "INSERT INTO cloud_album_photos VALUES(2,'other-album','missing')",+ ),+ ]),+ )+ assert.equal(await a.album('rolled-back'), null)+ await a.reserveUpload(reserve('collision'))+ await assert.rejects(a.completeUpload(complete('collision', 'same')))+ assert.equal(await a.photo('collision'), null)+ assert.equal((await a.upload('collision', 1))?.state, 'pending')+ await a.reserveUpload(reserve('expired', 'expired', 5))+ await assert.rejects(a.completeUpload(complete('expired')), {+ statusCode: 409,+ })+ await a.reserveUpload(reserve('cancelled'))+ await a.cancelUpload('cancelled', 2)+ assert.equal((await a.upload('cancelled', 1))?.state, 'pending')+ await a.cancelUpload('cancelled', 1)+ await assert.rejects(a.completeUpload(complete('cancelled')), {+ statusCode: 409,+ })+ await a.deletePhoto('same')+ assert.equal(await a.photo('same'), null)+ await assert.rejects(a.addToAlbum('album', 'same'), { statusCode: 404 })+ assert.equal((await a.job('same'))?.state, 'cancelled')+ assert.equal((await a.albumPhotos('album')).length, 0)+ assert.equal((await b.photo('same'))?.title, 'Other tenant')+ assert.equal(await a.completeUpload(complete('same')), 'same')+ assert.equal(await a.photo('same'), null) // retry cannot resurrect a tombstone+ await assert.rejects(+ a.reserveUpload({ ...reserve('bad'), expectedBytes: 0 }),+ { statusCode: 400 },+ )+ await assert.rejects(+ a.reserveUpload({ ...reserve('unknown'), ownerId: 999 }),+ )+ // Serializable session survives new repository instances (no closures in data).+ assert.equal(+ (await createCloudMediaRepository(db, { id: 1 }).upload('collision', 1))+ ?.object_key,+ 'originals/collision',+ )+ } finally {+ sqlite?.close()+ await mf?.dispose()+ }+ })+}++test('D1 media sessions and queued jobs survive a workerd restart', async () => {+ const directory = await mkdtemp(join(tmpdir(), 'chronoframe-media-'))+ const options = () => ({+ ...convertV4MiniflareOptions({+ modules: true,+ compatibilityDate: '2026-10-22',+ script: 'export default {fetch(){return new Response("test")}}',+ d1Databases: ['DB'],+ }),+ resourcePersistencePath: directory,+ })+ let mf = new Miniflare(options())+ try {+ let db = await mf.getD1Database('DB')+ for (const sql of [+ 'CREATE TABLE platform_sites(id INTEGER PRIMARY KEY)',+ 'CREATE TABLE platform_users(id INTEGER PRIMARY KEY)',+ 'INSERT INTO platform_sites VALUES(1)',+ 'INSERT INTO platform_users VALUES(1)',+ ])+ await db.prepare(sql).run()+ const migration = await readFile(+ new URL(+ '../packages/platform/migrations/0005_media_catalog.sql',+ import.meta.url,+ ),+ 'utf8',+ )+ for (const sql of migration+ .replace(/^--.*$/gm, '')+ .split(';')+ .map((s) => s.trim())+ .filter(Boolean))+ await db.prepare(sql).run()+ let repository = createCloudMediaRepository(db, { id: 1 })+ await repository.reserveUpload({+ id: 'upload',+ ownerId: 1,+ photoId: 'photo',+ sourceVersion: 'v1',+ connectionId: 'storage',+ objectKey: 'originals/photo',+ providerUploadId: 'provider-id',+ expectedBytes: 32,+ expiresAt: 1000,+ })+ await mf.dispose()+ mf = new Miniflare(options())+ db = await mf.getD1Database('DB')+ repository = createCloudMediaRepository(db, { id: 1 })+ assert.equal(+ (await repository.upload('upload', 1))?.provider_upload_id,+ 'provider-id',+ )+ await repository.completeUpload({+ id: 'upload',+ ownerId: 1,+ jobId: 'job',+ pipelineVersion: 'p1',+ title: 'Photo',+ now: 2,+ })+ await mf.dispose()+ mf = new Miniflare(options())+ db = await mf.getD1Database('DB')+ repository = createCloudMediaRepository(db, { id: 1 })+ assert.equal((await repository.upload('upload', 1))?.state, 'completed')+ assert.equal((await repository.job('job'))?.state, 'queued')+ assert.equal((await repository.photo('photo'))?.source_version, 'v1')+ } finally {+ await mf.dispose()+ await rm(directory, { recursive: true, force: true })+ }+})