Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
fix(processor): address configuration, privacy and maintenance review findings
Timothy Yinea2fed479fbc
diff --git a/apps/cloud/server/api/internal/worker/[action].post.ts b/apps/cloud/server/api/internal/worker/[action].post.tsindex 7ec401cd..dc27df80 100644--- a/apps/cloud/server/api/internal/worker/[action].post.ts+++ b/apps/cloud/server/api/internal/worker/[action].post.ts@@ -36,6 +36,11 @@ export default definePlatformHandler(async (event) => {),action = getRouterParam(event, 'action'),now = Date.now()+ if (+ ['register', 'status', 'claim'].includes(action ?? '') &&+ (typeof body.nonce !== 'string' || !/^[a-f0-9-]{36}$/.test(body.nonce))+ )+ throw createError({ statusCode: 400 })const limits = parseProcessorLimits(JSON.parse(node.limits_json))if (action === 'register' || action === 'status') {const short = (v: unknown, max = 128) =>@@ -63,7 +68,14 @@ export default definePlatformHandler(async (event) => {node.id,).run()- return processorEnvelope(raw, config.storageSecret, token, node, now)+ return processorEnvelope(+ raw,+ config.storageSecret,+ token,+ node,+ now,+ String(body.nonce),+ )}await raw.prepare(@@ -226,6 +238,7 @@ export default definePlatformHandler(async (event) => {token,node,now,+ String(body.nonce),),etag: sanitized ? undefined : detail?.verified_etag,companion: detail?.video_keydiff --git a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts b/apps/cloud/server/api/sites/[siteId]/media/[...operation].tsindex f49ad650..b15929ee 100644--- a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts+++ b/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts@@ -220,7 +220,8 @@ export default definePlatformHandler(async (event) => {row.revision,)results.push({ id: row.id, queued: true })- } catch {+ } catch (error) {+ if ((error as { statusCode?: number })?.statusCode !== 409) throw errorresults.push({ id: row.id, queued: false })}}diff --git a/apps/cloud/server/utils/cloud-live-attachment.ts b/apps/cloud/server/utils/cloud-live-attachment.tsindex dd9ea8c3..45fbebeb 100644--- a/apps/cloud/server/utils/cloud-live-attachment.ts+++ b/apps/cloud/server/utils/cloud-live-attachment.ts@@ -4,10 +4,26 @@ import {workspaceEntitlements,workspaceUsageSQL,} from '@chronoframe/platform/entitlements'+interface AttachmentSource {+ id: string+ connection_id: string+ connection_revision: number+ source_version: string+ photo_revision: number+}+interface CompanionUpload {+ id: string+ state: string+ expires_at: number+ connection_id: string+ connection_revision: number+ object_key: string+ expected_bytes: number+}export async function cloudLiveAttachment(ctx: Awaited<ReturnType<typeof cloudMedia>>,photoId: string,- body: Record<string, any>,+ body: Record<string, unknown>,now: number,) {const { raw, siteId, userId, site } = ctx@@ -20,12 +36,18 @@ export async function cloudLiveAttachment(.first()if (done) return { ok: true }}+ if (+ typeof body.revision !== 'number' ||+ !Number.isSafeInteger(body.revision) ||+ body.revision < 1+ )+ throw createError({ statusCode: 400 })const source = await raw.prepare("SELECT u.*,p.revision AS photo_revision FROM cloud_uploads u JOIN cloud_processing_jobs j ON j.site_id=u.site_id AND j.upload_id=u.id JOIN cloud_photos p ON p.site_id=j.site_id AND p.id=j.photo_id AND p.source_version=j.source_version WHERE p.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND j.state IN('succeeded','failed','cancelled')",).bind(siteId, photoId)- .first<any>()+ .first<AttachmentSource>()if (!source || source.photo_revision !== body.revision)throw createError({ statusCode: 409 })const { s3 } = await ctx.storage(@@ -49,6 +71,7 @@ export async function cloudLiveAttachment(typeof body.fileName !== 'string' ||!body.fileName.toLowerCase().endsWith('.mov') ||body.fileName.length > 255 ||+ typeof body.bytes !== 'number' ||!Number.isSafeInteger(body.bytes) ||body.bytes < 1 ||body.bytes > plan.limits.fileBytes ||@@ -95,7 +118,7 @@ export async function cloudLiveAttachment("SELECT * FROM cloud_uploads WHERE site_id=? AND id=? AND photo_id=? AND owner_id=? AND source_version=? AND content_type='video/quicktime'",).bind(siteId, body.uploadId, photoId, userId, source.source_version)- .first<any>()+ .first<CompanionUpload>()if (!upload ||upload.state !== 'pending' ||@@ -111,6 +134,9 @@ export async function cloudLiveAttachment()throw createError({ statusCode: 400 })const pipeline = `p_${String(Number(body.revision) + 1).padStart(16, '0')}_${crypto.randomUUID()}`+ const currentPair =+ "EXISTS(SELECT 1 FROM cloud_processing_jobs j JOIN cloud_photo_companions c ON c.site_id=j.site_id AND c.photo_id=j.photo_id WHERE j.site_id=? AND j.photo_id=? AND j.pipeline_version=? AND j.state='queued' AND c.upload_id=?)"+ const pairValues = [siteId, photoId, pipeline, upload.id] as constconst result = await raw.batch([raw.prepare(@@ -144,6 +170,24 @@ export async function cloudLiveAttachment(photoId,source.source_version,),+ raw+ .prepare(`INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at)+ SELECT site_id,'superseded-companion-'||id,connection_id,connection_revision,object_key,? FROM cloud_uploads+ WHERE site_id=? AND photo_id=? AND content_type='video/quicktime' AND id<>? AND state='completed' AND ${currentPair} ON CONFLICT DO NOTHING`)+ .bind(now + 360000, siteId, photoId, upload.id, ...pairValues),+ raw+ .prepare(`UPDATE cloud_uploads SET state='cancelled' WHERE site_id=? AND photo_id=? AND content_type='video/quicktime' AND id<>? AND state='completed' AND ${currentPair}+ AND EXISTS(SELECT 1 FROM cloud_cleanup_requests cleanup WHERE cleanup.site_id=cloud_uploads.site_id AND cleanup.id='superseded-companion-'||cloud_uploads.id AND cleanup.object_key=cloud_uploads.object_key)`)+ .bind(siteId, photoId, upload.id, ...pairValues),+ raw+ .prepare(`INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at)+ SELECT site_id,'superseded-source-video-'||id,connection_id,connection_revision,video_key,? FROM cloud_uploads+ WHERE site_id=? AND id=? AND video_key IS NOT NULL AND ${currentPair} ON CONFLICT DO NOTHING`)+ .bind(now + 360000, siteId, source.id, ...pairValues),+ raw+ .prepare(`UPDATE cloud_uploads SET video_key=NULL,video_etag=NULL,expected_video_bytes=0 WHERE site_id=? AND id=? AND video_key IS NOT NULL AND ${currentPair}+ AND EXISTS(SELECT 1 FROM cloud_cleanup_requests cleanup WHERE cleanup.site_id=cloud_uploads.site_id AND cleanup.id='superseded-source-video-'||cloud_uploads.id AND cleanup.object_key=cloud_uploads.video_key)`)+ .bind(siteId, source.id, ...pairValues),])if (!result[4]?.meta.changes) throw createError({ statusCode: 409 })return { ok: true }diff --git a/apps/shared/platform/media.ts b/apps/shared/platform/media.tsindex 36707e23..b79161de 100644--- a/apps/shared/platform/media.ts+++ b/apps/shared/platform/media.ts@@ -68,7 +68,7 @@ export async function privateMedia(const raw = platformDatabase(event)const source = await raw.prepare(- `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=p.site_id AND j.photo_id=p.id AND j.source_version=p.source_version AND j.state='succeeded') LIMIT 1`,+ `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_media_assets a WHERE a.site_id=p.site_id AND a.photo_id=p.id AND a.source_version=p.source_version AND a.role='thumbnail') LIMIT 1`,).bind(siteId, photoId).first<{diff --git a/apps/worker/src/configuration.ts b/apps/worker/src/configuration.tsnew file mode 100644index 00000000..18b8b9ab--- /dev/null+++ b/apps/worker/src/configuration.ts@@ -0,0 +1,39 @@+import {+ processorCipher,+ parseProcessorLimits,+ parseProcessorGeocoding,+ type ProcessorConfiguration,+ type ProcessorEnvelope,+} from '@chronoframe/contracts/processor'+/** A fresh request nonce authenticates the server clock; monotonic round-trip time bounds transit age. */+export async function openProcessorConfiguration(+ key: string,+ value: ProcessorEnvelope,+ nonce: string,+ elapsedMs: number,+): Promise<ProcessorConfiguration> {+ if (+ !value ||+ value.nodeId !== key.split('.')[0]?.slice(3) ||+ value.nonce !== nonce ||+ !Number.isSafeInteger(value.issuedAt) ||+ !Number.isSafeInteger(value.expiresAt) ||+ value.issuedAt < 1 ||+ value.expiresAt <= value.issuedAt ||+ value.expiresAt - value.issuedAt > 360000 ||+ !Number.isFinite(elapsedMs) ||+ elapsedMs < 0 ||+ elapsedMs >= value.expiresAt - value.issuedAt+ )+ throw new Error('invalid_configuration')+ const decoded = await (+ await processorCipher(key)+ ).open<ProcessorConfiguration>(+ value.envelope,+ `${value.nodeId}:${value.revision}:${value.issuedAt}:${value.expiresAt}:${nonce}`,+ )+ return {+ limits: parseProcessorLimits(decoded.limits),+ geocoding: parseProcessorGeocoding(decoded.geocoding),+ }+}diff --git a/apps/worker/src/geocoding.ts b/apps/worker/src/geocoding.tsindex eda6fb6d..2fbeb8ea 100644--- a/apps/worker/src/geocoding.ts+++ b/apps/worker/src/geocoding.ts@@ -12,6 +12,7 @@ export function processingGeocoder(const logger = { info() {}, warn() {}, error() {}, success() {}, debug() {} }const localSlot = createGeocodingRateLimiter()const services = createGeocodingServices({+ throwOnError: true,getLanguage: async () => config.language,request: (url, init) =>fetch(url, {@@ -38,8 +39,6 @@ export function processingGeocoder(if (!provider) return nullconst result = await provider.reverseGeocode(lat, lon)signal.throwIfAborted()- // Explicitly configured providers must succeed; do not silently publish incomplete locations.- if (!result) throw new Error('reverse_geocoding_unavailable')return result},}diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex d0dac377..f7721929 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -1,10 +1,9 @@import os from 'node:os'import {- processorCipher,- parseProcessorLimits,- parseProcessorGeocoding,type ProcessorConfiguration,+ type ProcessorEnvelope,} from '@chronoframe/contracts/processor'+import { openProcessorConfiguration } from './configuration.ts'import { randomUUID } from 'node:crypto'import { createWorkerGateway } from './gateway.ts'import { setTimeout as delay } from 'node:timers/promises'@@ -15,46 +14,33 @@ const connectionKey = process.env.CHRONOFRAME_PROCESSOR_KEY ?? ''const version = process.env.CHRONOFRAME_PROCESSOR_VERSION ?? 'development'let configuration!: ProcessorConfigurationlet lastStatus = 0-async function acceptConfiguration(value: {- nodeId: string- revision: string- expiresAt: number- envelope: string-}) {- if (- !value ||- value.nodeId !== connectionKey.split('.')[0]?.slice(3) ||- value.expiresAt <= Date.now() ||- value.expiresAt > Date.now() + 360000- )- throw new Error('invalid_configuration')- const decoded = await (- await processorCipher(connectionKey)- ).open<ProcessorConfiguration>(- value.envelope,- `${value.nodeId}:${value.revision}:${value.expiresAt}`,- )- configuration = {- limits: parseProcessorLimits(decoded.limits),- geocoding: parseProcessorGeocoding(decoded.geocoding),- }-}async function reportStatus(register = false) {- if (!register && Date.now() - lastStatus < 30000) return- const result = await rpc<any>(register ? 'register' : 'status', {- instanceId,- version,- hostname: os.hostname(),- metrics: {- cpuCount: os.cpus().length,- load1: os.loadavg()[0],- rss: process.memoryUsage().rss,- totalMemory: os.totalmem(),- freeMemory: os.freemem(),+ if (!register && performance.now() - lastStatus < 30000) return+ const nonce = randomUUID(),+ started = performance.now()+ const result = await rpc<ProcessorEnvelope>(+ register ? 'register' : 'status',+ {+ nonce,+ instanceId,+ version,+ hostname: os.hostname(),+ metrics: {+ cpuCount: os.cpus().length,+ load1: os.loadavg()[0],+ rss: process.memoryUsage().rss,+ totalMemory: os.totalmem(),+ freeMemory: os.freemem(),+ },},- })- await acceptConfiguration(result)- lastStatus = Date.now()+ )+ configuration = await openProcessorConfiguration(+ connectionKey,+ result,+ nonce,+ performance.now() - started,+ )+ lastStatus = performance.now()}const rpc = createWorkerGateway({origin: process.env.CHRONOFRAME_PORTAL_ORIGIN ?? '',@@ -72,23 +58,19 @@ for (const signal of ['SIGINT', 'SIGTERM'] as const)while (!stopping) {try {await reportStatus(!configuration)+ const nonce = randomUUID(),+ started = performance.now()const item = await rpc<- (WorkItem & { configuration: any }) | { lease: null }- >('claim', {})+ (WorkItem & { configuration: ProcessorEnvelope }) | { lease: null }+ >('claim', { nonce })if (!item.lease) {await rpc('cleanup', {})await delay(3000, undefined, { signal: shutdown.signal }).catch(() => {})continue}- await acceptConfiguration(- (item as WorkItem & { configuration: any }).configuration,- )- const work = item as WorkItem,- controller = new AbortController(),- timeout = setTimeout(- () => controller.abort('processing_timeout'),- configuration.limits.timeoutMs,- )+ const work = item as WorkItem & { configuration: ProcessorEnvelope },+ controller = new AbortController()+ let timeout: ReturnType<typeof setTimeout> | undefinedlet heartbeatRunning = falseconst timer = setInterval(() => {if (heartbeatRunning) return@@ -104,6 +86,20 @@ while (!stopping) {})}, 30000)try {+ try {+ configuration = await openProcessorConfiguration(+ connectionKey,+ work.configuration,+ nonce,+ performance.now() - started,+ )+ } catch {+ throw new WorkerProcessingError('configuration_invalid', true)+ }+ timeout = setTimeout(+ () => controller.abort('processing_timeout'),+ configuration.limits.timeoutMs,+ )const workConfiguration = configurationconst processed = await processWork(work,@@ -180,7 +176,9 @@ while (!stopping) {? error.requiredBytes: undefined,retryable: aborted- ? controller.signal.reason === 'processing_timeout'+ ? ['processing_timeout', 'heartbeat_unavailable'].includes(+ String(controller.signal.reason),+ ): error instanceof WorkerProcessingError? error.retryable: true,diff --git a/apps/worker/src/process.ts b/apps/worker/src/process.tsindex b2ed153a..fee41fc9 100644--- a/apps/worker/src/process.ts+++ b/apps/worker/src/process.ts@@ -12,6 +12,7 @@ import { ExifTool } from 'exiftool-vendored'import {createPhotoProcessor,EXIF_LOCATION_KEYS,+ stripLocationFromExif,} from '@chronoframe/processing'import { createExifService } from '@chronoframe/media'import type { StorageProvider } from '@chronoframe/storage'@@ -369,7 +370,19 @@ export async function processWork(cleanMetadata.exif = previous.exifcleanMetadata.dateTaken = previous.dateTaken}- if (eraseLocation) cleanMetadata.locationErased = true+ if (eraseLocation) {+ for (const key of [+ 'latitude',+ 'longitude',+ 'country',+ 'city',+ 'locationName',+ 'geocoding',+ ])+ cleanMetadata[key] = null+ cleanMetadata.exif = stripLocationFromExif(cleanMetadata.exif)+ cleanMetadata.locationErased = true+ }const metadataBytes = Buffer.byteLength(JSON.stringify(cleanMetadata))if (metadataBytes > limits.maxMetadataBytes)throw new WorkerProcessingError(diff --git a/packages/contracts/src/processor.ts b/packages/contracts/src/processor.tsindex df690422..8f257f0d 100644--- a/packages/contracts/src/processor.ts+++ b/packages/contracts/src/processor.ts@@ -85,6 +85,14 @@ export function parseProcessorGeocoding(value: unknown): ProcessorGeocoding {language: result.language,}}+export interface ProcessorEnvelope {+ nodeId: string+ revision: string+ issuedAt: number+ expiresAt: number+ nonce: string+ envelope: string+}export interface ProcessorConfiguration {limits: ProcessorLimitsgeocoding: ProcessorGeocodingdiff --git a/packages/geocoding/src/index.ts b/packages/geocoding/src/index.tsindex 5279f1b9..95c0f94d 100644--- a/packages/geocoding/src/index.ts+++ b/packages/geocoding/src/index.ts@@ -29,6 +29,8 @@ export interface RetryOptions {delayStrategy?: 'exponential' | 'linear' | 'fixed'}export interface GeocodingDependencies {+ /** Processing queues can distinguish provider failures from successful empty results. */+ throwOnError?: booleangetLanguage(): Promise<string>request: typeof fetchwaitForGeocodingSlot(@@ -48,6 +50,7 @@ export interface GeocodingDependencies {/** Create reverse-geocoding providers using caller-owned transport, retry, language and rate limits. */export function createGeocodingServices({getLanguage,+ throwOnError = false,request,waitForGeocodingSlot,withRetry,@@ -138,6 +141,7 @@ export function createGeocodingServices({'Mapbox reverse geocoding failed after all retries:',error,)+ if (throwOnError) throw errorreturn null}}@@ -183,6 +187,10 @@ export function createGeocodingServices({},})+ if (response.status === 404) {+ const empty = await response.json().catch(() => null)+ if (empty?.error === 'Unable to geocode') return null+ }if (!response.ok) {throw new Error(`Nominatim API error: ${response.status} ${response.statusText}`,@@ -191,9 +199,9 @@ export function createGeocodingServices({const data = await response.json()- if (!data || data.error) {- throw new Error(`Nominatim API returned error: ${data?.error}`)- }+ if (!data || data.error === 'Unable to geocode') return null+ if (data.error)+ throw new Error(`Nominatim API returned error: ${data.error}`)const geocoding = normalizeNominatim(data, language)if (!geocoding) return null@@ -218,6 +226,7 @@ export function createGeocodingServices({'Nominatim reverse geocoding failed after all retries:',error,)+ if (throwOnError) throw errorreturn null}}diff --git a/packages/platform/src/processors.ts b/packages/platform/src/processors.tsindex f162d32e..668119e1 100644--- a/packages/platform/src/processors.ts+++ b/packages/platform/src/processors.ts@@ -43,6 +43,7 @@ export async function processorEnvelope(token: string,node: { id: string; limits_json: string; revision: number },now: number,+ nonce: string,) {const geocoding = await processingGeocoding(db, secret)const revision = `${node.revision}:${geocoding.revision}`,@@ -59,8 +60,10 @@ export async function processorEnvelope(nodeId: node.id,revision,expiresAt,+ issuedAt: now,+ nonce,envelope: await (await processorCipher(token)- ).seal(config, `${node.id}:${revision}:${expiresAt}`),+ ).seal(config, `${node.id}:${revision}:${now}:${expiresAt}:${nonce}`),}}diff --git a/scripts/processor-environment.mjs b/scripts/processor-environment.mjsnew file mode 100644index 00000000..35f9fad4--- /dev/null+++ b/scripts/processor-environment.mjs@@ -0,0 +1,16 @@+/** Preserve a registered node key and optional Access settings when setup is rerun. */+export function processorEnvironment(portal, suppliedKey, existing = '') {+ const stored =+ /^CHRONOFRAME_PROCESSOR_KEY=(.*)$/m.exec(existing)?.[1].trim() ?? ''+ const unquote = (value) =>+ /^(['"]).*\1$/.test(value) ? value.slice(1, -1) : value+ const key = unquote(suppliedKey?.trim() || stored)+ if (key && !/^pn_[a-f0-9-]{36}\.[A-Za-z0-9_-]{43}$/.test(key))+ throw new Error('Invalid processing node connection key')+ const extra = existing+ .split(/\r?\n/)+ .filter((line) => !/^CHRONOFRAME_(PORTAL_ORIGIN|PROCESSOR_KEY)=/.test(line))+ .join('\n')+ .trim()+ return `CHRONOFRAME_PORTAL_ORIGIN=${portal}\nCHRONOFRAME_PROCESSOR_KEY=${key}\n${extra ? extra + '\n' : ''}`+}diff --git a/scripts/setup-cloud-media.mjs b/scripts/setup-cloud-media.mjsindex 28126268..e2131f17 100644--- a/scripts/setup-cloud-media.mjs+++ b/scripts/setup-cloud-media.mjs@@ -1,3 +1,4 @@+import { processorEnvironment } from './processor-environment.mjs'import { randomBytes } from 'node:crypto'import { mkdir, readFile, writeFile, realpath, chmod } from 'node:fs/promises'import path from 'node:path'@@ -74,9 +75,19 @@ gh(secrets.storage,)const workerEnv = path.join(target, environment + '-processor.env')+let existingEnv = ''+try {+ existingEnv = await readFile(workerEnv, 'utf8')+} catch (error) {+ if (error.code !== 'ENOENT') throw error+}await writeFile(workerEnv,- `CHRONOFRAME_PORTAL_ORIGIN=${portal}\n# Register a node in the administration panel and paste its one-time connection key.\nCHRONOFRAME_PROCESSOR_KEY=${process.env.CHRONOFRAME_PROCESSOR_KEY || ''}\n`,+ processorEnvironment(+ portal,+ process.env.CHRONOFRAME_PROCESSOR_KEY,+ existingEnv,+ ),{ mode: 0o600 },)await chmod(workerEnv, 0o600)diff --git a/tests/cloud-live-attachment.test.ts b/tests/cloud-live-attachment.test.tsindex d74d71f8..b4c471f1 100644--- a/tests/cloud-live-attachment.test.ts+++ b/tests/cloud-live-attachment.test.ts@@ -44,6 +44,16 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev"INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One')",).run()+ await db+ .prepare(+ "INSERT INTO platform_storage_pools(id,name,enabled,checked_at) VALUES('pool','Pool',1,1)",+ )+ .run()+ await db+ .prepare(+ "INSERT INTO cloud_storage_connections(site_id,id,current_revision,pool_id) VALUES(1,'s3',1,'pool')",+ )+ .run()const media = createCloudMediaRepository(db, { id: 1 }),queue = createCloudExecutionRepository(db, { id: 1 })await media.reserveUpload({@@ -84,6 +94,11 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev},20,)+ await db+ .prepare(+ "UPDATE cloud_uploads SET video_key='old-initial.MOV',expected_video_bytes=40 WHERE id='photo'",+ )+ .run()const revision = (await db.prepare("SELECT revision FROM cloud_photos WHERE id='photo'")@@ -197,6 +212,114 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev(await db.prepare('SELECT COUNT(*) n FROM cloud_usage_events').first()).n,2,)++ assert.equal(+ (+ await db+ .prepare("SELECT video_key FROM cloud_uploads WHERE id='photo'")+ .first()+ ).video_key,+ null,+ )+ assert(+ await db+ .prepare(+ "SELECT id FROM cloud_cleanup_requests WHERE object_key='old-initial.MOV'",+ )+ .first(),+ )+ const firstVideoLease = await queue.claim('node', 'live-attempt', 70, 1000)+ assert(firstVideoLease)+ const prefix = `chronoframe/1/derived/photo/1/${firstVideoLease.pipelineVersion}/live-attempt/`+ assert(+ await queue.complete(+ firstVideoLease,+ {+ width: 16,+ height: 12,+ metadata: {},+ assets: [+ {+ role: 'thumbnail',+ objectKey: prefix + 'thumbnail.webp',+ bytes: 5,+ contentType: 'image/webp',+ },+ {+ role: 'live-video',+ objectKey: prefix + 'live-video.mov',+ bytes: 100,+ contentType: 'video/quicktime',+ },+ ],+ },+ 80,+ ),+ )+ const nextRevision = (+ await db+ .prepare("SELECT revision FROM cloud_photos WHERE id='photo'")+ .first()+ ).revision+ const replacement = await attach(+ ctx,+ 'photo',+ {+ phase: 'prepare',+ revision: nextRevision,+ fileName: 'replacement.mov',+ bytes: 120,+ },+ 90,+ )+ bytes = 120+ assert(+ (+ await attach(+ ctx,+ 'photo',+ {+ phase: 'complete',+ revision: nextRevision,+ uploadId: replacement.id,+ },+ 100,+ )+ ).ok,+ )+ assert.equal(+ (+ await db+ .prepare('SELECT state FROM cloud_uploads WHERE id=?')+ .bind(prepared.id)+ .first()+ ).state,+ 'cancelled',+ )+ assert.equal(+ (await db.prepare('SELECT upload_id FROM cloud_photo_companions').first())+ .upload_id,+ replacement.id,+ )+ const cleanup = await db+ .prepare(+ 'SELECT object_key,available_at FROM cloud_cleanup_requests WHERE id=?',+ )+ .bind('superseded-companion-' + prepared.id)+ .first()+ assert.equal(cleanup.object_key, signed[0])+ assert(cleanup.available_at > 100)+ const storage = (+ await db+ .prepare(`SELECT ${workspaceUsageSQL(1).storage} AS bytes`)+ .bind(100)+ .first()+ ).bytes+ assert.equal(+ storage,+ 250,+ 'retained published derivatives count, but the superseded video source does not',+ )} finally {await mf.dispose()}diff --git a/tests/processor-media-access.test.ts b/tests/processor-media-access.test.tsnew file mode 100644index 00000000..8fea52dc--- /dev/null+++ b/tests/processor-media-access.test.ts@@ -0,0 +1,93 @@+import test from 'node:test'+import assert from 'node:assert/strict'+import Database from 'better-sqlite3'+import { readFile, readdir } from 'node:fs/promises'+import ts from 'typescript'+test('published photo streams remain available throughout maintenance and after failure or cancellation', async () => {+ const sqlite = new Database(':memory:'),+ originalFetch = globalThis.fetch+ try {+ for (const file of (await readdir('packages/platform/migrations'))+ .filter((f) => f.endsWith('.sql'))+ .sort())+ sqlite.exec(+ await readFile('packages/platform/migrations/' + file, 'utf8'),+ )+ sqlite.exec(+ "INSERT INTO platform_users(id,email,created_at) VALUES(1,'[email protected]',1);INSERT INTO platform_workspaces(id,name) VALUES(1,'One');INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One');INSERT INTO cloud_photos(site_id,id,source_version,title,metadata_json) VALUES(1,'photo','1','Photo','{}');INSERT INTO cloud_uploads(site_id,id,owner_id,photo_id,source_version,connection_id,object_key,expected_bytes,expires_at,state,content_type) VALUES(1,'upload',1,'photo','1','storage','original.jpg',3,100000,'completed','image/jpeg');INSERT INTO cloud_processing_jobs(site_id,id,upload_id,photo_id,source_version,pipeline_version,created_at,state) VALUES(1,'job','upload','photo','1','1',1,'succeeded');INSERT INTO cloud_media_assets(site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type) VALUES(1,'photo','1','1','thumbnail','storage','thumb.webp',3,'image/webp'),(1,'photo','1','1','live-video','storage','video.mov',3,'video/quicktime')",+ )+ const db = {+ prepare: (sql: string) => {+ let values: any[] = []+ const statement = {+ bind: (...v: any[]) => {+ values = v+ return statement+ },+ first: async () => sqlite.prepare(sql).get(...values),+ }+ return statement+ },+ }+ const source = (+ await readFile('apps/shared/platform/media.ts', 'utf8')+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')+ const code = ts.transpileModule(source, {+ compilerOptions: {+ module: ts.ModuleKind.CommonJS,+ target: ts.ScriptTarget.ES2022,+ },+ }).outputText+ const deps = {+ exports: {},+ platformDatabase: () => db,+ useRuntimeConfig: () => ({+ platform: {+ storageSecret: 's'.repeat(32),+ s3EndpointHosts: 'storage.example',+ },+ }),+ createStorageCipher: async () => ({}),+ createStorageVault: () => ({+ resolve: async () => ({+ config: { endpoint: 'https://storage.example' },+ credentials: {},+ }),+ }),+ createCloudS3: () => ({+ signGet: async (key: string) => 'https://storage.example/' + key,+ }),+ getHeader: () => undefined,+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),+ }+ const media = new Function(+ ...Object.keys(deps),+ code + ';return exports.privateMedia',+ )(...Object.values(deps))+ globalThis.fetch = async () =>+ new Response('img', {+ headers: { 'content-length': '3', 'content-type': 'image/jpeg' },+ })+ for (const state of [+ 'succeeded',+ 'queued',+ 'running',+ 'retry_wait',+ 'failed',+ 'cancelled',+ ]) {+ sqlite.prepare('UPDATE cloud_processing_jobs SET state=?').run(state)+ for (const role of ['original', 'display', 'thumbnail', 'live-video'])+ assert.equal(await (await media({}, 1, 'photo', role)).text(), 'img')+ }+ sqlite.exec('UPDATE cloud_photos SET metadata_json=NULL')+ await assert.rejects(media({}, 1, 'photo', 'thumbnail'), {+ statusCode: 404,+ })+ sqlite.exec("UPDATE cloud_photos SET metadata_json='{}',deleted=1")+ await assert.rejects(media({}, 1, 'photo', 'original'), { statusCode: 404 })+ } finally {+ globalThis.fetch = originalFetch+ sqlite.close()+ }+})diff --git a/tests/processor-media.test.ts b/tests/processor-media.test.tsindex 9431fded..e71224ed 100644--- a/tests/processor-media.test.ts+++ b/tests/processor-media.test.ts@@ -157,3 +157,58 @@ test('explicit MOV attachment overrides an embedded motion video and preserves eglobalThis.fetch = originalFetch}})++for (const operation of ['reindex', 'live-photo', 'geocoding'])+ test(`location erasure wins over restored ${operation} metadata`, async () => {+ const originalFetch = globalThis.fetch+ try {+ const input = await sharp({+ create: { width: 16, height: 12, channels: 3, background: '#123456' },+ })+ .jpeg()+ .toBuffer()+ globalThis.fetch = async () => new Response(new Uint8Array(input))+ const result = await processWork(+ {+ lease: { ...lease, expectedBytes: input.length },+ sourceUrl: 'https://storage.example/source',+ fileName: 'photo.jpg',+ contentType: 'image/jpeg',+ eraseLocation: true,+ options: { operation, options_json: '{}' },+ priorMetadata: {+ title: 'Edited',+ metadata_json: JSON.stringify({+ latitude: 31,+ longitude: 121,+ country: 'Country',+ city: 'City',+ locationName: 'Place',+ geocoding: { version: 1 },+ exif: {+ GPSLatitude: 31,+ GPSLongitude: 121,+ GPSCoordinates: '31,121',+ Rating: 5,+ },+ }),+ },+ },+ new AbortController().signal,+ )+ for (const key of [+ 'latitude',+ 'longitude',+ 'country',+ 'city',+ 'locationName',+ 'geocoding',+ ])+ assert.equal(result.result.metadata[key], null)+ for (const key of ['GPSLatitude', 'GPSLongitude', 'GPSCoordinates'])+ assert.equal((result.result.metadata.exif as any)?.[key], undefined)+ assert.equal(result.result.metadata.locationErased, true)+ } finally {+ globalThis.fetch = originalFetch+ }+ })diff --git a/tests/processor-nodes.test.ts b/tests/processor-nodes.test.tsindex 2453668f..9061ef47 100644--- a/tests/processor-nodes.test.ts+++ b/tests/processor-nodes.test.ts@@ -159,6 +159,7 @@ test('registered node auth revokes old keys; full schema preserves assets and feaction: 'register',key,body: {+ nonce: crypto.randomUUID(),version: 'test-build',instanceId: 'instance-a',hostname: 'host',@@ -196,10 +197,20 @@ test('registered node auth revokes old keys; full schema preserves assets and fe}),{ statusCode: 403 },)- const config = await processorEnvelope(db, 's'.repeat(32), key, node, 100)+ const config = await processorEnvelope(+ db,+ 's'.repeat(32),+ key,+ node,+ 100,+ crypto.randomUUID(),+ )const decoded = await (await processorCipher(key)- ).open<any>(config.envelope, `${id}:${config.revision}:${config.expiresAt}`)+ ).open<any>(+ config.envelope,+ `${id}:${config.revision}:${config.issuedAt}:${config.expiresAt}:${config.nonce}`,+ )assert.equal(decoded.limits.maxPixels, DEFAULT_PROCESSOR_LIMITS.maxPixels)await db.prepare('UPDATE platform_processor_nodes SET secret_hash=? WHERE id=?')diff --git a/tests/processor-review.test.ts b/tests/processor-review.test.tsnew file mode 100644index 00000000..7b118509--- /dev/null+++ b/tests/processor-review.test.ts@@ -0,0 +1,285 @@+import test from 'node:test'+import assert from 'node:assert/strict'+import { readFile } from 'node:fs/promises'+import ts from 'typescript'+import { processorEnvironment } from '../scripts/processor-environment.mjs'+import {+ processorCipher,+ DEFAULT_PROCESSOR_LIMITS,+ DEFAULT_PROCESSOR_GEOCODING,+ type ProcessorEnvelope,+} from '../packages/contracts/src/processor.ts'+import { openProcessorConfiguration } from '../apps/worker/src/configuration.ts'+import { WorkerProcessingError } from '../apps/worker/src/process.ts'+import { processingGeocoder } from '../apps/worker/src/geocoding.ts'+import { createGeocodingServices } from '../packages/geocoding/src/index.ts'+const nodeId = '11111111-1111-4111-8111-111111111111',+ key = `pn_${nodeId}.${'a'.repeat(43)}`++test('configuration uses authenticated server time and request nonce without widening its six-minute window', async () => {+ const nonce = crypto.randomUUID(),+ issuedAt = 1800000000000,+ expiresAt = issuedAt + 300000,+ revision = '1:1'+ const config = {+ limits: DEFAULT_PROCESSOR_LIMITS,+ geocoding: DEFAULT_PROCESSOR_GEOCODING,+ }+ const value: ProcessorEnvelope = {+ nodeId,+ issuedAt,+ expiresAt,+ revision,+ nonce,+ envelope: await (+ await processorCipher(key)+ ).seal(config, `${nodeId}:${revision}:${issuedAt}:${expiresAt}:${nonce}`),+ }+ const now = Date.now+ try {+ for (const skew of [-86400000, 86400000]) {+ Date.now = () => issuedAt + skew+ assert.deepEqual(+ await openProcessorConfiguration(key, value, nonce, 100),+ config,+ )+ }+ } finally {+ Date.now = now+ }+ await assert.rejects(+ openProcessorConfiguration(key, value, crypto.randomUUID(), 10),+ )+ await assert.rejects(openProcessorConfiguration(key, value, nonce, 300001))+ await assert.rejects(+ openProcessorConfiguration(+ key,+ { ...value, expiresAt: issuedAt + 360001 },+ nonce,+ 1,+ ),+ )+ await assert.rejects(+ openProcessorConfiguration(+ key,+ { ...value, issuedAt: issuedAt + 1 },+ nonce,+ 1,+ ),+ )+})++test('a bad post-claim configuration promptly fails the claimed attempt instead of leaking its lease', async () => {+ const source = (await readFile('apps/worker/src/index.ts', 'utf8')).replace(+ /^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm,+ '',+ )+ const code = ts.transpileModule(source, {+ compilerOptions: {+ module: ts.ModuleKind.CommonJS,+ target: ts.ScriptTarget.ES2022,+ },+ }).outputText+ const handlers: Record<string, () => void> = {},+ failures: any[] = []+ let opened = 0,+ processed = false+ const lease = { id: 'job', siteId: 1, workerId: nodeId, attemptId: 'attempt' }+ const rpc = async (action: string, body: any) => {+ if (action === 'register') return {}+ if (action === 'claim') return { lease, configuration: {} }+ if (action === 'fail') {+ failures.push(body)+ handlers.SIGTERM!()+ return { ok: true }+ }+ throw new Error('Unexpected action ' + action)+ }+ const deps = {+ exports: {},+ os: {+ hostname: () => '',+ cpus: () => [],+ loadavg: () => [0],+ totalmem: () => 0,+ freemem: () => 0,+ },+ process: {+ env: {+ CHRONOFRAME_PROCESSOR_KEY: key,+ CHRONOFRAME_PORTAL_ORIGIN: 'https://app.example',+ },+ on: (signal: string, fn: () => void) => {+ handlers[signal] = fn+ },+ memoryUsage: () => ({ rss: 0 }),+ },+ randomUUID: () => crypto.randomUUID(),+ createWorkerGateway: () => rpc,+ openProcessorConfiguration: async () => {+ if (++opened > 1) throw Error('invalid_configuration')+ return {+ limits: DEFAULT_PROCESSOR_LIMITS,+ geocoding: DEFAULT_PROCESSOR_GEOCODING,+ }+ },+ WorkerProcessingError,+ processWork: async () => {+ processed = true+ },+ delay: async () => {},+ setInterval: () => 1,+ clearInterval: () => {},+ setTimeout: () => 1,+ clearTimeout: () => {},+ console: { log: () => {} },+ }+ await new Function(+ ...Object.keys(deps),+ 'return (async()=>{' + code + '})()',+ )(...Object.values(deps))+ assert.equal(processed, false)+ assert.equal(failures.length, 1)+ assert.deepEqual(failures[0].lease, lease)+ assert.equal(failures[0].code, 'configuration_invalid')+ assert.equal(failures[0].retryable, true)+})++test('provider empty results stay successful nulls while transport failures can propagate to processing queues', async () => {+ const logger = { info() {}, warn() {}, error() {}, success() {} }+ for (const provider of ['mapbox', 'nominatim'] as const) {+ const originalFetch = globalThis.fetch+ try {+ globalThis.fetch = async () =>+ new Response(+ JSON.stringify(+ provider === 'mapbox'+ ? { features: [] }+ : { error: 'Unable to geocode' },+ ),+ )+ const processor = processingGeocoder(+ { ...DEFAULT_PROCESSOR_GEOCODING, provider, token: 'provider-token' },+ new AbortController().signal,+ async () => {},+ )+ assert.equal(await processor.reverseGeocode(31, 121), null)+ } finally {+ globalThis.fetch = originalFetch+ }+ for (const throwOnError of [false, true]) {+ const services = createGeocodingServices({+ throwOnError,+ getLanguage: async () => 'en',+ request: async () => {+ throw Error('network unavailable')+ },+ waitForGeocodingSlot: async () => {},+ withRetry: async (op) => op(new AbortController().signal),+ retryOptions: { maxAttempts: 1 },+ logger,+ })+ const service =+ provider === 'mapbox'+ ? new services.MapboxGeocodingProvider('credential')+ : new services.NominatimGeocodingProvider()+ if (throwOnError)+ await assert.rejects(+ service.reverseGeocode(31, 121),+ /network unavailable/,+ )+ else assert.equal(await service.reverseGeocode(31, 121), null)+ }+ }+})++test('rerunning media setup retains its registered key and optional Access credentials', () => {+ const existing = `CHRONOFRAME_PORTAL_ORIGIN=https://old.example\nCHRONOFRAME_PROCESSOR_KEY=${key}\nCHRONOFRAME_ACCESS_CLIENT_SECRET=private-access-example\n`+ for (const supplied of [undefined, '']) {+ const result = processorEnvironment(+ 'https://new.example',+ supplied,+ existing,+ )+ assert(result.includes('CHRONOFRAME_PROCESSOR_KEY=' + key))+ assert(+ result.includes(+ 'CHRONOFRAME_ACCESS_CLIENT_SECRET=private-access-example',+ ),+ )+ assert(result.includes('CHRONOFRAME_PORTAL_ORIGIN=https://new.example'))+ }+ const replacement = `pn_${nodeId}.${'b'.repeat(43)}`+ const rotated = processorEnvironment(+ 'https://new.example',+ replacement,+ existing,+ )+ assert(rotated.includes(replacement))+ assert(!rotated.includes(key))+ assert.throws(() =>+ processorEnvironment(+ 'https://new.example',+ 'bad\nINJECTED=value',+ existing,+ ),+ )+})++test('batch processing reports conflicts but propagates unexpected queue failures', async () => {+ const source = (+ await readFile(+ 'apps/cloud/server/api/sites/[siteId]/media/[...operation].ts',+ 'utf8',+ )+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')+ const code = ts.transpileModule(source, {+ compilerOptions: {+ module: ts.ModuleKind.CommonJS,+ target: ts.ScriptTarget.ES2022,+ },+ }).outputText+ let queueError: Error = Object.assign(new Error('Already queued'), {+ statusCode: 409,+ })+ const statement = {+ bind: () => statement,+ all: async () => ({ results: [{ id: 'photo', revision: 1 }] }),+ },+ db = { prepare: () => statement }+ const deps = {+ exports: {},+ definePlatformHandler: (f: any) => f,+ cloudMedia: async () => ({+ siteId: 1,+ userId: 1,+ site: { workspaceId: 1 },+ db,+ raw: db,+ }),+ getRouterParam: () => 'photos/process-batch',+ siteManagementBody: async (e: any) => e.body,+ queuePhotoProcessing: async () => {+ throw queueError+ },+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),+ }+ const api = new Function(+ ...Object.keys(deps),+ code + ';return exports.default',+ )(...Object.values(deps))+ const event = {+ method: 'POST',+ body: { photoIds: ['photo'], operation: 'reindex' },+ }+ await assert.rejects(+ api(event),+ (e: any) => e.statusCode === 409 && e.data.results[0].queued === false,+ )+ queueError = new Error('Database unavailable')+ await assert.rejects(api(event), (e: any) => e === queueError)+ queueError = Object.assign(new Error('Unexpected failure'), {+ statusCode: 500,+ })+ await assert.rejects(api(event), (e: any) => e === queueError)+})