Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

fix(processor): address configuration, privacy and maintenance review findings

Timothy Yinea2fed479fbc
diff --git a/apps/cloud/server/api/internal/worker/[action].post.ts b/apps/cloud/server/api/internal/worker/[action].post.ts
index 7ec401cd..dc27df80 100644
--- a/apps/cloud/server/api/internal/worker/[action].post.ts
+++ b/apps/cloud/server/api/internal/worker/[action].post.ts
@@ -36,6 +36,11 @@ export default definePlatformHandler(async (event) => {
),
action = getRouterParam(event, 'action'),
now = Date.now()
+ if (
+ ['register', 'status', 'claim'].includes(action ?? '') &&
+ (typeof body.nonce !== 'string' || !/^[a-f0-9-]{36}$/.test(body.nonce))
+ )
+ throw createError({ statusCode: 400 })
const limits = parseProcessorLimits(JSON.parse(node.limits_json))
if (action === 'register' || action === 'status') {
const short = (v: unknown, max = 128) =>
@@ -63,7 +68,14 @@ export default definePlatformHandler(async (event) => {
node.id,
)
.run()
- return processorEnvelope(raw, config.storageSecret, token, node, now)
+ return processorEnvelope(
+ raw,
+ config.storageSecret,
+ token,
+ node,
+ now,
+ String(body.nonce),
+ )
}
await raw
.prepare(
@@ -226,6 +238,7 @@ export default definePlatformHandler(async (event) => {
token,
node,
now,
+ String(body.nonce),
),
etag: sanitized ? undefined : detail?.verified_etag,
companion: detail?.video_key
diff --git a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts b/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
index f49ad650..b15929ee 100644
--- a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
+++ b/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
@@ -220,7 +220,8 @@ export default definePlatformHandler(async (event) => {
row.revision,
)
results.push({ id: row.id, queued: true })
- } catch {
+ } catch (error) {
+ if ((error as { statusCode?: number })?.statusCode !== 409) throw error
results.push({ id: row.id, queued: false })
}
}
diff --git a/apps/cloud/server/utils/cloud-live-attachment.ts b/apps/cloud/server/utils/cloud-live-attachment.ts
index dd9ea8c3..45fbebeb 100644
--- a/apps/cloud/server/utils/cloud-live-attachment.ts
+++ b/apps/cloud/server/utils/cloud-live-attachment.ts
@@ -4,10 +4,26 @@ import {
workspaceEntitlements,
workspaceUsageSQL,
} from '@chronoframe/platform/entitlements'
+interface AttachmentSource {
+ id: string
+ connection_id: string
+ connection_revision: number
+ source_version: string
+ photo_revision: number
+}
+interface CompanionUpload {
+ id: string
+ state: string
+ expires_at: number
+ connection_id: string
+ connection_revision: number
+ object_key: string
+ expected_bytes: number
+}
export async function cloudLiveAttachment(
ctx: Awaited<ReturnType<typeof cloudMedia>>,
photoId: string,
- body: Record<string, any>,
+ body: Record<string, unknown>,
now: number,
) {
const { raw, siteId, userId, site } = ctx
@@ -20,12 +36,18 @@ export async function cloudLiveAttachment(
.first()
if (done) return { ok: true }
}
+ if (
+ typeof body.revision !== 'number' ||
+ !Number.isSafeInteger(body.revision) ||
+ body.revision < 1
+ )
+ throw createError({ statusCode: 400 })
const source = await raw
.prepare(
"SELECT u.*,p.revision AS photo_revision FROM cloud_uploads u JOIN cloud_processing_jobs j ON j.site_id=u.site_id AND j.upload_id=u.id JOIN cloud_photos p ON p.site_id=j.site_id AND p.id=j.photo_id AND p.source_version=j.source_version WHERE p.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND j.state IN('succeeded','failed','cancelled')",
)
.bind(siteId, photoId)
- .first<any>()
+ .first<AttachmentSource>()
if (!source || source.photo_revision !== body.revision)
throw createError({ statusCode: 409 })
const { s3 } = await ctx.storage(
@@ -49,6 +71,7 @@ export async function cloudLiveAttachment(
typeof body.fileName !== 'string' ||
!body.fileName.toLowerCase().endsWith('.mov') ||
body.fileName.length > 255 ||
+ typeof body.bytes !== 'number' ||
!Number.isSafeInteger(body.bytes) ||
body.bytes < 1 ||
body.bytes > plan.limits.fileBytes ||
@@ -95,7 +118,7 @@ export async function cloudLiveAttachment(
"SELECT * FROM cloud_uploads WHERE site_id=? AND id=? AND photo_id=? AND owner_id=? AND source_version=? AND content_type='video/quicktime'",
)
.bind(siteId, body.uploadId, photoId, userId, source.source_version)
- .first<any>()
+ .first<CompanionUpload>()
if (
!upload ||
upload.state !== 'pending' ||
@@ -111,6 +134,9 @@ export async function cloudLiveAttachment(
)
throw createError({ statusCode: 400 })
const pipeline = `p_${String(Number(body.revision) + 1).padStart(16, '0')}_${crypto.randomUUID()}`
+ const currentPair =
+ "EXISTS(SELECT 1 FROM cloud_processing_jobs j JOIN cloud_photo_companions c ON c.site_id=j.site_id AND c.photo_id=j.photo_id WHERE j.site_id=? AND j.photo_id=? AND j.pipeline_version=? AND j.state='queued' AND c.upload_id=?)"
+ const pairValues = [siteId, photoId, pipeline, upload.id] as const
const result = await raw.batch([
raw
.prepare(
@@ -144,6 +170,24 @@ export async function cloudLiveAttachment(
photoId,
source.source_version,
),
+ raw
+ .prepare(`INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at)
+ SELECT site_id,'superseded-companion-'||id,connection_id,connection_revision,object_key,? FROM cloud_uploads
+ WHERE site_id=? AND photo_id=? AND content_type='video/quicktime' AND id<>? AND state='completed' AND ${currentPair} ON CONFLICT DO NOTHING`)
+ .bind(now + 360000, siteId, photoId, upload.id, ...pairValues),
+ raw
+ .prepare(`UPDATE cloud_uploads SET state='cancelled' WHERE site_id=? AND photo_id=? AND content_type='video/quicktime' AND id<>? AND state='completed' AND ${currentPair}
+ AND EXISTS(SELECT 1 FROM cloud_cleanup_requests cleanup WHERE cleanup.site_id=cloud_uploads.site_id AND cleanup.id='superseded-companion-'||cloud_uploads.id AND cleanup.object_key=cloud_uploads.object_key)`)
+ .bind(siteId, photoId, upload.id, ...pairValues),
+ raw
+ .prepare(`INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at)
+ SELECT site_id,'superseded-source-video-'||id,connection_id,connection_revision,video_key,? FROM cloud_uploads
+ WHERE site_id=? AND id=? AND video_key IS NOT NULL AND ${currentPair} ON CONFLICT DO NOTHING`)
+ .bind(now + 360000, siteId, source.id, ...pairValues),
+ raw
+ .prepare(`UPDATE cloud_uploads SET video_key=NULL,video_etag=NULL,expected_video_bytes=0 WHERE site_id=? AND id=? AND video_key IS NOT NULL AND ${currentPair}
+ AND EXISTS(SELECT 1 FROM cloud_cleanup_requests cleanup WHERE cleanup.site_id=cloud_uploads.site_id AND cleanup.id='superseded-source-video-'||cloud_uploads.id AND cleanup.object_key=cloud_uploads.video_key)`)
+ .bind(siteId, source.id, ...pairValues),
])
if (!result[4]?.meta.changes) throw createError({ statusCode: 409 })
return { ok: true }
diff --git a/apps/shared/platform/media.ts b/apps/shared/platform/media.ts
index 36707e23..b79161de 100644
--- a/apps/shared/platform/media.ts
+++ b/apps/shared/platform/media.ts
@@ -68,7 +68,7 @@ export async function privateMedia(
const raw = platformDatabase(event)
const source = await raw
.prepare(
- `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=p.site_id AND j.photo_id=p.id AND j.source_version=p.source_version AND j.state='succeeded') LIMIT 1`,
+ `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_media_assets a WHERE a.site_id=p.site_id AND a.photo_id=p.id AND a.source_version=p.source_version AND a.role='thumbnail') LIMIT 1`,
)
.bind(siteId, photoId)
.first<{
diff --git a/apps/worker/src/configuration.ts b/apps/worker/src/configuration.ts
new file mode 100644
index 00000000..18b8b9ab
--- /dev/null
+++ b/apps/worker/src/configuration.ts
@@ -0,0 +1,39 @@
+import {
+ processorCipher,
+ parseProcessorLimits,
+ parseProcessorGeocoding,
+ type ProcessorConfiguration,
+ type ProcessorEnvelope,
+} from '@chronoframe/contracts/processor'
+/** A fresh request nonce authenticates the server clock; monotonic round-trip time bounds transit age. */
+export async function openProcessorConfiguration(
+ key: string,
+ value: ProcessorEnvelope,
+ nonce: string,
+ elapsedMs: number,
+): Promise<ProcessorConfiguration> {
+ if (
+ !value ||
+ value.nodeId !== key.split('.')[0]?.slice(3) ||
+ value.nonce !== nonce ||
+ !Number.isSafeInteger(value.issuedAt) ||
+ !Number.isSafeInteger(value.expiresAt) ||
+ value.issuedAt < 1 ||
+ value.expiresAt <= value.issuedAt ||
+ value.expiresAt - value.issuedAt > 360000 ||
+ !Number.isFinite(elapsedMs) ||
+ elapsedMs < 0 ||
+ elapsedMs >= value.expiresAt - value.issuedAt
+ )
+ throw new Error('invalid_configuration')
+ const decoded = await (
+ await processorCipher(key)
+ ).open<ProcessorConfiguration>(
+ value.envelope,
+ `${value.nodeId}:${value.revision}:${value.issuedAt}:${value.expiresAt}:${nonce}`,
+ )
+ return {
+ limits: parseProcessorLimits(decoded.limits),
+ geocoding: parseProcessorGeocoding(decoded.geocoding),
+ }
+}
diff --git a/apps/worker/src/geocoding.ts b/apps/worker/src/geocoding.ts
index eda6fb6d..2fbeb8ea 100644
--- a/apps/worker/src/geocoding.ts
+++ b/apps/worker/src/geocoding.ts
@@ -12,6 +12,7 @@ export function processingGeocoder(
const logger = { info() {}, warn() {}, error() {}, success() {}, debug() {} }
const localSlot = createGeocodingRateLimiter()
const services = createGeocodingServices({
+ throwOnError: true,
getLanguage: async () => config.language,
request: (url, init) =>
fetch(url, {
@@ -38,8 +39,6 @@ export function processingGeocoder(
if (!provider) return null
const result = await provider.reverseGeocode(lat, lon)
signal.throwIfAborted()
- // Explicitly configured providers must succeed; do not silently publish incomplete locations.
- if (!result) throw new Error('reverse_geocoding_unavailable')
return result
},
}
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index d0dac377..f7721929 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -1,10 +1,9 @@
import os from 'node:os'
import {
- processorCipher,
- parseProcessorLimits,
- parseProcessorGeocoding,
type ProcessorConfiguration,
+ type ProcessorEnvelope,
} from '@chronoframe/contracts/processor'
+import { openProcessorConfiguration } from './configuration.ts'
import { randomUUID } from 'node:crypto'
import { createWorkerGateway } from './gateway.ts'
import { setTimeout as delay } from 'node:timers/promises'
@@ -15,46 +14,33 @@ const connectionKey = process.env.CHRONOFRAME_PROCESSOR_KEY ?? ''
const version = process.env.CHRONOFRAME_PROCESSOR_VERSION ?? 'development'
let configuration!: ProcessorConfiguration
let lastStatus = 0
-async function acceptConfiguration(value: {
- nodeId: string
- revision: string
- expiresAt: number
- envelope: string
-}) {
- if (
- !value ||
- value.nodeId !== connectionKey.split('.')[0]?.slice(3) ||
- value.expiresAt <= Date.now() ||
- value.expiresAt > Date.now() + 360000
- )
- throw new Error('invalid_configuration')
- const decoded = await (
- await processorCipher(connectionKey)
- ).open<ProcessorConfiguration>(
- value.envelope,
- `${value.nodeId}:${value.revision}:${value.expiresAt}`,
- )
- configuration = {
- limits: parseProcessorLimits(decoded.limits),
- geocoding: parseProcessorGeocoding(decoded.geocoding),
- }
-}
async function reportStatus(register = false) {
- if (!register && Date.now() - lastStatus < 30000) return
- const result = await rpc<any>(register ? 'register' : 'status', {
- instanceId,
- version,
- hostname: os.hostname(),
- metrics: {
- cpuCount: os.cpus().length,
- load1: os.loadavg()[0],
- rss: process.memoryUsage().rss,
- totalMemory: os.totalmem(),
- freeMemory: os.freemem(),
+ if (!register && performance.now() - lastStatus < 30000) return
+ const nonce = randomUUID(),
+ started = performance.now()
+ const result = await rpc<ProcessorEnvelope>(
+ register ? 'register' : 'status',
+ {
+ nonce,
+ instanceId,
+ version,
+ hostname: os.hostname(),
+ metrics: {
+ cpuCount: os.cpus().length,
+ load1: os.loadavg()[0],
+ rss: process.memoryUsage().rss,
+ totalMemory: os.totalmem(),
+ freeMemory: os.freemem(),
+ },
},
- })
- await acceptConfiguration(result)
- lastStatus = Date.now()
+ )
+ configuration = await openProcessorConfiguration(
+ connectionKey,
+ result,
+ nonce,
+ performance.now() - started,
+ )
+ lastStatus = performance.now()
}
const rpc = createWorkerGateway({
origin: process.env.CHRONOFRAME_PORTAL_ORIGIN ?? '',
@@ -72,23 +58,19 @@ for (const signal of ['SIGINT', 'SIGTERM'] as const)
while (!stopping) {
try {
await reportStatus(!configuration)
+ const nonce = randomUUID(),
+ started = performance.now()
const item = await rpc<
- (WorkItem & { configuration: any }) | { lease: null }
- >('claim', {})
+ (WorkItem & { configuration: ProcessorEnvelope }) | { lease: null }
+ >('claim', { nonce })
if (!item.lease) {
await rpc('cleanup', {})
await delay(3000, undefined, { signal: shutdown.signal }).catch(() => {})
continue
}
- await acceptConfiguration(
- (item as WorkItem & { configuration: any }).configuration,
- )
- const work = item as WorkItem,
- controller = new AbortController(),
- timeout = setTimeout(
- () => controller.abort('processing_timeout'),
- configuration.limits.timeoutMs,
- )
+ const work = item as WorkItem & { configuration: ProcessorEnvelope },
+ controller = new AbortController()
+ let timeout: ReturnType<typeof setTimeout> | undefined
let heartbeatRunning = false
const timer = setInterval(() => {
if (heartbeatRunning) return
@@ -104,6 +86,20 @@ while (!stopping) {
})
}, 30000)
try {
+ try {
+ configuration = await openProcessorConfiguration(
+ connectionKey,
+ work.configuration,
+ nonce,
+ performance.now() - started,
+ )
+ } catch {
+ throw new WorkerProcessingError('configuration_invalid', true)
+ }
+ timeout = setTimeout(
+ () => controller.abort('processing_timeout'),
+ configuration.limits.timeoutMs,
+ )
const workConfiguration = configuration
const processed = await processWork(
work,
@@ -180,7 +176,9 @@ while (!stopping) {
? error.requiredBytes
: undefined,
retryable: aborted
- ? controller.signal.reason === 'processing_timeout'
+ ? ['processing_timeout', 'heartbeat_unavailable'].includes(
+ String(controller.signal.reason),
+ )
: error instanceof WorkerProcessingError
? error.retryable
: true,
diff --git a/apps/worker/src/process.ts b/apps/worker/src/process.ts
index b2ed153a..fee41fc9 100644
--- a/apps/worker/src/process.ts
+++ b/apps/worker/src/process.ts
@@ -12,6 +12,7 @@ import { ExifTool } from 'exiftool-vendored'
import {
createPhotoProcessor,
EXIF_LOCATION_KEYS,
+ stripLocationFromExif,
} from '@chronoframe/processing'
import { createExifService } from '@chronoframe/media'
import type { StorageProvider } from '@chronoframe/storage'
@@ -369,7 +370,19 @@ export async function processWork(
cleanMetadata.exif = previous.exif
cleanMetadata.dateTaken = previous.dateTaken
}
- if (eraseLocation) cleanMetadata.locationErased = true
+ if (eraseLocation) {
+ for (const key of [
+ 'latitude',
+ 'longitude',
+ 'country',
+ 'city',
+ 'locationName',
+ 'geocoding',
+ ])
+ cleanMetadata[key] = null
+ cleanMetadata.exif = stripLocationFromExif(cleanMetadata.exif)
+ cleanMetadata.locationErased = true
+ }
const metadataBytes = Buffer.byteLength(JSON.stringify(cleanMetadata))
if (metadataBytes > limits.maxMetadataBytes)
throw new WorkerProcessingError(
diff --git a/packages/contracts/src/processor.ts b/packages/contracts/src/processor.ts
index df690422..8f257f0d 100644
--- a/packages/contracts/src/processor.ts
+++ b/packages/contracts/src/processor.ts
@@ -85,6 +85,14 @@ export function parseProcessorGeocoding(value: unknown): ProcessorGeocoding {
language: result.language,
}
}
+export interface ProcessorEnvelope {
+ nodeId: string
+ revision: string
+ issuedAt: number
+ expiresAt: number
+ nonce: string
+ envelope: string
+}
export interface ProcessorConfiguration {
limits: ProcessorLimits
geocoding: ProcessorGeocoding
diff --git a/packages/geocoding/src/index.ts b/packages/geocoding/src/index.ts
index 5279f1b9..95c0f94d 100644
--- a/packages/geocoding/src/index.ts
+++ b/packages/geocoding/src/index.ts
@@ -29,6 +29,8 @@ export interface RetryOptions {
delayStrategy?: 'exponential' | 'linear' | 'fixed'
}
export interface GeocodingDependencies {
+ /** Processing queues can distinguish provider failures from successful empty results. */
+ throwOnError?: boolean
getLanguage(): Promise<string>
request: typeof fetch
waitForGeocodingSlot(
@@ -48,6 +50,7 @@ export interface GeocodingDependencies {
/** Create reverse-geocoding providers using caller-owned transport, retry, language and rate limits. */
export function createGeocodingServices({
getLanguage,
+ throwOnError = false,
request,
waitForGeocodingSlot,
withRetry,
@@ -138,6 +141,7 @@ export function createGeocodingServices({
'Mapbox reverse geocoding failed after all retries:',
error,
)
+ if (throwOnError) throw error
return null
}
}
@@ -183,6 +187,10 @@ export function createGeocodingServices({
},
})
+ if (response.status === 404) {
+ const empty = await response.json().catch(() => null)
+ if (empty?.error === 'Unable to geocode') return null
+ }
if (!response.ok) {
throw new Error(
`Nominatim API error: ${response.status} ${response.statusText}`,
@@ -191,9 +199,9 @@ export function createGeocodingServices({
const data = await response.json()
- if (!data || data.error) {
- throw new Error(`Nominatim API returned error: ${data?.error}`)
- }
+ if (!data || data.error === 'Unable to geocode') return null
+ if (data.error)
+ throw new Error(`Nominatim API returned error: ${data.error}`)
const geocoding = normalizeNominatim(data, language)
if (!geocoding) return null
@@ -218,6 +226,7 @@ export function createGeocodingServices({
'Nominatim reverse geocoding failed after all retries:',
error,
)
+ if (throwOnError) throw error
return null
}
}
diff --git a/packages/platform/src/processors.ts b/packages/platform/src/processors.ts
index f162d32e..668119e1 100644
--- a/packages/platform/src/processors.ts
+++ b/packages/platform/src/processors.ts
@@ -43,6 +43,7 @@ export async function processorEnvelope(
token: string,
node: { id: string; limits_json: string; revision: number },
now: number,
+ nonce: string,
) {
const geocoding = await processingGeocoding(db, secret)
const revision = `${node.revision}:${geocoding.revision}`,
@@ -59,8 +60,10 @@ export async function processorEnvelope(
nodeId: node.id,
revision,
expiresAt,
+ issuedAt: now,
+ nonce,
envelope: await (
await processorCipher(token)
- ).seal(config, `${node.id}:${revision}:${expiresAt}`),
+ ).seal(config, `${node.id}:${revision}:${now}:${expiresAt}:${nonce}`),
}
}
diff --git a/scripts/processor-environment.mjs b/scripts/processor-environment.mjs
new file mode 100644
index 00000000..35f9fad4
--- /dev/null
+++ b/scripts/processor-environment.mjs
@@ -0,0 +1,16 @@
+/** Preserve a registered node key and optional Access settings when setup is rerun. */
+export function processorEnvironment(portal, suppliedKey, existing = '') {
+ const stored =
+ /^CHRONOFRAME_PROCESSOR_KEY=(.*)$/m.exec(existing)?.[1].trim() ?? ''
+ const unquote = (value) =>
+ /^(['"]).*\1$/.test(value) ? value.slice(1, -1) : value
+ const key = unquote(suppliedKey?.trim() || stored)
+ if (key && !/^pn_[a-f0-9-]{36}\.[A-Za-z0-9_-]{43}$/.test(key))
+ throw new Error('Invalid processing node connection key')
+ const extra = existing
+ .split(/\r?\n/)
+ .filter((line) => !/^CHRONOFRAME_(PORTAL_ORIGIN|PROCESSOR_KEY)=/.test(line))
+ .join('\n')
+ .trim()
+ return `CHRONOFRAME_PORTAL_ORIGIN=${portal}\nCHRONOFRAME_PROCESSOR_KEY=${key}\n${extra ? extra + '\n' : ''}`
+}
diff --git a/scripts/setup-cloud-media.mjs b/scripts/setup-cloud-media.mjs
index 28126268..e2131f17 100644
--- a/scripts/setup-cloud-media.mjs
+++ b/scripts/setup-cloud-media.mjs
@@ -1,3 +1,4 @@
+import { processorEnvironment } from './processor-environment.mjs'
import { randomBytes } from 'node:crypto'
import { mkdir, readFile, writeFile, realpath, chmod } from 'node:fs/promises'
import path from 'node:path'
@@ -74,9 +75,19 @@ gh(
secrets.storage,
)
const workerEnv = path.join(target, environment + '-processor.env')
+let existingEnv = ''
+try {
+ existingEnv = await readFile(workerEnv, 'utf8')
+} catch (error) {
+ if (error.code !== 'ENOENT') throw error
+}
await writeFile(
workerEnv,
- `CHRONOFRAME_PORTAL_ORIGIN=${portal}\n# Register a node in the administration panel and paste its one-time connection key.\nCHRONOFRAME_PROCESSOR_KEY=${process.env.CHRONOFRAME_PROCESSOR_KEY || ''}\n`,
+ processorEnvironment(
+ portal,
+ process.env.CHRONOFRAME_PROCESSOR_KEY,
+ existingEnv,
+ ),
{ mode: 0o600 },
)
await chmod(workerEnv, 0o600)
diff --git a/tests/cloud-live-attachment.test.ts b/tests/cloud-live-attachment.test.ts
index d74d71f8..b4c471f1 100644
--- a/tests/cloud-live-attachment.test.ts
+++ b/tests/cloud-live-attachment.test.ts
@@ -44,6 +44,16 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev
"INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One')",
)
.run()
+ await db
+ .prepare(
+ "INSERT INTO platform_storage_pools(id,name,enabled,checked_at) VALUES('pool','Pool',1,1)",
+ )
+ .run()
+ await db
+ .prepare(
+ "INSERT INTO cloud_storage_connections(site_id,id,current_revision,pool_id) VALUES(1,'s3',1,'pool')",
+ )
+ .run()
const media = createCloudMediaRepository(db, { id: 1 }),
queue = createCloudExecutionRepository(db, { id: 1 })
await media.reserveUpload({
@@ -84,6 +94,11 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev
},
20,
)
+ await db
+ .prepare(
+ "UPDATE cloud_uploads SET video_key='old-initial.MOV',expected_video_bytes=40 WHERE id='photo'",
+ )
+ .run()
const revision = (
await db
.prepare("SELECT revision FROM cloud_photos WHERE id='photo'")
@@ -197,6 +212,114 @@ test('post-upload MOV pairing verifies ownership, object metadata, quota and rev
(await db.prepare('SELECT COUNT(*) n FROM cloud_usage_events').first()).n,
2,
)
+
+ assert.equal(
+ (
+ await db
+ .prepare("SELECT video_key FROM cloud_uploads WHERE id='photo'")
+ .first()
+ ).video_key,
+ null,
+ )
+ assert(
+ await db
+ .prepare(
+ "SELECT id FROM cloud_cleanup_requests WHERE object_key='old-initial.MOV'",
+ )
+ .first(),
+ )
+ const firstVideoLease = await queue.claim('node', 'live-attempt', 70, 1000)
+ assert(firstVideoLease)
+ const prefix = `chronoframe/1/derived/photo/1/${firstVideoLease.pipelineVersion}/live-attempt/`
+ assert(
+ await queue.complete(
+ firstVideoLease,
+ {
+ width: 16,
+ height: 12,
+ metadata: {},
+ assets: [
+ {
+ role: 'thumbnail',
+ objectKey: prefix + 'thumbnail.webp',
+ bytes: 5,
+ contentType: 'image/webp',
+ },
+ {
+ role: 'live-video',
+ objectKey: prefix + 'live-video.mov',
+ bytes: 100,
+ contentType: 'video/quicktime',
+ },
+ ],
+ },
+ 80,
+ ),
+ )
+ const nextRevision = (
+ await db
+ .prepare("SELECT revision FROM cloud_photos WHERE id='photo'")
+ .first()
+ ).revision
+ const replacement = await attach(
+ ctx,
+ 'photo',
+ {
+ phase: 'prepare',
+ revision: nextRevision,
+ fileName: 'replacement.mov',
+ bytes: 120,
+ },
+ 90,
+ )
+ bytes = 120
+ assert(
+ (
+ await attach(
+ ctx,
+ 'photo',
+ {
+ phase: 'complete',
+ revision: nextRevision,
+ uploadId: replacement.id,
+ },
+ 100,
+ )
+ ).ok,
+ )
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT state FROM cloud_uploads WHERE id=?')
+ .bind(prepared.id)
+ .first()
+ ).state,
+ 'cancelled',
+ )
+ assert.equal(
+ (await db.prepare('SELECT upload_id FROM cloud_photo_companions').first())
+ .upload_id,
+ replacement.id,
+ )
+ const cleanup = await db
+ .prepare(
+ 'SELECT object_key,available_at FROM cloud_cleanup_requests WHERE id=?',
+ )
+ .bind('superseded-companion-' + prepared.id)
+ .first()
+ assert.equal(cleanup.object_key, signed[0])
+ assert(cleanup.available_at > 100)
+ const storage = (
+ await db
+ .prepare(`SELECT ${workspaceUsageSQL(1).storage} AS bytes`)
+ .bind(100)
+ .first()
+ ).bytes
+ assert.equal(
+ storage,
+ 250,
+ 'retained published derivatives count, but the superseded video source does not',
+ )
} finally {
await mf.dispose()
}
diff --git a/tests/processor-media-access.test.ts b/tests/processor-media-access.test.ts
new file mode 100644
index 00000000..8fea52dc
--- /dev/null
+++ b/tests/processor-media-access.test.ts
@@ -0,0 +1,93 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import Database from 'better-sqlite3'
+import { readFile, readdir } from 'node:fs/promises'
+import ts from 'typescript'
+test('published photo streams remain available throughout maintenance and after failure or cancellation', async () => {
+ const sqlite = new Database(':memory:'),
+ originalFetch = globalThis.fetch
+ try {
+ for (const file of (await readdir('packages/platform/migrations'))
+ .filter((f) => f.endsWith('.sql'))
+ .sort())
+ sqlite.exec(
+ await readFile('packages/platform/migrations/' + file, 'utf8'),
+ )
+ sqlite.exec(
+ "INSERT INTO platform_users(id,email,created_at) VALUES(1,'[email protected]',1);INSERT INTO platform_workspaces(id,name) VALUES(1,'One');INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One');INSERT INTO cloud_photos(site_id,id,source_version,title,metadata_json) VALUES(1,'photo','1','Photo','{}');INSERT INTO cloud_uploads(site_id,id,owner_id,photo_id,source_version,connection_id,object_key,expected_bytes,expires_at,state,content_type) VALUES(1,'upload',1,'photo','1','storage','original.jpg',3,100000,'completed','image/jpeg');INSERT INTO cloud_processing_jobs(site_id,id,upload_id,photo_id,source_version,pipeline_version,created_at,state) VALUES(1,'job','upload','photo','1','1',1,'succeeded');INSERT INTO cloud_media_assets(site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type) VALUES(1,'photo','1','1','thumbnail','storage','thumb.webp',3,'image/webp'),(1,'photo','1','1','live-video','storage','video.mov',3,'video/quicktime')",
+ )
+ const db = {
+ prepare: (sql: string) => {
+ let values: any[] = []
+ const statement = {
+ bind: (...v: any[]) => {
+ values = v
+ return statement
+ },
+ first: async () => sqlite.prepare(sql).get(...values),
+ }
+ return statement
+ },
+ }
+ const source = (
+ await readFile('apps/shared/platform/media.ts', 'utf8')
+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')
+ const code = ts.transpileModule(source, {
+ compilerOptions: {
+ module: ts.ModuleKind.CommonJS,
+ target: ts.ScriptTarget.ES2022,
+ },
+ }).outputText
+ const deps = {
+ exports: {},
+ platformDatabase: () => db,
+ useRuntimeConfig: () => ({
+ platform: {
+ storageSecret: 's'.repeat(32),
+ s3EndpointHosts: 'storage.example',
+ },
+ }),
+ createStorageCipher: async () => ({}),
+ createStorageVault: () => ({
+ resolve: async () => ({
+ config: { endpoint: 'https://storage.example' },
+ credentials: {},
+ }),
+ }),
+ createCloudS3: () => ({
+ signGet: async (key: string) => 'https://storage.example/' + key,
+ }),
+ getHeader: () => undefined,
+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),
+ }
+ const media = new Function(
+ ...Object.keys(deps),
+ code + ';return exports.privateMedia',
+ )(...Object.values(deps))
+ globalThis.fetch = async () =>
+ new Response('img', {
+ headers: { 'content-length': '3', 'content-type': 'image/jpeg' },
+ })
+ for (const state of [
+ 'succeeded',
+ 'queued',
+ 'running',
+ 'retry_wait',
+ 'failed',
+ 'cancelled',
+ ]) {
+ sqlite.prepare('UPDATE cloud_processing_jobs SET state=?').run(state)
+ for (const role of ['original', 'display', 'thumbnail', 'live-video'])
+ assert.equal(await (await media({}, 1, 'photo', role)).text(), 'img')
+ }
+ sqlite.exec('UPDATE cloud_photos SET metadata_json=NULL')
+ await assert.rejects(media({}, 1, 'photo', 'thumbnail'), {
+ statusCode: 404,
+ })
+ sqlite.exec("UPDATE cloud_photos SET metadata_json='{}',deleted=1")
+ await assert.rejects(media({}, 1, 'photo', 'original'), { statusCode: 404 })
+ } finally {
+ globalThis.fetch = originalFetch
+ sqlite.close()
+ }
+})
diff --git a/tests/processor-media.test.ts b/tests/processor-media.test.ts
index 9431fded..e71224ed 100644
--- a/tests/processor-media.test.ts
+++ b/tests/processor-media.test.ts
@@ -157,3 +157,58 @@ test('explicit MOV attachment overrides an embedded motion video and preserves e
globalThis.fetch = originalFetch
}
})
+
+for (const operation of ['reindex', 'live-photo', 'geocoding'])
+ test(`location erasure wins over restored ${operation} metadata`, async () => {
+ const originalFetch = globalThis.fetch
+ try {
+ const input = await sharp({
+ create: { width: 16, height: 12, channels: 3, background: '#123456' },
+ })
+ .jpeg()
+ .toBuffer()
+ globalThis.fetch = async () => new Response(new Uint8Array(input))
+ const result = await processWork(
+ {
+ lease: { ...lease, expectedBytes: input.length },
+ sourceUrl: 'https://storage.example/source',
+ fileName: 'photo.jpg',
+ contentType: 'image/jpeg',
+ eraseLocation: true,
+ options: { operation, options_json: '{}' },
+ priorMetadata: {
+ title: 'Edited',
+ metadata_json: JSON.stringify({
+ latitude: 31,
+ longitude: 121,
+ country: 'Country',
+ city: 'City',
+ locationName: 'Place',
+ geocoding: { version: 1 },
+ exif: {
+ GPSLatitude: 31,
+ GPSLongitude: 121,
+ GPSCoordinates: '31,121',
+ Rating: 5,
+ },
+ }),
+ },
+ },
+ new AbortController().signal,
+ )
+ for (const key of [
+ 'latitude',
+ 'longitude',
+ 'country',
+ 'city',
+ 'locationName',
+ 'geocoding',
+ ])
+ assert.equal(result.result.metadata[key], null)
+ for (const key of ['GPSLatitude', 'GPSLongitude', 'GPSCoordinates'])
+ assert.equal((result.result.metadata.exif as any)?.[key], undefined)
+ assert.equal(result.result.metadata.locationErased, true)
+ } finally {
+ globalThis.fetch = originalFetch
+ }
+ })
diff --git a/tests/processor-nodes.test.ts b/tests/processor-nodes.test.ts
index 2453668f..9061ef47 100644
--- a/tests/processor-nodes.test.ts
+++ b/tests/processor-nodes.test.ts
@@ -159,6 +159,7 @@ test('registered node auth revokes old keys; full schema preserves assets and fe
action: 'register',
key,
body: {
+ nonce: crypto.randomUUID(),
version: 'test-build',
instanceId: 'instance-a',
hostname: 'host',
@@ -196,10 +197,20 @@ test('registered node auth revokes old keys; full schema preserves assets and fe
}),
{ statusCode: 403 },
)
- const config = await processorEnvelope(db, 's'.repeat(32), key, node, 100)
+ const config = await processorEnvelope(
+ db,
+ 's'.repeat(32),
+ key,
+ node,
+ 100,
+ crypto.randomUUID(),
+ )
const decoded = await (
await processorCipher(key)
- ).open<any>(config.envelope, `${id}:${config.revision}:${config.expiresAt}`)
+ ).open<any>(
+ config.envelope,
+ `${id}:${config.revision}:${config.issuedAt}:${config.expiresAt}:${config.nonce}`,
+ )
assert.equal(decoded.limits.maxPixels, DEFAULT_PROCESSOR_LIMITS.maxPixels)
await db
.prepare('UPDATE platform_processor_nodes SET secret_hash=? WHERE id=?')
diff --git a/tests/processor-review.test.ts b/tests/processor-review.test.ts
new file mode 100644
index 00000000..7b118509
--- /dev/null
+++ b/tests/processor-review.test.ts
@@ -0,0 +1,285 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { readFile } from 'node:fs/promises'
+import ts from 'typescript'
+import { processorEnvironment } from '../scripts/processor-environment.mjs'
+import {
+ processorCipher,
+ DEFAULT_PROCESSOR_LIMITS,
+ DEFAULT_PROCESSOR_GEOCODING,
+ type ProcessorEnvelope,
+} from '../packages/contracts/src/processor.ts'
+import { openProcessorConfiguration } from '../apps/worker/src/configuration.ts'
+import { WorkerProcessingError } from '../apps/worker/src/process.ts'
+import { processingGeocoder } from '../apps/worker/src/geocoding.ts'
+import { createGeocodingServices } from '../packages/geocoding/src/index.ts'
+const nodeId = '11111111-1111-4111-8111-111111111111',
+ key = `pn_${nodeId}.${'a'.repeat(43)}`
+
+test('configuration uses authenticated server time and request nonce without widening its six-minute window', async () => {
+ const nonce = crypto.randomUUID(),
+ issuedAt = 1800000000000,
+ expiresAt = issuedAt + 300000,
+ revision = '1:1'
+ const config = {
+ limits: DEFAULT_PROCESSOR_LIMITS,
+ geocoding: DEFAULT_PROCESSOR_GEOCODING,
+ }
+ const value: ProcessorEnvelope = {
+ nodeId,
+ issuedAt,
+ expiresAt,
+ revision,
+ nonce,
+ envelope: await (
+ await processorCipher(key)
+ ).seal(config, `${nodeId}:${revision}:${issuedAt}:${expiresAt}:${nonce}`),
+ }
+ const now = Date.now
+ try {
+ for (const skew of [-86400000, 86400000]) {
+ Date.now = () => issuedAt + skew
+ assert.deepEqual(
+ await openProcessorConfiguration(key, value, nonce, 100),
+ config,
+ )
+ }
+ } finally {
+ Date.now = now
+ }
+ await assert.rejects(
+ openProcessorConfiguration(key, value, crypto.randomUUID(), 10),
+ )
+ await assert.rejects(openProcessorConfiguration(key, value, nonce, 300001))
+ await assert.rejects(
+ openProcessorConfiguration(
+ key,
+ { ...value, expiresAt: issuedAt + 360001 },
+ nonce,
+ 1,
+ ),
+ )
+ await assert.rejects(
+ openProcessorConfiguration(
+ key,
+ { ...value, issuedAt: issuedAt + 1 },
+ nonce,
+ 1,
+ ),
+ )
+})
+
+test('a bad post-claim configuration promptly fails the claimed attempt instead of leaking its lease', async () => {
+ const source = (await readFile('apps/worker/src/index.ts', 'utf8')).replace(
+ /^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm,
+ '',
+ )
+ const code = ts.transpileModule(source, {
+ compilerOptions: {
+ module: ts.ModuleKind.CommonJS,
+ target: ts.ScriptTarget.ES2022,
+ },
+ }).outputText
+ const handlers: Record<string, () => void> = {},
+ failures: any[] = []
+ let opened = 0,
+ processed = false
+ const lease = { id: 'job', siteId: 1, workerId: nodeId, attemptId: 'attempt' }
+ const rpc = async (action: string, body: any) => {
+ if (action === 'register') return {}
+ if (action === 'claim') return { lease, configuration: {} }
+ if (action === 'fail') {
+ failures.push(body)
+ handlers.SIGTERM!()
+ return { ok: true }
+ }
+ throw new Error('Unexpected action ' + action)
+ }
+ const deps = {
+ exports: {},
+ os: {
+ hostname: () => '',
+ cpus: () => [],
+ loadavg: () => [0],
+ totalmem: () => 0,
+ freemem: () => 0,
+ },
+ process: {
+ env: {
+ CHRONOFRAME_PROCESSOR_KEY: key,
+ CHRONOFRAME_PORTAL_ORIGIN: 'https://app.example',
+ },
+ on: (signal: string, fn: () => void) => {
+ handlers[signal] = fn
+ },
+ memoryUsage: () => ({ rss: 0 }),
+ },
+ randomUUID: () => crypto.randomUUID(),
+ createWorkerGateway: () => rpc,
+ openProcessorConfiguration: async () => {
+ if (++opened > 1) throw Error('invalid_configuration')
+ return {
+ limits: DEFAULT_PROCESSOR_LIMITS,
+ geocoding: DEFAULT_PROCESSOR_GEOCODING,
+ }
+ },
+ WorkerProcessingError,
+ processWork: async () => {
+ processed = true
+ },
+ delay: async () => {},
+ setInterval: () => 1,
+ clearInterval: () => {},
+ setTimeout: () => 1,
+ clearTimeout: () => {},
+ console: { log: () => {} },
+ }
+ await new Function(
+ ...Object.keys(deps),
+ 'return (async()=>{' + code + '})()',
+ )(...Object.values(deps))
+ assert.equal(processed, false)
+ assert.equal(failures.length, 1)
+ assert.deepEqual(failures[0].lease, lease)
+ assert.equal(failures[0].code, 'configuration_invalid')
+ assert.equal(failures[0].retryable, true)
+})
+
+test('provider empty results stay successful nulls while transport failures can propagate to processing queues', async () => {
+ const logger = { info() {}, warn() {}, error() {}, success() {} }
+ for (const provider of ['mapbox', 'nominatim'] as const) {
+ const originalFetch = globalThis.fetch
+ try {
+ globalThis.fetch = async () =>
+ new Response(
+ JSON.stringify(
+ provider === 'mapbox'
+ ? { features: [] }
+ : { error: 'Unable to geocode' },
+ ),
+ )
+ const processor = processingGeocoder(
+ { ...DEFAULT_PROCESSOR_GEOCODING, provider, token: 'provider-token' },
+ new AbortController().signal,
+ async () => {},
+ )
+ assert.equal(await processor.reverseGeocode(31, 121), null)
+ } finally {
+ globalThis.fetch = originalFetch
+ }
+ for (const throwOnError of [false, true]) {
+ const services = createGeocodingServices({
+ throwOnError,
+ getLanguage: async () => 'en',
+ request: async () => {
+ throw Error('network unavailable')
+ },
+ waitForGeocodingSlot: async () => {},
+ withRetry: async (op) => op(new AbortController().signal),
+ retryOptions: { maxAttempts: 1 },
+ logger,
+ })
+ const service =
+ provider === 'mapbox'
+ ? new services.MapboxGeocodingProvider('credential')
+ : new services.NominatimGeocodingProvider()
+ if (throwOnError)
+ await assert.rejects(
+ service.reverseGeocode(31, 121),
+ /network unavailable/,
+ )
+ else assert.equal(await service.reverseGeocode(31, 121), null)
+ }
+ }
+})
+
+test('rerunning media setup retains its registered key and optional Access credentials', () => {
+ const existing = `CHRONOFRAME_PORTAL_ORIGIN=https://old.example\nCHRONOFRAME_PROCESSOR_KEY=${key}\nCHRONOFRAME_ACCESS_CLIENT_SECRET=private-access-example\n`
+ for (const supplied of [undefined, '']) {
+ const result = processorEnvironment(
+ 'https://new.example',
+ supplied,
+ existing,
+ )
+ assert(result.includes('CHRONOFRAME_PROCESSOR_KEY=' + key))
+ assert(
+ result.includes(
+ 'CHRONOFRAME_ACCESS_CLIENT_SECRET=private-access-example',
+ ),
+ )
+ assert(result.includes('CHRONOFRAME_PORTAL_ORIGIN=https://new.example'))
+ }
+ const replacement = `pn_${nodeId}.${'b'.repeat(43)}`
+ const rotated = processorEnvironment(
+ 'https://new.example',
+ replacement,
+ existing,
+ )
+ assert(rotated.includes(replacement))
+ assert(!rotated.includes(key))
+ assert.throws(() =>
+ processorEnvironment(
+ 'https://new.example',
+ 'bad\nINJECTED=value',
+ existing,
+ ),
+ )
+})
+
+test('batch processing reports conflicts but propagates unexpected queue failures', async () => {
+ const source = (
+ await readFile(
+ 'apps/cloud/server/api/sites/[siteId]/media/[...operation].ts',
+ 'utf8',
+ )
+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')
+ const code = ts.transpileModule(source, {
+ compilerOptions: {
+ module: ts.ModuleKind.CommonJS,
+ target: ts.ScriptTarget.ES2022,
+ },
+ }).outputText
+ let queueError: Error = Object.assign(new Error('Already queued'), {
+ statusCode: 409,
+ })
+ const statement = {
+ bind: () => statement,
+ all: async () => ({ results: [{ id: 'photo', revision: 1 }] }),
+ },
+ db = { prepare: () => statement }
+ const deps = {
+ exports: {},
+ definePlatformHandler: (f: any) => f,
+ cloudMedia: async () => ({
+ siteId: 1,
+ userId: 1,
+ site: { workspaceId: 1 },
+ db,
+ raw: db,
+ }),
+ getRouterParam: () => 'photos/process-batch',
+ siteManagementBody: async (e: any) => e.body,
+ queuePhotoProcessing: async () => {
+ throw queueError
+ },
+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),
+ }
+ const api = new Function(
+ ...Object.keys(deps),
+ code + ';return exports.default',
+ )(...Object.values(deps))
+ const event = {
+ method: 'POST',
+ body: { photoIds: ['photo'], operation: 'reindex' },
+ }
+ await assert.rejects(
+ api(event),
+ (e: any) => e.statusCode === 409 && e.data.results[0].queued === false,
+ )
+ queueError = new Error('Database unavailable')
+ await assert.rejects(api(event), (e: any) => e === queueError)
+ queueError = Object.assign(new Error('Unexpected failure'), {
+ statusCode: 500,
+ })
+ await assert.rejects(api(event), (e: any) => e === queueError)
+})